use dnsdumpster api (#1484)

* use dnsdumpster api

* remove debug code

v2/pkg/passive/sources_test.go

@@ -67,7 +67,7 @@ var (
 		"chinaz",
 		"crtsh",
 		"digitorus",
-		// "dnsdumpster", //failing with "unexpected status code 403 received"
+		"dnsdumpster",
 		"dnsrepo",
 		"fofa",
 		"fullhunt",
@@ -96,7 +96,6 @@ var (
 		"bufferover",
 		"certspotter",
 		"crtsh",
-		"dnsdumpster",
 		"dnsdb",
 		"digitorus",
 		"hackertarget",

v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go

@@ -3,67 +3,29 @@ package dnsdumpster
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
-	"io"
-	"net/url"
-	"regexp"
-	"strings"
 	"time"
 
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
 )
 
-// CSRFSubMatchLength CSRF regex submatch length
-const CSRFSubMatchLength = 2
-
-var re = regexp.MustCompile("<input type=\"hidden\" name=\"csrfmiddlewaretoken\" value=\"(.*)\">")
-
-// getCSRFToken gets the CSRF Token from the page
-func getCSRFToken(page string) string {
-	if subs := re.FindStringSubmatch(page); len(subs) == CSRFSubMatchLength {
-		return strings.TrimSpace(subs[1])
-	}
-	return ""
-}
-
-// postForm posts a form for a domain and returns the response
-func postForm(ctx context.Context, session *subscraping.Session, token, domain string) (string, error) {
-	params := url.Values{
-		"csrfmiddlewaretoken": {token},
-		"targetip":            {domain},
-		"user":                {"free"},
-	}
-
-	resp, err := session.HTTPRequest(
-		ctx,
-		"POST",
-		"https://dnsdumpster.com/",
-		fmt.Sprintf("csrftoken=%s; Domain=dnsdumpster.com", token),
-		map[string]string{
-			"Content-Type": "application/x-www-form-urlencoded",
-			"Referer":      "https://dnsdumpster.com",
-			"X-CSRF-Token": token,
-		},
-		strings.NewReader(params.Encode()),
-		subscraping.BasicAuth{},
-	)
-
-	if err != nil {
-		session.DiscardHTTPResponse(resp)
-		return "", err
-	}
-
-	// Now, grab the entire page
-	in, err := io.ReadAll(resp.Body)
-	resp.Body.Close()
-	return string(in), err
+type response struct {
+	A []struct {
+		Host string `json:"host"`
+	} `json:"a"`
+	Ns []struct {
+		Host string `json:"host"`
+	} `json:"ns"`
 }
 
 // Source is the passive scraping agent
 type Source struct {
+	apiKeys   []string
 	timeTaken time.Duration
 	errors    int
 	results   int
+	skipped   bool
 }
 
 // Run function returns all subdomains found with the service
@@ -78,35 +40,35 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 			close(results)
 		}(time.Now())
 
-		resp, err := session.SimpleGet(ctx, "https://dnsdumpster.com/")
-		if err != nil {
-			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
-			s.errors++
-			session.DiscardHTTPResponse(resp)
+		randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
+		if randomApiKey == "" {
+			s.skipped = true
 			return
 		}
 
-		body, err := io.ReadAll(resp.Body)
+		resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdumpster.com/domain/%s", domain), "", map[string]string{"X-API-Key": randomApiKey})
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
 			s.errors++
-			resp.Body.Close()
+			session.DiscardHTTPResponse(resp)
 			return
 		}
-		resp.Body.Close()
+		defer resp.Body.Close()
 
-		csrfToken := getCSRFToken(string(body))
-		data, err := postForm(ctx, session, csrfToken, domain)
+		var response response
+		err = json.NewDecoder(resp.Body).Decode(&response)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
 			s.errors++
+			resp.Body.Close()
 			return
 		}
 
-		for _, subdomain := range session.Extractor.Extract(data) {
-			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
+		for _, record := range append(response.A, response.Ns...) {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host}
 			s.results++
 		}
+
 	}()
 
 	return results
@@ -118,25 +80,26 @@ func (s *Source) Name() string {
 }
 
 func (s *Source) IsDefault() bool {
-	return false
+	return true
 }
 
 func (s *Source) HasRecursiveSupport() bool {
-	return true
+	return false
 }
 
 func (s *Source) NeedsKey() bool {
-	return false
+	return true
 }
 
-func (s *Source) AddApiKeys(_ []string) {
-	// no key needed
+func (s *Source) AddApiKeys(keys []string) {
+	s.apiKeys = keys
 }
 
 func (s *Source) Statistics() subscraping.Statistics {
 	return subscraping.Statistics{
 		Errors:    s.errors,
 		Results:   s.results,
 		TimeTaken: s.timeTaken,
+		Skipped:   s.skipped,
 	}
 }