feat/lazy lookup auth

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: "1.21"
       - name: Build
         run: |
           go version
@@ -46,7 +46,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.23'
+          go-version: "1.23"
       - name: Other lint
         run: |
           go install golang.org/x/tools/cmd/goimports@latest

README.md

@@ -311,7 +311,7 @@ result verbosity levels: `short`, `normal` (default), `long`, and `trace`:
 
 Users can also include specific additional fields using the `--include-fields`
 flag and specifying a list of fields, e.g., `--include-fields=flags,resolver`.
-Additional fields are: class, protocol, ttl, resolver, flags.
+Additional fields are: class, protocol, ttl, resolver, flags, dnssec.
 
 Name Server Mode
 ----------------

go.mod

@@ -1,6 +1,6 @@
 module github.com/zmap/zdns
 
-go 1.20
+go 1.21.1
 
 require (
 	github.com/hashicorp/go-version v1.7.0
@@ -10,6 +10,7 @@ require (
 	github.com/schollz/progressbar/v3 v3.15.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.9.0
+	github.com/zmap/go-dns-root-anchors v0.0.0-20241116225636-aa592d6ee59f
 	github.com/zmap/go-iptree v0.0.0-20210731043055-d4e632617837
 	github.com/zmap/zcrypto v0.0.0-20240803002437-3a861682ac77
 	github.com/zmap/zflags v1.4.0-beta.1.0.20200204220219-9d95409821b6

go.sum

@@ -269,6 +269,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/zmap/dns v1.1.63-zdns1 h1:vaZIXXLZqjmZTGcpu9qPDcunqWfxeRMh0OwLiCxcjsI=
 github.com/zmap/dns v1.1.63-zdns1/go.mod h1:L50pXblXGxDFLaon9W4vGSfC1rGIcBL29sS7sNvNKuI=
+github.com/zmap/go-dns-root-anchors v0.0.0-20241116225636-aa592d6ee59f h1:XffIjsyncaK1BH7GpceohPRnesxBaO7lduwbXG05ICo=
+github.com/zmap/go-dns-root-anchors v0.0.0-20241116225636-aa592d6ee59f/go.mod h1:AkYq/HWOOEFPx6YgVn2oYjaZjLw2R/HV5WUWfJrFQ6s=
 github.com/zmap/go-iptree v0.0.0-20210731043055-d4e632617837 h1:DjHnADS2r2zynZ3WkCFAQ+PNYngMSNceRROi0pO6c3M=
 github.com/zmap/go-iptree v0.0.0-20210731043055-d4e632617837/go.mod h1:9vp0bxqozzQwcjBwenEXfKVq8+mYbwHkQ1NF9Ap0DMw=
 github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=

src/cli/cli.go

@@ -64,6 +64,7 @@ type QueryOptions struct {
 	ClassString        string `long:"class" default:"INET" description:"DNS class to query. Options: INET, CSNET, CHAOS, HESIOD, NONE, ANY."`
 	ClientSubnetString string `long:"client-subnet" description:"Client subnet in CIDR format for EDNS0."`
 	Dnssec             bool   `long:"dnssec" description:"Requests DNSSEC records by setting the DNSSEC OK (DO) bit"`
+	ValidateDNSSEC     bool   `long:"validate-dnssec" description:"Validate DNSSEC records, only applicable with --iterative"`
 	UseNSID            bool   `long:"nsid" description:"Request NSID."`
 }
 
@@ -90,7 +91,7 @@ type InputOutputOptions struct {
 	BlacklistFilePath            string `long:"blacklist-file" description:"blacklist file for servers to exclude from lookups"`
 	DNSConfigFilePath            string `long:"conf-file" default:"/etc/resolv.conf" description:"config file for DNS servers"`
 	MultipleModuleConfigFilePath string `short:"c" long:"multi-config-file" description:"config file path for multiple module"`
-	IncludeInOutput              string `long:"include-fields" description:"Comma separated list of fields to additionally output beyond result verbosity. Options: class, protocol, ttl, resolver, flags"`
+	IncludeInOutput              string `long:"include-fields" description:"Comma separated list of fields to additionally output beyond result verbosity. Options: class, protocol, ttl, resolver, flags, dnssec"`
 	InputFilePath                string `short:"f" long:"input-file" default:"-" description:"names to read, defaults to stdin"`
 	LogFilePath                  string `long:"log-file" default:"-" description:"where should JSON logs be saved, defaults to stderr"`
 	MetadataFilePath             string `long:"metadata-file" description:"where should JSON metadata be saved, defaults to no metadata output. Use '-' for stderr."`

src/cli/worker_manager.go

@@ -211,7 +211,17 @@ func populateResolverConfig(gc *CLIConf) *zdns.ResolverConfig {
 	config.MaxDepth = gc.MaxDepth
 	config.CheckingDisabledBit = gc.CheckingDisabled
 	config.ShouldRecycleSockets = !gc.DisableRecycleSockets
-	config.DNSSecEnabled = gc.Dnssec
+
+	config.ShouldValidateDNSSEC = gc.ValidateDNSSEC
+	if config.ShouldValidateDNSSEC {
+		config.DNSSecEnabled = true
+		if !gc.IterativeResolution {
+			log.Fatal("DNSSEC validation is only supported with iterative resolution")
+		}
+	} else {
+		config.DNSSecEnabled = gc.Dnssec
+	}
+
 	config.DNSConfigFilePath = gc.DNSConfigFilePath
 
 	config.LogLevel = log.Level(gc.Verbosity)

src/zdns/answers.go

@@ -78,6 +78,21 @@ type DNSKEYAnswer struct {
 	PublicKey string `json:"public_key" groups:"short,normal,long,trace"`
 }
 
+func (r *DNSKEYAnswer) ToVanillaType() *dns.DNSKEY {
+	return &dns.DNSKEY{
+		Hdr: dns.RR_Header{
+			Name:   dns.CanonicalName(r.Name),
+			Rrtype: r.RrType,
+			Class:  dns.StringToClass[r.Class],
+			Ttl:    r.TTL,
+		},
+		Flags:     r.Flags,
+		Protocol:  r.Protocol,
+		Algorithm: r.Algorithm,
+		PublicKey: r.PublicKey,
+	}
+}
+
 type DSAnswer struct {
 	Answer
 	KeyTag     uint16 `json:"key_tag" groups:"short,normal,long,trace"`
@@ -86,6 +101,22 @@ type DSAnswer struct {
 	Digest     string `json:"digest" groups:"short,normal,long,trace"`
 }
 
+func (r *DSAnswer) ToVanillaType() *dns.DS {
+	return &dns.DS{
+		Hdr: dns.RR_Header{
+
+			Name:   dns.CanonicalName(r.Name),
+			Rrtype: r.RrType,
+			Class:  dns.StringToClass[r.Class],
+			Ttl:    r.TTL,
+		},
+		KeyTag:     r.KeyTag,
+		Algorithm:  r.Algorithm,
+		DigestType: r.DigestType,
+		Digest:     r.Digest,
+	}
+}
+
 type GPOSAnswer struct {
 	Answer
 	Longitude string `json:"preference" groups:"short,normal,long,trace"`
@@ -186,6 +217,36 @@ type RRSIGAnswer struct {
 	Signature   string `json:"signature" groups:"short,normal,long,trace"`
 }
 
+func (r *RRSIGAnswer) ToVanillaType() *dns.RRSIG {
+	expiration, err := dns.StringToTime(r.Expiration)
+	if err != nil {
+		panic("failed to parse expiration time: " + r.Expiration)
+	}
+
+	inception, err := dns.StringToTime(r.Inception)
+	if err != nil {
+		panic("failed to parse inception time: " + r.Inception)
+	}
+
+	return &dns.RRSIG{
+		Hdr: dns.RR_Header{
+			Name:   dns.CanonicalName(r.Name),
+			Rrtype: r.RrType,
+			Class:  dns.StringToClass[r.Class],
+			Ttl:    r.TTL,
+		},
+		TypeCovered: r.TypeCovered,
+		Algorithm:   r.Algorithm,
+		Labels:      r.Labels,
+		OrigTtl:     r.OriginalTTL,
+		Expiration:  expiration,
+		Inception:   inception,
+		KeyTag:      r.KeyTag,
+		SignerName:  r.SignerName,
+		Signature:   r.Signature,
+	}
+}
+
 type RPAnswer struct {
 	Answer
 	Mbox string `json:"mbox" groups:"short,normal,long,trace"`

src/zdns/cache.go

@@ -309,7 +309,38 @@ func (s *Cache) SafeAddCachedAuthority(res *SingleQueryResult, ns *NameServer, d
 		nsString = ns.String()
 	}
 
-	cachedRes := s.buildCachedResult(res, depth, layer)
+	// Referrals may contain DS records in the authority section. These need to be cached under the child name.
+	delegateToDSRRs := make(map[string][]interface{})
+	var otherRRs []interface{}
+	for _, rr := range res.Authorities {
+		if dsRR, ok := rr.(DSAnswer); ok {
+			delegateName := removeTrailingDotIfNotRoot(dsRR.BaseAns().Name)
+			delegateToDSRRs[delegateName] = append(delegateToDSRRs[delegateName], dsRR)
+		} else {
+			otherRRs = append(otherRRs, rr)
+		}
+	}
+
+	if len(delegateToDSRRs) > 0 {
+		s.VerboseLog(depth+1, "SafeAddCachedAuthority: found DS records in authority section, caching under child names")
+
+		for delegateName, dsRRs := range delegateToDSRRs {
+			dsRes := &SingleQueryResult{
+				Answers:            dsRRs,
+				Protocol:           res.Protocol,
+				Resolver:           res.Resolver,
+				Flags:              res.Flags,
+				TLSServerHandshake: res.TLSServerHandshake,
+			}
+			dsRes.Flags.Authoritative = true
+			dsCachedRes := s.buildCachedResult(dsRes, depth, layer)
+			s.addCachedAnswer(Question{Name: delegateName, Type: dns.TypeDS, Class: dns.ClassINET}, nsString, false, dsCachedRes, depth)
+		}
+	}
+
+	copiedRes := *res
+	copiedRes.Authorities = otherRRs
+	cachedRes := s.buildCachedResult(&copiedRes, depth, layer)
 	if len(cachedRes.Answers) == 0 && len(cachedRes.Authorities) == 0 && len(cachedRes.Additionals) == 0 {
 		s.VerboseLog(depth+1, "SafeAddCachedAnswer: no cacheable records found, aborting")
 		return

src/zdns/conf.go

@@ -49,8 +49,17 @@ const (
 	StatusIterTimeout  Status = "ITERATIVE_TIMEOUT"
 	StatusNoAuth       Status = "NOAUTH"
 	StatusNoNeededGlue Status = "NONEEDEDGLUE" // When a nameserver is authoritative for itself and the parent nameserver doesn't provide the glue to look it up
+	StatusCircular     Status = "CIRCULAR"     // When circular query dependencies are detected
 )
 
+func isStatusRetryable(status Status) bool {
+	switch status {
+	case StatusServFail, StatusNXDomain, StatusRefused, StatusTruncated, StatusError, StatusTimeout, StatusIterTimeout:
+		return true
+	}
+	return false
+}
+
 var RootServersV4 = []NameServer{
 	{IP: net.ParseIP("198.41.0.4"), Port: 53},     // A
 	{IP: net.ParseIP("170.247.170.2"), Port: 53},  // B - Changed several times, this is current as of July '24