diff --git a/Makefile b/Makefile index e1ee516cf6..b3aed71bde 100644 --- a/Makefile +++ b/Makefile @@ -137,5 +137,5 @@ dsl-docs: template-validate: build template-validate: ./bin/nuclei -ut - ./bin/nuclei -validate - ./bin/nuclei -validate -w workflows \ No newline at end of file + ./bin/nuclei -validate -et http/technologies + ./bin/nuclei -validate -w workflows -et http/technologies \ No newline at end of file diff --git a/cmd/integration-test/integration-test.go b/cmd/integration-test/integration-test.go index ca77533c49..a35d7f92a9 100644 --- a/cmd/integration-test/integration-test.go +++ b/cmd/integration-test/integration-test.go @@ -115,6 +115,10 @@ func main() { fmt.Println("::group::Failed integration tests in debug mode") _ = runTests(failedTestTemplatePaths) fmt.Println("::endgroup::") + } else { + fmt.Println("::group::All tests passed") + fmt.Println("::endgroup::") + os.Exit(0) } } diff --git a/cmd/integration-test/ssl.go b/cmd/integration-test/ssl.go index 24f6e9b903..de7a91a947 100644 --- a/cmd/integration-test/ssl.go +++ b/cmd/integration-test/ssl.go @@ -123,7 +123,11 @@ func (h *sslWithVars) Execute(filePath string) error { type sslMultiReq struct{} func (h *sslMultiReq) Execute(filePath string) error { - ts := testutils.NewTCPServer(&tls.Config{}, defaultStaticPort, func(conn net.Conn) { + //nolint:staticcheck // SSLv3 is intentionally used for testing purposes + ts := testutils.NewTCPServer(&tls.Config{ + MinVersion: tls.VersionSSL30, + MaxVersion: tls.VersionTLS11, + }, defaultStaticPort, func(conn net.Conn) { defer conn.Close() data := make([]byte, 4) if _, err := conn.Read(data); err != nil { diff --git a/go.mod b/go.mod index 566e3df290..c3c2f1d151 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/projectdiscovery/rawhttp v0.1.84 github.com/projectdiscovery/retryabledns v1.0.94 github.com/projectdiscovery/retryablehttp-go v1.0.97 - github.com/projectdiscovery/yamldoc-go v1.0.4 + github.com/projectdiscovery/yamldoc-go v1.0.5 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 github.com/segmentio/ksuid v1.0.4 diff --git a/go.sum b/go.sum index b3f61f555a..1f54b4692d 100644 --- a/go.sum +++ b/go.sum @@ -927,8 +927,8 @@ github.com/projectdiscovery/utils v0.4.8 h1:/Xd38fP8xc6kifZayjrhcYALenJrjO3sHO7l github.com/projectdiscovery/utils v0.4.8/go.mod h1:S314NzLcXVCbLbwYCoorAJYcnZEwv7Uhw2d3aF5fJ4s= github.com/projectdiscovery/wappalyzergo v0.2.13 h1:yrzmcT/Zq+H+qpbAjHKKFVVQA7uei+sAYlmS2HLpkGI= github.com/projectdiscovery/wappalyzergo v0.2.13/go.mod h1:3vtvQCSYpU+Ilk0qy09WYT9BH0Stut5Qon7KJJ78GKw= -github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= -github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= +github.com/projectdiscovery/yamldoc-go v1.0.5 h1:On5D9F5Qe59Tek9SX05uSbjahiRTvDqUSIoG/gDuRd4= +github.com/projectdiscovery/yamldoc-go v1.0.5/go.mod h1:R5lWrNzP+7Oyn77NDVPnBsxx2/FyQZBBkIAaSaCQFxw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= diff --git a/pkg/protocols/offlinehttp/request.go b/pkg/protocols/offlinehttp/request.go index 772f21ce20..07a0fc2dc4 100644 --- a/pkg/protocols/offlinehttp/request.go +++ b/pkg/protocols/offlinehttp/request.go @@ -1,7 +1,9 @@ package offlinehttp import ( + "fmt" "io" + "net/http" "net/http/httputil" "os" @@ -36,7 +38,7 @@ var RawInputMode = false // ExecuteWithResults executes the protocol requests and returns results instead of writing them. func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, previous output.InternalEvent, callback protocols.OutputEventCallback) error { if RawInputMode { - return request.executeRawInput(input.MetaInput.Input, input, callback) + return request.executeRawInput(input.MetaInput.Input, "", input, callback) } wg, err := syncutil.New(syncutil.WithSize(request.options.Options.BulkSize)) @@ -74,7 +76,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, } dataStr := conversion.String(buffer) - if err := request.executeRawInput(dataStr, input, callback); err != nil { + if err := request.executeRawInput(dataStr, data, input, callback); err != nil { gologger.Error().Msgf("Could not execute raw input %s: %s\n", data, err) return } @@ -90,7 +92,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, return nil } -func (request *Request) executeRawInput(data string, input *contextargs.Context, callback protocols.OutputEventCallback) error { +func (request *Request) executeRawInput(data, inputString string, input *contextargs.Context, callback protocols.OutputEventCallback) error { resp, err := readResponseFromString(data) if err != nil { return errors.Wrap(err, "could not read raw response") @@ -111,8 +113,12 @@ func (request *Request) executeRawInput(data string, input *contextargs.Context, if err != nil { return errors.Wrap(err, "could not read raw http response body") } + reqURL := inputString + if inputString == "" { + reqURL = getURLFromRequest(resp.Request) + } - outputEvent := request.responseToDSLMap(resp, data, data, data, conversion.String(dumpedResponse), conversion.String(body), utils.HeadersToString(resp.Header), 0, nil) + outputEvent := request.responseToDSLMap(resp, data, reqURL, data, conversion.String(dumpedResponse), conversion.String(body), utils.HeadersToString(resp.Header), 0, nil) // add response fields to template context and merge templatectx variables to output event request.options.AddTemplateVars(input.MetaInput, request.Type(), request.GetID(), outputEvent) if request.options.HasTemplateCtx(input.MetaInput) { @@ -124,3 +130,10 @@ func (request *Request) executeRawInput(data string, input *contextargs.Context, callback(event) return nil } + +func getURLFromRequest(req *http.Request) string { + if req.URL.Scheme == "" { + req.URL.Scheme = "https" + } + return fmt.Sprintf("%s://%s%s", req.URL.Scheme, req.Host, req.URL.Path) +} diff --git a/pkg/templates/parser.go b/pkg/templates/parser.go index f76352e76c..a139e0cffd 100644 --- a/pkg/templates/parser.go +++ b/pkg/templates/parser.go @@ -33,6 +33,13 @@ func NewParser() *Parser { return p } +func NewParserWithParsedCache(cache *Cache) *Parser { + return &Parser{ + parsedTemplatesCache: cache, + compiledTemplatesCache: NewCache(), + } +} + // Cache returns the parsed templates cache func (p *Parser) Cache() *Cache { return p.parsedTemplatesCache