Feature 3141 implement first version of secret validator application

gradle/projects.gradle

@@ -51,7 +51,9 @@ projectType = [
                     /* special wrapper applications which are also spring boot applications
                        to have dependency injection and access to some other spring boot parts */
                     project(':sechub-wrapper-checkmarx'),
-                    project(':sechub-wrapper-prepare')
+                    project(':sechub-wrapper-prepare'),
+
+                    project(':sechub-wrapper-secret-validator'),
             ],
 
             /* adapter projects - have simple spring dependencies, but know only sechub-adapter as base */
@@ -96,7 +98,8 @@ projectType = [
                    project(':sechub-pds'),
                    project(':sechub-wrapper-checkmarx'),
                    project(':sechub-wrapper-prepare'),
-                   project(':sechub-webui')
+                   project(':sechub-webui'),
+                   project(':sechub-wrapper-secret-validator'),
             ],
 
             springBootWebUIProjects:[

sechub-integrationtest/pds/product-scripts/pds-solutions-gitleaks-mocked.sh

@@ -9,4 +9,18 @@ echo "PDS solutions GITLEAKS mock starting"
 cp "./../sechub-pds-solutions/gitleaks/docker/mocks/mock.sarif.json" "$PDS_JOB_RESULT_FILE"
 
 warnMessage "mocked result"
-infoMessage "product:gitleaks"
+infoMessage "product:gitleaks"
+
+# Here we test the sechub wrapper secret validator application with the results of "mock.sarif.json"
+export PDS_INTEGRATIONTEST_ENABLED=true
+export TOOL_FOLDER=./../sechub-integrationtest/build/pds-tools
+
+# Export the config file necessary for the sechub wrapper secret validator application
+# Besides the config file the wrapper application will automatically use the PDS_JOB_RESULT_FILE,
+# which is already available in this context
+export SECRET_VALIDATOR_CONFIGFILE="./../sechub-pds-solutions/gitleaks/docker/sechub-wrapper-secret-validator-config.json"
+
+# Uses the original gitleaks.sh script from the pds gitleaks
+# Since 'PDS_INTEGRATIONTEST_ENABLED=true' gitleaks will no be executed.
+# The wrapper application starts with the 'integrationtest' profile and will not perform real web reuqests
+source ./../sechub-pds-solutions/gitleaks/docker/scripts/gitleaks.sh

sechub-integrationtest/src/main/resources/pds-config-integrationtest.json

@@ -229,7 +229,7 @@
   }, {
     "id" : "PDS_SOLUTION_GITLEAKS_MOCKED",
     "path" : "./../sechub-integrationtest/pds/product-scripts/pds-solutions-gitleaks-mocked.sh",
-    "scanType" : "secretSCan",
+    "scanType" : "secretScan",
     "description" : "Returns mock data, but from real product"
   }, {
     "id" : "PDS_SOLUTION_ZAP_MOCKED",

sechub-integrationtest/src/test/java/com/mercedesbenz/sechub/integrationtest/scenario21/PDSSolutionMockModeScenario21IntTest.java

@@ -32,6 +32,8 @@
 import com.mercedesbenz.sechub.commons.model.SecHubSourceDataConfiguration;
 import com.mercedesbenz.sechub.commons.model.SecHubStatus;
 import com.mercedesbenz.sechub.commons.model.SecHubWebScanConfiguration;
+import com.mercedesbenz.sechub.commons.model.Severity;
+import com.mercedesbenz.sechub.commons.model.TrafficLight;
 import com.mercedesbenz.sechub.integrationtest.api.IntegrationTestSetup;
 import com.mercedesbenz.sechub.integrationtest.api.TestAPI;
 import com.mercedesbenz.sechub.integrationtest.api.TestProject;
@@ -72,7 +74,16 @@ public void pds_solution_scancode_spdx_mocked_report_in_json_and_html_available(
 
     @Test
     public void pds_solution_gitleaks_mocked_report_in_json_and_html_available() throws Exception {
-        executePDSSolutionJobAndStoreReports(ScanType.SECRET_SCAN, PROJECT_6, "gitleaks");
+        SecHubReportModel report = executePDSSolutionJobAndStoreReports(ScanType.SECRET_SCAN, PROJECT_6, "gitleaks");
+        /* @formatter:off */
+        assertReportUnordered(report.toJSON())
+                    .hasTrafficLight(TrafficLight.RED)
+                           .finding()
+                           .severity(Severity.CRITICAL)
+                           .scanType(ScanType.SECRET_SCAN)
+                           .description("github-pat has detected secret for file UnSAFE_Bank/iOS/Source Code/Pods/README.adoc.")
+                           .isContained();
+        /* @formatter:on */
     }
 
     @Test
@@ -100,7 +111,7 @@ public void pds_solution_findsecuritybugs_mocked_report_in_json_and_html_availab
         executePDSSolutionJobAndStoreReports(ScanType.CODE_SCAN, PROJECT_10, "findsecuritybugs");
     }
 
-    private void executePDSSolutionJobAndStoreReports(ScanType scanType, TestProject project, String solutionName) {
+    private SecHubReportModel executePDSSolutionJobAndStoreReports(ScanType scanType, TestProject project, String solutionName) {
         SecHubConfigurationModel model = createTestModelFor(scanType, project);
         UUID jobUUID = as(USER_1).createJobAndReturnJobUUID(project, model);
 
@@ -162,6 +173,7 @@ private void executePDSSolutionJobAndStoreReports(ScanType scanType, TestProject
             String spdxReport = as(USER_1).getSpdxReport(project, jobUUID);
             storeTestReport(reportName + ".spdx.json", spdxReport);
         }
+        return report;
     }
 
     private SecHubConfigurationModel createTestModelFor(ScanType type, TestProject project) {

sechub-pds-solutions/gitleaks/docker/mocks/mock.sarif.json

@@ -1100,6 +1100,37 @@
       "date": "",
       "commitMessage": ""
      }
+    },
+    {
+     "message": {
+      "text": "github-pat has detected secret for file UnSAFE_Bank/iOS/Source Code/Pods/README.adoc."
+     },
+     "ruleId": "github-pat",
+     "locations": [
+      {
+       "physicalLocation": {
+        "artifactLocation": {
+         "uri": "UnSAFE_Bank/iOS/Source Code/Pods/README.adoc"
+        },
+        "region": {
+         "startLine": 1,
+         "startColumn": 1,
+         "endLine": 1,
+         "endColumn": 40,
+         "snippet": {
+          "text": "invalid-token"
+         }
+        }
+       }
+      }
+     ],
+     "partialFingerprints": {
+      "commitSha": "",
+      "email": "",
+      "author": "",
+      "date": "",
+      "commitMessage": ""
+     }
     }
    ]
   }

sechub-pds-solutions/gitleaks/docker/scripts/gitleaks.sh

@@ -1,6 +1,18 @@
-#!/bin/sh
+#!/usr/bin/bash
 # SPDX-License-Identifier: MIT
 
+declare -r secretvalidation_wrapper="$TOOL_FOLDER/sechub-wrapper-secret-validator.jar"
+
+if [[ "$PDS_INTEGRATIONTEST_ENABLED" = "true" ]]; then
+    echo "Integrationtest will be performed. Gitleaks will not be executed."
+
+    # Execute the wrapper using the 'integrationtest' profile
+    java -jar "-Dspring.profiles.active=integrationtest" "$secretvalidation_wrapper"
+
+    exit $?
+fi
+
+
 . "$SCRIPT_FOLDER/common.sh"
 
 echo "Look for potential .git folder to perform history scan."

sechub-pds-solutions/gitleaks/docker/sechub-wrapper-secret-validator-config.json

@@ -0,0 +1,21 @@
+{
+  "validatorConfigList" : [ {
+    "ruleId" : "github-pat",
+    "categorization" : {
+      "defaultSeverity" : "high",
+      "validationFailedSeverity" : "medium",
+      "validationSuccessSeverity" : "critical"
+    },
+    "requests" : [ {
+      "proxyRequired" : true,
+      "url" : "https://api.github.com",
+      "headers" : [ {
+        "name" : "Authorization",
+        "valuePrefix" : "token"
+      } ],
+      "expectedResponse" : {
+        "httpStatus" : 200
+      }
+    } ]
+  } ]
+}

sechub-sereco/src/main/java/com/mercedesbenz/sechub/sereco/importer/GitleaksSarifImportWorkaround.java

@@ -1,11 +1,18 @@
 // SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.sereco.importer;
 
+import java.util.List;
 import java.util.Map;
 
 import org.springframework.stereotype.Component;
 
+import com.mercedesbenz.sechub.sereco.metadata.SerecoSeverity;
+
+import de.jcup.sarif_2_1_0.model.Location;
 import de.jcup.sarif_2_1_0.model.PartialFingerprints;
+import de.jcup.sarif_2_1_0.model.PhysicalLocation;
+import de.jcup.sarif_2_1_0.model.PropertyBag;
+import de.jcup.sarif_2_1_0.model.Region;
 import de.jcup.sarif_2_1_0.model.ReportingDescriptor;
 import de.jcup.sarif_2_1_0.model.Result;
 import de.jcup.sarif_2_1_0.model.Run;
@@ -42,6 +49,39 @@ public String resolveFindingRevisionId(Result result, Run run) {
         return null;
     }
 
+    @Override
+    public SerecoSeverity resolveCustomSerecoSeverity(Result result, Run run) {
+        if (result == null) {
+            return null;
+        }
+        if (!isGitleaksRun(run)) {
+            return null;
+        }
+        List<Location> locations = result.getLocations();
+        if (locations == null || locations.isEmpty()) {
+            return null;
+        }
+        PhysicalLocation physicalLocation = locations.get(0).getPhysicalLocation();
+        if (physicalLocation == null) {
+            return null;
+        }
+        Region region = physicalLocation.getRegion();
+        if (region == null) {
+            return null;
+        }
+        PropertyBag properties = region.getProperties();
+        if (properties == null) {
+            return null;
+        }
+        Map<String, Object> additionalProperties = properties.getAdditionalProperties();
+        if (additionalProperties == null) {
+            return null;
+        }
+        String severityKey = SarifImporterKeys.SECRETSCAN_SERECO_SEVERITY.getKey();
+        String severityValue = (String) additionalProperties.get(severityKey);
+        return SerecoSeverity.fromString(severityValue);
+    }
+
     private boolean isGitleaksRun(Run run) {
         if (run == null) {
             return false;

sechub-sereco/src/main/java/com/mercedesbenz/sechub/sereco/importer/SarifImportProductWorkaround.java

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.sereco.importer;
 
+import com.mercedesbenz.sechub.sereco.metadata.SerecoSeverity;
+
 import de.jcup.sarif_2_1_0.model.ReportingDescriptor;
 import de.jcup.sarif_2_1_0.model.Result;
 import de.jcup.sarif_2_1_0.model.Run;
@@ -22,4 +24,8 @@ public default String resolveType(ReportingDescriptor rule, Run run) {
     public default String resolveFindingRevisionId(Result result, Run run) {
         return null;
     }
+
+    public default SerecoSeverity resolveCustomSerecoSeverity(Result result, Run run) {
+        return null;
+    }
 }

sechub-sereco/src/main/java/com/mercedesbenz/sechub/sereco/importer/SarifImportProductWorkaroundSupport.java

@@ -7,6 +7,8 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import com.mercedesbenz.sechub.sereco.metadata.SerecoSeverity;
+
 import de.jcup.sarif_2_1_0.model.ReportingDescriptor;
 import de.jcup.sarif_2_1_0.model.Result;
 import de.jcup.sarif_2_1_0.model.Run;
@@ -58,6 +60,16 @@ private <R, E> R visitAllWorkaroundsAndUseFirstResultNotNull(E element, Run run,
         return null;
     }
 
+    public SerecoSeverity resolveCustomSechubSeverity(Result result, Run run) {
+        return visitAllWorkaroundsAndUseFirstResultNotNull(result, run, new WorkaroundVisitor<SerecoSeverity, Result>() {
+
+            @Override
+            public SerecoSeverity visit(Result element, Run run, SarifImportProductWorkaround workaround) {
+                return workaround.resolveCustomSerecoSeverity(result, run);
+            }
+        });
+    }
+
     public interface WorkaroundVisitor<R, E> {
 
         public R visit(E element, Run run, SarifImportProductWorkaround workaround);

sechub-sereco/src/main/java/com/mercedesbenz/sechub/sereco/importer/SarifImporterKeys.java

@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: MIT
+package com.mercedesbenz.sechub.sereco.importer;
+
+/*
+ * If this needs to get changed, make sure to change
+ * com.mercedesbenz.sechub.sereco.importer.SarifImporterKeys accordingly
+ */
+public enum SarifImporterKeys {
+
+    SECRETSCAN_SERECO_SEVERITY("secretscan.sereco.severity", "The key for the sereco severity which is more precise than the SARIF Level enum."),
+
+    SECRETSCAN_VALIDATED_BY_URL("secretscan.validated.by.url", "The key for the URL the secret was validated with."),
+
+    ;
+
+    private String key;
+    private String description;
+
+    private SarifImporterKeys(String key, String description) {
+        this.key = key;
+        this.description = description;
+    }
+
+    public String getKey() {
+        return key;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+}

sechub-sereco/src/main/java/com/mercedesbenz/sechub/sereco/importer/SarifV1JSONImporter.java

@@ -285,8 +285,12 @@ private void handleWebRequest(Result result, SerecoWeb serecoWeb) {
     }
 
     private SerecoSeverity resolveSeverity(Result result, Run run) {
-        Level level = sarifSchema210LogicSupport.resolveLevel(result, run);
-        return mapToSeverity(level);
+        SerecoSeverity serecoSeverity = workaroundSupport.resolveCustomSechubSeverity(result, run);
+        if (serecoSeverity == null) {
+            Level level = sarifSchema210LogicSupport.resolveLevel(result, run);
+            return mapToSeverity(level);
+        }
+        return serecoSeverity;
     }
 
     private class ResultData {