Feature 3141 implement first version of secret validator application

gradle/projects.gradle

@@ -53,10 +53,7 @@ projectType = [
                     project(':sechub-wrapper-checkmarx'),
                     project(':sechub-wrapper-prepare'),
 
-                    /* special application to validate and categorize results of secret scans */
                     project(':sechub-wrapper-secret-validator'),
-
-
             ],
 
             /* adapter projects - have simple spring dependencies, but know only sechub-adapter as base */
@@ -101,7 +98,8 @@ projectType = [
                    project(':sechub-pds'),
                    project(':sechub-wrapper-checkmarx'),
                    project(':sechub-wrapper-prepare'),
-                   project(':sechub-webui')
+                   project(':sechub-webui'),
+                   project(':sechub-wrapper-secret-validator'),
             ],
 
             springBootWebUIProjects:[

sechub-integrationtest/pds/product-scripts/pds-solutions-gitleaks-mocked.sh

@@ -9,4 +9,18 @@ echo "PDS solutions GITLEAKS mock starting"
 cp "./../sechub-pds-solutions/gitleaks/docker/mocks/mock.sarif.json" "$PDS_JOB_RESULT_FILE"
 
 warnMessage "mocked result"
-infoMessage "product:gitleaks"
+infoMessage "product:gitleaks"
+
+# Here we test the sechub wrapper secret validator application with the results of "mock.sarif.json"
+export PDS_INTEGRATIONTEST_ENABLED=true
+export TOOL_FOLDER=./../sechub-integrationtest/build/pds-tools
+
+# Export the config file necessary for the sechub wrapper secret validator application
+# Besides the config file the wrapper application will automatically use the PDS_JOB_RESULT_FILE,
+# which is already available in this context
+export SECRET_VALIDATOR_CONFIGFILE="./../sechub-pds-solutions/gitleaks/docker/sechub-wrapper-secret-validator-config.json"
+
+# Uses the original gitleaks.sh script from the pds gitleaks
+# Since 'PDS_INTEGRATIONTEST_ENABLED=true' gitleaks will no be executed.
+# The wrapper application starts with the 'integrationtest' profile and will not perform real web reuqests
+source ./../sechub-pds-solutions/gitleaks/docker/scripts/gitleaks.sh

sechub-integrationtest/src/main/resources/pds-config-integrationtest.json

@@ -229,7 +229,7 @@
   }, {
     "id" : "PDS_SOLUTION_GITLEAKS_MOCKED",
     "path" : "./../sechub-integrationtest/pds/product-scripts/pds-solutions-gitleaks-mocked.sh",
-    "scanType" : "secretSCan",
+    "scanType" : "secretScan",
     "description" : "Returns mock data, but from real product"
   }, {
     "id" : "PDS_SOLUTION_ZAP_MOCKED",

sechub-integrationtest/src/test/java/com/mercedesbenz/sechub/integrationtest/scenario21/PDSSolutionMockModeScenario21IntTest.java

@@ -32,6 +32,8 @@
 import com.mercedesbenz.sechub.commons.model.SecHubSourceDataConfiguration;
 import com.mercedesbenz.sechub.commons.model.SecHubStatus;
 import com.mercedesbenz.sechub.commons.model.SecHubWebScanConfiguration;
+import com.mercedesbenz.sechub.commons.model.Severity;
+import com.mercedesbenz.sechub.commons.model.TrafficLight;
 import com.mercedesbenz.sechub.integrationtest.api.IntegrationTestSetup;
 import com.mercedesbenz.sechub.integrationtest.api.TestAPI;
 import com.mercedesbenz.sechub.integrationtest.api.TestProject;
@@ -72,7 +74,16 @@ public void pds_solution_scancode_spdx_mocked_report_in_json_and_html_available(
 
     @Test
     public void pds_solution_gitleaks_mocked_report_in_json_and_html_available() throws Exception {
-        executePDSSolutionJobAndStoreReports(ScanType.SECRET_SCAN, PROJECT_6, "gitleaks");
+        SecHubReportModel report = executePDSSolutionJobAndStoreReports(ScanType.SECRET_SCAN, PROJECT_6, "gitleaks");
+        /* @formatter:off */
+        assertReportUnordered(report.toJSON())
+                    .hasTrafficLight(TrafficLight.RED)
+                           .finding()
+                           .severity(Severity.CRITICAL)
+                           .scanType(ScanType.SECRET_SCAN)
+                           .description("github-pat has detected secret for file UnSAFE_Bank/iOS/Source Code/Pods/README.adoc.")
+                           .isContained();
+        /* @formatter:on */
     }
 
     @Test
@@ -100,7 +111,7 @@ public void pds_solution_findsecuritybugs_mocked_report_in_json_and_html_availab
         executePDSSolutionJobAndStoreReports(ScanType.CODE_SCAN, PROJECT_10, "findsecuritybugs");
     }
 
-    private void executePDSSolutionJobAndStoreReports(ScanType scanType, TestProject project, String solutionName) {
+    private SecHubReportModel executePDSSolutionJobAndStoreReports(ScanType scanType, TestProject project, String solutionName) {
         SecHubConfigurationModel model = createTestModelFor(scanType, project);
         UUID jobUUID = as(USER_1).createJobAndReturnJobUUID(project, model);
 
@@ -162,6 +173,7 @@ private void executePDSSolutionJobAndStoreReports(ScanType scanType, TestProject
             String spdxReport = as(USER_1).getSpdxReport(project, jobUUID);
             storeTestReport(reportName + ".spdx.json", spdxReport);
         }
+        return report;
     }
 
     private SecHubConfigurationModel createTestModelFor(ScanType type, TestProject project) {

sechub-pds-solutions/gitleaks/docker/mocks/mock.sarif.json

@@ -1100,6 +1100,37 @@
       "date": "",
       "commitMessage": ""
      }
+    },
+    {
+     "message": {
+      "text": "github-pat has detected secret for file UnSAFE_Bank/iOS/Source Code/Pods/README.adoc."
+     },
+     "ruleId": "github-pat",
+     "locations": [
+      {
+       "physicalLocation": {
+        "artifactLocation": {
+         "uri": "UnSAFE_Bank/iOS/Source Code/Pods/README.adoc"
+        },
+        "region": {
+         "startLine": 1,
+         "startColumn": 1,
+         "endLine": 1,
+         "endColumn": 40,
+         "snippet": {
+          "text": "invalid-token"
+         }
+        }
+       }
+      }
+     ],
+     "partialFingerprints": {
+      "commitSha": "",
+      "email": "",
+      "author": "",
+      "date": "",
+      "commitMessage": ""
+     }
     }
    ]
   }

sechub-pds-solutions/gitleaks/docker/scripts/gitleaks.sh

@@ -1,6 +1,18 @@
-#!/bin/sh
+#!/usr/bin/bash
 # SPDX-License-Identifier: MIT
 
+declare -r secretvalidation_wrapper="$TOOL_FOLDER/sechub-wrapper-secret-validator.jar"
+
+if [[ "$PDS_INTEGRATIONTEST_ENABLED" = "true" ]]; then
+    echo "Integrationtest will be performed. Gitleaks will not be executed."
+
+    # Execute the wrapper using the 'integrationtest' profile
+    java -jar "-Dspring.profiles.active=integrationtest" "$secretvalidation_wrapper"
+
+    exit $?
+fi
+
+
 . "$SCRIPT_FOLDER/common.sh"
 
 echo "Look for potential .git folder to perform history scan."

sechub-pds-solutions/gitleaks/docker/sechub-wrapper-secret-validator-config.json

@@ -0,0 +1,21 @@
+{
+  "validatorConfigList" : [ {
+    "ruleId" : "github-pat",
+    "categorization" : {
+      "defaultSeverity" : "high",
+      "validationFailedSeverity" : "medium",
+      "validationSuccessSeverity" : "critical"
+    },
+    "requests" : [ {
+      "proxyRequired" : true,
+      "url" : "https://api.github.com",
+      "headers" : [ {
+        "name" : "Authorization",
+        "valuePrefix" : "token"
+      } ],
+      "expectedResponse" : {
+        "httpStatus" : 200
+      }
+    } ]
+  } ]
+}

sechub-sereco/src/main/java/com/mercedesbenz/sechub/sereco/importer/GitleaksSarifImportWorkaround.java

@@ -6,7 +6,18 @@
 
 import org.springframework.stereotype.Component;
 
-import de.jcup.sarif_2_1_0.model.*;
+import com.mercedesbenz.sechub.sereco.metadata.SerecoSeverity;
+
+import de.jcup.sarif_2_1_0.model.Location;
+import de.jcup.sarif_2_1_0.model.PartialFingerprints;
+import de.jcup.sarif_2_1_0.model.PhysicalLocation;
+import de.jcup.sarif_2_1_0.model.PropertyBag;
+import de.jcup.sarif_2_1_0.model.Region;
+import de.jcup.sarif_2_1_0.model.ReportingDescriptor;
+import de.jcup.sarif_2_1_0.model.Result;
+import de.jcup.sarif_2_1_0.model.Run;
+import de.jcup.sarif_2_1_0.model.Tool;
+import de.jcup.sarif_2_1_0.model.ToolComponent;
 
 @Component
 public class GitleaksSarifImportWorkaround implements SarifImportProductWorkaround {
@@ -39,7 +50,7 @@ public String resolveFindingRevisionId(Result result, Run run) {
     }
 
     @Override
-    public String resolveCustomSechubSeverity(Result result, Run run) {
+    public SerecoSeverity resolveCustomSerecoSeverity(Result result, Run run) {
         if (result == null) {
             return null;
         }
@@ -66,8 +77,9 @@ public String resolveCustomSechubSeverity(Result result, Run run) {
         if (additionalProperties == null) {
             return null;
         }
-        String severityKey = SarifImporterKeys.SECRETSCAN_SECHUB_SEVERITY.getKey();
-        return (String) additionalProperties.get(severityKey);
+        String severityKey = SarifImporterKeys.SECRETSCAN_SERECO_SEVERITY.getKey();
+        String severityValue = (String) additionalProperties.get(severityKey);
+        return SerecoSeverity.fromString(severityValue);
     }
 
     private boolean isGitleaksRun(Run run) {

sechub-sereco/src/main/java/com/mercedesbenz/sechub/sereco/importer/SarifImportProductWorkaround.java

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.sereco.importer;
 
+import com.mercedesbenz.sechub.sereco.metadata.SerecoSeverity;
+
 import de.jcup.sarif_2_1_0.model.ReportingDescriptor;
 import de.jcup.sarif_2_1_0.model.Result;
 import de.jcup.sarif_2_1_0.model.Run;
@@ -23,7 +25,7 @@ public default String resolveFindingRevisionId(Result result, Run run) {
         return null;
     }
 
-    public default String resolveCustomSechubSeverity(Result result, Run run) {
+    public default SerecoSeverity resolveCustomSerecoSeverity(Result result, Run run) {
         return null;
     }
 }

sechub-sereco/src/main/java/com/mercedesbenz/sechub/sereco/importer/SarifImportProductWorkaroundSupport.java

@@ -7,6 +7,8 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+import com.mercedesbenz.sechub.sereco.metadata.SerecoSeverity;
+
 import de.jcup.sarif_2_1_0.model.ReportingDescriptor;
 import de.jcup.sarif_2_1_0.model.Result;
 import de.jcup.sarif_2_1_0.model.Run;
@@ -58,12 +60,12 @@ private <R, E> R visitAllWorkaroundsAndUseFirstResultNotNull(E element, Run run,
         return null;
     }
 
-    public String resolveCustomSechubSeverity(Result result, Run run) {
-        return visitAllWorkaroundsAndUseFirstResultNotNull(result, run, new WorkaroundVisitor<String, Result>() {
+    public SerecoSeverity resolveCustomSechubSeverity(Result result, Run run) {
+        return visitAllWorkaroundsAndUseFirstResultNotNull(result, run, new WorkaroundVisitor<SerecoSeverity, Result>() {
 
             @Override
-            public String visit(Result element, Run run, SarifImportProductWorkaround workaround) {
-                return workaround.resolveCustomSechubSeverity(result, run);
+            public SerecoSeverity visit(Result element, Run run, SarifImportProductWorkaround workaround) {
+                return workaround.resolveCustomSerecoSeverity(result, run);
             }
         });
     }

sechub-sereco/src/main/java/com/mercedesbenz/sechub/sereco/importer/SarifImporterKeys.java

@@ -3,11 +3,11 @@
 
 /*
  * If this needs to get changed, make sure to change
- * com.mercedesbenz.sechub.wrapper.secret.validator.support.SarifImporterKeys accordingly
+ * com.mercedesbenz.sechub.sereco.importer.SarifImporterKeys accordingly
  */
 public enum SarifImporterKeys {
 
-    SECRETSCAN_SECHUB_SEVERITY("secretscan.sechub.severity", "The key for the sechub severity which is more precise than the SARIF Level enum."),
+    SECRETSCAN_SERECO_SEVERITY("secretscan.sereco.severity", "The key for the sereco severity which is more precise than the SARIF Level enum."),
 
     SECRETSCAN_VALIDATED_BY_URL("secretscan.validated.by.url", "The key for the URL the secret was validated with."),
 

sechub-sereco/src/main/java/com/mercedesbenz/sechub/sereco/importer/SarifV1JSONImporter.java

@@ -285,8 +285,7 @@ private void handleWebRequest(Result result, SerecoWeb serecoWeb) {
     }
 
     private SerecoSeverity resolveSeverity(Result result, Run run) {
-        String customSecHubSeverity = workaroundSupport.resolveCustomSechubSeverity(result, run);
-        SerecoSeverity serecoSeverity = severityFromString(customSecHubSeverity);
+        SerecoSeverity serecoSeverity = workaroundSupport.resolveCustomSechubSeverity(result, run);
         if (serecoSeverity == null) {
             Level level = sarifSchema210LogicSupport.resolveLevel(result, run);
             return mapToSeverity(level);
@@ -528,10 +527,6 @@ private SerecoSeverity mapToSeverity(Level level) {
         }
     }
 
-    private SerecoSeverity severityFromString(String customSecHubSeverity) {
-        return SerecoSeverity.fromString(customSecHubSeverity);
-    }
-
     @Override
     public boolean isAbleToImportForProduct(ImportParameter param) {
         /* first we do the simple check... */

sechub-sereco/src/test/resources/sarif/sarif_2.1.0_gitleaks_8.0-with-invalid-validator-severity-properties.json

@@ -802,7 +802,7 @@
               "text" : "531486b2bf646636a6a1bba61e78ec4a4a54efbd"
             },
             "properties" : {
-              "secretscan.sechub.severity" : "invalid-value"
+              "secretscan.sereco.severity" : "invalid-value"
             }
           }
         }
@@ -833,7 +833,7 @@
               "text" : "531486b2bf646636a6a1bba61e78ec4a4a54efbd"
             },
             "properties" : {
-              "secretscan.sechub.severity" : "another-invalid-value-1234"
+              "secretscan.sereco.severity" : "another-invalid-value-1234"
             }
           }
         }

sechub-sereco/src/test/resources/sarif/sarif_2.1.0_gitleaks_8.0-with-validator-severity-properties.json

@@ -802,7 +802,7 @@
               "text" : "531486b2bf646636a6a1bba61e78ec4a4a54efbd"
             },
             "properties" : {
-              "secretscan.sechub.severity" : "info"
+              "secretscan.sereco.severity" : "info"
             }
           }
         }
@@ -833,7 +833,7 @@
               "text" : "531486b2bf646636a6a1bba61e78ec4a4a54efbd"
             },
             "properties" : {
-              "secretscan.sechub.severity" : "unclassified"
+              "secretscan.sereco.severity" : "unclassified"
             }
           }
         }
@@ -864,7 +864,7 @@
               "text" : "9bbc0d79e686e847bc305c9bd4cc2ea6"
             },
             "properties" : {
-              "secretscan.sechub.severity" : "low"
+              "secretscan.sereco.severity" : "low"
             }
           }
         }
@@ -895,7 +895,7 @@
               "text" : "9bbc0d79e686e847bc305c9bd4cc2ea6"
             },
             "properties" : {
-              "secretscan.sechub.severity" : "medium"
+              "secretscan.sereco.severity" : "medium"
             }
           }
         }
@@ -926,7 +926,7 @@
               "text" : "b3816fddcf28aa29d94b10ec305cd52be14c472b"
             },
             "properties" : {
-              "secretscan.sechub.severity" : "high"
+              "secretscan.sereco.severity" : "high"
             }
           }
         }
@@ -957,7 +957,7 @@
               "text" : "b3816fddcf28aa29d94b10ec305cd52be14c472b"
             },
             "properties" : {
-              "secretscan.sechub.severity" : "critical"
+              "secretscan.sereco.severity" : "critical"
             }
           }
         }