Merge pull request #3846 from mercedes-benz/feature-3834-enable-handl…

…ing-of-pac-files

Support PAC file in wrapper and pds solution #3834

sechub-pds-solutions/owaspzap/README.adoc

@@ -22,10 +22,12 @@ SecHub will automatically create the directory `$PDS_JOB_EXTRACTED_ASSETS_FOLDER
 When creating a template with an asset for webscan authentication and assigning it to a SecHub project, ensure that 
 
 - `PDS_OWASP_ZAP.zip` is available as asset file inside the used asset
-- `PDS_OWASP_ZAP.zip` contains `script.groovy` at root level
+- `PDS_OWASP_ZAP.zip` can optionally contain `script.groovy` at root level for authentication
+- `PDS_OWASP_ZAP.zip` can optionally contain `proxy.pac` at root level for dynamic proxy handling during authentication
 
-The PDS will extract the file at runtime to `$PDS_JOB_EXTRACTED_ASSETS_FOLDER/webscan-login/script.groovy`.
-If the extracted file exists by the time the script `owasp-zap.sh` is executed, the groovy script will be used for automated login.
+The PDS will extract the file at runtime to `$PDS_JOB_EXTRACTED_ASSETS_FOLDER/webscan-login/script.groovy` and `$PDS_JOB_EXTRACTED_ASSETS_FOLDER/webscan-login/proxy.pac`.
+If the extracted file `$PDS_JOB_EXTRACTED_ASSETS_FOLDER/webscan-login/script.groovy` exists by the time the script `owasp-zap.sh` is executed, the groovy script will be used for automated login.
+If the extracted file `$PDS_JOB_EXTRACTED_ASSETS_FOLDER/webscan-login/proxy.pac` exists by the time the script `owasp-zap.sh` is executed, the PAC file will be used by the browser during the authentication.
 Details about the configuration of the wrapper application that is used to configure the ZAP, can be found at link:https://github.com/mercedes-benz/sechub/blob/develop/sechub-wrapper-owasp-zap/README.adoc[https://github.com/mercedes-benz/sechub/blob/develop/sechub-wrapper-owasp-zap/README.adoc].
 
 === Script authentication template data variables

sechub-pds-solutions/owaspzap/docker/scripts/owasp-zap.sh

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: MIT
 
 target_script_file="$PDS_JOB_EXTRACTED_ASSETS_FOLDER/webscan-login/script.groovy"
+target_pac_file="$PDS_JOB_EXTRACTED_ASSETS_FOLDER/webscan-login/proxy.pac"
 
 shutdownZAP() {
 	# --full: to specify the process by looking at full command line including the parameters
@@ -130,6 +131,13 @@ else
     echo "No script login file was found"
 fi
 
+if [ -f "$target_pac_file" ] ; then
+    export ZAP_LOGIN_PAC_FILE_PATH="$target_pac_file"
+    echo "Use PAC file: $ZAP_LOGIN_PAC_FILE_PATH"
+else
+    echo "No PAC file was found"
+fi
+
 echo ""
 echo "Start scanning"
 echo ""

sechub-wrapper-owasp-zap/README.adoc

@@ -59,6 +59,11 @@ Usage: ZapWrapper [options]
       Maximum number of times the wrapper tries to reach each URL. Including 
       each URL constructed from the sechub includes.
       Default: 3
+    --pacFilePath
+      PAC file the ZAP wrapper uses for script based authentication for the 
+      browsers profile, when templates are defined. You can also set the 
+      environment variable ZAP_LOGIN_PAC_FILE_PATH, instead of using this 
+      parameter. 
     --pdsEventFolder
       Folder where the ZAP wrapper listens for events of the PDS, like cancel 
       requests for the current job. You can also set the environment variable 
@@ -90,7 +95,7 @@ Usage: ZapWrapper [options]
       value cannot be less than 1000 milliseconds.
       Default: 1000
     --sechubConfigfile
-      The SecHub config file, containing additonal configurations for the 
+      The SecHub config file, containing additional configurations for the 
       scan. 
   * --targetURL
       Specifies the target url to be scanned.

sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/cli/CommandLineSettings.java

@@ -43,7 +43,7 @@ public String getJobUUID() {
         return jobUUID;
     }
 
-    @Parameter(names = { "--sechubConfigfile" }, description = "The SecHub config file, containing additonal configurations for the scan.", required = false)
+    @Parameter(names = { "--sechubConfigfile" }, description = "The SecHub config file, containing additional configurations for the scan.", required = false)
     private String sechubConfigFile;
 
     public File getSecHubConfigFile() {
@@ -205,4 +205,13 @@ public String getPDSEventFolder() {
     public String getGroovyLoginScriptFile() {
         return groovyLoginScriptFile;
     }
+
+    @Parameter(names = {
+            "--pacFilePath" }, description = "PAC file the ZAP wrapper uses for script based authentication for the browsers profile, when templates are defined. You can also set the environment variable "
+                    + EnvironmentVariableConstants.ZAP_LOGIN_PAC_FILE_PATH + ", instead of using this parameter.", required = false)
+    private String pacFilePath;
+
+    public String getPacFilePath() {
+        return pacFilePath;
+    }
 }

sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContext.java

@@ -49,6 +49,7 @@ public class ZapScanContext {
 
     private File groovyScriptLoginFile;
     private Map<String, String> templateVariables = new LinkedHashMap<>();
+    private File pacFilePath;
 
     private ZapScanContext() {
     }
@@ -155,6 +156,10 @@ public Map<String, String> getTemplateVariables() {
         return Collections.unmodifiableMap(templateVariables);
     }
 
+    public File getPacFilePath() {
+        return pacFilePath;
+    }
+
     public static ZapScanContextBuilder builder() {
         return new ZapScanContextBuilder();
     }
@@ -204,6 +209,8 @@ public static class ZapScanContextBuilder {
 
         private Map<String, String> templateVariables = new LinkedHashMap<>();
 
+        private File pacFilePath;
+
         public ZapScanContextBuilder setServerConfig(ZapServerConfiguration serverConfig) {
             this.serverConfig = serverConfig;
             return this;
@@ -331,6 +338,11 @@ public ZapScanContextBuilder setTemplateVariables(Map<String, String> templateVa
             return this;
         }
 
+        public ZapScanContextBuilder setPacFilePath(File pacFilePath) {
+            this.pacFilePath = pacFilePath;
+            return this;
+        }
+
         public ZapScanContext build() {
             ZapScanContext zapScanContext = new ZapScanContext();
             zapScanContext.serverConfig = this.serverConfig;
@@ -370,6 +382,8 @@ public ZapScanContext build() {
             zapScanContext.groovyScriptLoginFile = this.groovyScriptLoginFile;
             zapScanContext.templateVariables = this.templateVariables;
 
+            zapScanContext.pacFilePath = this.pacFilePath;
+
             return zapScanContext;
         }
 

sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/config/ZapScanContextFactory.java

@@ -15,7 +15,6 @@
 import com.mercedesbenz.sechub.commons.model.template.TemplateType;
 import com.mercedesbenz.sechub.zapwrapper.cli.CommandLineSettings;
 import com.mercedesbenz.sechub.zapwrapper.cli.ZapWrapperExitCode;
-import com.mercedesbenz.sechub.zapwrapper.cli.ZapWrapperRuntimeException;
 import com.mercedesbenz.sechub.zapwrapper.helper.*;
 import com.mercedesbenz.sechub.zapwrapper.util.EnvironmentVariableConstants;
 import com.mercedesbenz.sechub.zapwrapper.util.EnvironmentVariableReader;
@@ -111,6 +110,7 @@ public ZapScanContext create(CommandLineSettings settings) throws ZapWrapperCont
 												.setZapPDSEventHandler(zapEventHandler)
 												.setGroovyScriptLoginFile(groovyScriptFile)
 												.setTemplateVariables(templateVariables)
+												.setPacFilePath(fetchPacFilePath(settings))
 											  .build();
 		/* @formatter:on */
         return scanContext;
@@ -302,7 +302,6 @@ private Map<String, String> fetchTemplateVariables(SecHubScanConfiguration sechu
      * @param templateVariables
      * @throws ZapWrapperContextCreationException
      *
-     * @throws ZapWrapperRuntimeException
      */
     private void assertValidScriptLoginConfiguration(File groovyScriptFile, Map<String, String> templateVariables) throws ZapWrapperContextCreationException {
         // no script login was defined
@@ -333,4 +332,36 @@ private void assertValidScriptLoginConfiguration(File groovyScriptFile, Map<Stri
             }
         }
     }
+
+    /**
+     * This method returns a PAC file for script based authentication. If a file is
+     * specified it must exist on the filesystem. As always, command line parameters
+     * take precedence over environment variables.
+     *
+     * @param settings
+     * @return a file that exists on the filesystem or <code>null</code> if nothing
+     *         was specified.
+     *
+     *
+     * @throws ZapWrapperContextCreationException in case the specified file does
+     *                                            not exist on the filesystem
+     */
+    private File fetchPacFilePath(CommandLineSettings settings) throws ZapWrapperContextCreationException {
+        String pacFilePath = settings.getPacFilePath();
+
+        if (pacFilePath == null) {
+            pacFilePath = environmentVariableReader.readAsString(EnvironmentVariableConstants.ZAP_LOGIN_PAC_FILE_PATH);
+        }
+        if (pacFilePath == null) {
+            return null;
+        }
+        File pacFile = new File(pacFilePath);
+        if (!pacFile.isFile()) {
+            throw new ZapWrapperContextCreationException(
+                    "A pac file was specified for script login, that does not exist on the filesystem!\n:Pac file path was:  " + pacFilePath,
+                    ZapWrapperExitCode.UNSUPPORTED_CONFIGURATION);
+        }
+        return pacFile;
+    }
+
 }

sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/scan/ZapScanner.java

@@ -25,7 +25,6 @@
 import com.mercedesbenz.sechub.zapwrapper.cli.ZapWrapperRuntimeException;
 import com.mercedesbenz.sechub.zapwrapper.config.ProxyInformation;
 import com.mercedesbenz.sechub.zapwrapper.config.ZapScanContext;
-import com.mercedesbenz.sechub.zapwrapper.config.ZapTemplateDataVariableKeys;
 import com.mercedesbenz.sechub.zapwrapper.config.auth.ZapAuthenticationType;
 import com.mercedesbenz.sechub.zapwrapper.config.auth.ZapSessionManagementType;
 import com.mercedesbenz.sechub.zapwrapper.helper.ZapPDSEventHandler;
@@ -43,6 +42,9 @@ public class ZapScanner implements ZapScan {
     private static final int DEFAULT_MAX_DEPTH_AJAX_SPIDER = 10;
     private static final int DEFAULT_MAX_DEPTH_SPIDER = 5;
 
+    // all kinds of logout calls that might show up
+    private static final String DEFAULT_EXCLUDE = "(?i).*(log[\\s_+-]*out|log[\\s_+-]*off|sign[\\s_+-]*out|sign[\\s_+-]*off|abmelden|ausloggen).*";
+
     private final ClientApiWrapper clientApiWrapper;
     private final ZapScanContext scanContext;
 
@@ -81,6 +83,7 @@ public void scan() throws ZapWrapperRuntimeException {
             int zapContextId = createContext();
             addXSecHubDASTHeader();
             addReplacerRulesForHeaders();
+            addDefaultExcludes();
 
             /* ZAP setup with access to target */
             // The order of the following method calls is important. We want to load the
@@ -306,6 +309,29 @@ void executeScan(int zapContextId) throws ClientApiException {
     /**
      * Configure login according to the sechub webscan config.
      *
+     * <p>
+     * A future use case with script authentication could be multiple users scan
+     * with different sessions. See also
+     * <a href="https://github.com/zaproxy/zaproxy/issues/6342">
+     * https://github.com/zaproxy/zaproxy/issues/6342</a>
+     * </p>
+     *
+     * <pre>
+     * {@code
+     * // Example how to set up one user with a specific session
+     *
+     * String zapAuthSessionName = scriptLogin.login(scanContext, clientApiWrapper);
+     * String username = scanContext.getTemplateVariables().get(ZapTemplateDataVariableKeys.USERNAME_KEY);
+     * LOG.info("For scan {}: Setup scan user in ZAP to use authenticated session.",
+     * scanContext.getContextName()); StringBuilder authCredentialsConfigParams =
+     * new StringBuilder();
+     * authCredentialsConfigParams.append("username=").append(urlEncodeUTF8(username))
+     * .append("&sessionName=").append(urlEncodeUTF8(zapAuthSessionName));
+     * clientApiWrapper.addIncludeUrlPatternToContext(scanContext.getContextName(),
+     * "^.*"+scanContext.getTargetUrl().getHost()+".*"); UserInformation userInfo =
+     * setupScanUserForZapContext(zapContextId, username,
+     * authCredentialsConfigParams.toString()); }
+     *
      * @param zapContextId
      * @return UserInformation containing userName and zapUserId or
      *         <code>null</code> if nothing could be configured.
@@ -328,17 +354,14 @@ UserInformation setupLoginInsideZapContext(int zapContextId) throws ClientApiExc
             setupAuthenticationAndSessionManagementMethodForScriptLogin(zapContextId);
 
             LOG.info("For scan {}: Performing script authentication.", scanContext.getContextName());
-            String zapAuthSessionName = scriptLogin.login(scanContext, clientApiWrapper);
-
-            String username = scanContext.getTemplateVariables().get(ZapTemplateDataVariableKeys.USERNAME_KEY);
-            /* @formatter:off */
-            LOG.info("For scan {}: Setup scan user in ZAP to use authenticated session.", scanContext.getContextName());
-            StringBuilder authCredentialsConfigParams = new StringBuilder();
-            authCredentialsConfigParams.append("username=").append(urlEncodeUTF8(username))
-                                       .append("&sessionName=").append(urlEncodeUTF8(zapAuthSessionName));
-            /* @formatter:on */
-            UserInformation userInfo = setupScanUserForZapContext(zapContextId, username, authCredentialsConfigParams.toString());
-            return userInfo;
+            // we only want to scan with one valid session
+            // this means it is enough to login and set the session to the active session
+            // ZAP will then use this session for all requests, no user setup is needed
+            // A user setup with ZAP's manual authentication mode is only necessary if
+            // multiple users with different sessions must be used
+            // See the JavaDoc for an example if this use case appears.
+            scriptLogin.login(scanContext, clientApiWrapper);
+            return null;
         }
         return null;
     }
@@ -842,6 +865,10 @@ private boolean scriptLoginConfigured() {
         return scanContext.getGroovyScriptLoginFile() != null && !scanContext.getTemplateVariables().isEmpty();
     }
 
+    private void addDefaultExcludes() throws ClientApiException {
+        clientApiWrapper.addExcludeUrlPatternToContext(scanContext.getContextName(), DEFAULT_EXCLUDE);
+    }
+
     record UserInformation(String userName, int zapuserId) {
     }
 

sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/scan/login/ZapScriptLoginWebDriverFactory.java

@@ -1,10 +1,13 @@
 // SPDX-License-Identifier: MIT
 package com.mercedesbenz.sechub.zapwrapper.scan.login;
 
+import java.io.File;
+
 import org.openqa.selenium.Dimension;
 import org.openqa.selenium.Proxy;
 import org.openqa.selenium.firefox.FirefoxDriver;
 import org.openqa.selenium.firefox.FirefoxOptions;
+import org.openqa.selenium.firefox.FirefoxProfile;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -15,7 +18,7 @@ public class ZapScriptLoginWebDriverFactory {
 
     private static final Dimension DEFAULT_WEBDRIVER_RESOLUTION = new Dimension(1920, 1080);
 
-    public FirefoxDriver createFirefoxWebdriver(ProxyInformation proxyInformation, boolean headless) {
+    public FirefoxDriver createFirefoxWebdriver(ProxyInformation proxyInformation, File pacFilePath, boolean headless) {
 
         FirefoxOptions options = new FirefoxOptions();
         if (headless) {
@@ -32,6 +35,16 @@ public FirefoxDriver createFirefoxWebdriver(ProxyInformation proxyInformation, b
             proxy.setHttpProxy(proxyString);
             proxy.setSslProxy(proxyString);
             options.setProxy(proxy);
+        } else if (pacFilePath != null) {
+            String pacFilePathUrlAsString = "file://" + pacFilePath.getAbsolutePath();
+            LOG.info("Adding PAC file: {} to firefox browser profile and add profile to firefox options.", pacFilePathUrlAsString);
+            FirefoxProfile profile = new FirefoxProfile();
+            // see: http://kb.mozillazine.org/Network.proxy.type
+            profile.setPreference("network.proxy.type", 2);
+            // see: http://kb.mozillazine.org/Network.proxy.autoconfig_url
+            profile.setPreference("network.proxy.autoconfig_url", pacFilePathUrlAsString);
+
+            options.setProfile(profile);
         }
         LOG.info("Creating selenium firefox driver.");
         FirefoxDriver firefox = new FirefoxDriver(options);

sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/scan/login/ZapWrapperGroovyScriptExecutor.java

@@ -51,7 +51,7 @@ public ZapWrapperGroovyScriptExecutor() {
     }
 
     public ScriptLoginResult executeScript(File scriptFile, ZapScanContext scanContext) {
-        FirefoxDriver firefox = webDriverFactory.createFirefoxWebdriver(scanContext.getProxyInformation(), true);
+        FirefoxDriver firefox = webDriverFactory.createFirefoxWebdriver(scanContext.getProxyInformation(), scanContext.getPacFilePath(), true);
         WebDriverWait wait = new WebDriverWait(firefox, Duration.ofSeconds(webdriverTimeoutInSeconds));
 
         ScriptEngine scriptEngine = new GroovyScriptEngineFactory().getScriptEngine();

sechub-wrapper-owasp-zap/src/main/java/com/mercedesbenz/sechub/zapwrapper/util/EnvironmentVariableConstants.java

@@ -21,5 +21,6 @@ public class EnvironmentVariableConstants {
     public static final String PDS_SCAN_CONFIGURATION = "PDS_SCAN_CONFIGURATION";
 
     public static final String ZAP_GROOVY_LOGIN_SCRIPT_FILE = "ZAP_GROOVY_LOGIN_SCRIPT_FILE";
+    public static final String ZAP_LOGIN_PAC_FILE_PATH = "ZAP_LOGIN_PAC_FILE_PATH";
 
 }