SecHub data encryption #3250 (#3254)

* SecHub
  - described concept of data encryption #3250
  - Introduced sechub-encryption #3273 + update bouncy castle version #3275
  - encryption implementation are now inside own gradle sub module "sechub-encryption"
  - refacotred sechub encryption library #3274

  - implemented data encryption inside SecHub #3250
  - restricted access and storage, avoid using configuration when not
    absolut necessary
  - created dedicated job message which contains unencrypted configuration
    at runtime. Only one message uses this one -> clear not accidently
    used on another code location
  - created migration scripts, seperated pool id generation for h2 and
    postgres because of binary type. Also postgres will migrate old
    data automatically to NoneCipher variant (means no real encryption,
    but admin will be able to rotate keys...)
  - wrote tests
  - introduced new usecases
  - new  REST APIs introduced
  - added integration test for encryption rotation
  - added developer admin ui actions

  - auto cleanup does also auto clean old unused encryption pool data
 - Scheduler now only executes for accepted encryption pool ids #3250
  -  Updated open api file for encryption parts #3250

*  PDS 
  - implemented data encryption + documentation #3264
  - NONE is default cipher encryption, means startup possible without
  encryption 
  - summary log service shows encryption algorithm
  - handled encryption out of sync problems on PDS side and
  at SecHub side

gradle/libraries.gradle

@@ -88,8 +88,12 @@ ext {
       cycloneDX_core:                          "8.0.0",
       cyclonedx_gradle_plugin:                 "1.7.4",
 
-     /* Prepare wrapper */
-     jgit_core:                                "6.9.0.202403050737-r"
+      /* Prepare wrapper */
+      jgit_core:                                "6.9.0.202403050737-r",
+
+      /* encryption */
+      // https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk18on
+      bouncy_castle_bcprov_jdk8:                "1.78.1"
 
    ]    
 
@@ -187,7 +191,10 @@ ext {
         /* cycloneDX core for Xray and sechub importer */
         cycloneDX_core:                         "org.cyclonedx:cyclonedx-core-java:${libraryVersion.cycloneDX_core}",
 
-        jgit_core:                              "org.eclipse.jgit:org.eclipse.jgit:${libraryVersion.jgit_core}"
+        jgit_core:                              "org.eclipse.jgit:org.eclipse.jgit:${libraryVersion.jgit_core}",
+
+        bouncy_castle_bcprov_jdk8:              "org.bouncycastle:bcprov-jdk18on:${libraryVersion.bouncy_castle_bcprov_jdk8}"
+
    ]
 
 

gradle/projects.gradle

@@ -19,6 +19,7 @@ projectType = [
                project(':sechub-commons-model-testframework'),
                project(':sechub-commons-pds'),
                project(':sechub-commons-archive'),
+               project(':sechub-commons-encryption'),
                project(':sechub-storage-core'),
                project(':sechub-wrapper-owasp-zap'),
                project(':sechub-pds-commons-core'),

sechub-adapter-pds/src/main/java/com/mercedesbenz/sechub/adapter/pds/PDSAdapterV1.java

@@ -169,12 +169,15 @@ private void waitForJobDone(PDSContext context) throws Exception {
             /* see PDSJobStatusState.java */
             jobstatus = context.getResilientJobStatusResultExecutor().executeResilient(() -> getJobStatus(context));
 
-            PDSJobStatusState state = jobstatus.state;
+            PDSJobStatusState state = jobstatus.getState();
             switch (state) {
             case DONE:
                 jobEnded = true;
                 break;
             case FAILED:
+                if (jobstatus.isEncryptionOutOfSync()) {
+                    throw new PDSEncryptionOutOfSyncException();
+                }
                 throw asAdapterException("PDS job execution failed: TimeOut=" + timeOutCheck.wasTimeOut() + ",JobEnded=" + jobEnded, config);
             case CANCELED:
             case CANCEL_REQUESTED:
@@ -454,7 +457,7 @@ private AdapterExecutionResult handleExecutionTypeRestart(PDSContext context, Ad
             /* check job status */
             try {
                 PDSJobStatus currentPdsJobStatus = getJobStatus(context, UUID.fromString(pdsJobUUID));
-                currentPdsJobState = currentPdsJobStatus.state;
+                currentPdsJobState = currentPdsJobStatus.getState();
 
                 if (currentPdsJobState == null) {
                     throw new IllegalStateException("PDS job state null is not supported!");

sechub-adapter-pds/src/main/java/com/mercedesbenz/sechub/adapter/pds/PDSEncryptionOutOfSyncException.java

@@ -0,0 +1,7 @@
+package com.mercedesbenz.sechub.adapter.pds;
+
+public class PDSEncryptionOutOfSyncException extends Exception {
+
+    private static final long serialVersionUID = 1L;
+
+}

sechub-adapter-pds/src/test/java/com/mercedesbenz/sechub/adapter/pds/PDSAdapterV1Test.java

@@ -249,7 +249,7 @@ private void preparePDSJobCreation(UUID pdsJobUUID) throws AdapterException {
 
     private void preparePDSJobStatus(UUID pdsJobUUID, PDSJobStatusState state) {
         PDSJobStatus jobStatus = new PDSJobStatus();
-        jobStatus.state = state;
+        jobStatus.setState(state);
         ResponseEntity<PDSJobStatus> responseEntity = new ResponseEntity<>(jobStatus, HttpStatus.OK);
         when(restOperations.getForEntity(eq("null/api/job/" + pdsJobUUID.toString() + "/status"), eq(PDSJobStatus.class))).thenReturn(responseEntity);
     }

sechub-administration/build.gradle

@@ -7,6 +7,7 @@
  */
 dependencies {
     implementation project(':sechub-shared-kernel')
+    implementation project(':sechub-commons-encryption') // necessary to validate
     implementation library.springboot_starter_thymeleaf
 
     testImplementation project(':sechub-testframework')

sechub-administration/src/main/java/com/mercedesbenz/sechub/domain/administration/AdministrationAPIConstants.java

@@ -106,6 +106,12 @@ private AdministrationAPIConstants() {
 
     public static final String API_CHANGE_PROJECT_ACCESSLEVEL = API_ADMINISTRATION + "project/{projectId}/accesslevel/{projectAccessLevel}";
 
+    /* +-----------------------------------------------------------------------+ */
+    /* +............................ Encryption................................+ */
+    /* +-----------------------------------------------------------------------+ */
+    public static final String API_ADMIN_ENCRYPTION_ROTATION = API_ADMINISTRATION + "encryption/rotate";
+    public static final String API_ADMIN_ENCRYPTION_STATUS = API_ADMINISTRATION + "encryption/status";
+
     /* +-----------------------------------------------------------------------+ */
     /* +............................ Anonymous ................................+ */
     /* +-----------------------------------------------------------------------+ */

sechub-administration/src/main/java/com/mercedesbenz/sechub/domain/administration/encryption/AdministrationEncryptionRotationService.java

@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: MIT
+package com.mercedesbenz.sechub.domain.administration.encryption;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.mercedesbenz.sechub.sharedkernel.Step;
+import com.mercedesbenz.sechub.sharedkernel.UserContextService;
+import com.mercedesbenz.sechub.sharedkernel.encryption.SecHubEncryptionData;
+import com.mercedesbenz.sechub.sharedkernel.encryption.SecHubEncryptionDataValidator;
+import com.mercedesbenz.sechub.sharedkernel.logging.AuditLogService;
+import com.mercedesbenz.sechub.sharedkernel.messaging.DomainMessage;
+import com.mercedesbenz.sechub.sharedkernel.messaging.DomainMessageService;
+import com.mercedesbenz.sechub.sharedkernel.messaging.IsSendingAsyncMessage;
+import com.mercedesbenz.sechub.sharedkernel.messaging.MessageDataKeys;
+import com.mercedesbenz.sechub.sharedkernel.messaging.MessageID;
+import com.mercedesbenz.sechub.sharedkernel.usecases.encryption.UseCaseAdminStartsEncryptionRotation;
+
+@Service
+public class AdministrationEncryptionRotationService {
+
+    @Autowired
+    DomainMessageService domainMessageService;
+
+    @Autowired
+    SecHubEncryptionDataValidator validator;
+
+    @Autowired
+    AuditLogService auditLogService;
+
+    @Autowired
+    UserContextService userContextService;
+
+    @UseCaseAdminStartsEncryptionRotation(@Step(number = 2, name = "Service call", description = "Triggers rotation of encryption via domain message"))
+    @IsSendingAsyncMessage(MessageID.START_ENCRYPTION_ROTATION)
+    public void rotateEncryption(SecHubEncryptionData data) {
+        if (data == null) {
+            throw new IllegalArgumentException("data may not be null!");
+        }
+        auditLogService.log("started encryption rotation. New cipher algorithm will be: {}, datasource type:{}, datasource: {}", data.getAlgorithm(),
+                data.getPasswordSourceType(), data.getPasswordSourceData());
+
+        String executedBy = userContextService.getUserId();
+
+        DomainMessage message = new DomainMessage(MessageID.START_ENCRYPTION_ROTATION);
+        message.set(MessageDataKeys.SECHUB_ENCRYPT_ROTATION_DATA, data);
+        message.set(MessageDataKeys.EXECUTED_BY, executedBy);
+
+        domainMessageService.sendAsynchron(message);
+    }
+
+}

sechub-administration/src/main/java/com/mercedesbenz/sechub/domain/administration/encryption/AdministrationEncryptionStatusService.java

@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: MIT
+package com.mercedesbenz.sechub.domain.administration.encryption;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import com.mercedesbenz.sechub.sharedkernel.Step;
+import com.mercedesbenz.sechub.sharedkernel.encryption.SecHubDomainEncryptionStatus;
+import com.mercedesbenz.sechub.sharedkernel.encryption.SecHubEncryptionStatus;
+import com.mercedesbenz.sechub.sharedkernel.logging.AuditLogService;
+import com.mercedesbenz.sechub.sharedkernel.messaging.DomainMessage;
+import com.mercedesbenz.sechub.sharedkernel.messaging.DomainMessageService;
+import com.mercedesbenz.sechub.sharedkernel.messaging.DomainMessageSynchronousResult;
+import com.mercedesbenz.sechub.sharedkernel.messaging.IsSendingSyncMessage;
+import com.mercedesbenz.sechub.sharedkernel.messaging.MessageDataKeys;
+import com.mercedesbenz.sechub.sharedkernel.messaging.MessageID;
+import com.mercedesbenz.sechub.sharedkernel.usecases.encryption.UseCaseAdminFetchesEncryptionStatus;
+
+@Service
+public class AdministrationEncryptionStatusService {
+
+    @Autowired
+    DomainMessageService domainMessageService;
+
+    @Autowired
+    AuditLogService auditLogService;
+
+    @UseCaseAdminFetchesEncryptionStatus(@Step(number = 1, name = "Service call", description = "Services collects encryption status from domains via event bus"))
+    public SecHubEncryptionStatus fetchStatus() {
+        auditLogService.log("starts collecting encryption status");
+
+        SecHubEncryptionStatus sechubEncryptionStatus = new SecHubEncryptionStatus();
+        collectScheduleEncryptionStatus(sechubEncryptionStatus);
+
+        return sechubEncryptionStatus;
+
+    }
+
+    @IsSendingSyncMessage(MessageID.GET_ENCRYPTION_STATUS_SCHEDULE_DOMAIN)
+    private void collectScheduleEncryptionStatus(SecHubEncryptionStatus status) {
+        DomainMessage message = new DomainMessage(MessageID.GET_ENCRYPTION_STATUS_SCHEDULE_DOMAIN);
+
+        DomainMessageSynchronousResult result = domainMessageService.sendSynchron(message);
+        SecHubDomainEncryptionStatus schedulerStatus = result.get(MessageDataKeys.SECHUB_DOMAIN_ENCRYPTION_STATUS);
+
+        status.getDomains().add(schedulerStatus);
+    }
+
+}

sechub-administration/src/main/java/com/mercedesbenz/sechub/domain/administration/encryption/EncryptionAdministrationRestController.java

@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: MIT
+package com.mercedesbenz.sechub.domain.administration.encryption;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.context.annotation.Profile;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.WebDataBinder;
+import org.springframework.web.bind.annotation.InitBinder;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.mercedesbenz.sechub.domain.administration.AdministrationAPIConstants;
+import com.mercedesbenz.sechub.sharedkernel.Profiles;
+import com.mercedesbenz.sechub.sharedkernel.RoleConstants;
+import com.mercedesbenz.sechub.sharedkernel.Step;
+import com.mercedesbenz.sechub.sharedkernel.encryption.SecHubEncryptionData;
+import com.mercedesbenz.sechub.sharedkernel.encryption.SecHubEncryptionDataValidator;
+import com.mercedesbenz.sechub.sharedkernel.encryption.SecHubEncryptionStatus;
+import com.mercedesbenz.sechub.sharedkernel.usecases.encryption.UseCaseAdminFetchesEncryptionStatus;
+import com.mercedesbenz.sechub.sharedkernel.usecases.encryption.UseCaseAdminStartsEncryptionRotation;
+
+import jakarta.annotation.security.RolesAllowed;
+import jakarta.validation.Valid;
+
+/**
+ * The rest api for encryption done by a super admin.
+ *
+ * @author Albert Tregnaghi
+ *
+ */
+@RestController
+@EnableAutoConfiguration
+@RolesAllowed(RoleConstants.ROLE_SUPERADMIN)
+@Profile({ Profiles.TEST, Profiles.ADMIN_ACCESS })
+public class EncryptionAdministrationRestController {
+
+    @Autowired
+    AdministrationEncryptionRotationService administrationEncryptionRotationService;
+
+    @Autowired
+    AdministrationEncryptionStatusService administrationStatusService;
+
+    @Autowired
+    SecHubEncryptionDataValidator encryptionDataValidator;
+
+    /* @formatter:off */
+	@UseCaseAdminStartsEncryptionRotation(@Step(number=1,name="Rest call",description="Admin triggers rotation of encryption via REST", needsRestDoc =true))
+	@RequestMapping(path = AdministrationAPIConstants.API_ADMIN_ENCRYPTION_ROTATION, method = RequestMethod.POST, produces= {MediaType.APPLICATION_JSON_VALUE})
+	public void rotateEncryption(@RequestBody @Valid SecHubEncryptionData data) {
+		/* @formatter:on */
+        administrationEncryptionRotationService.rotateEncryption(data);
+    }
+
+    /* @formatter:off */
+	@UseCaseAdminFetchesEncryptionStatus(@Step(number=1,name="Rest call",description="Admin fetches encryption status from domains via REST", needsRestDoc =true))
+	@RequestMapping(path = AdministrationAPIConstants.API_ADMIN_ENCRYPTION_STATUS, method = RequestMethod.GET, produces= {MediaType.APPLICATION_JSON_VALUE})
+	public SecHubEncryptionStatus fetchEncryptionStatus() {
+	    /* @formatter:on */
+        return administrationStatusService.fetchStatus();
+    }
+
+    @InitBinder
+    protected void initBinder(WebDataBinder binder) {
+        binder.setValidator(encryptionDataValidator);
+    }
+
+}

sechub-administration/src/main/java/com/mercedesbenz/sechub/domain/administration/job/JobInformation.java

@@ -44,11 +44,6 @@ public class JobInformation {
      * This is a free text field and shows up some information about
      */
     public static final String COLUMN_INFO = "INFO";
-    /**
-     * JSON configuration for this job. Interesting for administrators because of
-     * support
-     */
-    public static final String COLUMN_CONFIGURATION = "CONFIGURATION";
 
     /* +-----------------------------------------------------------------------+ */
     /* +............................ JPQL .....................................+ */
@@ -61,7 +56,6 @@ public class JobInformation {
     public static final String PROPERTY_PROJECT_ID = "projectId";
     public static final String PROPERTY_OWNER = "owner";
     public static final String PROPERTY_SINCE = "since";
-    public static final String PROPERTY_CONFIGURATION = "configuration";
 
     public static final String QUERY_FIND_ALL_RUNNING_JOBS = "SELECT j FROM JobInformation j where j.status = com.mercedesbenz.sechub.domain.administration.job.JobStatus.RUNNING";
     public static final String QUERY_DELETE_JOBINFORMATION_FOR_JOBUUID = "DELETE FROM JobInformation j WHERE j.jobUUID=:jobUUID";
@@ -95,9 +89,6 @@ public JobInformation(UUID jobUUID) {
     @Column(name = COLUMN_SINCE) // remark: we setup hibernate to use UTC settings - see application.properties
     LocalDateTime since;
 
-    @Column(name = COLUMN_CONFIGURATION)
-    String configuration;
-
     @Column(name = COLUMN_INFO)
     String info;
 
@@ -113,14 +104,6 @@ public void setOwner(String owner) {
         this.owner = owner;
     }
 
-    public String getConfiguration() {
-        return configuration;
-    }
-
-    public void setConfiguration(String jsonConfiguration) {
-        this.configuration = jsonConfiguration;
-    }
-
     public void setProjectId(String projectId) {
         this.projectId = projectId;
     }

sechub-administration/src/main/java/com/mercedesbenz/sechub/domain/administration/job/JobInformationCreateService.java

@@ -37,26 +37,25 @@ public void createByMessage(JobMessage message, JobStatus status) {
 
         LOG.debug("Creating a new job information entry for project: {}, SecHub job: {}", projectId, jobUUID);
 
-        JobInformation entity = null;
+        JobInformation jobInformation = null;
 
         Optional<JobInformation> existingInfo = repository.findById(jobUUID);
         if (existingInfo.isPresent()) {
-            entity = existingInfo.get();
+            jobInformation = existingInfo.get();
 
             LOG.warn("There was an existing information entity about SecHub job: {}. The entity will be reused and updated.", jobUUID);
 
         } else {
-            entity = new JobInformation(jobUUID);
+            jobInformation = new JobInformation(jobUUID);
         }
 
-        entity.setProjectId(projectId);
-        entity.setConfiguration(message.getConfiguration());
-        entity.setOwner(message.getOwner());
-        entity.setSince(message.getSince());
+        jobInformation.setProjectId(projectId);
+        jobInformation.setOwner(message.getOwner());
+        jobInformation.setSince(message.getSince());
 
-        entity.setStatus(status);
+        jobInformation.setStatus(status);
 
-        entity = repository.save(entity);
+        jobInformation = repository.save(jobInformation);
 
         LOG.debug("Saved job information entry for project: {}, SecHub job: {}, ", projectId, jobUUID);
     }

sechub-administration/src/test/java/com/mercedesbenz/sechub/domain/administration/job/JobInformationCreateServiceTest.java

@@ -39,7 +39,6 @@ void beforeEach() {
     @Test
     void createByMessage_no_existing_entity_new_entry_will_be_created_and_saved() {
         /* prepare */
-        String configuration = "{ dummy }";
 
         when(repository.findById(jobUUID)).thenReturn(Optional.empty());
 
@@ -50,7 +49,6 @@ void createByMessage_no_existing_entity_new_entry_will_be_created_and_saved() {
         message.setOwner("newOwner");
         message.setProjectId("newProjectId");
         message.setSince(since);
-        message.setConfiguration(configuration);
 
         JobStatus status = JobStatus.RUNNING;
 
@@ -69,7 +67,6 @@ void createByMessage_no_existing_entity_new_entry_will_be_created_and_saved() {
         assertEquals("newOwner", savedJobInformation.getOwner());
         assertEquals("newProjectId", savedJobInformation.getProjectId());
         assertEquals(since, savedJobInformation.getSince());
-        assertEquals(configuration, savedJobInformation.getConfiguration());
         assertEquals(null, savedJobInformation.version);
 
         assertEquals(status, savedJobInformation.getStatus());
@@ -93,7 +90,6 @@ void createByMessage_an_existing_entity_entry_will_be_reused_and_saved() {
         message.setOwner("newOwner");
         message.setProjectId("newProjectId");
         message.setSince(since);
-        message.setConfiguration(configuration);
 
         JobStatus status = JobStatus.RUNNING;
 
@@ -113,7 +109,6 @@ void createByMessage_an_existing_entity_entry_will_be_reused_and_saved() {
         assertEquals("newOwner", savedJobInformation.getOwner());
         assertEquals("newProjectId", savedJobInformation.getProjectId());
         assertEquals(since, savedJobInformation.getSince());
-        assertEquals(configuration, savedJobInformation.getConfiguration());
         assertEquals(42, savedJobInformation.version.intValue());
 
         assertEquals(status, savedJobInformation.getStatus());