From 550b0b5345c0c19b84c2d6f649d9f6a6fdcc1b81 Mon Sep 17 00:00:00 2001 From: Albert Tregnaghi Date: Fri, 15 Nov 2024 15:17:34 +0100 Subject: [PATCH] Made flaky integration test TemplateScenario1IntTest stable #3523 - introduced TestAPI.executeResilient which is a short term for executeRunnableAndAcceptAssertionsMaximumTimes(4, runnable, 500); - all time relevant parts inside integration test TemplateScenario1IntTest do now use this for testing --- .../sechub/integrationtest/api/TestAPI.java | 11 ++++ .../scenario1/TemplateScenario1IntTest.java | 58 +++++++++---------- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/sechub-integrationtest/src/main/java/com/mercedesbenz/sechub/integrationtest/api/TestAPI.java b/sechub-integrationtest/src/main/java/com/mercedesbenz/sechub/integrationtest/api/TestAPI.java index f3c593b82..08289ece9 100644 --- a/sechub-integrationtest/src/main/java/com/mercedesbenz/sechub/integrationtest/api/TestAPI.java +++ b/sechub-integrationtest/src/main/java/com/mercedesbenz/sechub/integrationtest/api/TestAPI.java @@ -589,6 +589,17 @@ public static void executeUntilSuccessOrTimeout(TestExecutable testExecutable) { return; } + /** + * Tries to execute runnable with default maximum time and retry (4 times a 500 + * milliseconds) Shortcut for + * executeRunnableAndAcceptAssertionsMaximumTimes(4,runnable, 500); + * + * @param runnable + */ + public static void executeResilient(Runnable runnable) { + executeRunnableAndAcceptAssertionsMaximumTimes(4, runnable, 500); + } + public static void executeRunnableAndAcceptAssertionsMaximumTimes(int tries, Runnable runnable, int millisBeforeNextRetry) { executeCallableAndAcceptAssertionsMaximumTimes(tries, () -> { runnable.run(); diff --git a/sechub-integrationtest/src/test/java/com/mercedesbenz/sechub/integrationtest/scenario1/TemplateScenario1IntTest.java b/sechub-integrationtest/src/test/java/com/mercedesbenz/sechub/integrationtest/scenario1/TemplateScenario1IntTest.java index c439492e0..50f6a74cb 100644 --- a/sechub-integrationtest/src/test/java/com/mercedesbenz/sechub/integrationtest/scenario1/TemplateScenario1IntTest.java +++ b/sechub-integrationtest/src/test/java/com/mercedesbenz/sechub/integrationtest/scenario1/TemplateScenario1IntTest.java @@ -15,7 +15,6 @@ import com.mercedesbenz.sechub.commons.model.template.TemplateDefinition; import com.mercedesbenz.sechub.commons.model.template.TemplateDefinition.TemplateVariable; import com.mercedesbenz.sechub.commons.model.template.TemplateType; -import com.mercedesbenz.sechub.domain.administration.project.ProjectDetailInformation; import com.mercedesbenz.sechub.domain.scan.project.ScanProjectConfig; import com.mercedesbenz.sechub.domain.scan.project.ScanProjectConfigID; import com.mercedesbenz.sechub.integrationtest.api.IntegrationTestSetup; @@ -106,15 +105,13 @@ public void template_crud_test() { private void assertTemplateNotInsideTemplateList() { List templateIds = as(SUPER_ADMIN).fetchTemplateList(); - assertThat(templateIds).doesNotContain(templateId); + executeResilient(() -> assertThat(templateIds).doesNotContain(templateId)); } private void assertTemplateExistsInTemplateListAndCanBeFetched() { - List templateIds; - // check template list still contains the test templte */ - templateIds = as(SUPER_ADMIN).fetchTemplateList(); - assertThat(templateIds).contains(templateId); - assertThat(as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId)).isNotNull(); + // check template list still contains the test template */ + executeResilient(() -> assertThat(as(SUPER_ADMIN).fetchTemplateList()).contains(templateId)); + executeResilient(() -> assertThat(as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId)).isNotNull()); } private void assertProjectDeleteDoesPurgeTemplateAssignment() { @@ -122,8 +119,8 @@ private void assertProjectDeleteDoesPurgeTemplateAssignment() { as(SUPER_ADMIN).deleteProject(Scenario1.PROJECT_1); /* test 7 - configuration for project is removed */ - List configurations = fetchScanProjectConfigurations(Scenario1.PROJECT_1); - assertThat(configurations).isEmpty(); + executeResilient(() -> assertThat(fetchScanProjectConfigurations(Scenario1.PROJECT_1)).isEmpty()); + } private void assertTemplateCanBeRecreatedWithSameId() { @@ -131,7 +128,7 @@ private void assertTemplateCanBeRecreatedWithSameId() { as(SUPER_ADMIN).createOrUpdateTemplate(templateId, createDefinition); /* test 6 - template is recreated */ - assertThat(as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId)).isNotNull(); + executeResilient(() -> assertThat(as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId)).isNotNull()); } private void assertTemplateCanBeDeletedAndAssignmentIsPurged() { @@ -139,25 +136,19 @@ private void assertTemplateCanBeDeletedAndAssignmentIsPurged() { as(SUPER_ADMIN).deleteTemplate(templateId); /* test 5.1 check delete unassigns template */ - ProjectDetailInformation info = as(SUPER_ADMIN).fetchProjectDetailInformation(Scenario1.PROJECT_1); - assertThat(info.getTemplates()).contains(templateId); + executeResilient(() -> assertThat(as(SUPER_ADMIN).fetchProjectDetailInformation(Scenario1.PROJECT_1).getTemplates()).contains(templateId)); /* test 5.2 check template no longer exists */ - assertThat(as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId)).isNull(); + executeResilient(() -> assertThat(as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId)).isNull()); } private void assertTemplateCanBeUnassignedFromProject() { - List configurations; /* execute 4 - unassign */ as(SUPER_ADMIN).unassignTemplateFromProject(templateId, Scenario1.PROJECT_1); /* test 4 - check assignment */ - ProjectDetailInformation info; - info = as(SUPER_ADMIN).fetchProjectDetailInformation(Scenario1.PROJECT_1); - assertThat(info.getTemplates()).isEmpty(); - - configurations = fetchScanProjectConfigurations(Scenario1.PROJECT_1); - assertThat(configurations).isEmpty(); + executeResilient(() -> assertThat(as(SUPER_ADMIN).fetchProjectDetailInformation(Scenario1.PROJECT_1).getTemplates()).isEmpty()); + executeResilient(() -> assertThat(fetchScanProjectConfigurations(Scenario1.PROJECT_1)).isEmpty()); } private void assertTemplateCanBeAssignedToProject() { @@ -166,14 +157,15 @@ private void assertTemplateCanBeAssignedToProject() { as(SUPER_ADMIN).assignTemplateToProject(templateId, Scenario1.PROJECT_1); /* test 3.1 - check assignment by project details in domain administration */ - ProjectDetailInformation info = as(SUPER_ADMIN).fetchProjectDetailInformation(Scenario1.PROJECT_1); - assertThat(info.getTemplates()).contains(templateId); + executeResilient(() -> assertThat(as(SUPER_ADMIN).fetchProjectDetailInformation(Scenario1.PROJECT_1).getTemplates()).contains(templateId)); /* test 3.2 - check project scan configuration in domain scan */ - List configurations = fetchScanProjectConfigurations(Scenario1.PROJECT_1); - assertThat(configurations).isNotEmpty().hasSize(1) - .contains(new ScanProjectConfig(ScanProjectConfigID.TEMPLATE_WEBSCAN_LOGIN, Scenario1.PROJECT_1.getProjectId())); - assertThat(configurations.iterator().next().getData()).isEqualTo(templateId); + executeResilient(() -> { + List configurations = fetchScanProjectConfigurations(Scenario1.PROJECT_1); + assertThat(configurations).isNotEmpty().hasSize(1) + .contains(new ScanProjectConfig(ScanProjectConfigID.TEMPLATE_WEBSCAN_LOGIN, Scenario1.PROJECT_1.getProjectId())); + assertThat(configurations.iterator().next().getData()).isEqualTo(templateId); + }); } private void assertTemplateCanBeUpdated() { @@ -184,10 +176,12 @@ private void assertTemplateCanBeUpdated() { as(SUPER_ADMIN).createOrUpdateTemplate(templateId, updateDefinition); /* test 2 - update works */ - TemplateDefinition loadedTemplate = as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId); - assertThat(loadedTemplate.getAssetId()).isEqualTo("asset2"); - assertThat(loadedTemplate.getType()).isEqualTo(TemplateType.WEBSCAN_LOGIN); - assertThat(loadedTemplate.getId()).isEqualTo(templateId); + executeResilient(() -> { + TemplateDefinition loadedTemplate = as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId); + assertThat(loadedTemplate.getAssetId()).isEqualTo("asset2"); + assertThat(loadedTemplate.getType()).isEqualTo(TemplateType.WEBSCAN_LOGIN); + assertThat(loadedTemplate.getId()).isEqualTo(templateId); + }); } private void assertTemplateCanBeCreated() { @@ -195,8 +189,8 @@ private void assertTemplateCanBeCreated() { as(SUPER_ADMIN).createOrUpdateTemplate(templateId, createDefinition); /* test 1 - created definition has content as expected and contains id */ - TemplateDefinition loadedTemplate = as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId); - assertThat(loadedTemplate.toFormattedJSON()).isEqualTo(definitionWithId.toFormattedJSON()); + executeResilient( + () -> assertThat(as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId).toFormattedJSON()).isEqualTo(definitionWithId.toFormattedJSON())); } }