Skip to content

Commit

Permalink
Made flaky integration test TemplateScenario1IntTest stable #3523
Browse files Browse the repository at this point in the history 
- introduced TestAPI.executeResilient which is a short term for
  executeRunnableAndAcceptAssertionsMaximumTimes(4, runnable, 500);
- all time relevant parts inside integration test
  TemplateScenario1IntTest do now use this for testing
  • Loading branch information
@de-jcup
de-jcup committed Nov 15, 2024
1 parent e346ecf commit 550b0b5
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 32 deletions.
11 changes: 11 additions & 0 deletions ...ub-integrationtest/src/main/java/com/mercedesbenz/sechub/integrationtest/api/TestAPI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -589,6 +589,17 @@ public static void executeUntilSuccessOrTimeout(TestExecutable testExecutable) {
return;
}

/**
* Tries to execute runnable with default maximum time and retry (4 times a 500
* milliseconds) Shortcut for
* <code>executeRunnableAndAcceptAssertionsMaximumTimes(4,runnable, 500);</code>
*
* @param runnable
*/
public static void executeResilient(Runnable runnable) {
executeRunnableAndAcceptAssertionsMaximumTimes(4, runnable, 500);
}

public static void executeRunnableAndAcceptAssertionsMaximumTimes(int tries, Runnable runnable, int millisBeforeNextRetry) {
executeCallableAndAcceptAssertionsMaximumTimes(tries, () -> {
runnable.run();
Expand Down
58 changes: 26 additions & 32 deletions ...test/java/com/mercedesbenz/sechub/integrationtest/scenario1/TemplateScenario1IntTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
import com.mercedesbenz.sechub.commons.model.template.TemplateDefinition;
import com.mercedesbenz.sechub.commons.model.template.TemplateDefinition.TemplateVariable;
import com.mercedesbenz.sechub.commons.model.template.TemplateType;
import com.mercedesbenz.sechub.domain.administration.project.ProjectDetailInformation;
import com.mercedesbenz.sechub.domain.scan.project.ScanProjectConfig;
import com.mercedesbenz.sechub.domain.scan.project.ScanProjectConfigID;
import com.mercedesbenz.sechub.integrationtest.api.IntegrationTestSetup;
Expand Down Expand Up @@ -106,58 +105,50 @@ public void template_crud_test() {

private void assertTemplateNotInsideTemplateList() {
List<String> templateIds = as(SUPER_ADMIN).fetchTemplateList();
assertThat(templateIds).doesNotContain(templateId);
executeResilient(() -> assertThat(templateIds).doesNotContain(templateId));
}

private void assertTemplateExistsInTemplateListAndCanBeFetched() {
List<String> templateIds;
// check template list still contains the test templte */
templateIds = as(SUPER_ADMIN).fetchTemplateList();
assertThat(templateIds).contains(templateId);
assertThat(as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId)).isNotNull();
// check template list still contains the test template */
executeResilient(() -> assertThat(as(SUPER_ADMIN).fetchTemplateList()).contains(templateId));
executeResilient(() -> assertThat(as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId)).isNotNull());
}

private void assertProjectDeleteDoesPurgeTemplateAssignment() {
/* execute 7 - delete project */
as(SUPER_ADMIN).deleteProject(Scenario1.PROJECT_1);

/* test 7 - configuration for project is removed */
List<ScanProjectConfig> configurations = fetchScanProjectConfigurations(Scenario1.PROJECT_1);
assertThat(configurations).isEmpty();
executeResilient(() -> assertThat(fetchScanProjectConfigurations(Scenario1.PROJECT_1)).isEmpty());

}

private void assertTemplateCanBeRecreatedWithSameId() {
/* execute 6 - create template with same id again */
as(SUPER_ADMIN).createOrUpdateTemplate(templateId, createDefinition);

/* test 6 - template is recreated */
assertThat(as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId)).isNotNull();
executeResilient(() -> assertThat(as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId)).isNotNull());
}

private void assertTemplateCanBeDeletedAndAssignmentIsPurged() {
/* execute 5 - delete template */
as(SUPER_ADMIN).deleteTemplate(templateId);

/* test 5.1 check delete unassigns template */
ProjectDetailInformation info = as(SUPER_ADMIN).fetchProjectDetailInformation(Scenario1.PROJECT_1);
assertThat(info.getTemplates()).contains(templateId);
executeResilient(() -> assertThat(as(SUPER_ADMIN).fetchProjectDetailInformation(Scenario1.PROJECT_1).getTemplates()).contains(templateId));

/* test 5.2 check template no longer exists */
assertThat(as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId)).isNull();
executeResilient(() -> assertThat(as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId)).isNull());
}

private void assertTemplateCanBeUnassignedFromProject() {
List<ScanProjectConfig> configurations;
/* execute 4 - unassign */
as(SUPER_ADMIN).unassignTemplateFromProject(templateId, Scenario1.PROJECT_1);

/* test 4 - check assignment */
ProjectDetailInformation info;
info = as(SUPER_ADMIN).fetchProjectDetailInformation(Scenario1.PROJECT_1);
assertThat(info.getTemplates()).isEmpty();

configurations = fetchScanProjectConfigurations(Scenario1.PROJECT_1);
assertThat(configurations).isEmpty();
executeResilient(() -> assertThat(as(SUPER_ADMIN).fetchProjectDetailInformation(Scenario1.PROJECT_1).getTemplates()).isEmpty());
executeResilient(() -> assertThat(fetchScanProjectConfigurations(Scenario1.PROJECT_1)).isEmpty());
}

private void assertTemplateCanBeAssignedToProject() {
Expand All @@ -166,14 +157,15 @@ private void assertTemplateCanBeAssignedToProject() {
as(SUPER_ADMIN).assignTemplateToProject(templateId, Scenario1.PROJECT_1);

/* test 3.1 - check assignment by project details in domain administration */
ProjectDetailInformation info = as(SUPER_ADMIN).fetchProjectDetailInformation(Scenario1.PROJECT_1);
assertThat(info.getTemplates()).contains(templateId);
executeResilient(() -> assertThat(as(SUPER_ADMIN).fetchProjectDetailInformation(Scenario1.PROJECT_1).getTemplates()).contains(templateId));

/* test 3.2 - check project scan configuration in domain scan */
List<ScanProjectConfig> configurations = fetchScanProjectConfigurations(Scenario1.PROJECT_1);
assertThat(configurations).isNotEmpty().hasSize(1)
.contains(new ScanProjectConfig(ScanProjectConfigID.TEMPLATE_WEBSCAN_LOGIN, Scenario1.PROJECT_1.getProjectId()));
assertThat(configurations.iterator().next().getData()).isEqualTo(templateId);
executeResilient(() -> {
List<ScanProjectConfig> configurations = fetchScanProjectConfigurations(Scenario1.PROJECT_1);
assertThat(configurations).isNotEmpty().hasSize(1)
.contains(new ScanProjectConfig(ScanProjectConfigID.TEMPLATE_WEBSCAN_LOGIN, Scenario1.PROJECT_1.getProjectId()));
assertThat(configurations.iterator().next().getData()).isEqualTo(templateId);
});
}

private void assertTemplateCanBeUpdated() {
Expand All @@ -184,19 +176,21 @@ private void assertTemplateCanBeUpdated() {
as(SUPER_ADMIN).createOrUpdateTemplate(templateId, updateDefinition);

/* test 2 - update works */
TemplateDefinition loadedTemplate = as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId);
assertThat(loadedTemplate.getAssetId()).isEqualTo("asset2");
assertThat(loadedTemplate.getType()).isEqualTo(TemplateType.WEBSCAN_LOGIN);
assertThat(loadedTemplate.getId()).isEqualTo(templateId);
executeResilient(() -> {
TemplateDefinition loadedTemplate = as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId);
assertThat(loadedTemplate.getAssetId()).isEqualTo("asset2");
assertThat(loadedTemplate.getType()).isEqualTo(TemplateType.WEBSCAN_LOGIN);
assertThat(loadedTemplate.getId()).isEqualTo(templateId);
});
}

private void assertTemplateCanBeCreated() {
/* execute 1 - create */
as(SUPER_ADMIN).createOrUpdateTemplate(templateId, createDefinition);

/* test 1 - created definition has content as expected and contains id */
TemplateDefinition loadedTemplate = as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId);
assertThat(loadedTemplate.toFormattedJSON()).isEqualTo(definitionWithId.toFormattedJSON());
executeResilient(
() -> assertThat(as(SUPER_ADMIN).fetchTemplateDefinitionOrNull(templateId).toFormattedJSON()).isEqualTo(definitionWithId.toFormattedJSON()));
}

}

0 comments on commit 550b0b5

Please sign in to comment.