Parallelize tests by filename

README.md

@@ -56,15 +56,19 @@ The following is the list of environment variables that controls the behavior
 of running the test suites:
 
 ```bash
-SKIP_TEARDOWN=yes # don't remove the testing namespace
-TEST_NAMESPACE=test # k8s namespace to use
+SKIP_TEARDOWN=yes # don't remove the testing namespace(s)
+TEST_NAMESPACE=test # k8s namespace to use, setting this env to any value forces tests to run in sequence
 # HELM_VALUES_EXTRA_FILE is a default file containing global helm
 # options that can be optionally applied on helm install/upgrade
 # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl
 # if not passed.
 HELM_VALUES_EXTRA_FILE=./path/to/your/default/values.yaml
 # The BATS formatter to use: https://bats-core.readthedocs.io/en/stable/usage.html
 BATS_FORMATTER=tap
+#BATS_JOBS_PARAMS (default below) will run 8 tests in parallel,
+# this can be disabled by changing this variable OR by setting a TEST_NAMESPACE env
+# which will force jobs to run in sequence
+BATS_JOBS_PARAMS="--jobs 8 --no-parallelize-within-files"
 # The Fluent Bit image to use
 FLUENTBIT_IMAGE_REPOSITORY=ghcr.io/fluent/fluent-bit
 FLUENTBIT_IMAGE_TAG=latest

helpers/test-helpers.bash

@@ -100,4 +100,17 @@ function wait_for_url() {
     fi
     # shellcheck disable=SC2086
     wait_for_curl "$MAX_ATTEMPTS" "$URL" $extra_args
+}
+
+function create_helm_extra_values_file() {
+    # HELM_VALUES_EXTRA_FILE is a default file containing global helm
+    # options that can be optionally applied on helm install/upgrade
+    # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl
+    # if not passed.
+    if [ -e  "${HELM_VALUES_EXTRA_FILE}" ]; then
+      # shellcheck disable=SC2086,SC2155
+      envsubst < "${HELM_VALUES_EXTRA_FILE}" > "$(dirname ${HELM_VALUES_EXTRA_FILE})/${TEST_NAMESPACE}.values.yaml"
+      # shellcheck disable=SC2086,SC2155
+      export HELM_VALUES_EXTRA_FILE="$(dirname ${HELM_VALUES_EXTRA_FILE})/${TEST_NAMESPACE}.values.yaml"
+    fi
 }

run-tests.sh

@@ -25,12 +25,12 @@ export RESOURCES_ROOT="${SCRIPT_DIR}/resources/"
 export BATS_FORMATTER=${BATS_FORMATTER:-tap}
 export BATS_ROOT=${BATS_ROOT:-$SCRIPT_DIR/tools/bats}
 export BATS_ARGS=${BATS_ARGS:---timing --verbose-run}
+export BATS_JOBS_PARAMS=${BATS_JOBS_PARAMS:---jobs 8 --no-parallelize-within-files}
 
 export BATS_FILE_ROOT=$BATS_ROOT/lib/bats-file
 export BATS_SUPPORT_ROOT=$BATS_ROOT/lib/bats-support
 export BATS_ASSERT_ROOT=$BATS_ROOT/lib/bats-assert
 export BATS_DETIK_ROOT=$BATS_ROOT/lib/bats-detik
-export TEST_NAMESPACE=${TEST_NAMESPACE:-test}
 
 export FLUENTBIT_IMAGE_REPOSITORY=${FLUENTBIT_IMAGE_REPOSITORY:-ghcr.io/fluent/fluent-bit}
 export FLUENTBIT_IMAGE_TAG=${FLUENTBIT_IMAGE_TAG:-latest}
@@ -79,11 +79,18 @@ function run_tests() {
     echo
     echo
 
+    # If TEST_NAMESPACE is not set (the default), we run jobs in parallel
+    # otherwise, if it is set we can only run 1 job at a time and need to remove
+    # all BATS_JOBS_PARAMS
+    if [[ -n "${TEST_NAMESPACE:-}" ]]; then
+        BATS_JOBS_PARAMS=""
+    fi
+
     # We run BATS in a subshell to prevent it from inheriting our exit/err trap, which can mess up its internals
     # We set +exu because unbound variables can cause test failures with zero context
     set +xeu
     # shellcheck disable=SC2086
-    (bats --formatter "${BATS_FORMATTER}" $run $BATS_ARGS)
+    (bats $BATS_JOBS_PARAMS --formatter "${BATS_FORMATTER}" $run $BATS_ARGS)
     local bats_retval=$?
 
     echo

tests/chunk-rollover/basic.bats

@@ -2,27 +2,21 @@
 
 load "$HELPERS_ROOT/test-helpers.bash"
 
-ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG ELASTICSEARCH_IMAGE_REPOSITORY ELASTICSEARCH_IMAGE_TAG
+ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG ELASTICSEARCH_IMAGE_REPOSITORY ELASTICSEARCH_IMAGE_TAG
 
 load "$BATS_DETIK_ROOT/utils.bash"
 load "$BATS_DETIK_ROOT/linter.bash"
 load "$BATS_DETIK_ROOT/detik.bash"
 load "$BATS_SUPPORT_ROOT/load.bash"
 load "$BATS_ASSERT_ROOT/load.bash"
 load "$BATS_FILE_ROOT/load.bash"
-
+ 
 setup_file() {
+    export TEST_NAMESPACE=${TEST_NAMESPACE:-chunk-rollover-basic}
     echo "recreating namespace $TEST_NAMESPACE"
     run kubectl delete namespace "$TEST_NAMESPACE"
     run kubectl create namespace "$TEST_NAMESPACE"
-    # HELM_VALUES_EXTRA_FILE is a default file containing global helm
-    # options that can be optionally applied on helm install/upgrade
-    # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl
-    # if not passed.
-    if [ -e  "${HELM_VALUES_EXTRA_FILE}" ]; then
-      envsubst < "${HELM_VALUES_EXTRA_FILE}" > "${HELM_VALUES_EXTRA_FILE%.*}"
-      export HELM_VALUES_EXTRA_FILE="${HELM_VALUES_EXTRA_FILE%.*}"
-    fi
+    create_helm_extra_values_file
 }
 
 teardown_file() {
@@ -31,6 +25,7 @@ teardown_file() {
         run kubectl delete namespace "$TEST_NAMESPACE"
         rm -f ${HELM_VALUES_EXTRA_FILE}
     fi
+    unset TEST_NAMESPACE
 }
 
 function teardown() {

tests/chunk-rollover/resources/helm/fluentbit-basic.yaml

@@ -7,9 +7,6 @@ extraVolumes:
     sizeLimit: 3Mi
 rbac:
   create: true
-# fullnameOverride is required so resources don't conflict if we are running
-# in a hosted cluster like gke where the default resource names will already exist
-fullnameOverride: fluentbit-ci-tests
 
 config:
   service: |

tests/defaults/values.yaml.tpl

@@ -1,2 +1,3 @@
 image:
   pullPolicy: ${HELM_IMAGE_PULL_POLICY}
+fullnameOverride: fluent-bit-ci-test-${TEST_NAMESPACE}

tests/elasticsearch/basic.bats

@@ -2,7 +2,7 @@
 
 load "$HELPERS_ROOT/test-helpers.bash"
 
-ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG ELASTICSEARCH_IMAGE_REPOSITORY ELASTICSEARCH_IMAGE_TAG
+ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG ELASTICSEARCH_IMAGE_REPOSITORY ELASTICSEARCH_IMAGE_TAG
 
 load "$BATS_DETIK_ROOT/utils.bash"
 load "$BATS_DETIK_ROOT/linter.bash"
@@ -12,17 +12,11 @@ load "$BATS_ASSERT_ROOT/load.bash"
 load "$BATS_FILE_ROOT/load.bash"
 
 setup_file() {
+    export TEST_NAMESPACE=${TEST_NAMESPACE:-elasticsearch-basic}
     echo "recreating namespace $TEST_NAMESPACE"
     run kubectl delete namespace "$TEST_NAMESPACE"
     run kubectl create namespace "$TEST_NAMESPACE"
-    # HELM_VALUES_EXTRA_FILE is a default file containing global helm
-    # options that can be optionally applied on helm install/upgrade
-    # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl
-    # if not passed.
-    if [ -e  "${HELM_VALUES_EXTRA_FILE}" ]; then
-      envsubst < "${HELM_VALUES_EXTRA_FILE}" > "${HELM_VALUES_EXTRA_FILE%.*}"
-      export HELM_VALUES_EXTRA_FILE="${HELM_VALUES_EXTRA_FILE%.*}"
-    fi
+    create_helm_extra_values_file
 }
 
 teardown_file() {
@@ -32,6 +26,7 @@ teardown_file() {
         run kubectl delete namespace "$TEST_NAMESPACE"
         rm -f ${HELM_VALUES_EXTRA_FILE}
     fi
+    unset TEST_NAMESPACE
 }
 
 # These are required for bats-detik
@@ -49,7 +44,6 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}"
     helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" elasticsearch elastic/elasticsearch \
         --values ${BATS_TEST_DIRNAME}/resources/helm/elasticsearch-basic.yaml \
         --set image=${ELASTICSEARCH_IMAGE_REPOSITORY} --set imageTag=${ELASTICSEARCH_IMAGE_TAG} \
-        --values "$HELM_VALUES_EXTRA_FILE" \
         --timeout "${HELM_DEFAULT_TIMEOUT:-10m0s}" \
         --wait
 

tests/elasticsearch/compress.bats

@@ -2,7 +2,7 @@
 
 load "$HELPERS_ROOT/test-helpers.bash"
 
-ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG ELASTICSEARCH_IMAGE_REPOSITORY ELASTICSEARCH_IMAGE_TAG
+ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG ELASTICSEARCH_IMAGE_REPOSITORY ELASTICSEARCH_IMAGE_TAG
 
 load "$BATS_DETIK_ROOT/utils.bash"
 load "$BATS_DETIK_ROOT/linter.bash"
@@ -12,17 +12,11 @@ load "$BATS_ASSERT_ROOT/load.bash"
 load "$BATS_FILE_ROOT/load.bash"
 
 setup_file() {
+    export TEST_NAMESPACE=${TEST_NAMESPACE:-elasticsearch-compress}
     echo "recreating namespace $TEST_NAMESPACE"
     run kubectl delete namespace "$TEST_NAMESPACE"
     run kubectl create namespace "$TEST_NAMESPACE"
-    # HELM_VALUES_EXTRA_FILE is a default file containing global helm
-    # options that can be optionally applied on helm install/upgrade
-    # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl
-    # if not passed.
-    if [ -e  "${HELM_VALUES_EXTRA_FILE}" ]; then
-      envsubst < "${HELM_VALUES_EXTRA_FILE}" > "${HELM_VALUES_EXTRA_FILE%.*}"
-      export HELM_VALUES_EXTRA_FILE="${HELM_VALUES_EXTRA_FILE%.*}"
-    fi
+    create_helm_extra_values_file
 }
 
 teardown_file() {
@@ -32,6 +26,7 @@ teardown_file() {
         run kubectl delete namespace "$TEST_NAMESPACE"
         rm -f ${HELM_VALUES_EXTRA_FILE}
     fi
+    unset TEST_NAMESPACE
 }
 
 # These are required for bats-detik
@@ -49,7 +44,6 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}"
     helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" elasticsearch elastic/elasticsearch \
         --values ${BATS_TEST_DIRNAME}/resources/helm/elasticsearch-compress.yaml \
         --set image=${ELASTICSEARCH_IMAGE_REPOSITORY} --set imageTag=${ELASTICSEARCH_IMAGE_TAG} \
-        --values "$HELM_VALUES_EXTRA_FILE" \
         --timeout "${HELM_DEFAULT_TIMEOUT:-10m0s}" \
         --wait
 

tests/kubernetes-plugins/basic.bats

@@ -2,7 +2,7 @@
 
 load "$HELPERS_ROOT/test-helpers.bash"
 
-ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG
+ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG
 
 load "$BATS_DETIK_ROOT/utils.bash"
 load "$BATS_DETIK_ROOT/linter.bash"
@@ -20,17 +20,11 @@ FLUENTBIT_POD_NAME=""
 TEST_POD_NAME=""
 
 setup_file() {
+    export TEST_NAMESPACE=${TEST_NAMESPACE:-kubernetes-plugins-basic}
     echo "recreating namespace $TEST_NAMESPACE"
     run kubectl delete namespace "$TEST_NAMESPACE"
     run kubectl create namespace "$TEST_NAMESPACE"
-    # HELM_VALUES_EXTRA_FILE is a default file containing global helm
-    # options that can be optionally applied on helm install/upgrade
-    # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl
-    # if not passed.
-    if [ -e  "${HELM_VALUES_EXTRA_FILE}" ]; then
-      envsubst < "${HELM_VALUES_EXTRA_FILE}" > "${HELM_VALUES_EXTRA_FILE%.*}"
-      export HELM_VALUES_EXTRA_FILE="${HELM_VALUES_EXTRA_FILE%.*}"
-    fi
+    create_helm_extra_values_file
 
     helm repo add fluent https://fluent.github.io/helm-charts/ || helm repo add fluent https://fluent.github.io/helm-charts
     helm repo update --fail-on-repo-update-fail
@@ -50,6 +44,7 @@ teardown_file() {
         run kubectl delete namespace "$TEST_NAMESPACE"
         rm -f ${HELM_VALUES_EXTRA_FILE}
     fi
+    unset TEST_NAMESPACE
 }
 
 setup() {
@@ -68,8 +63,8 @@ teardown() {
 
 function set_fluent_bit_pod_name() {
     try "at most 30 times every 2s " \
-        "to find 1 pods named 'fluentbit-ci-tests' " \
-        "with 'status' being 'Running'"
+        "to find 1 pods named 'fluent-bit' " \
+        "with 'status' being 'running'"
 
     FLUENTBIT_POD_NAME=$(kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/name=fluent-bit" --no-headers | awk '{ print $1 }')
     if [ -z "$FLUENTBIT_POD_NAME" ]; then

tests/kubernetes-plugins/full.bats

@@ -2,7 +2,7 @@
 
 load "$HELPERS_ROOT/test-helpers.bash"
 
-ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG
+ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG
 
 load "$BATS_DETIK_ROOT/utils.bash"
 load "$BATS_DETIK_ROOT/linter.bash"
@@ -20,14 +20,8 @@ FLUENTBIT_POD_NAME=""
 TEST_POD_NAME=""
 
 setup_file() {
-    # HELM_VALUES_EXTRA_FILE is a default file containing global helm
-    # options that can be optionally applied on helm install/upgrade
-    # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl
-    # if not passed.
-    if [ -e  "${HELM_VALUES_EXTRA_FILE}" ]; then
-      envsubst < "${HELM_VALUES_EXTRA_FILE}" > "${HELM_VALUES_EXTRA_FILE%.*}"
-      export HELM_VALUES_EXTRA_FILE="${HELM_VALUES_EXTRA_FILE%.*}"
-    fi
+    export TEST_NAMESPACE=${TEST_NAMESPACE:-kubernetes-plugins-full}
+    create_helm_extra_values_file
 
     # First check that we should run these conditional tests at all
     run docker run --rm -t $FLUENTBIT_IMAGE_REPOSITORY:$FLUENTBIT_IMAGE_TAG /fluent-bit/bin/fluent-bit -F kubernetes --help
@@ -60,6 +54,7 @@ teardown_file() {
         run kubectl delete namespace "$TEST_NAMESPACE"
         rm -f ${HELM_VALUES_EXTRA_FILE}
     fi
+    unset TEST_NAMESPACE
 }
 
 setup() {
@@ -77,9 +72,9 @@ teardown() {
 
 function set_fluent_bit_pod_name() {
     try "at most 30 times every 2s " \
-        "to find 1 pods named 'fluentbit-ci-tests' " \
-        "with 'status' being 'Running'"
-    
+        "to find 1 pods named 'fluent-bit' " \
+        "with 'status' being 'running'"
+
     FLUENTBIT_POD_NAME=$(kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/name=fluent-bit" --no-headers | awk '{ print $1 }')
     if [ -z "$FLUENTBIT_POD_NAME" ]; then
         fail "Unable to get running fluent-bit pod's name"

tests/kubernetes-plugins/resources/fluentbit-basic.yaml

@@ -1,8 +1,5 @@
 kind: Deployment
 replicaCount: 1
-# fullnameOverride is required so resources don't conflict if we are running
-# in a hosted cluster like gke where the default resource names will already exist
-fullnameOverride: fluentbit-ci-tests
 rbac:
   create: true
 extraVolumeMounts:

tests/kubernetes-plugins/resources/fluentbit-full.yaml

@@ -1,8 +1,5 @@
 kind: Deployment
 replicaCount: 1
-# fullnameOverride is required so resources don't conflict if we are running
-# in a hosted cluster like gke where the default resource names will already exist
-fullnameOverride: fluentbit-ci-tests
 rbac:
   create: true
   nodeAccess: true

tests/opensearch/basic.bats

@@ -2,7 +2,7 @@
 
 load "$HELPERS_ROOT/test-helpers.bash"
 
-ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG OPENSEARCH_IMAGE_REPOSITORY OPENSEARCH_IMAGE_TAG
+ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG OPENSEARCH_IMAGE_REPOSITORY OPENSEARCH_IMAGE_TAG
 
 load "$BATS_DETIK_ROOT/utils.bash"
 load "$BATS_DETIK_ROOT/linter.bash"
@@ -12,17 +12,11 @@ load "$BATS_ASSERT_ROOT/load.bash"
 load "$BATS_FILE_ROOT/load.bash"
 
 function setup_file() {
+    export TEST_NAMESPACE=${TEST_NAMESPACE:-opensearch-basic}
     echo "recreating namespace $TEST_NAMESPACE"
     run kubectl delete namespace "$TEST_NAMESPACE"
     run kubectl create namespace "$TEST_NAMESPACE"
-    # HELM_VALUES_EXTRA_FILE is a default file containing global helm
-    # options that can be optionally applied on helm install/upgrade
-    # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl
-    # if not passed.
-    if [ -e  "${HELM_VALUES_EXTRA_FILE}" ]; then
-      envsubst < "${HELM_VALUES_EXTRA_FILE}" > "${HELM_VALUES_EXTRA_FILE%.*}"
-      export HELM_VALUES_EXTRA_FILE="${HELM_VALUES_EXTRA_FILE%.*}"
-    fi
+    create_helm_extra_values_file
 }
 
 function teardown_file() {
@@ -32,6 +26,7 @@ function teardown_file() {
         run kubectl delete namespace "$TEST_NAMESPACE"
         rm -f ${HELM_VALUES_EXTRA_FILE}
     fi
+    unset TEST_NAMESPACE
 }
 
 function teardown() {
@@ -66,7 +61,6 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}"
     helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" opensearch opensearch/opensearch \
         --values ${BATS_TEST_DIRNAME}/resources/helm/opensearch-basic.yaml \
         --set image.repository=${OPENSEARCH_IMAGE_REPOSITORY},image.tag=${OPENSEARCH_IMAGE_TAG} \
-        --values "$HELM_VALUES_EXTRA_FILE" \
         --timeout "${HELM_DEFAULT_TIMEOUT:-10m0s}" \
         --wait
 
@@ -85,7 +79,7 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}"
     try "at most 15 times every 2s " \
         "to find 1 pods named 'fluent-bit' " \
         "with 'status' being 'running'"
-
+        
     attempt=0
     while true; do
     	run kubectl exec -q -n $TEST_NAMESPACE opensearch-cluster-master-0 -- curl --insecure -s -w "%{http_code}" https://admin:admin@localhost:9200/fluentbit/_search/ -o /dev/null