From ae75b34a80da2947c9b0f810ee302d2d37086a8b Mon Sep 17 00:00:00 2001 From: ryanohnemus Date: Sun, 28 Jan 2024 14:27:31 -0600 Subject: [PATCH 1/5] parallelize tests by filename Signed-off-by: ryanohnemus --- helpers/test-helpers.bash | 11 +++++++++++ run-tests.sh | 10 ++++++++-- tests/chunk-rollover/basic.bats | 15 +++++---------- .../resources/helm/fluentbit-basic.yaml | 3 --- tests/defaults/values.yaml.tpl | 1 + tests/elasticsearch/basic.bats | 18 +++++------------- tests/elasticsearch/compress.bats | 18 +++++------------- tests/kubernetes-plugins/basic.bats | 17 +++++------------ tests/kubernetes-plugins/full.bats | 17 +++++------------ .../resources/fluentbit-basic.yaml | 3 --- .../resources/fluentbit-full.yaml | 3 --- tests/opensearch/basic.bats | 18 +++++------------- tests/opensearch/hosted.bats | 17 +++++------------ 13 files changed, 55 insertions(+), 96 deletions(-) diff --git a/helpers/test-helpers.bash b/helpers/test-helpers.bash index 93140ce..41cccd7 100755 --- a/helpers/test-helpers.bash +++ b/helpers/test-helpers.bash @@ -100,4 +100,15 @@ function wait_for_url() { fi # shellcheck disable=SC2086 wait_for_curl "$MAX_ATTEMPTS" "$URL" $extra_args +} + +function create_helm_extra_values_file() { + # HELM_VALUES_EXTRA_FILE is a default file containing global helm + # options that can be optionally applied on helm install/upgrade + # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl + # if not passed. + if [ -e "${HELM_VALUES_EXTRA_FILE}" ]; then + envsubst < "${HELM_VALUES_EXTRA_FILE}" > "$(dirname ${HELM_VALUES_EXTRA_FILE})/${TEST_NAMESPACE}.values.yaml" + export HELM_VALUES_EXTRA_FILE="$(dirname ${HELM_VALUES_EXTRA_FILE})/${TEST_NAMESPACE}.values.yaml" + fi } \ No newline at end of file diff --git a/run-tests.sh b/run-tests.sh index 55ca010..e7bb3b4 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -30,7 +30,6 @@ export BATS_FILE_ROOT=$BATS_ROOT/lib/bats-file export BATS_SUPPORT_ROOT=$BATS_ROOT/lib/bats-support export BATS_ASSERT_ROOT=$BATS_ROOT/lib/bats-assert export BATS_DETIK_ROOT=$BATS_ROOT/lib/bats-detik -export TEST_NAMESPACE=${TEST_NAMESPACE:-test} export FLUENTBIT_IMAGE_REPOSITORY=${FLUENTBIT_IMAGE_REPOSITORY:-ghcr.io/fluent/fluent-bit} export FLUENTBIT_IMAGE_TAG=${FLUENTBIT_IMAGE_TAG:-latest} @@ -79,11 +78,18 @@ function run_tests() { echo echo + # If TEST_NAMESPACE is not set (the default), we run 8 jobs in parallel + # otherwise, if it is set we can only run 1 job at a time + BATS_JOBS_PARAMS="" + if [[ -z "$TEST_NAMESPACE" ]]; then + BATS_JOBS_PARAMS="--jobs $BATS_JOBS_COUNT --no-parallelize-within-files" + fi + # We run BATS in a subshell to prevent it from inheriting our exit/err trap, which can mess up its internals # We set +exu because unbound variables can cause test failures with zero context set +xeu # shellcheck disable=SC2086 - (bats --formatter "${BATS_FORMATTER}" $run $BATS_ARGS) + (bats $BATS_JOBS_PARAMS --formatter "${BATS_FORMATTER}" $run $BATS_ARGS) local bats_retval=$? echo diff --git a/tests/chunk-rollover/basic.bats b/tests/chunk-rollover/basic.bats index fda60b5..c0f7556 100644 --- a/tests/chunk-rollover/basic.bats +++ b/tests/chunk-rollover/basic.bats @@ -2,7 +2,7 @@ load "$HELPERS_ROOT/test-helpers.bash" -ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG ELASTICSEARCH_IMAGE_REPOSITORY ELASTICSEARCH_IMAGE_TAG +ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG ELASTICSEARCH_IMAGE_REPOSITORY ELASTICSEARCH_IMAGE_TAG load "$BATS_DETIK_ROOT/utils.bash" load "$BATS_DETIK_ROOT/linter.bash" @@ -10,19 +10,13 @@ load "$BATS_DETIK_ROOT/detik.bash" load "$BATS_SUPPORT_ROOT/load.bash" load "$BATS_ASSERT_ROOT/load.bash" load "$BATS_FILE_ROOT/load.bash" - + setup_file() { + export TEST_NAMESPACE=${TEST_NAMESPACE:-chunk-rollover-basic} echo "recreating namespace $TEST_NAMESPACE" run kubectl delete namespace "$TEST_NAMESPACE" run kubectl create namespace "$TEST_NAMESPACE" - # HELM_VALUES_EXTRA_FILE is a default file containing global helm - # options that can be optionally applied on helm install/upgrade - # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl - # if not passed. - if [ -e "${HELM_VALUES_EXTRA_FILE}" ]; then - envsubst < "${HELM_VALUES_EXTRA_FILE}" > "${HELM_VALUES_EXTRA_FILE%.*}" - export HELM_VALUES_EXTRA_FILE="${HELM_VALUES_EXTRA_FILE%.*}" - fi + create_helm_extra_values_file } teardown_file() { @@ -31,6 +25,7 @@ teardown_file() { run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi + unset TEST_NAMESPACE } function teardown() { diff --git a/tests/chunk-rollover/resources/helm/fluentbit-basic.yaml b/tests/chunk-rollover/resources/helm/fluentbit-basic.yaml index ec83514..2fcc06a 100644 --- a/tests/chunk-rollover/resources/helm/fluentbit-basic.yaml +++ b/tests/chunk-rollover/resources/helm/fluentbit-basic.yaml @@ -7,9 +7,6 @@ extraVolumes: sizeLimit: 3Mi rbac: create: true -# fullnameOverride is required so resources don't conflict if we are running -# in a hosted cluster like gke where the default resource names will already exist -fullnameOverride: fluentbit-ci-tests config: service: | diff --git a/tests/defaults/values.yaml.tpl b/tests/defaults/values.yaml.tpl index 99546f1..7edebf9 100644 --- a/tests/defaults/values.yaml.tpl +++ b/tests/defaults/values.yaml.tpl @@ -1,2 +1,3 @@ image: pullPolicy: ${HELM_IMAGE_PULL_POLICY} +fullnameOverride: fluentbit-ci-test-${TEST_NAMESPACE} \ No newline at end of file diff --git a/tests/elasticsearch/basic.bats b/tests/elasticsearch/basic.bats index 05a3bff..67af29f 100644 --- a/tests/elasticsearch/basic.bats +++ b/tests/elasticsearch/basic.bats @@ -2,7 +2,7 @@ load "$HELPERS_ROOT/test-helpers.bash" -ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG ELASTICSEARCH_IMAGE_REPOSITORY ELASTICSEARCH_IMAGE_TAG +ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG ELASTICSEARCH_IMAGE_REPOSITORY ELASTICSEARCH_IMAGE_TAG load "$BATS_DETIK_ROOT/utils.bash" load "$BATS_DETIK_ROOT/linter.bash" @@ -12,17 +12,11 @@ load "$BATS_ASSERT_ROOT/load.bash" load "$BATS_FILE_ROOT/load.bash" setup_file() { + export TEST_NAMESPACE=${TEST_NAMESPACE:-elasticsearch-basic} echo "recreating namespace $TEST_NAMESPACE" run kubectl delete namespace "$TEST_NAMESPACE" run kubectl create namespace "$TEST_NAMESPACE" - # HELM_VALUES_EXTRA_FILE is a default file containing global helm - # options that can be optionally applied on helm install/upgrade - # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl - # if not passed. - if [ -e "${HELM_VALUES_EXTRA_FILE}" ]; then - envsubst < "${HELM_VALUES_EXTRA_FILE}" > "${HELM_VALUES_EXTRA_FILE%.*}" - export HELM_VALUES_EXTRA_FILE="${HELM_VALUES_EXTRA_FILE%.*}" - fi + create_helm_extra_values_file } teardown_file() { @@ -32,6 +26,7 @@ teardown_file() { run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi + unset TEST_NAMESPACE } # These are required for bats-detik @@ -49,7 +44,6 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" elasticsearch elastic/elasticsearch \ --values ${BATS_TEST_DIRNAME}/resources/helm/elasticsearch-basic.yaml \ --set image=${ELASTICSEARCH_IMAGE_REPOSITORY} --set imageTag=${ELASTICSEARCH_IMAGE_TAG} \ - --values "$HELM_VALUES_EXTRA_FILE" \ --timeout "${HELM_DEFAULT_TIMEOUT:-10m0s}" \ --wait @@ -64,9 +58,7 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" --timeout "${HELM_FB_TIMEOUT:-5m0s}" \ --wait - try "at most 15 times every 2s " \ - "to find 1 pods named 'fluent-bit' " \ - "with 'status' being 'running'" + kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s attempt=0 while true; do diff --git a/tests/elasticsearch/compress.bats b/tests/elasticsearch/compress.bats index ed8aaea..9acc4ef 100644 --- a/tests/elasticsearch/compress.bats +++ b/tests/elasticsearch/compress.bats @@ -2,7 +2,7 @@ load "$HELPERS_ROOT/test-helpers.bash" -ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG ELASTICSEARCH_IMAGE_REPOSITORY ELASTICSEARCH_IMAGE_TAG +ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG ELASTICSEARCH_IMAGE_REPOSITORY ELASTICSEARCH_IMAGE_TAG load "$BATS_DETIK_ROOT/utils.bash" load "$BATS_DETIK_ROOT/linter.bash" @@ -12,17 +12,11 @@ load "$BATS_ASSERT_ROOT/load.bash" load "$BATS_FILE_ROOT/load.bash" setup_file() { + export TEST_NAMESPACE=${TEST_NAMESPACE:-elasticsearch-compress} echo "recreating namespace $TEST_NAMESPACE" run kubectl delete namespace "$TEST_NAMESPACE" run kubectl create namespace "$TEST_NAMESPACE" - # HELM_VALUES_EXTRA_FILE is a default file containing global helm - # options that can be optionally applied on helm install/upgrade - # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl - # if not passed. - if [ -e "${HELM_VALUES_EXTRA_FILE}" ]; then - envsubst < "${HELM_VALUES_EXTRA_FILE}" > "${HELM_VALUES_EXTRA_FILE%.*}" - export HELM_VALUES_EXTRA_FILE="${HELM_VALUES_EXTRA_FILE%.*}" - fi + create_helm_extra_values_file } teardown_file() { @@ -32,6 +26,7 @@ teardown_file() { run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi + unset TEST_NAMESPACE } # These are required for bats-detik @@ -49,7 +44,6 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" elasticsearch elastic/elasticsearch \ --values ${BATS_TEST_DIRNAME}/resources/helm/elasticsearch-compress.yaml \ --set image=${ELASTICSEARCH_IMAGE_REPOSITORY} --set imageTag=${ELASTICSEARCH_IMAGE_TAG} \ - --values "$HELM_VALUES_EXTRA_FILE" \ --timeout "${HELM_DEFAULT_TIMEOUT:-10m0s}" \ --wait @@ -64,9 +58,7 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" --timeout "${HELM_DEFAULT_TIMEOUT:-10m0s}" \ --wait - try "at most 15 times every 2s " \ - "to find 1 pods named 'fluent-bit' " \ - "with 'status' being 'running'" + kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s attempt=0 while true; do diff --git a/tests/kubernetes-plugins/basic.bats b/tests/kubernetes-plugins/basic.bats index 78198d6..359e3b4 100644 --- a/tests/kubernetes-plugins/basic.bats +++ b/tests/kubernetes-plugins/basic.bats @@ -2,7 +2,7 @@ load "$HELPERS_ROOT/test-helpers.bash" -ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG +ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG load "$BATS_DETIK_ROOT/utils.bash" load "$BATS_DETIK_ROOT/linter.bash" @@ -20,17 +20,11 @@ FLUENTBIT_POD_NAME="" TEST_POD_NAME="" setup_file() { + export TEST_NAMESPACE=${TEST_NAMESPACE:-kubernetes-plugins-basic} echo "recreating namespace $TEST_NAMESPACE" run kubectl delete namespace "$TEST_NAMESPACE" run kubectl create namespace "$TEST_NAMESPACE" - # HELM_VALUES_EXTRA_FILE is a default file containing global helm - # options that can be optionally applied on helm install/upgrade - # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl - # if not passed. - if [ -e "${HELM_VALUES_EXTRA_FILE}" ]; then - envsubst < "${HELM_VALUES_EXTRA_FILE}" > "${HELM_VALUES_EXTRA_FILE%.*}" - export HELM_VALUES_EXTRA_FILE="${HELM_VALUES_EXTRA_FILE%.*}" - fi + create_helm_extra_values_file helm repo add fluent https://fluent.github.io/helm-charts/ || helm repo add fluent https://fluent.github.io/helm-charts helm repo update --fail-on-repo-update-fail @@ -50,6 +44,7 @@ teardown_file() { run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi + unset TEST_NAMESPACE } setup() { @@ -67,9 +62,7 @@ teardown() { function set_fluent_bit_pod_name() { - try "at most 30 times every 2s " \ - "to find 1 pods named 'fluentbit-ci-tests' " \ - "with 'status' being 'Running'" + kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s FLUENTBIT_POD_NAME=$(kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/name=fluent-bit" --no-headers | awk '{ print $1 }') if [ -z "$FLUENTBIT_POD_NAME" ]; then diff --git a/tests/kubernetes-plugins/full.bats b/tests/kubernetes-plugins/full.bats index a2b35e1..7534bb1 100644 --- a/tests/kubernetes-plugins/full.bats +++ b/tests/kubernetes-plugins/full.bats @@ -2,7 +2,7 @@ load "$HELPERS_ROOT/test-helpers.bash" -ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG +ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG load "$BATS_DETIK_ROOT/utils.bash" load "$BATS_DETIK_ROOT/linter.bash" @@ -20,14 +20,8 @@ FLUENTBIT_POD_NAME="" TEST_POD_NAME="" setup_file() { - # HELM_VALUES_EXTRA_FILE is a default file containing global helm - # options that can be optionally applied on helm install/upgrade - # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl - # if not passed. - if [ -e "${HELM_VALUES_EXTRA_FILE}" ]; then - envsubst < "${HELM_VALUES_EXTRA_FILE}" > "${HELM_VALUES_EXTRA_FILE%.*}" - export HELM_VALUES_EXTRA_FILE="${HELM_VALUES_EXTRA_FILE%.*}" - fi + export TEST_NAMESPACE=${TEST_NAMESPACE:-kubernetes-plugins-full} + create_helm_extra_values_file # First check that we should run these conditional tests at all run docker run --rm -t $FLUENTBIT_IMAGE_REPOSITORY:$FLUENTBIT_IMAGE_TAG /fluent-bit/bin/fluent-bit -F kubernetes --help @@ -60,6 +54,7 @@ teardown_file() { run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi + unset TEST_NAMESPACE } setup() { @@ -76,9 +71,7 @@ teardown() { } function set_fluent_bit_pod_name() { - try "at most 30 times every 2s " \ - "to find 1 pods named 'fluentbit-ci-tests' " \ - "with 'status' being 'Running'" + kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s FLUENTBIT_POD_NAME=$(kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/name=fluent-bit" --no-headers | awk '{ print $1 }') if [ -z "$FLUENTBIT_POD_NAME" ]; then diff --git a/tests/kubernetes-plugins/resources/fluentbit-basic.yaml b/tests/kubernetes-plugins/resources/fluentbit-basic.yaml index d5f24e3..cc41fdf 100644 --- a/tests/kubernetes-plugins/resources/fluentbit-basic.yaml +++ b/tests/kubernetes-plugins/resources/fluentbit-basic.yaml @@ -1,8 +1,5 @@ kind: Deployment replicaCount: 1 -# fullnameOverride is required so resources don't conflict if we are running -# in a hosted cluster like gke where the default resource names will already exist -fullnameOverride: fluentbit-ci-tests rbac: create: true extraVolumeMounts: diff --git a/tests/kubernetes-plugins/resources/fluentbit-full.yaml b/tests/kubernetes-plugins/resources/fluentbit-full.yaml index d826277..8e60ff8 100644 --- a/tests/kubernetes-plugins/resources/fluentbit-full.yaml +++ b/tests/kubernetes-plugins/resources/fluentbit-full.yaml @@ -1,8 +1,5 @@ kind: Deployment replicaCount: 1 -# fullnameOverride is required so resources don't conflict if we are running -# in a hosted cluster like gke where the default resource names will already exist -fullnameOverride: fluentbit-ci-tests rbac: create: true nodeAccess: true diff --git a/tests/opensearch/basic.bats b/tests/opensearch/basic.bats index 3d9e74c..8b89fa8 100644 --- a/tests/opensearch/basic.bats +++ b/tests/opensearch/basic.bats @@ -2,7 +2,7 @@ load "$HELPERS_ROOT/test-helpers.bash" -ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG OPENSEARCH_IMAGE_REPOSITORY OPENSEARCH_IMAGE_TAG +ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG OPENSEARCH_IMAGE_REPOSITORY OPENSEARCH_IMAGE_TAG load "$BATS_DETIK_ROOT/utils.bash" load "$BATS_DETIK_ROOT/linter.bash" @@ -12,17 +12,11 @@ load "$BATS_ASSERT_ROOT/load.bash" load "$BATS_FILE_ROOT/load.bash" function setup_file() { + export TEST_NAMESPACE=${TEST_NAMESPACE:-opensearch-basic} echo "recreating namespace $TEST_NAMESPACE" run kubectl delete namespace "$TEST_NAMESPACE" run kubectl create namespace "$TEST_NAMESPACE" - # HELM_VALUES_EXTRA_FILE is a default file containing global helm - # options that can be optionally applied on helm install/upgrade - # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl - # if not passed. - if [ -e "${HELM_VALUES_EXTRA_FILE}" ]; then - envsubst < "${HELM_VALUES_EXTRA_FILE}" > "${HELM_VALUES_EXTRA_FILE%.*}" - export HELM_VALUES_EXTRA_FILE="${HELM_VALUES_EXTRA_FILE%.*}" - fi + create_helm_extra_values_file } function teardown_file() { @@ -32,6 +26,7 @@ function teardown_file() { run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi + unset TEST_NAMESPACE } function teardown() { @@ -66,7 +61,6 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" opensearch opensearch/opensearch \ --values ${BATS_TEST_DIRNAME}/resources/helm/opensearch-basic.yaml \ --set image.repository=${OPENSEARCH_IMAGE_REPOSITORY},image.tag=${OPENSEARCH_IMAGE_TAG} \ - --values "$HELM_VALUES_EXTRA_FILE" \ --timeout "${HELM_DEFAULT_TIMEOUT:-10m0s}" \ --wait @@ -82,9 +76,7 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" --timeout "${HELM_FB_TIMEOUT:-5m0s}" \ --wait - try "at most 15 times every 2s " \ - "to find 1 pods named 'fluent-bit' " \ - "with 'status' being 'running'" + kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s attempt=0 while true; do diff --git a/tests/opensearch/hosted.bats b/tests/opensearch/hosted.bats index 1a84716..c7449bc 100644 --- a/tests/opensearch/hosted.bats +++ b/tests/opensearch/hosted.bats @@ -2,7 +2,7 @@ load "$HELPERS_ROOT/test-helpers.bash" -ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_TAG HOSTED_OPENSEARCH_HOST HOSTED_OPENSEARCH_PORT HOSTED_OPENSEARCH_USERNAME HOSTED_OPENSEARCH_PASSWORD +ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT FLUENTBIT_IMAGE_TAG HOSTED_OPENSEARCH_HOST HOSTED_OPENSEARCH_PORT HOSTED_OPENSEARCH_USERNAME HOSTED_OPENSEARCH_PASSWORD load "$BATS_DETIK_ROOT/utils.bash" load "$BATS_DETIK_ROOT/linter.bash" @@ -12,17 +12,11 @@ load "$BATS_ASSERT_ROOT/load.bash" load "$BATS_FILE_ROOT/load.bash" setup_file() { + export TEST_NAMESPACE=${TEST_NAMESPACE:-opensearch-hosted} echo "recreating namespace $TEST_NAMESPACE" run kubectl delete namespace "$TEST_NAMESPACE" run kubectl create namespace "$TEST_NAMESPACE" - # HELM_VALUES_EXTRA_FILE is a default file containing global helm - # options that can be optionally applied on helm install/upgrade - # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl - # if not passed. - if [ -e "${HELM_VALUES_EXTRA_FILE}" ]; then - envsubst < "${HELM_VALUES_EXTRA_FILE}" > "${HELM_VALUES_EXTRA_FILE%.*}" - export HELM_VALUES_EXTRA_FILE="${HELM_VALUES_EXTRA_FILE%.*}" - fi + create_helm_extra_values_file } teardown_file() { @@ -34,6 +28,7 @@ teardown_file() { rm -f ${BATS_TEST_DIRNAME}/resources/helm/fluentbit-hosted.yaml fi fi + unset TEST_NAMESPACE } function teardown() { @@ -64,9 +59,7 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" --timeout "${HELM_DEFAULT_TIMEOUT:-10m0s}" \ --wait - try "at most 15 times every 2s " \ - "to find 1 pods named 'fluent-bit' " \ - "with 'status' being 'running'" + kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s attempt=0 while true; do From 38f78e0a80afb26612378a5543b1ab195ef82276 Mon Sep 17 00:00:00 2001 From: ryanohnemus Date: Sat, 3 Feb 2024 07:53:31 -0600 Subject: [PATCH 2/5] Address comments, add doc Signed-off-by: ryanohnemus --- README.md | 8 ++++++-- run-tests.sh | 11 ++++++----- tests/defaults/values.yaml.tpl | 2 +- tests/elasticsearch/basic.bats | 4 +++- tests/elasticsearch/compress.bats | 4 +++- tests/kubernetes-plugins/basic.bats | 4 +++- tests/kubernetes-plugins/full.bats | 6 ++++-- tests/opensearch/basic.bats | 6 ++++-- tests/opensearch/hosted.bats | 6 ++++-- 9 files changed, 34 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 695f031..b4dfcd5 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,8 @@ The following is the list of environment variables that controls the behavior of running the test suites: ```bash -SKIP_TEARDOWN=yes # don't remove the testing namespace -TEST_NAMESPACE=test # k8s namespace to use +SKIP_TEARDOWN=yes # don't remove the testing namespace(s) +TEST_NAMESPACE=test # k8s namespace to use, setting this env to any value forces tests to run in sequence # HELM_VALUES_EXTRA_FILE is a default file containing global helm # options that can be optionally applied on helm install/upgrade # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl @@ -65,6 +65,10 @@ TEST_NAMESPACE=test # k8s namespace to use HELM_VALUES_EXTRA_FILE=./path/to/your/default/values.yaml # The BATS formatter to use: https://bats-core.readthedocs.io/en/stable/usage.html BATS_FORMATTER=tap +#BATS_JOBS_PARAMS (default below) will run 8 tests in parallel, +# this can be disabled by changing this variable OR by setting a TEST_NAMESPACE env +# which will force jobs to run in sequence +BATS_JOBS_PARAMS="--jobs 8 --no-parallelize-within-files" # The Fluent Bit image to use FLUENTBIT_IMAGE_REPOSITORY=ghcr.io/fluent/fluent-bit FLUENTBIT_IMAGE_TAG=latest diff --git a/run-tests.sh b/run-tests.sh index e7bb3b4..3d38b9c 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -25,6 +25,7 @@ export RESOURCES_ROOT="${SCRIPT_DIR}/resources/" export BATS_FORMATTER=${BATS_FORMATTER:-tap} export BATS_ROOT=${BATS_ROOT:-$SCRIPT_DIR/tools/bats} export BATS_ARGS=${BATS_ARGS:---timing --verbose-run} +export BATS_JOBS_PARAMS=${BATS_JOBS_PARAMS:---jobs 8 --no-parallelize-within-files} export BATS_FILE_ROOT=$BATS_ROOT/lib/bats-file export BATS_SUPPORT_ROOT=$BATS_ROOT/lib/bats-support @@ -78,11 +79,11 @@ function run_tests() { echo echo - # If TEST_NAMESPACE is not set (the default), we run 8 jobs in parallel - # otherwise, if it is set we can only run 1 job at a time - BATS_JOBS_PARAMS="" - if [[ -z "$TEST_NAMESPACE" ]]; then - BATS_JOBS_PARAMS="--jobs $BATS_JOBS_COUNT --no-parallelize-within-files" + # If TEST_NAMESPACE is not set (the default), we run jobs in parallel + # otherwise, if it is set we can only run 1 job at a time and need to remove + # all BATS_JOBS_PARAMS + if [[ ! -z "${TEST_NAMESPACE:-}" ]]; then + BATS_JOBS_PARAMS="" fi # We run BATS in a subshell to prevent it from inheriting our exit/err trap, which can mess up its internals diff --git a/tests/defaults/values.yaml.tpl b/tests/defaults/values.yaml.tpl index 7edebf9..9bfab3e 100644 --- a/tests/defaults/values.yaml.tpl +++ b/tests/defaults/values.yaml.tpl @@ -1,3 +1,3 @@ image: pullPolicy: ${HELM_IMAGE_PULL_POLICY} -fullnameOverride: fluentbit-ci-test-${TEST_NAMESPACE} \ No newline at end of file +fullnameOverride: fluent-bit-ci-test-${TEST_NAMESPACE} \ No newline at end of file diff --git a/tests/elasticsearch/basic.bats b/tests/elasticsearch/basic.bats index 67af29f..ae5790d 100644 --- a/tests/elasticsearch/basic.bats +++ b/tests/elasticsearch/basic.bats @@ -58,7 +58,9 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" --timeout "${HELM_FB_TIMEOUT:-5m0s}" \ --wait - kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s + try "at most 15 times every 2s " \ + "to find 1 pods named 'fluent-bit' " \ + "with 'status' being 'running'" attempt=0 while true; do diff --git a/tests/elasticsearch/compress.bats b/tests/elasticsearch/compress.bats index 9acc4ef..c7be5db 100644 --- a/tests/elasticsearch/compress.bats +++ b/tests/elasticsearch/compress.bats @@ -58,7 +58,9 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" --timeout "${HELM_DEFAULT_TIMEOUT:-10m0s}" \ --wait - kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s + try "at most 15 times every 2s " \ + "to find 1 pods named 'fluent-bit' " \ + "with 'status' being 'running'" attempt=0 while true; do diff --git a/tests/kubernetes-plugins/basic.bats b/tests/kubernetes-plugins/basic.bats index 359e3b4..3d8be85 100644 --- a/tests/kubernetes-plugins/basic.bats +++ b/tests/kubernetes-plugins/basic.bats @@ -62,7 +62,9 @@ teardown() { function set_fluent_bit_pod_name() { - kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s + try "at most 30 times every 2s " \ + "to find 1 pods named 'fluent-bit' " \ + "with 'status' being 'running'" FLUENTBIT_POD_NAME=$(kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/name=fluent-bit" --no-headers | awk '{ print $1 }') if [ -z "$FLUENTBIT_POD_NAME" ]; then diff --git a/tests/kubernetes-plugins/full.bats b/tests/kubernetes-plugins/full.bats index 7534bb1..53233f4 100644 --- a/tests/kubernetes-plugins/full.bats +++ b/tests/kubernetes-plugins/full.bats @@ -71,8 +71,10 @@ teardown() { } function set_fluent_bit_pod_name() { - kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s - + try "at most 30 times every 2s " \ + "to find 1 pods named 'fluent-bit' " \ + "with 'status' being 'running'" + FLUENTBIT_POD_NAME=$(kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/name=fluent-bit" --no-headers | awk '{ print $1 }') if [ -z "$FLUENTBIT_POD_NAME" ]; then fail "Unable to get running fluent-bit pod's name" diff --git a/tests/opensearch/basic.bats b/tests/opensearch/basic.bats index 8b89fa8..86336a8 100644 --- a/tests/opensearch/basic.bats +++ b/tests/opensearch/basic.bats @@ -76,8 +76,10 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" --timeout "${HELM_FB_TIMEOUT:-5m0s}" \ --wait - kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s - + try "at most 15 times every 2s " \ + "to find 1 pods named 'fluent-bit' " \ + "with 'status' being 'running'" + attempt=0 while true; do run kubectl exec -q -n $TEST_NAMESPACE opensearch-cluster-master-0 -- curl --insecure -s -w "%{http_code}" https://admin:admin@localhost:9200/fluentbit/_search/ -o /dev/null diff --git a/tests/opensearch/hosted.bats b/tests/opensearch/hosted.bats index c7449bc..61e083c 100644 --- a/tests/opensearch/hosted.bats +++ b/tests/opensearch/hosted.bats @@ -59,8 +59,10 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" --timeout "${HELM_DEFAULT_TIMEOUT:-10m0s}" \ --wait - kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s - + try "at most 15 times every 2s " \ + "to find 1 pods named 'fluent-bit' " \ + "with 'status' being 'running'" + attempt=0 while true; do run curl -XGET --header 'Content-Type: application/json' --insecure -s -w "%{http_code}" https://${HOSTED_OPENSEARCH_USERNAME}:${HOSTED_OPENSEARCH_PASSWORD}@${HOSTED_OPENSEARCH_HOST}/fluentbit/_search/ -d '{ "query": { "range": { "timestamp": { "gte": "now-15s" }}}}' -o /dev/null From 0cb622f329ec78cee05b1c026f24bf67310c8470 Mon Sep 17 00:00:00 2001 From: ryanohnemus Date: Sat, 3 Feb 2024 07:58:41 -0600 Subject: [PATCH 3/5] fixing lints Signed-off-by: ryanohnemus --- helpers/test-helpers.bash | 2 ++ run-tests.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/helpers/test-helpers.bash b/helpers/test-helpers.bash index 41cccd7..61c4739 100755 --- a/helpers/test-helpers.bash +++ b/helpers/test-helpers.bash @@ -108,7 +108,9 @@ function create_helm_extra_values_file() { # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl # if not passed. if [ -e "${HELM_VALUES_EXTRA_FILE}" ]; then + # shellcheck disable=SC2086,SC2155 envsubst < "${HELM_VALUES_EXTRA_FILE}" > "$(dirname ${HELM_VALUES_EXTRA_FILE})/${TEST_NAMESPACE}.values.yaml" + # shellcheck disable=SC2086,SC2155 export HELM_VALUES_EXTRA_FILE="$(dirname ${HELM_VALUES_EXTRA_FILE})/${TEST_NAMESPACE}.values.yaml" fi } \ No newline at end of file diff --git a/run-tests.sh b/run-tests.sh index 3d38b9c..e0ec08d 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -82,7 +82,7 @@ function run_tests() { # If TEST_NAMESPACE is not set (the default), we run jobs in parallel # otherwise, if it is set we can only run 1 job at a time and need to remove # all BATS_JOBS_PARAMS - if [[ ! -z "${TEST_NAMESPACE:-}" ]]; then + if [[ -n "${TEST_NAMESPACE:-}" ]]; then BATS_JOBS_PARAMS="" fi From 4398fae04f8ab24974ed2e1dd37d776317b6ba2d Mon Sep 17 00:00:00 2001 From: ryanohnemus Date: Mon, 5 Feb 2024 07:43:21 -0600 Subject: [PATCH 4/5] remove unset TEST_NAMESPACE Signed-off-by: ryanohnemus --- tests/chunk-rollover/basic.bats | 3 +-- tests/elasticsearch/basic.bats | 1 - tests/elasticsearch/compress.bats | 1 - tests/kubernetes-plugins/basic.bats | 1 - tests/kubernetes-plugins/full.bats | 1 - tests/opensearch/basic.bats | 3 +-- tests/opensearch/hosted.bats | 8 +++----- 7 files changed, 5 insertions(+), 13 deletions(-) diff --git a/tests/chunk-rollover/basic.bats b/tests/chunk-rollover/basic.bats index c0f7556..6fc740f 100644 --- a/tests/chunk-rollover/basic.bats +++ b/tests/chunk-rollover/basic.bats @@ -25,7 +25,6 @@ teardown_file() { run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi - unset TEST_NAMESPACE } function teardown() { @@ -68,7 +67,7 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" COUNTER=0 - kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s + kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=60s while [ $COUNTER -lt $TOTAL_TIME ]; do # Get the number of Fluent Bit DaemonSet pods that are not in the "Running" status diff --git a/tests/elasticsearch/basic.bats b/tests/elasticsearch/basic.bats index ae5790d..7a6e11f 100644 --- a/tests/elasticsearch/basic.bats +++ b/tests/elasticsearch/basic.bats @@ -26,7 +26,6 @@ teardown_file() { run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi - unset TEST_NAMESPACE } # These are required for bats-detik diff --git a/tests/elasticsearch/compress.bats b/tests/elasticsearch/compress.bats index c7be5db..d11ef6b 100644 --- a/tests/elasticsearch/compress.bats +++ b/tests/elasticsearch/compress.bats @@ -26,7 +26,6 @@ teardown_file() { run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi - unset TEST_NAMESPACE } # These are required for bats-detik diff --git a/tests/kubernetes-plugins/basic.bats b/tests/kubernetes-plugins/basic.bats index 3d8be85..07d13cd 100644 --- a/tests/kubernetes-plugins/basic.bats +++ b/tests/kubernetes-plugins/basic.bats @@ -44,7 +44,6 @@ teardown_file() { run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi - unset TEST_NAMESPACE } setup() { diff --git a/tests/kubernetes-plugins/full.bats b/tests/kubernetes-plugins/full.bats index 53233f4..d9309ed 100644 --- a/tests/kubernetes-plugins/full.bats +++ b/tests/kubernetes-plugins/full.bats @@ -54,7 +54,6 @@ teardown_file() { run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi - unset TEST_NAMESPACE } setup() { diff --git a/tests/opensearch/basic.bats b/tests/opensearch/basic.bats index 86336a8..b8db933 100644 --- a/tests/opensearch/basic.bats +++ b/tests/opensearch/basic.bats @@ -26,7 +26,6 @@ function teardown_file() { run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} fi - unset TEST_NAMESPACE } function teardown() { @@ -79,7 +78,7 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" try "at most 15 times every 2s " \ "to find 1 pods named 'fluent-bit' " \ "with 'status' being 'running'" - + attempt=0 while true; do run kubectl exec -q -n $TEST_NAMESPACE opensearch-cluster-master-0 -- curl --insecure -s -w "%{http_code}" https://admin:admin@localhost:9200/fluentbit/_search/ -o /dev/null diff --git a/tests/opensearch/hosted.bats b/tests/opensearch/hosted.bats index 61e083c..10bbcf8 100644 --- a/tests/opensearch/hosted.bats +++ b/tests/opensearch/hosted.bats @@ -12,6 +12,9 @@ load "$BATS_ASSERT_ROOT/load.bash" load "$BATS_FILE_ROOT/load.bash" setup_file() { + if [[ $HOSTED_OPENSEARCH_HOST == "localhost" ]]; then + skip "Skipping Hosted OpenSearch When 'HOSTED_OPENSEARCH_HOST=localhost'" + fi export TEST_NAMESPACE=${TEST_NAMESPACE:-opensearch-hosted} echo "recreating namespace $TEST_NAMESPACE" run kubectl delete namespace "$TEST_NAMESPACE" @@ -28,7 +31,6 @@ teardown_file() { rm -f ${BATS_TEST_DIRNAME}/resources/helm/fluentbit-hosted.yaml fi fi - unset TEST_NAMESPACE } function teardown() { @@ -45,10 +47,6 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" @test "test fluent-bit forwards logs to AWS OpenSearch hosted service default index" { - if [[ $HOSTED_OPENSEARCH_HOST == "localhost" ]]; then - skip "Skipping Hosted OpenSearch When 'HOSTED_OPENSEARCH_HOST=localhost'" - fi - envsubst < "${BATS_TEST_DIRNAME}/resources/helm/fluentbit-hosted.yaml.tpl" > "${BATS_TEST_DIRNAME}/resources/helm/fluentbit-hosted.yaml" helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" fluent-bit fluent/fluent-bit \ From 50973fe65dc0d79e275b9d55bbc1e5cfac7bd817 Mon Sep 17 00:00:00 2001 From: ryanohnemus Date: Sat, 10 Feb 2024 07:55:37 -0600 Subject: [PATCH 5/5] address pr comments Signed-off-by: ryanohnemus --- tests/opensearch/hosted.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/opensearch/hosted.bats b/tests/opensearch/hosted.bats index 10bbcf8..4d6c069 100644 --- a/tests/opensearch/hosted.bats +++ b/tests/opensearch/hosted.bats @@ -60,7 +60,7 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" try "at most 15 times every 2s " \ "to find 1 pods named 'fluent-bit' " \ "with 'status' being 'running'" - + attempt=0 while true; do run curl -XGET --header 'Content-Type: application/json' --insecure -s -w "%{http_code}" https://${HOSTED_OPENSEARCH_USERNAME}:${HOSTED_OPENSEARCH_PASSWORD}@${HOSTED_OPENSEARCH_HOST}/fluentbit/_search/ -d '{ "query": { "range": { "timestamp": { "gte": "now-15s" }}}}' -o /dev/null @@ -75,4 +75,4 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" break fi done -} \ No newline at end of file +}