Address comments, add doc

Signed-off-by: ryanohnemus <[email protected]>

README.md

@@ -56,15 +56,19 @@ The following is the list of environment variables that controls the behavior
 of running the test suites:
 
 ```bash
-SKIP_TEARDOWN=yes # don't remove the testing namespace
-TEST_NAMESPACE=test # k8s namespace to use
+SKIP_TEARDOWN=yes # don't remove the testing namespace(s)
+TEST_NAMESPACE=test # k8s namespace to use, setting this env to any value forces tests to run in sequence
 # HELM_VALUES_EXTRA_FILE is a default file containing global helm
 # options that can be optionally applied on helm install/upgrade
 # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl
 # if not passed.
 HELM_VALUES_EXTRA_FILE=./path/to/your/default/values.yaml
 # The BATS formatter to use: https://bats-core.readthedocs.io/en/stable/usage.html
 BATS_FORMATTER=tap
+#BATS_JOBS_PARAMS (default below) will run 8 tests in parallel,
+# this can be disabled by changing this variable OR by setting a TEST_NAMESPACE env
+# which will force jobs to run in sequence
+BATS_JOBS_PARAMS="--jobs 8 --no-parallelize-within-files"
 # The Fluent Bit image to use
 FLUENTBIT_IMAGE_REPOSITORY=ghcr.io/fluent/fluent-bit
 FLUENTBIT_IMAGE_TAG=latest

run-tests.sh

@@ -25,6 +25,7 @@ export RESOURCES_ROOT="${SCRIPT_DIR}/resources/"
 export BATS_FORMATTER=${BATS_FORMATTER:-tap}
 export BATS_ROOT=${BATS_ROOT:-$SCRIPT_DIR/tools/bats}
 export BATS_ARGS=${BATS_ARGS:---timing --verbose-run}
+export BATS_JOBS_PARAMS=${BATS_JOBS_PARAMS:---jobs 8 --no-parallelize-within-files}
 
 export BATS_FILE_ROOT=$BATS_ROOT/lib/bats-file
 export BATS_SUPPORT_ROOT=$BATS_ROOT/lib/bats-support
@@ -78,11 +79,11 @@ function run_tests() {
     echo
     echo
 
-    # If TEST_NAMESPACE is not set (the default), we run 8 jobs in parallel
-    # otherwise, if it is set we can only run 1 job at a time
-    BATS_JOBS_PARAMS=""
-    if [[ -z "$TEST_NAMESPACE" ]]; then
-        BATS_JOBS_PARAMS="--jobs $BATS_JOBS_COUNT --no-parallelize-within-files"
+    # If TEST_NAMESPACE is not set (the default), we run jobs in parallel
+    # otherwise, if it is set we can only run 1 job at a time and need to remove
+    # all BATS_JOBS_PARAMS
+    if [[ ! -z "${TEST_NAMESPACE:-}" ]]; then
+        BATS_JOBS_PARAMS=""
     fi
 
     # We run BATS in a subshell to prevent it from inheriting our exit/err trap, which can mess up its internals

tests/defaults/values.yaml.tpl

@@ -1,3 +1,3 @@
 image:
   pullPolicy: ${HELM_IMAGE_PULL_POLICY}
-fullnameOverride: fluentbit-ci-test-${TEST_NAMESPACE}
+fullnameOverride: fluent-bit-ci-test-${TEST_NAMESPACE}

tests/elasticsearch/basic.bats

@@ -58,7 +58,9 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}"
         --timeout "${HELM_FB_TIMEOUT:-5m0s}" \
         --wait
 
-    kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s
+    try "at most 15 times every 2s " \
+        "to find 1 pods named 'fluent-bit' " \
+        "with 'status' being 'running'"
 
     attempt=0
     while true; do

tests/elasticsearch/compress.bats

@@ -58,7 +58,9 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}"
         --timeout "${HELM_DEFAULT_TIMEOUT:-10m0s}" \
         --wait
 
-    kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s
+    try "at most 15 times every 2s " \
+        "to find 1 pods named 'fluent-bit' " \
+        "with 'status' being 'running'"
 
     attempt=0
     while true; do

tests/kubernetes-plugins/basic.bats

@@ -62,7 +62,9 @@ teardown() {
 
 
 function set_fluent_bit_pod_name() {
-    kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s
+    try "at most 30 times every 2s " \
+        "to find 1 pods named 'fluent-bit' " \
+        "with 'status' being 'running'"
 
     FLUENTBIT_POD_NAME=$(kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/name=fluent-bit" --no-headers | awk '{ print $1 }')
     if [ -z "$FLUENTBIT_POD_NAME" ]; then

tests/kubernetes-plugins/full.bats

@@ -71,8 +71,10 @@ teardown() {
 }
 
 function set_fluent_bit_pod_name() {
-    kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s
-
+    try "at most 30 times every 2s " \
+        "to find 1 pods named 'fluent-bit' " \
+        "with 'status' being 'running'"
+
     FLUENTBIT_POD_NAME=$(kubectl get pods -n "$TEST_NAMESPACE" -l "app.kubernetes.io/name=fluent-bit" --no-headers | awk '{ print $1 }')
     if [ -z "$FLUENTBIT_POD_NAME" ]; then
         fail "Unable to get running fluent-bit pod's name"

tests/opensearch/basic.bats

@@ -76,8 +76,10 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}"
         --timeout "${HELM_FB_TIMEOUT:-5m0s}" \
         --wait
 
-    kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s
-
+    try "at most 15 times every 2s " \
+        "to find 1 pods named 'fluent-bit' " \
+        "with 'status' being 'running'"
+
     attempt=0
     while true; do
     	run kubectl exec -q -n $TEST_NAMESPACE opensearch-cluster-master-0 -- curl --insecure -s -w "%{http_code}" https://admin:admin@localhost:9200/fluentbit/_search/ -o /dev/null

tests/opensearch/hosted.bats

@@ -59,8 +59,10 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}"
         --timeout "${HELM_DEFAULT_TIMEOUT:-10m0s}" \
         --wait
 
-    kubectl wait pods -n "$TEST_NAMESPACE" -l app.kubernetes.io/name=fluent-bit --for condition=Ready --timeout=30s
-
+    try "at most 15 times every 2s " \
+        "to find 1 pods named 'fluent-bit' " \
+        "with 'status' being 'running'"
+
     attempt=0
     while true; do
         run curl -XGET --header 'Content-Type: application/json' --insecure -s -w "%{http_code}" https://${HOSTED_OPENSEARCH_USERNAME}:${HOSTED_OPENSEARCH_PASSWORD}@${HOSTED_OPENSEARCH_HOST}/fluentbit/_search/ -d '{ "query": { "range": { "timestamp": { "gte": "now-15s" }}}}' -o /dev/null