From 475cb7cd026bf27c2f37b7625df9115a29058092 Mon Sep 17 00:00:00 2001 From: Pedro Chambino Date: Tue, 13 Feb 2024 17:13:40 +0000 Subject: [PATCH] Raise jobs API limit to 100, and better handle missing "front_of_queue_pipeline_number" If front_of_queue_pipeline_number is missing, it means there are more jobs runnning than the ones the API returned. I guess the behaviour here could be configurable, but I am choosing to wait for the number of jobs to reduce, since otherwise we can't know if we are front of the queue or not. --- scripts/loop.bash | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/scripts/loop.bash b/scripts/loop.bash index a7f56b6..74ba0d8 100755 --- a/scripts/loop.bash +++ b/scripts/loop.bash @@ -85,8 +85,8 @@ update_active_run_data(){ oldest_running_build_num=`jq ". | map(select(.workflows.job_name | test(\"${JOB_NAME}\";\"sx\"))) | sort_by(.pipeline_number)| .[0].build_num" $AUGMENTED_JOBSTATUS_PATH` front_of_queue_pipeline_number=`jq -r ". | map(select(.workflows.job_name | test(\"${JOB_NAME}\";\"sx\"))) | sort_by(.pipeline_number)| .[0].workflows.pipeline_number // empty" $AUGMENTED_JOBSTATUS_PATH` fi - if [ -z $front_of_queue_pipeline_number ];then - echo "API Call for existing jobs returned no matches. This means job is alone." + if [ -z "$front_of_queue_pipeline_number" ];then + echo "API Call for existing jobs returned no matches. This means we have more jobs running, than the API limit (100)." if [[ $DEBUG == "true" ]];then echo "All running jobs:" cat $SHALLOW_JOBSTATUS_PATH || exit 0 @@ -100,7 +100,7 @@ update_active_run_data(){ } fetch_filtered_active_builds(){ - JOB_API_SUFFIX="?filter=running&shallow=true" + JOB_API_SUFFIX="?filter=running&shallow=true&limit=100" jobs_api_url_template="${CIRCLECI_BASE_URL}/api/v1.1/project/${VCS_TYPE}/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}${JOB_API_SUFFIX}" if [ "${FILTER_BRANCH}" != "1" ];then echo "Orb parameter 'this-branch-only' is false, will block previous builds on any branch." @@ -207,9 +207,9 @@ while true; do echo "This Job's Pipeline #: $MY_PIPELINE_NUMBER" echo "Front of Queue (fifo) Pipeline #: $front_of_queue_pipeline_number" - if [[ -z "$front_of_queue_pipeline_number" ]] || [[ ! -z "$MY_PIPELINE_NUMBER" ]] && [[ "$front_of_queue_pipeline_number" == "$MY_PIPELINE_NUMBER" ]] ; then - # recent-jobs API does not include pending, so it is possible we queried in between a workflow transition, and we;re NOT really front of line. - if [ $confidence -lt $CONFIDENCE_THRESHOLD ];then + if [ "$front_of_queue_pipeline_number" = "$MY_PIPELINE_NUMBER" ]; then + # recent-jobs API does not include pending, so it is possible we queried in between a workflow transition, and we're NOT really front of line. + if [ $confidence -lt $CONFIDENCE_THRESHOLD ]; then # To grow confidence, we check again with a delay. confidence=$((confidence+1)) echo "API shows no conflicting jobs/workflows. However it is possible a previous workflow has pending jobs not yet visible in API. To avoid a race condition we will verify out place in queue." @@ -221,7 +221,11 @@ while true; do else # If we fail, reset confidence confidence=0 - echo "This build (${CIRCLE_BUILD_NUM}), pipeline (${MY_PIPELINE_NUMBER}) is queued, waiting for build(${oldest_running_build_num}) pipeline (${front_of_queue_pipeline_number}) to complete." + if [ -z "$front_of_queue_pipeline_number" ]; then + echo "This build (${CIRCLE_BUILD_NUM}), pipeline (${MY_PIPELINE_NUMBER}) is queued, waiting for total number of jobs to be under 100." + else + echo "This build (${CIRCLE_BUILD_NUM}), pipeline (${MY_PIPELINE_NUMBER}) is queued, waiting for build(${oldest_running_build_num}) pipeline (${front_of_queue_pipeline_number}) to complete." + fi echo "Total Queue time: ${wait_time} seconds." fi