chore: ci3 post-merge fixes

.devcontainer/dev/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "name": "Development",
-  "image": "aztecprotocol/devbox:1.0",
+  "image": "aztecprotocol/devbox:1.2",
   "features": {
     // Use custom fork with noble added to list of supported distros.
     "./docker-in-docker": {}

.github/ensure-builder/action.yml

@@ -16,8 +16,6 @@ runs:
   # define an action, runs in OS of caller
   using: composite
   steps:
-    - name: CI Setup
-      uses: ./.github/ci-setup-action
     - name: Select Instance Type and AMI
       id: select_instance
       shell: bash
@@ -80,73 +78,14 @@ runs:
         echo "BUILDER_SPOT_IP=${SPOT_IP}" >> $GITHUB_ENV
         echo "BUILDER_SPOT_KEY=${SPOT_KEY}" >> $GITHUB_ENV
     # Set up a context for this run
-    - name: Initialize Builder
-      env:
-        UNPACK: |
-          [ -d ~/run-$RUN_ID ] || (
-            mkdir -p ~/run-$RUN_ID
-            cd ~/run-$RUN_ID
-            git init &>/dev/null
-            git remote add origin https://github.com/aztecprotocol/aztec-packages
-            git fetch --depth 1 origin $GIT_COMMIT
-            git checkout FETCH_HEAD >/dev/null
-          )
-      shell: bash
-      run: |
-        # need to call run_on_builder as the action version assumes git is set up
-        run_on_builder "sudo shutdown -P ${{ steps.select_instance.outputs.ttl }}; flock /var/lock/git.lock bash -c '$UNPACK'"
-
-    - name: Setup Graceful Exit
-      uses: ./.github/run-on-builder
-      with:
-        run: |
-          [ -x /usr/local/bin/earthly ] || ci3/earthly_install
-          # One-time config
-          if ! [ -f ~/maybe_exit_spot.sh ] ; then
-            cp scripts/ci/maybe_exit_spot.sh ~/maybe_exit_spot.sh
-            cp scripts/ci/spot_runner_graceful_exit.sh ~/spot_runner_graceful_exit.sh
-            # Run maybe_exit_spot.sh every minute
-            chmod +x ~/spot_runner_graceful_exit.sh
-            chmod +x ~/maybe_exit_spot.sh
-            echo "* * * * * ~/maybe_exit_spot.sh" | crontab -
-            echo "Configured graceful exit."
-          else
-            echo "Chron jobs already configured."
-          fi
-    - name: Docker Login
-      uses: ./.github/run-on-builder
-      with:
-        run: |
-          i=0
-          while ! systemctl is-active --quiet docker; do
-            sleep 2
-            if [ $(( i++ )) -gt 60 ]; then
-              echo "Docker service not found! Report this."
-              exit 1
-            fi
-          done
-          echo "Docker service is active!"
-          echo $DOCKERHUB_PASSWORD | docker login -u aztecprotocolci --password-stdin
+    # Run the test
     - name: Run On Builder
       id: test
-      env:
-        RUN_SCRIPT: |
-          cd ~/run-${{ env.RUN_ID }};
-          ci3/aws_handle_evict "${{ inputs.run }}"
       shell: bash
-      run:
-        set +e;
-        exit_code=254
-        scripts/run_on_builder "$RUN_SCRIPT";
-        exit_code=$?;
-        if [ $exit_code = 255 ]; then
-          echo "Treating ssh termination as spot eviction.";
-          exit_code=155;
-        fi;
-        echo "exit_code=$exit_code" >> $GITHUB_OUTPUT;
-        if [ $exit_code = 155 ]; then
-          echo "Spot eviction detected - retrying with on-demand.";
-        fi;
+      env:
+        INPUT: ${{ inputs.run }}
+      run: |
+        .github/ensure-builder/run ${{ steps.select_instance.outputs.ttl }}
 
     - name: Re-run with On-Demand
       uses: ./.github/ensure-builder

.github/ensure-builder/run

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+exit_code=254
+ttl=$1
+scripts/run_on_builder "
+  set -eu;
+  sudo shutdown -P $ttl;
+  if ! [ -d ~/run-$RUN_ID ]; then
+    mkdir -p ~/run-$RUN_ID;
+    cd ~/run-$RUN_ID;
+    git init >/dev/null 2>&1;
+    git remote add origin https://github.com/aztecprotocol/aztec-packages >/dev/null 2>&1;
+    git fetch --depth 1 origin $GIT_COMMIT >/dev/null 2>&1;
+    git checkout FETCH_HEAD >/dev/null 2>&1;
+  fi;
+  cd ~/run-$RUN_ID;
+  .github/ensure-builder/wrapper $DOCKERHUB_PASSWORD '$INPUT';
+"
+exit_code=$?
+if [ $exit_code = 255 ]; then
+  echo "Treating ssh termination as spot eviction.";
+  exit_code=155
+fi;
+echo "exit_code=$exit_code" >> $GITHUB_OUTPUT;
+if [ $exit_code = 155 ]; then
+  echo "Spot eviction detected - retrying with on-demand."
+fi

.github/ensure-builder/wrapper

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -eu
+# One-time config.
+if ! [ -f ~/maybe_exit_spot.sh ] ; then
+  cp scripts/ci/maybe_exit_spot.sh ~/maybe_exit_spot.sh
+  # Run maybe_exit_spot.sh every minute
+  chmod +x ~/maybe_exit_spot.sh
+  echo "* * * * * ~/maybe_exit_spot.sh" | crontab -
+  echo "Configured instance exit cron job."
+else
+  echo "Chron jobs already configured."
+fi
+set +x
+# Ensure docker is active.
+i=0
+while ! systemctl is-active --quiet docker; do
+  sleep 2
+  if [ $(( i++ )) -gt 60 ]; then
+    echo "Docker service not found! Report this."
+    exit 1
+  fi
+done
+echo $1 | ci3/dump_fail docker login -u aztecprotocolci --password-stdin
+if ! command -v yq >/dev/null; then
+  ci3/dump_fail wget https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_$(ci3/arch) -O ./yq
+  chmod +x ./yq
+  sudo mv ./yq /usr/bin/yq
+fi
+if ! command -v jq >/dev/null; then
+  ci3/dump_fail wget https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-$(ci3/arch) -O ./jq 2>&1 >/dev/null
+  chmod +x ./jq
+  sudo mv ./jq /usr/bin/jq
+fi
+function install_parallel() {
+  sudo apt update && sudo apt install parallel
+}
+export -f install_parallel
+if ! command -v parallel >/dev/null; then
+  DENOISE=1 ci3/denoise install_parallel
+fi
+[ -x /usr/local/bin/earthly ] || ci3/dump_fail ci3/earthly_install
+# Run the test.
+ci3/aws_handle_evict "set -eu; $2"

.github/ensure-tester/action.yml

@@ -31,8 +31,8 @@ runs:
           USERNAME="${{ inputs.username }}"
         fi
         echo "USERNAME=$USERNAME" >> $GITHUB_ENV
-        echo "runner_label=$USERNAME-$runner_type" >> $GITHUB_OUTPUT
-        echo "runner_label=$USERNAME-$runner_type" >> $GITHUB_OUTPUT
+        echo "runner_label=$USERNAME-${{ github.jobs[github.job].name }}" >> $GITHUB_OUTPUT
+        echo "runner_label=$USERNAME-${{ github.jobs[github.job].name }}" >> $GITHUB_OUTPUT
         echo "ami_id=ami-052a1e16394277fdf" >> $GITHUB_OUTPUT
         # no github runners, 'bare spot' in action code
         echo "runner_concurrency=0" >> $GITHUB_OUTPUT
@@ -80,80 +80,26 @@ runs:
         ec2_key_name: "build-instance"
         ec2_instance_tags: '[{"Key": "Keep-Alive", "Value": "true"}]'
 
-    # Set up a context for this run
-    - name: Initialize Tester
-      env:
-        UNPACK: |
-          [ -d ~/run-$RUN_ID ] || (
-            mkdir -p ~/run-$RUN_ID
-            cd ~/run-$RUN_ID
-            git init &>/dev/null
-            git remote add origin https://github.com/aztecprotocol/aztec-packages
-            git fetch --depth 1 origin $GIT_COMMIT
-            git checkout FETCH_HEAD >/dev/null
-          )
-      shell: bash
-      run: |
-        scripts/run_on_tester "sudo shutdown -P ${{ steps.select_instance.outputs.ttl }}; flock /var/lock/git.lock bash -c '$UNPACK'"
-        scripts/run_on_tester "
-          mkdir -p ~/.ssh
-          echo $BUILD_INSTANCE_SSH_KEY | base64 --decode > ~/.ssh/build_instance_key
-          chmod 600 ~/.ssh/build_instance_key
-          i=0
-          while ! systemctl is-active --quiet docker; do
-            sleep 2
-            if [ $(( i++ )) -gt 60 ]; then
-              echo "Docker service not found! Report this."
-              exit 1
-            fi
-          done
-          echo "Docker service is active!"
-          echo $DOCKERHUB_PASSWORD | docker login -u aztecprotocolci --password-stdin
-        "
-
-    - name: Setup yq and earthly
-      uses: ./.github/run-on-tester
-      with:
-        run: |
-          wget https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_amd64 -O ./yq 2>&1 >/dev/null
-          chmod +x ./yq
-          DENOISE=1 ci3/denoise sudo mv ./yq /usr/bin/yq
-          ci3/earthly_install
-          echo "Waiting for Docker service to become active..."
-          i=0
-
     - name: Ensure Tester Cleanup
       uses: gacts/run-and-post-run@v1
       with:
         post: scripts/run_on_tester "sudo shutdown now" || true
 
     - name: Run On Tester
       id: test
-      env:
-        RUN_SCRIPT: |
-          cd ~/run-${{ env.RUN_ID }};
-          ci3/aws_handle_evict "set -eu; ${{ inputs.run }}"
       shell: bash
-      run:
-        set +e;
-        scripts/run_on_tester "$RUN_SCRIPT";
-        exit_code=$?;
-        if [ $exit_code = 255 ]; then
-          echo "Treating ssh termination as spot eviction.";
-          exit_code=155;
-        fi;
-        echo "exit_code=$exit_code" >> $GITHUB_OUTPUT;
-        if [ $exit_code = 155 ]; then
-          echo "Spot eviction detected - retrying with on-demand.";
-        fi;
+      env:
+        INPUT: ${{ inputs.run }}
+      run: |
+        .github/ensure-tester/run ${{ steps.select_instance.outputs.ttl }}
 
     - name: Re-run with On-Demand
       uses: ./.github/ensure-tester
       if: steps.test.outputs.exit_code == '155' && inputs.spot_strategy != 'None'
       with:
         username: ${{ inputs.username }}
         runner_type: ${{ inputs.runner_type}}
-        spot_strategy: None
+        # spot_strategy: None
         ttl: ${{ inputs.ttl }}
         run: ${{ inputs.run }}
 

.github/ensure-tester/run

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+exit_code=254
+ttl=$1
+scripts/run_on_tester "
+  set -eu;
+  sudo shutdown -P $ttl;
+  if ! [ -d ~/run-$RUN_ID ]; then
+    mkdir -p ~/run-$RUN_ID;
+    cd ~/run-$RUN_ID;
+    git init >/dev/null 2>&1;
+    git remote add origin https://github.com/aztecprotocol/aztec-packages >/dev/null 2>&1;
+    git fetch --depth 1 origin $GIT_COMMIT >/dev/null 2>&1;
+    git checkout FETCH_HEAD >/dev/null 2>&1;
+  fi;
+  cd ~/run-$RUN_ID;
+  .github/ensure-tester/wrapper $DOCKERHUB_PASSWORD '$INPUT';
+"
+exit_code=$?
+if [ $exit_code = 255 ]; then
+  echo "Treating ssh termination as spot eviction."
+  exit_code=155
+fi
+echo "exit_code=$exit_code" >> $GITHUB_OUTPUT
+if [ $exit_code = 155 ]; then
+  echo "Spot eviction detected - retrying with on-demand."
+fi

.github/ensure-tester/wrapper

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -eu
+# One-time config.
+mkdir -p ~/.ssh
+echo $BUILD_INSTANCE_SSH_KEY | base64 --decode > ~/.ssh/build_instance_key
+chmod 600 ~/.ssh/build_instance_key
+set +x
+# Ensure docker is active.
+echo "Waiting for Docker service to become active..."
+i=0
+set +x
+while ! systemctl is-active --quiet docker; do
+  sleep 2
+  if [ $(( i++ )) -gt 60 ]; then
+    echo "Docker service not found! Report this."
+    exit 1
+  fi
+done
+echo $1 | ci3/dump_fail docker login -u aztecprotocolci --password-stdin
+ci3/dump_fail wget https://github.com/mikefarah/yq/releases/download/v4.44.3/yq_linux_$(ci3/arch) -O ./yq 2>&1 >/dev/null
+chmod +x ./yq
+sudo mv ./yq /usr/bin/yq
+ci3/dump_fail wget https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-$(ci3/arch) -O ./jq
+chmod +x ./jq
+sudo mv ./jq /usr/bin/jq
+ci3/dump_fail ci3/earthly_install
+function install_parallel() {
+  sudo apt update && sudo apt install parallel
+}
+export -f install_parallel
+DENOISE=1 ci3/denoise install_parallel
+# Run command.
+ci3/aws_handle_evict "set -eu; $2"