diff --git a/.github/actions/stage2-setup/action.yml b/.github/actions/stage2-setup/action.yml index 0e116300..727f4f1f 100644 --- a/.github/actions/stage2-setup/action.yml +++ b/.github/actions/stage2-setup/action.yml @@ -18,6 +18,10 @@ inputs: required: false type: boolean default: false + USE_NEXT: + required: false + type: boolean + default: false SATELLITE_NAME: required: false type: string @@ -67,16 +71,26 @@ runs: shell: bash - run: |- echo "Extracting earthly binary from stage1 of build" - BUILDKITD_IMAGE=docker.io/earthly/buildkitd-staging TAG=${GITHUB_SHA}-ubuntu-latest-${{inputs.BINARY}} ${{inputs.SUDO}} ./earthly upgrade + export TAG=${GITHUB_SHA}-ubuntu-latest-${{inputs.BINARY}} + if [ "${{inputs.USE_NEXT}}" = "true" ]; then export TAG="$TAG-ticktock"; fi + BUILDKITD_IMAGE=docker.io/earthly/buildkitd-staging ${{inputs.SUDO}} ./earthly upgrade ${{inputs.SUDO}} chown -R $USER ~/.earthly # restore non-sudo user ownership test -n "${{inputs.BUILT_EARTHLY_PATH}}" || (echo "BUILT_EARTHLY_PATH is empty" && exit 1) mkdir -p $(dirname "${{inputs.BUILT_EARTHLY_PATH}}") - ${{inputs.SUDO}} mv ${HOME}/.earthly/earthly-${GITHUB_SHA}-ubuntu-latest-${{inputs.BINARY}} "${{inputs.BUILT_EARTHLY_PATH}}" + ${{inputs.SUDO}} ls "${HOME}/.earthly/" + ${{inputs.SUDO}} mv "${HOME}/.earthly/earthly-$TAG" "${{inputs.BUILT_EARTHLY_PATH}}" echo "extracted ${{inputs.BUILT_EARTHLY_PATH}}" shell: bash + - if: ${{ inputs.USE_NEXT == 'true' }} + run: |- + export expected_buildkit_client_sha="$(cat earthly-next | head -c 12)" + test -n "$expected_buildkit_client_sha" || ( echo "expected_buildkit_client_sha is empty" && exit 1) + (strings ${{inputs.BUILT_EARTHLY_PATH}} | grep "$expected_buildkit_client_sha" ) || ( echo "expected to find $expected_buildkit_client_sha in earthly binary" && exit 1) + echo "correctly found $expected_buildkit_client_sha in earthly binary; this confirms earthly-next was used" + shell: bash - run: |- echo "Setting up mirror credentials in .arg and .secret" - export earthly=${{inputs.BUILT_EARTHLY_PATH}} + export earthly="${{inputs.BUILT_EARTHLY_PATH}}" # setup secrets echo "DOCKERHUB_MIRROR_USER=$($earthly secret --org earthly-technologies --project core get -n dockerhub-mirror/user || kill $$)" > .secret echo "DOCKERHUB_MIRROR_PASS=$($earthly secret --org earthly-technologies --project core get -n dockerhub-mirror/pass || kill $$)" >> .secret diff --git a/.github/workflows/build-earthly.yml b/.github/workflows/build-earthly.yml index 7e96f77e..7d6c72e3 100644 --- a/.github/workflows/build-earthly.yml +++ b/.github/workflows/build-earthly.yml @@ -19,11 +19,15 @@ on: required: false type: boolean default: false + USE_NEXT: + required: false + type: boolean + default: false jobs: build-earthly: - name: build (and push) earthly using ${{inputs.BINARY}} + name: build (and push) earthly using ${{inputs.BINARY}} and earthly-next=${{inputs.USE_NEXT}} if: ${{!inputs.SKIP_JOB}} runs-on: ${{inputs.RUNS_ON}} env: @@ -71,6 +75,9 @@ jobs: - name: Podman Login run: ${{inputs.SUDO}} ${{inputs.BINARY}} login docker.io --username "${{ secrets.DOCKERHUB_USERNAME }}" --password "${{ secrets.DOCKERHUB_TOKEN }}" if: inputs.BINARY == 'podman' + - name: Update Buildkit to earthly-next + if: inputs.USE_NEXT + run: ${{inputs.SUDO}} $(which earthly) +update-buildkit --BUILDKIT_GIT_SHA=$(cat earthly-next) - name: Build latest earthly using released earthly run: ${{inputs.SUDO}} $(which earthly) --use-inline-cache +for-linux - name: Earthly bootstrap using latest earthly build @@ -81,7 +88,10 @@ jobs: EARTHLY_VERSION_FLAG_OVERRIDES="$(tr -d '\n' < .earthly_version_flag_overrides)" echo "EARTHLY_VERSION_FLAG_OVERRIDES=$EARTHLY_VERSION_FLAG_OVERRIDES" >> "$GITHUB_ENV" - name: Build and push +ci-release using latest earthly build - run: ${{inputs.SUDO}} ./build/linux/amd64/earthly --ci --push +ci-release --TAG_SUFFIX="${{inputs.RUNS_ON}}-${{inputs.BINARY}}" + run: |- + export TAG_SUFFIX="${{inputs.RUNS_ON}}-${{inputs.BINARY}}" + if [ "${{inputs.USE_NEXT}}" = "true" ]; then export TAG_SUFFIX="$TAG_SUFFIX-ticktock"; fi + ${{inputs.SUDO}} ./build/linux/amd64/earthly --ci --push --build-arg TAG_SUFFIX +ci-release - name: Buildkit logs (runs on failure) run: ${{inputs.SUDO}} ${{inputs.BINARY}} logs earthly-buildkitd if: ${{ failure() }} diff --git a/.github/workflows/ci-earthly-next-docker-ubuntu.yml b/.github/workflows/ci-earthly-next-docker-ubuntu.yml new file mode 100644 index 00000000..1589d7e1 --- /dev/null +++ b/.github/workflows/ci-earthly-next-docker-ubuntu.yml @@ -0,0 +1,196 @@ +name: Earthly Next Docker CI Ubuntu + +on: + push: + branches: [ main ] + paths-ignore: + - 'docs/**' + - '**.md' + - '.github/renovate.json5' + - '.github/CODEOWNERS' + - 'LICENSE' + pull_request: + branches: [ main ] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build-earthly-with-next: + uses: ./.github/workflows/build-earthly.yml + with: + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group1: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group1" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group2: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group2" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group3: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group3" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group4: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group4" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group5: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group5" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group6: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group6" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group7: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group7" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group8: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group8" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group9: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group9" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group10: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group10" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group11: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group11" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit + + earthly-next-docker-tests-no-qemu-group12: + needs: build-earthly-with-next + uses: ./.github/workflows/reusable-test.yml + with: + TEST_TARGET: "+test-no-qemu-group12" + BUILT_EARTHLY_PATH: "./build/linux/amd64/earthly" + RUNS_ON: "ubuntu-latest" + BINARY: "docker" + USE_NEXT: true + SUDO: "" + SKIP_JOB: ${{ needs.build-earthly-with-next.result != 'success' }} + EXTRA_ARGS: "--auto-skip" + secrets: inherit diff --git a/.github/workflows/reusable-test.yml b/.github/workflows/reusable-test.yml index 474980ca..f320154d 100644 --- a/.github/workflows/reusable-test.yml +++ b/.github/workflows/reusable-test.yml @@ -38,6 +38,10 @@ on: required: false type: boolean default: false + USE_NEXT: + required: false + type: boolean + default: false jobs: test: @@ -67,6 +71,7 @@ jobs: BUILT_EARTHLY_PATH: "${{ inputs.BUILT_EARTHLY_PATH }}" BINARY: "${{ inputs.BINARY }}" USE_QEMU: "${{ inputs.USE_QEMU }}" + USE_NEXT: "${{ inputs.USE_NEXT }}" SUDO: "${{ inputs.SUDO }}" USE_SATELLITE: "${{ inputs.USE_SATELLITE }}" SATELLITE_NAME: "${{ inputs.SATELLITE_NAME }}" @@ -78,6 +83,12 @@ jobs: - name: Execute ${{ inputs.TEST_TARGET }} (Earthly Only) run: |- ${{inputs.SUDO}} ${{ inputs.BUILT_EARTHLY_PATH }} ${{inputs.EXTRA_ARGS}} --ci -P ${{inputs.TEST_TARGET}} + - name: Display buildkit version + run: |- + ${{inputs.SUDO}} ${{inputs.BINARY}} ps -a + ${{inputs.SUDO}} ${{inputs.BINARY}} logs earthly-buildkitd |& grep 'starting earthly-buildkit' + shell: bash + if: ${{ ! inputs.USE_SATELLITE }} - name: Execute fail test run: | ! ${{inputs.SUDO}} GITHUB_ACTIONS="" ${{ inputs.BUILT_EARTHLY_PATH }} ${{inputs.EXTRA_ARGS}} --ci ./tests/fail+test-fail