Update build.yml based on halfpipe continuous_integration.yml

.github/workflows/build.yml

@@ -10,152 +10,179 @@ on:
       - edited
 
 jobs:
-  docker_build:
+  build:
     runs-on: [self-hosted, ubuntu, x64]
-    outputs:
-      tag: ${{ steps.build_tag.outputs.tag }}
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          check-latest: true
+      - name: Install Python dependencies
+        run: |
+          python -m pip install \
+            awscli \
+            pygithub \
+            setuptools_scm
+
+      - name: Set up metadata
+        id: metadata
+        shell: python
+        run: |
+          import os
+
+          from github import Github
+          from setuptools_scm import get_version
+
+          pushed_at = "${{ github.event.repository.pushed_at }}"
+          ref = "${{ github.ref }}"
+          repository = "${{ github.repository }}"
+          sha = "${{ github.sha }}"
+          url = "${{ github.event.repository.html_url }}"
+
+          registry = "${{ secrets.REGISTRY }}"
+          token = "${{ secrets.GITHUB_TOKEN }}"
+
+          repository_data = Github(token).get_repo(repository)
+          repository_owner, repository_name = repository.lower().split("/")
+          reference, type, name = ref.split("/")[:3]
+          if reference != "refs":
+              raise ValueError(f"Unknown reference: {reference}")
+          if type == "heads":
+              tag = {"main": "latest"}.get(name, name)
+              push = True
+          elif type == "tags":
+              tag = name
+              push = True
+          else:
+              raise ValueError(f"Unknown reference type: {type}")
+
+          path = f"{registry}/{repository_name}"
+          cache_from = f"type=registry,ref={path}:buildcache"
+          output = dict(
+              cache_from=cache_from,
+              cache_to=f"{cache_from},compression=zstd,mode=max" if push else "",
+              labels=[
+                  f'org.opencontainers.image.created="{pushed_at}"',
+                  'org.opencontainers.image.authors="Lea Waller <[email protected]>"',
+                  f'org.opencontainers.image.url="{url}"',
+                  f'org.opencontainers.image.documentation="{url}"',
+                  f'org.opencontainers.image.source="{url}"',
+                  f'org.opencontainers.image.version="{get_version()}"',
+                  f'org.opencontainers.image.revision="{sha}"',
+                  f'org.opencontainers.image.licenses="{repository_data.license.spdx_id}"',
+                  f'org.opencontainers.image.title="{repository_name}"',
+                  f'org.opencontainers.image.description="{repository_data.description}"',
+              ],
+              push=str(push).lower(),
+              build_tag=f"{repository_name}:{tag}",
+              push_tags=[
+                  f"{path}:{tag}",
+                  f"docker.io/{repository_owner}/{repository_name}:{tag}",
+              ]
+              if push
+              else [],
+              singularity_name=f"{repository_name}-{tag}.sif",
+          )
+
+          with open(os.environ["GITHUB_OUTPUT"], "at") as file_handle:
+              for key, value in output.items():
+                  if isinstance(value, list):
+                      file_handle.write("\n".join([f"{key}<<eof", *value, "eof"]) + "\n")
+                  else:
+                      file_handle.write(f"{key}={value}\n")
+
       - name: Login to DockerHub
         uses: docker/login-action@v3
+        if: ${{ steps.metadata.outputs.push == 'true' }}
         with:
-          username: hippocampusgirl
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-
       - name: Login to container registry
         uses: docker/login-action@v3
+        if: ${{ steps.metadata.outputs.push == 'true' }}
         with:
           registry: ${{ secrets.REGISTRY }}
           username: ${{ secrets.REGISTRY_USERNAME }}
           password: ${{ secrets.REGISTRY_PASSWORD }}
 
-      - name: Set up build tag
-        id: build_tag
-        shell: bash
-        env:
-          github_ref: ${{ github.ref }}
-          github_repository: ${{ github.repository }}
-        run: |
-          version=$(echo "${github_ref}" | cut -d '/' -f 3)
-          if [[ "$version" == "main" ]]; then
-            version="latest"
-          fi
-          owner=$( \
-            echo "${github_repository}" | \
-            cut -d'/' -f1 | \
-            tr '[:upper:]' '[:lower:]' \
-          )
-          name=$( \
-            echo "${github_repository}" | \
-            cut -d'/' -f2 | \
-            sed -r 's/([A-Za-z0-9])([A-Z])([a-z0-9])/\1-\L\2\3/g' | \
-            tr '[:upper:]' '[:lower:]' \
-          )
-          echo "repo=${name}" >> ${GITHUB_OUTPUT}
-          echo "tag=${name}:${version}" >> ${GITHUB_OUTPUT}
-
-      - name: Checkout
-        uses: actions/checkout@v4
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
-
-      - name: Build and push to container registry
+      - name: Build container
         uses: docker/build-push-action@v6
         with:
+          cache-from: ${{ steps.metadata.outputs.cache_from }}
+          cache-to: ${{ steps.metadata.outputs.cache_to }}
           context: "."
           file: "./Dockerfile"
-          platforms: linux/amd64
-          cache-from: |
-            type=registry,ref=${{ secrets.REGISTRY }}/${{ steps.build_tag.outputs.repo }}:buildcache
-          cache-to: |
-            type=registry,ref=${{ secrets.REGISTRY }}/${{ steps.build_tag.outputs.repo }}:buildcache,compression=zstd,mode=max
-          outputs: |
-            type=image,name=${{ secrets.REGISTRY }}/${{ steps.build_tag.outputs.tag }},push=true,compression=gzip,compression-level=9,force-compression=true
           labels: |
-            org.opencontainers.image.title=${{ github.event.repository.name }}
-            org.opencontainers.image.url=${{ github.event.repository.html_url }}
-            org.opencontainers.image.source=${{ github.event.repository.html_url }}
-            org.opencontainers.image.revision=${{ github.sha }}
-            org.opencontainers.image.created=${{ github.event.repository.updated_at}}
+            ${{ steps.metadata.outputs.labels }}
+          load: true
+          platforms: linux/amd64
+          push: false
+          tags: |
+            ${{ steps.metadata.outputs.build_tag }}
 
-  singularity_build:
-    runs-on: self-hosted
-    needs:
-      - docker_build
-    strategy:
-      matrix:
-        singularity_version:
-          - "v3.11.5"
-          - "v2.6"
-    steps:
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - name: Push container to registry
+        if: ${{ steps.metadata.outputs.push == 'true' }}
+        uses: docker/build-push-action@v5
         with:
-          python-version: >
-            3.12
-          check-latest: true
-      - name: Install Python dependencies
-        run: |
-          python -m pip install \
-            awscli \
-            pygithub \
-            setuptools_scm
-
-      - name: Setup build container
-        run: |
-          mkdir -p -v "/home/runner/output"
-          docker run --privileged --rm --detach \
-            --name="build" \
-            --volume="/var/run/docker.sock:/var/run/docker.sock" \
-            --volume="/home/runner/output:/output" \
-            --entrypoint="tail" \
-            quay.io/singularity/docker2singularity:${{ matrix.singularity_version }} \
-            -f "/dev/null"
-
-      - name: Patch `docker2singularity` in build container
-        shell: docker exec build bash -x -e {0}
-        run: |
-          space=" "
-          patch /docker2singularity.sh <<EOF
-          --- docker2singularity.sh
-          +++ docker2singularity.sh
-          @@ -314,6 +314,8 @@
-          ${space}echo "(8/10) Stopping and removing the container..."
-          ${space}docker stop \$container_id >> /dev/null
-          ${space}docker rm \$container_id >> /dev/null
-          +docker container prune --force >> /dev/null
-          +docker image prune --all --force >> /dev/null
-          ${space}
-          ${space}# Build a final image from the sandbox
-          ${space}echo "(9/10) Building \${image_format} container..."
-          EOF
+          cache-from: ${{ steps.metadata.outputs.cache_from }}
+          context: "."
+          file: "./Dockerfile"
+          labels: |
+            ${{ steps.metadata.outputs.labels }}
+          platforms: linux/amd64
+          push: true
+          tags: |
+            ${{ steps.metadata.outputs.push_tags }}
 
-      - name: Run `docker2singularity` in build container
-        shell: docker exec build bash -x -e {0}
+      - name: Build Singularity 2.x container
+        if: ${{ steps.metadata.outputs.push == 'true' }}
         env:
-          docker_build_tag: ${{ needs.docker_build.outputs.tag }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }}
         run: |
-          singularity_image_name="$(echo -n ${docker_build_tag} | tr -c '[:alnum:]' '-')"
-          mkdir -p -v "/output"
-          docker-entrypoint.sh /docker2singularity.sh \
-            --name ${singularity_image_name} \
-            ${{ secrets.REGISTRY }}/${docker_build_tag}
+          mkdir --parents singularity-2
+          docker run \
+            --privileged \
+            --rm \
+            --volume="/var/run/docker.sock:/var/run/docker.sock" \
+            --volume="$(pwd)/singularity-2:/output" \
+            quay.io/singularity/docker2singularity:v2.6 \
+            --name ${{ steps.metadata.outputs.singularity_name }} \
+            ${{ steps.metadata.outputs.build_tag }}
+          image_file=$(find singularity-2 -type f | head --lines=1)
+          aws s3 mv \
+            --only-show-errors \
+            --acl public-read \
+            ${image_file} \
+            s3://download-gwas-science/singularity/
 
-      - name: Upload image file
-        shell: bash
+      - name: Build Singularity container
+        if: ${{ steps.metadata.outputs.push == 'true' }}
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.DIGITALOCEAN_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.DIGITALOCEAN_SECRET_KEY }}
-          AWS_ENDPOINT_URL: ${{ secrets.DIGITALOCEAN_REGION }}.digitaloceanspaces.com
-          DIGITALOCEAN_SPACE_NAME: ${{ secrets.DIGITALOCEAN_SPACE_NAME }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }}
         run: |
-          image_file=$(find "/home/runner/output" -type f | head -n1)
+          mkdir --parents singularity
+          docker run \
+            --rm \
+            --volume="/var/run/docker.sock:/var/run/docker.sock" \
+            --volume="$(pwd)/singularity:/output" \
+            quay.io/singularity/singularity:v4.0.1 \
+            build \
+            --disable-cache \
+            /output/${{ steps.metadata.outputs.singularity_name }} \
+            docker-daemon://${{ steps.metadata.outputs.build_tag }}
+          image_file=$(find singularity -type f | head --lines=1)
           aws s3 mv \
             --only-show-errors \
             --acl public-read \
             ${image_file} \
-            s3://${DIGITALOCEAN_SPACE_NAME}/singularity/
-
-      - name: Stop build container
-        run: |
-          docker kill build
+            s3://download-gwas-science/singularity/