diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ef41e83..fcda9b8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,152 +10,179 @@ on: - edited jobs: - docker_build: + build: runs-on: [self-hosted, ubuntu, x64] - outputs: - tag: ${{ steps.build_tag.outputs.tag }} steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + check-latest: true + - name: Install Python dependencies + run: | + python -m pip install \ + awscli \ + pygithub \ + setuptools_scm + + - name: Set up metadata + id: metadata + shell: python + run: | + import os + + from github import Github + from setuptools_scm import get_version + + pushed_at = "${{ github.event.repository.pushed_at }}" + ref = "${{ github.ref }}" + repository = "${{ github.repository }}" + sha = "${{ github.sha }}" + url = "${{ github.event.repository.html_url }}" + + registry = "${{ secrets.REGISTRY }}" + token = "${{ secrets.GITHUB_TOKEN }}" + + repository_data = Github(token).get_repo(repository) + repository_owner, repository_name = repository.lower().split("/") + reference, type, name = ref.split("/")[:3] + if reference != "refs": + raise ValueError(f"Unknown reference: {reference}") + if type == "heads": + tag = {"main": "latest"}.get(name, name) + push = True + elif type == "tags": + tag = name + push = True + else: + raise ValueError(f"Unknown reference type: {type}") + + path = f"{registry}/{repository_name}" + cache_from = f"type=registry,ref={path}:buildcache" + output = dict( + cache_from=cache_from, + cache_to=f"{cache_from},compression=zstd,mode=max" if push else "", + labels=[ + f'org.opencontainers.image.created="{pushed_at}"', + 'org.opencontainers.image.authors="Lea Waller "', + f'org.opencontainers.image.url="{url}"', + f'org.opencontainers.image.documentation="{url}"', + f'org.opencontainers.image.source="{url}"', + f'org.opencontainers.image.version="{get_version()}"', + f'org.opencontainers.image.revision="{sha}"', + f'org.opencontainers.image.licenses="{repository_data.license.spdx_id}"', + f'org.opencontainers.image.title="{repository_name}"', + f'org.opencontainers.image.description="{repository_data.description}"', + ], + push=str(push).lower(), + build_tag=f"{repository_name}:{tag}", + push_tags=[ + f"{path}:{tag}", + f"docker.io/{repository_owner}/{repository_name}:{tag}", + ] + if push + else [], + singularity_name=f"{repository_name}-{tag}.sif", + ) + + with open(os.environ["GITHUB_OUTPUT"], "at") as file_handle: + for key, value in output.items(): + if isinstance(value, list): + file_handle.write("\n".join([f"{key}<> ${GITHUB_OUTPUT} - echo "tag=${name}:${version}" >> ${GITHUB_OUTPUT} - - - name: Checkout - uses: actions/checkout@v4 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - - name: Build and push to container registry + - name: Build container uses: docker/build-push-action@v6 with: + cache-from: ${{ steps.metadata.outputs.cache_from }} + cache-to: ${{ steps.metadata.outputs.cache_to }} context: "." file: "./Dockerfile" - platforms: linux/amd64 - cache-from: | - type=registry,ref=${{ secrets.REGISTRY }}/${{ steps.build_tag.outputs.repo }}:buildcache - cache-to: | - type=registry,ref=${{ secrets.REGISTRY }}/${{ steps.build_tag.outputs.repo }}:buildcache,compression=zstd,mode=max - outputs: | - type=image,name=${{ secrets.REGISTRY }}/${{ steps.build_tag.outputs.tag }},push=true,compression=gzip,compression-level=9,force-compression=true labels: | - org.opencontainers.image.title=${{ github.event.repository.name }} - org.opencontainers.image.url=${{ github.event.repository.html_url }} - org.opencontainers.image.source=${{ github.event.repository.html_url }} - org.opencontainers.image.revision=${{ github.sha }} - org.opencontainers.image.created=${{ github.event.repository.updated_at}} + ${{ steps.metadata.outputs.labels }} + load: true + platforms: linux/amd64 + push: false + tags: | + ${{ steps.metadata.outputs.build_tag }} - singularity_build: - runs-on: self-hosted - needs: - - docker_build - strategy: - matrix: - singularity_version: - - "v3.11.5" - - "v2.6" - steps: - - name: Set up Python - uses: actions/setup-python@v5 + - name: Push container to registry + if: ${{ steps.metadata.outputs.push == 'true' }} + uses: docker/build-push-action@v5 with: - python-version: > - 3.12 - check-latest: true - - name: Install Python dependencies - run: | - python -m pip install \ - awscli \ - pygithub \ - setuptools_scm - - - name: Setup build container - run: | - mkdir -p -v "/home/runner/output" - docker run --privileged --rm --detach \ - --name="build" \ - --volume="/var/run/docker.sock:/var/run/docker.sock" \ - --volume="/home/runner/output:/output" \ - --entrypoint="tail" \ - quay.io/singularity/docker2singularity:${{ matrix.singularity_version }} \ - -f "/dev/null" - - - name: Patch `docker2singularity` in build container - shell: docker exec build bash -x -e {0} - run: | - space=" " - patch /docker2singularity.sh <> /dev/null - ${space}docker rm \$container_id >> /dev/null - +docker container prune --force >> /dev/null - +docker image prune --all --force >> /dev/null - ${space} - ${space}# Build a final image from the sandbox - ${space}echo "(9/10) Building \${image_format} container..." - EOF + cache-from: ${{ steps.metadata.outputs.cache_from }} + context: "." + file: "./Dockerfile" + labels: | + ${{ steps.metadata.outputs.labels }} + platforms: linux/amd64 + push: true + tags: | + ${{ steps.metadata.outputs.push_tags }} - - name: Run `docker2singularity` in build container - shell: docker exec build bash -x -e {0} + - name: Build Singularity 2.x container + if: ${{ steps.metadata.outputs.push == 'true' }} env: - docker_build_tag: ${{ needs.docker_build.outputs.tag }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }} run: | - singularity_image_name="$(echo -n ${docker_build_tag} | tr -c '[:alnum:]' '-')" - mkdir -p -v "/output" - docker-entrypoint.sh /docker2singularity.sh \ - --name ${singularity_image_name} \ - ${{ secrets.REGISTRY }}/${docker_build_tag} + mkdir --parents singularity-2 + docker run \ + --privileged \ + --rm \ + --volume="/var/run/docker.sock:/var/run/docker.sock" \ + --volume="$(pwd)/singularity-2:/output" \ + quay.io/singularity/docker2singularity:v2.6 \ + --name ${{ steps.metadata.outputs.singularity_name }} \ + ${{ steps.metadata.outputs.build_tag }} + image_file=$(find singularity-2 -type f | head --lines=1) + aws s3 mv \ + --only-show-errors \ + --acl public-read \ + ${image_file} \ + s3://download-gwas-science/singularity/ - - name: Upload image file - shell: bash + - name: Build Singularity container + if: ${{ steps.metadata.outputs.push == 'true' }} env: - AWS_ACCESS_KEY_ID: ${{ secrets.DIGITALOCEAN_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.DIGITALOCEAN_SECRET_KEY }} - AWS_ENDPOINT_URL: ${{ secrets.DIGITALOCEAN_REGION }}.digitaloceanspaces.com - DIGITALOCEAN_SPACE_NAME: ${{ secrets.DIGITALOCEAN_SPACE_NAME }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }} run: | - image_file=$(find "/home/runner/output" -type f | head -n1) + mkdir --parents singularity + docker run \ + --rm \ + --volume="/var/run/docker.sock:/var/run/docker.sock" \ + --volume="$(pwd)/singularity:/output" \ + quay.io/singularity/singularity:v4.0.1 \ + build \ + --disable-cache \ + /output/${{ steps.metadata.outputs.singularity_name }} \ + docker-daemon://${{ steps.metadata.outputs.build_tag }} + image_file=$(find singularity -type f | head --lines=1) aws s3 mv \ --only-show-errors \ --acl public-read \ ${image_file} \ - s3://${DIGITALOCEAN_SPACE_NAME}/singularity/ - - - name: Stop build container - run: | - docker kill build + s3://download-gwas-science/singularity/