diff --git a/.github/workflows/_update_terraform.yml b/.github/workflows/_update_terraform.yml new file mode 100644 index 0000000..572ea29 --- /dev/null +++ b/.github/workflows/_update_terraform.yml @@ -0,0 +1,59 @@ +name: Update Terraform +on: + workflow_call: + secrets: + PERSONAL_ACCESS_TOKEN: + required: true + inputs: + image_tag: + description: Tag for the image for docker/ghcr registries + required: true + type: string + deployment_environment: + description: The terraform target environment + required: true + type: string + default: staging +jobs: + deploy: + runs-on: ubuntu-latest + env: + GIT_SHA: ${{ github.sha }} + GIT_TAG: ${{ inputs.image_tag }} + steps: + - name: Checkout terraform config repo + uses: actions/checkout@v3 + with: + # public repo with terraform configuration + repository: 'datacite/mastino' + persist-credentials: false + - name: Setup dockerize and template parameters + # use go template in terraform config repository to update git sha and tag + # commit and push changes to trigger terraform workflow + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + wget https://github.com/jwilder/dockerize/releases/download/v0.6.0/dockerize-linux-amd64-v0.6.0.tar.gz + tar -xzvf dockerize-linux-amd64-v0.6.0.tar.gz + rm dockerize-linux-amd64-v0.6.0.tar.gz + - name: Conditionally update staging environment + if: ${{ (inputs.deployment_environment == 'staging') }} + run: | + ./dockerize -template stage/services/mds/_poodle.auto.tfvars.tmpl:stage/services/mds/_poodle.auto.tfvars + git add stage/services/mds/_poodle.auto.tfvars + git commit -m "Adding poodle git variables for commit ${{ steps.extract_variables.outputs.GIT_SHA }}" + - name: Conditionally update production/test environments + if: ${{ (inputs.deployment_environment == 'production') }} + run: | + ./dockerize -template prod-eu-west/services/mds/_poodle.auto.tfvars.tmpl:prod-eu-west/services/mds/_poodle.auto.tfvars + ./dockerize -template test/services/mds/_poodle.auto.tfvars.tmpl:test/services/mds/_poodle.auto.tfvars + git add prod-eu-west/services/mds/_poodle.auto.tfvars + git add test/services/mds/_poodle.auto.tfvars + git commit -m "Adding poodle git variables for tag ${{ steps.extract_variables.outputs.GIT_TAG }}" + - name: Push changes + uses: ad-m/github-push-action@v0.7.0 + with: + github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + repository: 'datacite/mastino' + branch: 'refs/heads/master' + tags: false diff --git a/.github/workflows/branch_to_staging.yml b/.github/workflows/branch_to_staging.yml new file mode 100644 index 0000000..45404ac --- /dev/null +++ b/.github/workflows/branch_to_staging.yml @@ -0,0 +1,21 @@ +name: Build/Deploy Branch to Staging +on: + workflow_dispatch: +jobs: + test: + uses: ./.github/workflows/ci.yml + secrets: inherit + call_build_and_push: + needs: test + uses: ./.github/workflows/build.yml + with: + image_name: ${{ github.repository }} + image_tag: ${{ github.ref_name }} + secrets: inherit + deploy: + needs: [test, call_build_and_push] + uses: ./.github/workflows/_update_terraform.yml + with: + image_tag: ${{ github.ref_name }} + deployment_environment: staging + secrets: inherit diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..0aaa418 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,55 @@ +name: Build and Tag +on: + workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: true + DOCKERHUB_TOKEN: + required: true + inputs: + image_name: + description: The name of the image for docker/ghcr registries + required: true + type: string + image_tag: + description: Tag for the image for docker/ghcr registries + required: true + type: string +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and Push + uses: docker/build-push-action@v4 + with: + context: . + file: ./Dockerfile + push: true + tags: | + ${{ inputs.image_name }}:${{ inputs.image_tag }} + ghcr.io/${{ inputs.image_name }}:${{ inputs.image_tag }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + # no-cache: true diff --git a/.github/workflows/test-branch.yml b/.github/workflows/ci.yml similarity index 68% rename from .github/workflows/test-branch.yml rename to .github/workflows/ci.yml index e75e9bf..18f503c 100644 --- a/.github/workflows/test-branch.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,13 @@ -name: Test Branch +name: CI on: - workflow_dispatch: - + workflow_call: + secrets: + MDS_USERNAME: + required: true + MDS_PASSWORD: + required: true + CC_TEST_REPORTER_ID: + required: true jobs: test: runs-on: ubuntu-latest @@ -27,14 +33,20 @@ jobs: restore-keys: | ${{ runner.os }}-gems- - name: Install + env: + MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} run: | gem install bundler bundle config path vendor/bundle bundle install --jobs 4 --retry 3 - - name: Lint and Test + - name: (Lint and) Test env: MEMCACHE_SERVERS: "localhost:11211" run: | # bundle exec rubocop bundle exec rspec echo $? + - name: Publish code coverage + uses: paambaati/codeclimate-action@v3.1.1 + env: + CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} \ No newline at end of file diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 69d33c4..0b96571 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,153 +1,23 @@ -name: Deploy +name: Deploy Main to Staging on: push: branches: - "master" jobs: test: - runs-on: ubuntu-latest - services: - memcached: - image: memcached:1.4.31 - ports: - - 11211/udp - env: - MEMCACHE_SERVERS: "localhost:11211" - MDS_USERNAME: ${{ secrets.MDS_USERNAME }} - MDS_PASSWORD: ${{ secrets.MDS_PASSWORD }} - steps: - - uses: actions/checkout@v3 - - name: Set up Ruby 2.6 - uses: ruby/setup-ruby@v1 - with: - ruby-version: '2.6.9' - - uses: actions/cache@v3 - with: - path: vendor/bundle - key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }} - restore-keys: | - ${{ runner.os }}-gems- - - name: Install - run: | - gem install bundler - bundle config path vendor/bundle - bundle install --jobs 4 --retry 3 - - name: Lint and Test - env: - MEMCACHE_SERVERS: "localhost:11211" - run: | - # bundle exec rubocop - bundle exec rspec - echo $? - - name: Publish code coverage - uses: paambaati/codeclimate-action@v3.1.1 - env: - CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} - - build: + uses: ./.github/workflows/ci.yml + secrets: inherit + call_build_and_push: needs: test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Push to Docker Hub - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - push: true - tags: ${{ github.repository }}:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - # no-cache: true - - name: Push to GitHub Packages - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile - push: true - tags: ghcr.io/${{ github.repository }}:latest - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - # no-cache: true - + uses: ./.github/workflows/build.yml + with: + image_name: ${{ github.repository }} + image_tag: main + secrets: inherit deploy: - needs: [test, build] - runs-on: ubuntu-latest - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Extract variables - shell: bash - run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//_/g')" >> $GITHUB_OUTPUT - echo "TAG=$(git tag --points-at HEAD)" >> $GITHUB_OUTPUT - echo "GIT_SHA=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - echo "GIT_SHA_SHORT=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - id: extract_variables - - - name: Checkout terraform config repo - uses: actions/checkout@v3 - with: - # public repo with terraform configuration - repository: 'datacite/mastino' - persist-credentials: false - - name: Commit changes to terraform config repository - # use go template in terraform config repository to update git sha and tag - # commit and push changes to trigger terraform workflow - run: | - export GIT_SHA=${{ steps.extract_variables.outputs.GIT_SHA_SHORT }} - export GIT_TAG=${{ steps.extract_variables.outputs.GIT_TAG }} - wget https://github.com/jwilder/dockerize/releases/download/v0.6.0/dockerize-linux-amd64-v0.6.0.tar.gz - tar -xzvf dockerize-linux-amd64-v0.6.0.tar.gz - rm dockerize-linux-amd64-v0.6.0.tar.gz - ./dockerize -template stage/services/mds/_poodle.auto.tfvars.tmpl:stage/services/mds/_poodle.auto.tfvars - - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git add stage/services/mds/_poodle.auto.tfvars - git commit -m "Adding poodle git variables for commit ${{ steps.extract_variables.outputs.GIT_SHA }}" - - name: Push changes - uses: ad-m/github-push-action@v0.6.0 - with: - github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - repository: 'datacite/mastino' - branch: 'refs/heads/master' - tags: false - - - name: Notify Slack - uses: edge/simple-slack-notify@master - with: - channel: '#ops' - color: 'good' - text: 'A new version of the is been deployed to stage.' - failure_text: '${env.GITHUB_WORKFLOW} (${env.GITHUB_RUN_NUMBER}) build failed' - fields: | - [{ "title": "Commit message", "value": "${{ github.event.head_commit.message }}" }, - { "title": "Committed by", "value": "", "short": true }, - { "title": "Commit SHA", "value": "", "short": true }, - { "title": "Repository", "value": "", "short": true }, - { "title": "Branch", "value": "", "short": true }] + needs: [test, call_build_and_push] + uses: ./.github/workflows/_update_terraform.yml + with: + image_tag: main + deployment_environment: staging + secrets: inherit diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 0000000..5cd06dc --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,11 @@ +name: Test Pull Request +on: + pull_request: + branches: + - master + workflow_dispatch: +jobs: + test: + uses: ./.github/workflows/ci.yml + secrets: inherit + \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0eecc55..59700ae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,159 +1,22 @@ -name: Release +name: Release to Production on: release: types: [published] jobs: test: - runs-on: ubuntu-latest - services: - memcached: - image: memcached:1.4.31 - ports: - - 11211/udp - env: - MEMCACHE_SERVERS: "localhost:11211" - MDS_USERNAME: ${{ secrets.MDS_USERNAME }} - MDS_PASSWORD: ${{ secrets.MDS_PASSWORD }} - steps: - - uses: actions/checkout@v3 - - name: Set up Ruby 2.6 - uses: ruby/setup-ruby@v1 - with: - ruby-version: '2.6.9' - - uses: actions/cache@v3 - with: - path: vendor/bundle - key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }} - restore-keys: | - ${{ runner.os }}-gems- - - name: Install - env: - MYSQL_PORT: ${{ job.services.mysql.ports[3306] }} - run: | - gem install bundler - bundle config path vendor/bundle - bundle install --jobs 4 --retry 3 - - name: Lint and Test - env: - MEMCACHE_SERVERS: "localhost:11211" - run: | - # bundle exec rubocop - bundle exec rspec - echo $? - - name: Publish code coverage - uses: paambaati/codeclimate-action@v3.1.1 - env: - CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} - - build: + uses: ./.github/workflows/ci.yml + secrets: inherit + call_build_and_push: needs: test - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Get git tag - run: | - echo "GIT_TAG=$(git describe --tags $(git rev-list --tags --max-count=1))" >> $GITHUB_OUTPUT - id: set_git_vars - - name: Push to Docker Hub - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - push: true - tags: ${{ github.repository }}:${{ steps.set_git_vars.outputs.GIT_TAG }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - # no-cache: true - - name: Push to GitHub Packages - uses: docker/build-push-action@v4 - with: - context: . - file: ./Dockerfile - push: true - tags: ghcr.io/${{ github.repository }}:${{ steps.set_git_vars.outputs.GIT_TAG }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - # no-cache: true - + uses: ./.github/workflows/build.yml + with: + image_name: ${{ github.repository }} + image_tag: ${{ github.ref_name }} + secrets: inherit deploy: - needs: [test, build] - runs-on: ubuntu-latest - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Extract variables - shell: bash - run: | - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//_/g')" >> $GITHUB_OUTPUT - echo "TAG=$(git tag --points-at HEAD)" >> $GITHUB_OUTPUT - echo "GIT_SHA=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - echo "GIT_SHA_SHORT=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - id: extract_variables - - - name: Checkout terraform config repo - uses: actions/checkout@v3 - with: - # public repo with terraform configuration - repository: 'datacite/mastino' - persist-credentials: false - - name: Commit changes to terraform config repository - # use go template in terraform config repository to update git sha and tag - # commit and push changes to trigger terraform workflow - run: | - export GIT_SHA=${{ steps.extract_variables.outputs.GIT_SHA_SHORT }} - export GIT_TAG=${{ steps.extract_variables.outputs.TAG }} - wget https://github.com/jwilder/dockerize/releases/download/v0.6.0/dockerize-linux-amd64-v0.6.0.tar.gz - tar -xzvf dockerize-linux-amd64-v0.6.0.tar.gz - rm dockerize-linux-amd64-v0.6.0.tar.gz - ./dockerize -template prod-eu-west/services/mds/_poodle.auto.tfvars.tmpl:prod-eu-west/services/mds/_poodle.auto.tfvars - ./dockerize -template test/services/mds/_poodle.auto.tfvars.tmpl:test/services/mds/_poodle.auto.tfvars - - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git add prod-eu-west/services/mds/_poodle.auto.tfvars - git add test/services/mds/_poodle.auto.tfvars - git commit -m "Adding poodle git variables for tag ${{ steps.extract_variables.outputs.GIT_TAG }}" - - name: Push changes - uses: ad-m/github-push-action@v0.7.0 - with: - github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - repository: 'datacite/mastino' - branch: 'refs/heads/master' - tags: false - - - name: Notify Slack - uses: edge/simple-slack-notify@master - with: - channel: '#ops' - color: 'good' - text: 'Version of the is being released to test and production.' - failure_text: '${env.GITHUB_WORKFLOW} (${env.GITHUB_RUN_NUMBER}) build failed' - fields: | - [{ "title": "Committed by", "value": "", "short": true }, - { "title": "Commit SHA", "value": "", "short": true }, - { "title": "Repository", "value": "", "short": true }, - { "title": "Release", "value": "", "short": true }] + needs: [test, call_build_and_push] + uses: ./.github/workflows/_update_terraform.yml + with: + image_tag: ${{ github.ref_name }} + deployment_environment: production + secrets: inherit diff --git a/.github/workflows/reset_staging.yml b/.github/workflows/reset_staging.yml new file mode 100644 index 0000000..aa9e039 --- /dev/null +++ b/.github/workflows/reset_staging.yml @@ -0,0 +1,10 @@ +name: Reset Staging to Main Branch +on: + workflow_dispatch: +jobs: + deploy: + uses: ./.github/workflows/_update_terraform.yml + with: + image_tag: main + deployment_environment: staging + secrets: inherit diff --git a/.github/workflows/test_branch.yml b/.github/workflows/test_branch.yml new file mode 100644 index 0000000..f5bb562 --- /dev/null +++ b/.github/workflows/test_branch.yml @@ -0,0 +1,8 @@ +name: Test Branch +on: + workflow_dispatch: +jobs: + test: + uses: ./.github/workflows/ci.yml + secrets: inherit + diff --git a/Gemfile b/Gemfile index 8297161..7475f29 100644 --- a/Gemfile +++ b/Gemfile @@ -9,7 +9,7 @@ gem "oj_mimic_json", "~> 1.0", ">= 1.0.1" gem "equivalent-xml", "~> 0.6.0" gem "nokogiri", ">= 1.10.4" gem "iso8601", "~> 0.9.0" -gem "bolognese", "~> 2.0.2" +gem "bolognese", "~> 2.0.3" gem "maremma", "~> 4.9.8" gem "faraday", "~> 0.17.6" gem "base32-url", "~> 0.5" diff --git a/Gemfile.lock b/Gemfile.lock index 9316eb0..d2c6039 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -56,7 +56,7 @@ GEM latex-decode (~> 0.0) binding_of_caller (1.0.0) debug_inspector (>= 0.0.1) - bolognese (2.0.2) + bolognese (2.0.3) activesupport (>= 4.2.5) benchmark_methods (~> 0.7) bibtex-ruby (>= 5.1.0) @@ -348,7 +348,7 @@ DEPENDENCIES base32-url (~> 0.5) better_errors binding_of_caller - bolognese (~> 2.0.2) + bolognese (~> 2.0.3) bootsnap (>= 1.1.0) byebug cancancan (~> 2.0)