change curation_ops and validate-suite branch in actions to main

ror-community · Sep 20, 2024 · 8bba9e0 · 8bba9e0
1 parent 15f6f89
commit 8bba9e0
Show file tree

Hide file tree

Showing 18 changed files with 949 additions and 9 deletions.
diff --git a/.github/workflows/generate_dump.yml b/.github/workflows/generate_dump.yml
@@ -44,7 +44,6 @@ jobs:
         with:
           repository: ror-community/curation_ops
           token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          ref: v2-crosswalk
           path: ./curation_ops
       - name: Set up Python environment
         uses: actions/setup-python@v2

diff --git a/.github/workflows/publish_dump_zenodo.yml b/.github/workflows/publish_dump_zenodo.yml
@@ -47,9 +47,9 @@ jobs:
         run: |
           cd ./ror-data
           python -m pip install --upgrade pip
-          curl https://raw.githubusercontent.com/ror-community/curation_ops/v2-crosswalk/upload_dump_zenodo/requirements.txt -o requirements.txt
+          curl https://raw.githubusercontent.com/ror-community/curation_ops/main/upload_dump_zenodo/requirements.txt -o requirements.txt
           pip install -r requirements.txt
-          curl https://raw.githubusercontent.com/ror-community/curation_ops/v2-crosswalk/upload_dump_zenodo/upload_dump_zenodo.py -o upload_dump_zenodo.py
+          curl https://raw.githubusercontent.com/ror-community/curation_ops/main/upload_dump_zenodo/upload_dump_zenodo.py -o upload_dump_zenodo.py
           python upload_dump_zenodo.py -r ${{github.event.inputs.release}} -p ${{github.event.inputs.record-id}} -e ${{github.event.inputs.zenodo-env}}
       - name: Notify Slack
         if: always()

diff --git a/.github/workflows/staging_manual_index.yml b/.github/workflows/staging_manual_index.yml
@@ -56,7 +56,6 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: ror-community/validation-suite
-          ref: schema-v2
           path: validation-suite
       - name: Validate files
         id: validatefiles

diff --git a/.github/workflows/staging_pull_request.yml b/.github/workflows/staging_pull_request.yml
@@ -49,7 +49,6 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: ror-community/validation-suite
-          ref: schema-v2
           path: validation-suite
       - name: Validate files
         id: validatefiles

diff --git a/.github/workflows/v2_temp/dev_index_dump.yml b/.github/workflows/v2_temp/dev_index_dump.yml
@@ -0,0 +1,77 @@
+name: DEV index full data dump
+on:
+  workflow_dispatch:
+    inputs:
+      release-dump:
+        type: string
+        description: Name of existing release dump file to index, without .zip extension (ex v1.41-2024-02-13-ror-data)
+      schema-version:
+        required: true
+        description: Schema version to index
+        type: choice
+        options:
+        - v1
+        - v2
+      data-env:
+        required: true
+        description: ROR data env (test uses ror-data-test repo, prod uses ror-data)
+        default: 'test'
+        type: choice
+        options:
+        - test
+        - prod
+
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+
+jobs:
+  check_permission:
+    runs-on: ubuntu-latest
+    if: github.event.ref == 'refs/heads/dev'
+    steps:
+      - name: Get Permission
+        uses: octokit/[email protected]
+        id: get_permission
+        with:
+          route: GET /repos/${{ github.repository }}/collaborators/${{ github.actor }}/permission
+        env:
+          GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+      - name: output
+        run:  |
+          export role_name=${{ fromJson(steps.get_permission.outputs.data).role_name }}
+          if [[ "$role_name" == "admin" ]]; then
+            exit 0
+          else
+            exit 1
+          fi
+  index-dump:
+    runs-on: ubuntu-latest
+    if: github.event.ref == 'refs/heads/dev'
+    needs: check_permission
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: Index dump file
+        id: indexdatadump
+        run: |
+          cd .github/workflows
+          python -m pip install --upgrade pip
+          pip install requests==2.23.0
+          if [[ ${{ github.event.inputs.schema-version }} == "v1" ]]; then
+            python index_dump.py -u ${{ secrets.INDEX_DUMP_DEV_API_URL }} -f ${{ github.event.inputs.release-dump }} -e  ${{ github.event.inputs.data-env }} -he ${{ secrets.INDEX_DEV_API_HEADERS }}
+          else
+            python index_dump.py -u ${{ secrets.INDEX_DUMP_DEV_API_URL_V2 }} -f ${{ github.event.inputs.release-dump }} -e  ${{ github.event.inputs.data-env }} -he ${{ secrets.INDEX_DEV_API_HEADERS }}
+          fi
+      - name: Notify Slack
+        if: always()
+        uses: edge/simple-slack-notify@master
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.CURATOR_SLACK_WEBHOOK_URL }}
+        with:
+          channel: '#ror-curation-releases'
+          color: 'good'
+          text: 'DEV full index from dump status: ${{ steps.indexdatadump.outcome }}. From file: ${{ github.event.inputs.release-dump }}. Schema version: ${{ github.event.inputs.schema-version }}. Data env: ${{ github.event.inputs.data-env }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+
diff --git a/.github/workflows/v2_temp/dev_manual_index.yml b/.github/workflows/v2_temp/dev_manual_index.yml
@@ -0,0 +1,144 @@
+name: DEV manual deploy from dev branch
+on:
+  workflow_dispatch:
+    inputs:
+      schema-version:
+            required: true
+            description: Schema version
+            type: choice
+            options:
+            - v1
+            - v2
+      directory-name:
+        type: string
+        required: true
+        description: Name of the directory you would like to deploy to DEV
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+
+jobs:
+  check_permission:
+    runs-on: ubuntu-latest
+    #if: github.event.ref == 'refs/heads/dev'
+    steps:
+      - name: Get Permission
+        uses: octokit/[email protected]
+        id: get_permission
+        with:
+          route: GET /repos/${{ github.repository }}/collaborators/${{ github.actor }}/permission
+        env:
+          GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+      - name: output
+        run:  |
+          export role_name=${{ fromJson(steps.get_permission.outputs.data).role_name }}
+          if [[ "$role_name" == "maintain"  || "$role_name" == "admin" ]]; then
+            exit 0
+          else
+            exit 1
+          fi
+  check_validation:
+    runs-on: ubuntu-latest
+    #if: github.event.ref == 'refs/heads/dev'
+    needs: check_permission
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.ref }}
+      - name: Set up Python environment
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: Checkout validation suite
+        uses: actions/checkout@v2
+        with:
+          repository: ror-community/validation-suite
+          ref: schema-v2
+          path: validation-suite
+      - name: Validate files
+        id: validatefiles
+        run: |
+          mkdir files
+          cp ./${{ github.event.inputs.directory-name }}/*.json files/
+          cd validation-suite
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            curl https://raw.githubusercontent.com/ror-community/ror-schema/master/ror_schema.json -o ror_schema.json
+            if [[ -f "../${{github.event.pull_request.head.ref}}/relationships.csv" ]]; then
+                python run_validations.py -i ../files -v 1 -s ror_schema.json -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
+              else
+                python run_validations.py -i ../files -v 1 -s ror_schema.json --no-geonames
+              fi
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            curl https://raw.githubusercontent.com/ror-community/ror-schema/master/ror_schema_v2_0.json -o ror_schema_v2_0.json
+            if [[ -f "../${{github.event.pull_request.head.ref}}/relationships.csv" ]]; then
+                python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json -f ../${{github.event.pull_request.head.ref}}/relationships.csv -p ../files/ --no-geonames
+              else
+                python run_validations.py -i ../files -v 2 -s ror_schema_v2_0.json --no-geonames
+              fi
+          fi
+      - name: Notify Slack
+        if: always()
+        uses: edge/simple-slack-notify@master
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.CURATOR_SLACK_WEBHOOK_URL }}
+        with:
+          channel: '#ror-curation-releases'
+          color: 'good'
+          text: 'Validation status during DEV manual index:  ${{ steps.validatefiles.outcome }}. In directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+  copy-files:
+    runs-on: ubuntu-latest
+    #if: github.event.ref == 'refs/heads/dev'
+    needs: check_validation
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.ref }}
+      - name: Install aws cli
+        id: install-aws-cli
+        uses: unfor19/[email protected]
+      - name: Zip and copy files
+        id: copyfiles
+        run: |
+          cd ./${{ github.event.inputs.directory-name }}
+          zip files.zip *.json
+          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_DEV }}/${{ github.event.inputs.directory-name }}-${{ github.event.inputs.schema-version }}/files.zip
+          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_DEV }}/${{ github.event.inputs.directory-name }}-${{ github.event.inputs.schema-version }}/files.zip
+      - name: Notify Slack
+        if: always()
+        uses: edge/simple-slack-notify@master
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.CURATOR_SLACK_WEBHOOK_URL }}
+        with:
+          channel: '#ror-curation-releases'
+          color: 'good'
+          text: 'Copy status to DEV S3 Bucket:  ${{ steps.copyfiles.outcome }}.  From directory: ${{ github.event.inputs.directory-name }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+      - name: Index file
+        id: indexdata
+        run: |
+          cd .github/workflows
+          python -m pip install --upgrade pip
+          pip install requests==2.23.0
+          if [[ ${{ github.event.inputs.schema-version }} == 'v1' ]]; then
+            python index_files.py -u ${{ secrets.INDEX_DEV_API_URL_V1 }} -d ${{ github.event.inputs.directory-name }}-v1 -he ${{ secrets.INDEX_DEV_API_HEADERS }}
+          fi
+          if [[ ${{ github.event.inputs.schema-version }} == 'v2' ]]; then
+            python index_files.py -u ${{ secrets.INDEX_DEV_API_URL_V2 }} -d ${{ github.event.inputs.directory-name }}-v2 -he ${{ secrets.INDEX_DEV_API_HEADERS }}
+          fi
+
+      - name: Notify Slack
+        if: always()
+        uses: edge/simple-slack-notify@master
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.CURATOR_SLACK_WEBHOOK_URL }}
+        with:
+          channel: '#ror-curation-releases'
+          color: 'good'
+          text: 'DEV index status: ${{ steps.indexdata.outcome }}. Using version ${{ github.event.inputs.schema-version }}. From directory: ${{ github.event.inputs.directory-name }}-${{ github.event.inputs.schema-version }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+
diff --git a/.github/workflows/v2_temp/dev_merge.yml b/.github/workflows/v2_temp/dev_merge.yml
@@ -0,0 +1,69 @@
+name: Deploy to DEV on PR merge to dev branch
+on:
+  push:
+    branches:
+      - 'dev'
+    paths-ignore:
+      - '**/README.md'
+      - '**/workflows/**'
+env:
+  MERGE_SHA: ${{ github.event.after }}
+  REPO_INFO: ${{ github.event.repository.full_name }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+
+jobs:
+  copy-files:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get Merge PR
+        uses: octokit/[email protected]
+        id: get_merge_pr
+        with:
+          route: GET /search/issues
+          q: 'repo:${{ github.event.repository.full_name }}+is:pr+is:closed+merge_sha_commit:${{ github.event.after }}'
+        env:
+          GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+      - name: Get branch name
+        run: |
+          export url=${{ fromJson(steps.get_merge_pr.outputs.data).items[0].pull_request.url }}
+          echo "release_branch=$(curl ${url} | jq -r .head.ref)" >> $GITHUB_ENV
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: Install aws cli
+        id: install-aws-cli
+        uses: unfor19/[email protected]
+      - name: Zip and copy files
+        id: copyfiles
+        run: |
+          cd ./${{ env.release_branch }}
+          zip files.zip *.json
+          aws s3 cp files.zip s3://${{ secrets.ROR_DATA_S3_BUCKET_DEV}}/${{ env.release_branch }}-v2/files.zip
+          aws s3 ls s3://${{ secrets.ROR_DATA_S3_BUCKET_DEV }}/${{ env.release_branch }}-v2/files.zip
+      - name: Notify Slack
+        if: always()
+        uses: edge/simple-slack-notify@master
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.CURATOR_SLACK_WEBHOOK_URL }}
+        with:
+          channel: '#ror-curation-releases'
+          color: 'good'
+          text: 'Copy status to DEV S3 Bucket:  ${{ steps.copyfiles.outcome }}. Using v2. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+      - name: Index file
+        id: indexdata
+        run: |
+          cd .github/workflows
+          python -m pip install --upgrade pip
+          pip install requests==2.23.0
+          python index_files.py -u ${{ secrets.INDEX_DEV_API_URL_V2 }} -d ${{ env.release_branch }}-v2 -he ${{ secrets.INDEX_DEV_API_HEADERS }}
+      - name: Notify Slack
+        if: always()
+        uses: edge/simple-slack-notify@master
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.CURATOR_SLACK_WEBHOOK_URL }}
+        with:
+          channel: '#ror-curation-releases'
+          color: 'good'
+          text: 'DEV index status: ${{ steps.indexdata.outcome }}. Using v2. On branch: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/tree/${env.GITHUB_REF_NAME}. Directory: ${{ env.release_branch }}. Please check: ${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}'
+