Merge pull request #2820 from wellcomecollection/Add-catalogue-graph

Add catalogue graph

builds/run_linting.sh

@@ -12,5 +12,5 @@ docker run --tty --rm \
 	--volume "$ROOT:/data" \
 	--workdir /data \
 	"$ECR_REGISTRY/wellcome/flake8:latest" \
-    --exclude .git,__pycache__,target,.terraform \
+    --exclude .git,__pycache__,target,.terraform,catalogue_graph \
     --ignore=E501,E122,E126,E203,W503

catalogue_graph/.github/workflows/autoformat-typecheck.yml

@@ -0,0 +1,59 @@
+# Runs auto-formatting and type checking on push to any branch
+name: "Python auto-formatting & type checking"
+
+on: push
+
+permissions:
+  id-token: write
+  contents: write
+
+jobs:
+  typecheck:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Setup CI
+        run: |
+          ./scripts/ci-setup.sh
+
+      - name: Run typecheck
+        run: |
+          ./scripts/typecheck.sh
+
+  autoformat:
+    runs-on: ubuntu-latest
+    needs: typecheck
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Setup CI
+        run: |
+          ./scripts/ci-setup.sh
+
+      - name: Run autoformat
+        run: |
+          ./scripts/autoformat.sh
+
+      - name: Check for formatting changes
+        id: check_formatting_changes
+        run: |
+          if [[ -n $(git status --porcelain) ]]; then
+              echo "changes=true" >> "$GITHUB_OUTPUT"; 
+          fi
+
+      - name: Commit and push formatting changes
+        if: steps.check_formatting_changes.outputs.changes == 'true'
+        run: |
+          git config user.name "Github on behalf of Wellcome Collection"
+          git config user.email "[email protected]"
+          git commit -am "Apply auto-formatting rules"
+          git push

catalogue_graph/.github/workflows/build.yml

@@ -0,0 +1,22 @@
+# Build deployment artifact 
+name: "Build deployment zip"
+
+on: push
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Setup CI
+        run: |
+          ./scripts/ci-setup.sh
+
+      - name: Build zip
+        run: |
+          ./scripts/build.sh

catalogue_graph/.github/workflows/ci.yml

@@ -0,0 +1,61 @@
+name: "Run CI pipeline"
+
+on: 
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  test: 
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - name: CI setup
+        run: |
+          ./scripts/ci-setup.sh
+      - name: Test
+        run: |
+          ./scripts/test.sh
+        
+  build:
+    runs-on: ubuntu-latest
+    needs: test
+    if: ${{ needs.test.result == 'success' }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-west-1
+          role-to-assume: ${{ secrets.CATALOGUE_GRAPH_CI_ROLE_ARN }}
+      - name: Log in to private ECR
+        uses: aws-actions/amazon-ecr-login@v2
+      - name: Build and push artefacts
+        run: |
+          ./scripts/build.sh --push
+  
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build
+    if: ${{ needs.build.result == 'success' }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-west-1
+          role-to-assume: ${{ secrets.CATALOGUE_GRAPH_CI_ROLE_ARN }}
+      - name: Deploy bulk-loader lambda
+        run: |
+          ./scripts/deploy_lambda_zip.sh catalogue-graph-bulk-loader
+      - name: Deploy bulk-load-poller lambda
+        run: |
+          ./scripts/deploy_lambda_zip.sh catalogue-graph-bulk-load-poller
+      - name: Deploy indexer lambda
+        run: |
+          ./scripts/deploy_lambda_zip.sh catalogue-graph-indexer 

catalogue_graph/.github/workflows/test.yml

@@ -0,0 +1,34 @@
+# Build deployment artifact 
+name: "Run tests"
+
+on: 
+    push:
+      branches:
+        - main
+    pull_request:
+    workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
+      - name: Setup CI
+        run: |
+          ./scripts/ci-setup.sh
+
+      - name: Run tests & generate coverage report
+        run: |
+          ./scripts/test.sh
+
+      - name: Get Cover 
+        if : ${{ github.event_name == 'pull_request' }}
+        uses: orgoro/[email protected]
+        with:
+          coverageFile: coverage.xml
+          token: ${{ secrets.GITHUB_TOKEN }}