github workflow for daily updates

identifiers-org · Nov 12, 2024 · ed147cb · ed147cb
1 parent a998bc0
commit ed147cb
Show file tree

Hide file tree

Showing 8 changed files with 80 additions and 28,023 deletions.
diff --git a/.github/workflows/ontop-k8s.yml b/.github/workflows/ontop-k8s.yml
@@ -10,14 +10,6 @@ on:
     branches:
       - main
 
-env:
-  PROJECT_ID: 'golden-tenure-196110'
-  GAR_LOCATION: 'europe-west1'
-  GKE_CLUSTER: 'idorg-production'
-  GKE_ZONE: 'europe-west1'
-  IMAGE: 'static-site'
-  WORKLOAD_IDENTITY_PROVIDER: 'projects/61618937397/locations/global/workloadIdentityPools/github/providers/my-repo'
-
 jobs:
   setup-build-publish-deploy:
     name: 'Setup and trigger job'
@@ -40,19 +32,25 @@ jobs:
         name: 'Authenticate to Google Cloud'
         uses: 'google-github-actions/auth@f112390a2df9932162083945e46d439060d66ec2' # google-github-actions/auth@v2
         with:
-          project_id: '${{ env.PROJECT_ID }}'
-          workload_identity_provider: '${{ env.WORKLOAD_IDENTITY_PROVIDER }}'
+          project_id: '${{ secrets.GCP_PROJECT_ID }}'
+          workload_identity_provider: '${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}'
 
 
       # Get the GKE credentials so we can deploy to the cluster
       - name: 'Set up GKE credentials'
         uses: 'google-github-actions/get-gke-credentials@6051de21ad50fbb1767bc93c11357a49082ad116' # google-github-actions/get-gke-credentials@v2
         with:
-          cluster_name: '${{ env.GKE_CLUSTER }}'
-          location: '${{ env.GKE_ZONE }}'
+          cluster_name: '${{ secrets.GKE_CLUSTER }}'
+          location: '${{ secrets.GCP_CLUSTER_LOCATION }}'
 
 
       # Deploy the Docker image to the GKE cluster
-      - name: 'Trigger the ontop-materialize job'
-        run: |-
-          kubectl get pods
+      - name: 'Update materialized triples job'
+        run: >-
+          kubectl -n hq delete -f ontop-k8s-job.yaml && 
+          kubectl -n hq apply  -f ontop-k8s-job.yaml && 
+          kubectl -n hq wait --for=condition=complete --timeout=60s -f ontop-k8s-job.yaml"
+
+      - name: 'Restart SPARQL server'
+        run: kubectl -n satellite rollout restart deployment/sparql
+
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
 tutorial/
 bins/
 .DS_Store
+*.iml
diff --git a/idorg-ontology/idorg-ontology-materialized.ttl b/idorg-ontology/idorg-ontology-materialized.ttl
diff --git a/idorg-ontology/idorg-ontology.q b/idorg-ontology/idorg-ontology.q
diff --git a/idorg-ontology/idorg-ontology.ttl b/idorg-ontology/idorg-ontology.ttl
@@ -248,21 +248,21 @@ idot:Registry rdf:type owl:NamedIndividual ,
               rdfs:seeAlso <https://docs.identifiers.org/pages/sparql.html> ;
               dcat:landingPage <https://registry.identifiers.org> .
 
-_:eq1 rdf:type owl:NamedIndividual, sh:SPARQLExecutable ;
+idot:eq1 rdf:type owl:NamedIndividual, sh:SPARQLExecutable ;
                    rdfs:comment "URL resolution using owl:sameAs via our virtual triple store" ;
                    sh:select """PREFIX owl: <http://www.w3.org/2002/07/owl#>
 SELECT * WHERE {
     <http://identifiers.org/uniprot:P12345> owl:sameAs ?obj .
 } LIMIT 10""" .
 
-_:eq2 rdf:type owl:NamedIndividual, sh:SPARQLExecutable ;
+idot:eq2 rdf:type owl:NamedIndividual, sh:SPARQLExecutable ;
                    rdfs:comment "Reverse resolution using owl:sameAs via our virtual triple store" ;
                    sh:select """PREFIX owl: <http://www.w3.org/2002/07/owl#>
 SELECT * WHERE {
     <http://purl.uniprot.org/uniprot/P12345> owl:sameAs ?obj .
 } LIMIT 10""" .
 
-_:eq3 rdf:type owl:NamedIndividual, sh:SPARQLExecutable ;
+idot:eq3 rdf:type owl:NamedIndividual, sh:SPARQLExecutable ;
                    rdfs:comment "Usage of id:active subgraph to limit resolution" ;
                    sh:select """PREFIX owl: <http://www.w3.org/2002/07/owl#>
 SELECT ?target
@@ -271,7 +271,7 @@ WHERE {
     <http://identifiers.org/uniprot:P12345> owl:sameAs ?target
 }""" .
 
-_:eq4 rdf:type owl:NamedIndividual, sh:SPARQLExecutable ;
+idot:eq4 rdf:type owl:NamedIndividual, sh:SPARQLExecutable ;
                    rdfs:comment "List namespaces and their names" ;
                    sh:select """PREFIX owl: <http://www.w3.org/2002/07/owl#>
 PREFIX idot: <http://identifiers.org/idot/>
@@ -283,7 +283,7 @@ WHERE {
 }
 LIMIT 10""" .
 
-_:eq5 rdf:type owl:NamedIndividual, sh:SPARQLExecutable ;
+idot:eq5 rdf:type owl:NamedIndividual, sh:SPARQLExecutable ;
                    rdfs:comment "List namespaces and their url patterns" ;
                    sh:select """PREFIX idot: <http://identifiers.org/idot/>
 PREFIX dcterms: <http://purl.org/dc/terms/>
@@ -296,7 +296,7 @@ WHERE {
 ORDER BY ASC(?namespace)
 LIMIT 10""" .
 
-_:eq6 rdf:type owl:NamedIndividual, sh:SPARQLExecutable ;
+idot:eq6 rdf:type owl:NamedIndividual, sh:SPARQLExecutable ;
                    rdfs:comment "Using DCAT to access registry" ;
                    sh:select """PREFIX dcterms: <http://purl.org/dc/terms/>
 PREFIX dcat: <http://www.w3.org/ns/dcat#>
@@ -305,6 +305,6 @@ WHERE {
     ?namespace a dcat:Dataset ; dcterms:title ?namespaceTitle .
 }""" .
 
-_:eq7 rdf:type owl:NamedIndividual, sh:SPARQLExecutable ;
+idot:eq7 rdf:type owl:NamedIndividual, sh:SPARQLExecutable ;
                    rdfs:comment "Get full description of namespace" ;
                    sh:select "DESCRIBE <http://identifiers.org/3dmet>" .
diff --git a/idorg-ontology/ontop.portal.toml b/idorg-ontology/ontop.portal.toml
diff --git a/ontop-k8s-job.yaml b/ontop-k8s-job.yaml
@@ -0,0 +1,44 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ontop-materializer
+  labels:
+    component: ontop
+    tier: api
+    app: idorg
+    deployment: hq
+    environment: staging
+spec:
+  template:
+    spec:
+      containers:
+      - name: ontop
+        image: identifiersorg/ontop:1.1.0 #ontop + postgresql jdbc driver + curl
+        workingDir: "/tmp"
+        command: ["/bin/sh", "-c"]
+        args: 
+          - >-
+            wget --no-check-certificate -q https://raw.githubusercontent.com/identifiers-org/ontop/refs/heads/main/update-triples-script.sh &&
+            bash update-triples-script.sh
+        env:
+          - name: ONTOP_DB_URL
+            value: "jdbc:postgresql://hq-postgresql:5432/dbhqregistry"
+          - name: ONTOP_DB_USER
+            valueFrom:
+              secretKeyRef:
+                name: hq-registry-database-credentials
+                key: username
+          - name: ONTOP_DB_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: hq-registry-database-credentials
+                key: password
+          - name: ONTOP_GH_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: github-credentials
+                key: auth-token
+          - name: ONTOP_LOG_LEVEL
+            value: ERROR
+      restartPolicy: Never
+  backoffLimit: 4
diff --git a/update-triples-script.sh b/update-triples-script.sh
@@ -4,20 +4,20 @@ wget --no-check-certificate -q https://raw.githubusercontent.com/identifiers-org
 wget --no-check-certificate -q https://raw.githubusercontent.com/identifiers-org/ontop/refs/heads/main/idorg-ontology/idorg-ontology.ttl
 wget --no-check-certificate -q https://raw.githubusercontent.com/identifiers-org/ontop/refs/heads/main/idorg-ontology/idorg-ontology.obda
 ontop materialize --db-user "$ONTOP_DB_USER" --db-password "$ONTOP_DB_PASSWORD" --db-url "$ONTOP_DB_URL" -m ./idorg-ontology.obda -t ./idorg-ontology.ttl -o ./new-idorg-ontology-materialized.rdf -p /dev/null
-# if diff -q new-idorg-ontology-materialized.rdf idorg-ontology-materialized.rdf > /dev/null; then
-echo Updating materialized file
+if diff -q new-idorg-ontology-materialized.rdf idorg-ontology-materialized.rdf > /dev/null; then
+    echo Updating materialized file
 
-echo -n "{\"message\":\"Sync materialized registry tuples with database\", \"committer\":{\"name\":\"Ontop bot\", \"email\":\"[email protected]\"}, \"content\":\"" > datafile
-cat new-idorg-ontology-materialized.rdf|base64 >> datafile
-old_sha=$(wget --no-check-certificate -O - -q https://api.github.com/repos/identifiers-org/ontop/contents/idorg-ontology/idorg-ontology-materialized.rdf | grep sha | cut -d\" -f4)
-echo "\", \"sha\": \"$old_sha\"}" >> datafile
+    echo -n "{\"message\":\"Sync triples with database [skip ci]\", \"committer\":{\"name\":\"Sync bot\", \"email\":\"[email protected]\"}, \"content\":\"" > datafile
+    cat new-idorg-ontology-materialized.rdf|base64 >> datafile
+    old_sha=$(wget --no-check-certificate -O - -q https://api.github.com/repos/identifiers-org/ontop/contents/idorg-ontology/idorg-ontology-materialized.rdf | grep sha | cut -d\" -f4)
+    echo "\", \"sha\": \"$old_sha\"}" >> datafile
 
-curl -X PUT --insecure \
-     -H "Accept: application/vnd.github+json" \
-     -H "Authorization: Bearer $ONTOP_GH_TOKEN" \
-     -H "X-GitHub-Api-Version: 2022-11-28" \
-     --data @./datafile \
-     https://api.github.com/repos/identifiers-org/ontop/contents/idorg-ontology/idorg-ontology-materialized.rdf
-# else
-#   echo No need to update materialized file
-# fi
+    curl -X PUT --insecure \
+         -H "Accept: application/vnd.github+json" \
+         -H "Authorization: Bearer $ONTOP_GH_TOKEN" \
+         -H "X-GitHub-Api-Version: 2022-11-28" \
+         --data @./datafile \
+         https://api.github.com/repos/identifiers-org/ontop/contents/idorg-ontology/idorg-ontology-materialized.rdf
+ else
+    echo No need to update materialized file
+ fi