Merge pull request #45 from polywrap/update-schema-pre-staging

feat: move github tables into indexing schema

ops/scripts/generate_types.py

@@ -54,9 +54,11 @@ def run():
     # Extract only the "CREATE TABLE" and "CREATE TYPE" statements
     create_table_statements = [stmt + ';' for stmt in statements if (
         stmt.strip().startswith('CREATE TABLE IF NOT EXISTS "public".') or
+        stmt.strip().startswith('CREATE TABLE IF NOT EXISTS "indexing".') or
         stmt.strip().startswith('CREATE TYPE "public".')
     )]
     create_table_statements = [stmt.replace('CREATE TABLE IF NOT EXISTS "public".', 'CREATE TABLE ') for stmt in create_table_statements]
+    create_table_statements = [stmt.replace('CREATE TABLE IF NOT EXISTS "indexing".', 'CREATE TABLE ') for stmt in create_table_statements]
     create_table_statements = [stmt.replace('"public".', '') for stmt in create_table_statements]
     # Remove some unsupported SQL features that break omymodels
     create_table_statements = [stmt.replace('DEFAULT "gen_random_uuid"() NOT NULL', '') for stmt in create_table_statements]

web/supabase/config.toml

@@ -8,7 +8,7 @@ enabled = true
 port = 54321
 # Schemas to expose in your API. Tables, views and stored procedures in this schema will get API
 # endpoints. public and storage are always included.
-schemas = ["public", "storage", "graphql_public"]
+schemas = ["public", "storage", "graphql_public", "indexing"]
 # Extra schemas to add to the search_path of every request. public is always included.
 extra_search_path = ["public", "extensions"]
 # The maximum number of rows returns from a view, table, or stored procedure. Limits payload size

web/supabase/dbTypes.ts

@@ -47,107 +47,6 @@ export interface Database {
           }
         ]
       }
-      gitcoin_applications: {
-        Row: {
-          created_at: number
-          data: Json
-          id: string
-          pointer: string
-          project_id: string
-          protocol: number
-          round_id: string
-        }
-        Insert: {
-          created_at: number
-          data: Json
-          id: string
-          pointer: string
-          project_id: string
-          protocol: number
-          round_id: string
-        }
-        Update: {
-          created_at?: number
-          data?: Json
-          id?: string
-          pointer?: string
-          project_id?: string
-          protocol?: number
-          round_id?: string
-        }
-        Relationships: [
-          {
-            foreignKeyName: "gitcoin_applications_project_id_fkey"
-            columns: ["project_id"]
-            isOneToOne: false
-            referencedRelation: "gitcoin_projects"
-            referencedColumns: ["id"]
-          }
-        ]
-      }
-      gitcoin_indexing_jobs: {
-        Row: {
-          created_at: string
-          error: string | null
-          id: string
-          is_failed: boolean
-          is_running: boolean
-          last_updated_at: string
-          network_id: number
-          skip_projects: number
-          skip_rounds: number
-          url: string
-        }
-        Insert: {
-          created_at?: string
-          error?: string | null
-          id?: string
-          is_failed?: boolean
-          is_running?: boolean
-          last_updated_at?: string
-          network_id: number
-          skip_projects?: number
-          skip_rounds?: number
-          url: string
-        }
-        Update: {
-          created_at?: string
-          error?: string | null
-          id?: string
-          is_failed?: boolean
-          is_running?: boolean
-          last_updated_at?: string
-          network_id?: number
-          skip_projects?: number
-          skip_rounds?: number
-          url?: string
-        }
-        Relationships: []
-      }
-      gitcoin_projects: {
-        Row: {
-          created_at: string
-          data: Json
-          id: string
-          pointer: string
-          protocol: number
-        }
-        Insert: {
-          created_at?: string
-          data: Json
-          id: string
-          pointer: string
-          protocol: number
-        }
-        Update: {
-          created_at?: string
-          data?: Json
-          id?: string
-          pointer?: string
-          protocol?: number
-        }
-        Relationships: []
-      }
       logs: {
         Row: {
           created_at: string

web/supabase/migrations/20240118150300_gitcoin.sql

@@ -1,15 +1,21 @@
-create table "public"."gitcoin_projects" (
+CREATE SCHEMA indexing;
+GRANT USAGE ON SCHEMA indexing TO service_role;
+GRANT ALL ON ALL TABLES IN SCHEMA indexing TO service_role;
+GRANT ALL ON ALL ROUTINES IN SCHEMA indexing TO service_role;
+GRANT ALL ON ALL SEQUENCES IN SCHEMA indexing TO service_role;
+
+create table "indexing"."gitcoin_projects" (
     "id" text not null,
     "created_at" timestamp with time zone not null default now(),
     "data" json not null,
     "protocol" int not null,
     "pointer" text not null,
     PRIMARY KEY ("id")
 );
-ALTER TABLE "public"."gitcoin_projects" OWNER TO "postgres";
-ALTER TABLE "public"."gitcoin_projects" enable row level security;
+ALTER TABLE "indexing"."gitcoin_projects" OWNER TO "postgres";
+ALTER TABLE "indexing"."gitcoin_projects" enable row level security;
 
-create table "public"."gitcoin_applications" (
+create table "indexing"."gitcoin_applications" (
     "id" text not null,
     "created_at" int not null,
     "data" json not null,
@@ -18,12 +24,12 @@ create table "public"."gitcoin_applications" (
     "round_id" text not null,
     "project_id" text not null,
     PRIMARY KEY ("id"),
-    FOREIGN KEY ("project_id") REFERENCES "public"."gitcoin_projects"("id") ON DELETE CASCADE
+    FOREIGN KEY ("project_id") REFERENCES "indexing"."gitcoin_projects"("id") ON DELETE CASCADE
 );
-ALTER TABLE "public"."gitcoin_applications" OWNER TO "postgres";
-ALTER TABLE "public"."gitcoin_applications" enable row level security;
+ALTER TABLE "indexing"."gitcoin_applications" OWNER TO "postgres";
+ALTER TABLE "indexing"."gitcoin_applications" enable row level security;
 
-create table "public"."gitcoin_indexing_jobs" (
+create table "indexing"."gitcoin_indexing_jobs" (
     "id" uuid not null default gen_random_uuid(),
     "created_at" timestamp with time zone not null default now(),
     "url" text not null,
@@ -36,10 +42,10 @@ create table "public"."gitcoin_indexing_jobs" (
     "error" text null,
     PRIMARY KEY ("id")
 );
-ALTER TABLE "public"."gitcoin_indexing_jobs" OWNER TO "postgres";
-ALTER TABLE "public"."gitcoin_indexing_jobs" enable row level security;
+ALTER TABLE "indexing"."gitcoin_indexing_jobs" OWNER TO "postgres";
+ALTER TABLE "indexing"."gitcoin_indexing_jobs" enable row level security;
 
-insert into "public"."gitcoin_indexing_jobs" ("url", "network_id") values 
+insert into "indexing"."gitcoin_indexing_jobs" ("url", "network_id") values 
     ('https://api.thegraph.com/subgraphs/name/allo-protocol/grants-round-polygon', 137), 
     ('https://api.thegraph.com/subgraphs/name/vacekj/allo-mainnet', 1),
     ('https://graph-gitcoin-mainnet.hirenodes.io/subgraphs/name/gitcoin/allo', 424),

workers/fund_public_goods/api/runs.py

@@ -1,6 +1,6 @@
 from fund_public_goods.inngest_client import inngest_client
 from fund_public_goods.workflows.create_strategy.events import CreateStrategyEvent
-from fund_public_goods.db import client, tables, entities
+from fund_public_goods.db import tables, entities
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
 
@@ -22,14 +22,13 @@ async def runs(worker_id: str, params: Params) -> Response:
     if prompt == "":
         raise HTTPException(status_code=400, detail="Prompt cannot be empty.")
 
-    db = client.create_admin()
-    worker_exists = tables.workers.exists(db, worker_id)
+    worker_exists = tables.workers.exists(worker_id)
     if not worker_exists:
         raise HTTPException(
             status_code=400, detail=f"Worker with ID: {worker_id} is not valid"
         )
 
-    run_id = tables.runs.insert(db, entities.Runs(
+    run_id = tables.runs.insert(entities.Runs(
         worker_id=worker_id,
         prompt=prompt
     ))

workers/fund_public_goods/api/workers.py

@@ -2,7 +2,7 @@
 from pydantic import BaseModel
 from fund_public_goods.inngest_client import inngest_client
 from fund_public_goods.workflows.create_strategy.events import CreateStrategyEvent
-from fund_public_goods.db import client, tables, entities
+from fund_public_goods.db import tables, entities
 
 router = APIRouter()
 
@@ -23,9 +23,8 @@ async def workers(params: Params) -> Response:
     if prompt == "":
         raise HTTPException(status_code=400, detail="Prompt cannot be empty.")
 
-    db = client.create_admin()
-    worker_id = tables.workers.insert(db)
-    run_id = tables.runs.insert(db, entities.Runs(
+    worker_id = tables.workers.insert()
+    run_id = tables.runs.insert(entities.Runs(
         worker_id=worker_id,
         prompt=prompt
     ))

workers/fund_public_goods/db/client.py

@@ -1,5 +1,7 @@
 import os
+from typing import Optional
 from supabase import create_client, Client
+from supabase.client import ClientOptions
 
 def create() -> Client:
     url: str | None = os.environ.get("NEXT_PUBLIC_SUPABASE_URL")
@@ -12,7 +14,7 @@ def create() -> Client:
 
     return create_client(url, key)
 
-def create_admin() -> Client:
+def create_admin(schema: Optional[str] = None) -> Client:
     url: str | None = os.environ.get("NEXT_PUBLIC_SUPABASE_URL")
     key: str | None = os.environ.get("SUPABASE_SERVICE_ROLE_KEY")
 
@@ -21,4 +23,8 @@ def create_admin() -> Client:
     if key is None:
         raise Exception("SUPABASE_SERVICE_ROLE_KEY is not set")
 
-    return create_client(url, key)
+    if schema:
+        options = ClientOptions(schema=schema)
+        return create_client(url, key, options)
+    else:
+        return create_client(url, key)

workers/fund_public_goods/db/entities.py

@@ -21,20 +21,6 @@ class StepStatus(str, Enum):
     NOT_STARTED = 'NOT_STARTED'
 
 
-class Applications(BaseModel):
-
-    id: str
-    created_at: int = Field(..., alias="createdAt")
-    recipient: str
-    network: int
-    round: str
-    answers: Optional[Json] = None
-    project_id: str = Field(..., alias="projectId")
-    model_config = ConfigDict(
-        populate_by_name=True
-    )
-
-
 class GitcoinApplications(BaseModel):
 
     id: str
@@ -80,6 +66,20 @@ class GitcoinProjects(BaseModel):
     )
 
 
+class Applications(BaseModel):
+
+    id: str
+    created_at: int = Field(..., alias="createdAt")
+    recipient: str
+    network: int
+    round: str
+    answers: Optional[Json] = None
+    project_id: str = Field(..., alias="projectId")
+    model_config = ConfigDict(
+        populate_by_name=True
+    )
+
+
 class Logs(BaseModel):
 
     id: Optional[UUID] = None

workers/fund_public_goods/db/tables/applications.py

@@ -1,10 +1,11 @@
-from supabase import Client, PostgrestAPIResponse
+from fund_public_goods.db.client import create_admin
 from fund_public_goods.db.entities import Applications
 
+
 def insert(
-    db: Client,
     row: Applications
 ):
+    db = create_admin()
     db.table("applications").insert({
         "id": row.id,
         "created_at": row.created_at,
@@ -16,9 +17,9 @@ def insert(
     }).execute()
 
 def get_applications(
-    db: Client,
     project_id: str
 ) -> list[Applications]:
+    db = create_admin()
     result = (db.table("applications")
         .select("id, created_at, recipient, network, round, answers, project_id")
         .eq("project_id", project_id)