Track Network & Address + Generate Pydantic DB Types

ops/pyproject.toml

@@ -0,0 +1,18 @@
+[tool.poetry]
+name = "scripts"
+version = "0.1.0"
+description = ""
+authors = []
+
+[tool.poetry.dependencies]
+python = "~3.10.6"
+
+[tool.poetry.group.dev.dependencies]
+omymodels = "^0.15.1"
+
+[tool.poetry.scripts]
+generate-types = "scripts.generate_types:run"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

ops/scripts/generate_types.py

@@ -0,0 +1,83 @@
+import subprocess
+import sys
+import re
+from omymodels import create_models
+
+def add_class_config(input_text):
+    # Regex pattern to match class definitions
+    class_pattern = r"(class\s+\w+\(BaseModel\):)([\s\S]+?)(\n\n|\Z)"
+    replacement = r"\1\2\n\n    model_config = ConfigDict(\n        populate_by_name=True\n    )\3"
+    return re.sub(class_pattern, replacement, input_text)
+
+def snake_to_camel(snake_str):
+    components = snake_str.split('_')
+    # Capitalize the first letter of each component except the first one
+    # and join them together.
+    return components[0] + ''.join(x.title() for x in components[1:])
+
+def add_alias_no_default(input_text):
+    # Regex pattern to match properties without a default value
+    property_pattern = r"(\s+)(\w+_\w+)(:\s+\w+)\s*\n"
+    def edit(match):
+        name, type_def = match.group(2), match.group(3)
+        camel_case = snake_to_camel(name)
+        return f"{match.group(1)}{name}{type_def} = Field(..., alias=\"{camel_case}\")\n"
+    return re.sub(property_pattern, edit, input_text)
+
+def add_alias_with_default(input_text):
+    # Regex pattern to match properties with a default value
+    property_with_default_pattern = r"(\s+)(\w+_\w+)(:\s+Optional\[\w+\.?\w*\]\s*=\s*None)\n"
+    def edit(match):
+        name, type_def = match.group(2), match.group(3)
+        # Extract the type without Optional and the default value
+        type_only = re.search(r'Optional\[(\w+\.?\w*)\]', type_def).group(1)
+        camel_case = snake_to_camel(name)
+        return f"{match.group(1)}{name}: Optional[{type_only}] = Field(default=None, alias=\"{camel_case}\")\n"
+    return re.sub(property_with_default_pattern, edit, input_text)
+
+def run():
+    # Run `supabase db dump --local` to get the db schema
+    result = subprocess.run(
+        ["npx", "supabase", "db", "dump", "--local"],
+        capture_output=True,
+        cwd="../web"
+    )
+    if result.returncode != 0:
+        print("Failed to run 'supabase db dump --local'")
+        print(result.stderr.decode())
+        sys.exit(1)
+
+    db_schema = result.stdout.decode()
+
+    # Split the schema by statement (ending in ;)
+    statements = [stmt.strip() for stmt in db_schema.split(';')]
+    # Extract only the "CREATE TABLE" statements
+    create_table_statements = [stmt + ';' for stmt in statements if stmt.strip().startswith('CREATE TABLE IF NOT EXISTS "public".')]
+    create_table_statements = [stmt.replace('CREATE TABLE IF NOT EXISTS "public".', 'CREATE TABLE ') for stmt in create_table_statements]
+    # Remove some unsupported SQL features that break omymodels
+    create_table_statements = [stmt.replace('DEFAULT "gen_random_uuid"() NOT NULL', '') for stmt in create_table_statements]
+    create_table_statements = [stmt.replace('with time zone DEFAULT "now"() NOT NULL', '') for stmt in create_table_statements]
+    create_table_statements = [re.sub(r'(?m)CONSTRAINT.*\n?', '', stmt) for stmt in create_table_statements]
+    db_schema = '\n\n'.join(create_table_statements)
+
+    # Generate pydantic types using omymodels
+    types = create_models(db_schema, models_type="pydantic")["code"]
+
+    # Convert "= false" and "= true" to proper Python
+    types = re.sub(r'= false', '= False', types)
+    types = re.sub(r'= true', '= Talse', types)
+
+    # Default Optional types = None
+    types = re.sub(r'Optional\[(.*?)\]', r'Optional[\1] = None', types)
+
+    # Add aliases for all snake case classes
+    types = add_class_config(types)
+    types = add_alias_no_default(types)
+    types = add_alias_with_default(types)
+    types = types.replace("from pydantic import BaseModel, Json", "from pydantic import BaseModel, Json, Field, ConfigDict")
+
+    # Write the types to a file
+    with open("../workers/fund_public_goods/db/entities.py", "w") as file:
+        file.write(types)
+
+    sys.exit(0)

package.json

@@ -4,16 +4,25 @@
     "web"
   ],
   "scripts": {
-    "postinstall": "cd workers && poetry install",
+    "postinstall": "yarn workers:install && yarn ops:install",
+    "codegen": "yarn web:codegen && yarn ops:codegen",
     "build": "yarn web:build && yarn workers:build",
     "dev": "npx concurrently \"yarn web:dev\" \"yarn workers:dev\" \"yarn events:dev\" -k -n web,workers,events",
+    "web:codegen": "cd web && yarn db:generate-types",
     "web:build": "yarn web:env && cd web && yarn build",
     "web:dev": "yarn web:env && cd web && yarn dev",
     "web:env": "if [ \"$CICD\" != \"true\" ]; then cp .env ./web/.env; fi",
+    "db:start": "cd web && yarn db:start",
+    "db:reset": "cd web && yarn db:reset",
+    "db:stop": "cd web && yarn db:stop",
+    "workers:install": "cd workers && poetry install",
     "workers:build": "cd workers && poetry run build-check",
     "workers:dev": "yarn workers:env && cd workers && poetry run python -m uvicorn fund_public_goods.main:app --reload",
     "workers:env": "if [ \"$CICD\" != \"true\" ]; then cp .env ./workers/.env; fi",
-    "events:dev": "npx inngest-cli dev"
+    "workers:types": "cd ops && poetry run generate-types",
+    "events:dev": "npx inngest-cli dev",
+    "ops:install": "cd ops && poetry install",
+    "ops:codegen": "cd ops && poetry run generate-types"
   },
   "dependencies": {
     "concurrently": "8.2.2",

web/app/actions/startWorker.ts

@@ -17,6 +17,10 @@ export const startWorker = async (
     },
   });
 
+  if (response.status !== 200) {
+    throw Error(`Error startring new worker. Status: ${response.status}\nMessage: ${response.statusText}`);
+  }
+
   const result = await response.json();
   if (!result.worker_id || !result.run_id) {
     throw new Error("Error starting new worker");

web/supabase/dbTypes.ts

@@ -9,6 +9,145 @@ export type Json =
 export interface Database {
   public: {
     Tables: {
+      applications: {
+        Row: {
+          answers: Json | null
+          created_at: number
+          id: string
+          network: number
+          project_id: string
+          recipient: string
+          round: string
+        }
+        Insert: {
+          answers?: Json | null
+          created_at: number
+          id: string
+          network: number
+          project_id: string
+          recipient: string
+          round: string
+        }
+        Update: {
+          answers?: Json | null
+          created_at?: number
+          id?: string
+          network?: number
+          project_id?: string
+          recipient?: string
+          round?: string
+        }
+        Relationships: [
+          {
+            foreignKeyName: "applications_project_id_fkey"
+            columns: ["project_id"]
+            isOneToOne: false
+            referencedRelation: "projects"
+            referencedColumns: ["id"]
+          }
+        ]
+      }
+      gitcoin_applications: {
+        Row: {
+          created_at: number
+          data: Json
+          id: string
+          pointer: string
+          project_id: string
+          protocol: number
+          round_id: string
+        }
+        Insert: {
+          created_at: number
+          data: Json
+          id: string
+          pointer: string
+          project_id: string
+          protocol: number
+          round_id: string
+        }
+        Update: {
+          created_at?: number
+          data?: Json
+          id?: string
+          pointer?: string
+          project_id?: string
+          protocol?: number
+          round_id?: string
+        }
+        Relationships: [
+          {
+            foreignKeyName: "gitcoin_applications_project_id_fkey"
+            columns: ["project_id"]
+            isOneToOne: false
+            referencedRelation: "gitcoin_projects"
+            referencedColumns: ["id"]
+          }
+        ]
+      }
+      gitcoin_indexing_jobs: {
+        Row: {
+          created_at: string
+          error: string | null
+          id: string
+          is_failed: boolean
+          is_running: boolean
+          last_updated_at: string
+          network_id: number
+          skip_projects: number
+          skip_rounds: number
+          url: string
+        }
+        Insert: {
+          created_at?: string
+          error?: string | null
+          id?: string
+          is_failed?: boolean
+          is_running?: boolean
+          last_updated_at?: string
+          network_id: number
+          skip_projects?: number
+          skip_rounds?: number
+          url: string
+        }
+        Update: {
+          created_at?: string
+          error?: string | null
+          id?: string
+          is_failed?: boolean
+          is_running?: boolean
+          last_updated_at?: string
+          network_id?: number
+          skip_projects?: number
+          skip_rounds?: number
+          url?: string
+        }
+        Relationships: []
+      }
+      gitcoin_projects: {
+        Row: {
+          created_at: string
+          data: Json
+          id: string
+          pointer: string
+          protocol: number
+        }
+        Insert: {
+          created_at?: string
+          data: Json
+          id: string
+          pointer: string
+          protocol: number
+        }
+        Update: {
+          created_at?: string
+          data?: Json
+          id?: string
+          pointer?: string
+          protocol?: number
+        }
+        Relationships: []
+      }
       logs: {
         Row: {
           created_at: string
@@ -42,22 +181,22 @@ export interface Database {
         Row: {
           description: string | null
           id: string
-          recipient: string | null
           title: string | null
+          updated_at: number
           website: string | null
         }
         Insert: {
           description?: string | null
-          id?: string
-          recipient?: string | null
+          id: string
           title?: string | null
+          updated_at: number
           website?: string | null
         }
         Update: {
           description?: string | null
           id?: string
-          recipient?: string | null
           title?: string | null
+          updated_at?: number
           website?: string | null
         }
         Relationships: []
@@ -123,6 +262,13 @@ export interface Database {
           weight?: number | null
         }
         Relationships: [
+          {
+            foreignKeyName: "strategy_entries_project_id_fkey"
+            columns: ["project_id"]
+            isOneToOne: false
+            referencedRelation: "projects"
+            referencedColumns: ["id"]
+          },
           {
             foreignKeyName: "strategy_entries_run_id_fkey"
             columns: ["run_id"]

web/supabase/migrations/20240118120046_init.sql

@@ -47,6 +47,7 @@ SELECT
 
 CREATE TABLE "public"."projects" (
     "id" text NOT NULL,
+    "updated_at" int NOT NULL,
     "title" TEXT,
     "description" TEXT,
     "website" TEXT,

web/supabase/migrations/20240118150300_gitcoin.sql

@@ -11,7 +11,7 @@ ALTER TABLE "public"."gitcoin_projects" enable row level security;
 
 create table "public"."gitcoin_applications" (
     "id" text not null,
-    "created_at" timestamp with time zone not null default now(),
+    "created_at" int not null,
     "data" json not null,
     "protocol" int not null,
     "pointer" text not null,
@@ -27,6 +27,7 @@ create table "public"."gitcoin_indexing_jobs" (
     "id" uuid not null default gen_random_uuid(),
     "created_at" timestamp with time zone not null default now(),
     "url" text not null,
+    "network_id" int not null,
     "is_running" boolean not null default false,
     "skip_rounds" int not null default 0,
     "skip_projects" int not null default 0,
@@ -38,14 +39,13 @@ create table "public"."gitcoin_indexing_jobs" (
 ALTER TABLE "public"."gitcoin_indexing_jobs" OWNER TO "postgres";
 ALTER TABLE "public"."gitcoin_indexing_jobs" enable row level security;
 
-insert into "public"."gitcoin_indexing_jobs" ("url") values 
-    ('https://api.thegraph.com/subgraphs/name/allo-protocol/grants-round-polygon'), 
-    ('https://api.thegraph.com/subgraphs/name/vacekj/allo-mainnet'),
-    ('https://graph-gitcoin-mainnet.hirenodes.io/subgraphs/name/gitcoin/allo'),
-    ('https://api.thegraph.com/subgraphs/name/gitcoinco/gitcoin-grants-arbitrum-one'),
-    ('https://api.thegraph.com/subgraphs/name/gitcoinco/grants-round-optimism-mainnet'),
-    ('https://api.studio.thegraph.com/query/45391/grants-round-base/v0.0.1'),
-    ('https://api.studio.thegraph.com/query/45391/grants-round-zkera/v0.0.2'),
-    ('https://api.thegraph.com/subgraphs/name/gitcoinco/grants-round-avalanche-mainnet'),
-    ('https://api.thegraph.com/subgraphs/name/gitcoinco/grants-round-fantom-mainnet'),
-    ('https://api.thegraph.com/subgraphs/name/gitcoinco/grants-round-optimism-mainnet');
+insert into "public"."gitcoin_indexing_jobs" ("url", "network_id") values 
+    ('https://api.thegraph.com/subgraphs/name/allo-protocol/grants-round-polygon', 137), 
+    ('https://api.thegraph.com/subgraphs/name/vacekj/allo-mainnet', 1),
+    ('https://graph-gitcoin-mainnet.hirenodes.io/subgraphs/name/gitcoin/allo', 424),
+    ('https://api.thegraph.com/subgraphs/name/gitcoinco/gitcoin-grants-arbitrum-one', 42161),
+    ('https://api.thegraph.com/subgraphs/name/gitcoinco/grants-round-optimism-mainnet', 10),
+    ('https://api.studio.thegraph.com/query/45391/grants-round-base/v0.0.1', 8453),
+    ('https://api.studio.thegraph.com/query/45391/grants-round-zkera/v0.0.2', 324),
+    ('https://api.thegraph.com/subgraphs/name/gitcoinco/grants-round-avalanche-mainnet', 43114),
+    ('https://api.thegraph.com/subgraphs/name/gitcoinco/grants-round-fantom-mainnet', 250);