diff --git a/ops/models.py b/ops/models.py deleted file mode 100644 index 68bcffa..0000000 --- a/ops/models.py +++ /dev/null @@ -1,91 +0,0 @@ -from uuid import UUID -import datetime -from typing import Optional -from pydantic import BaseModel, Json - - -class Applications(BaseModel): - - id: str - created_at: int - recipient: str - network: int - round: str - answers: Optional[Json] - project_id: str - - -class GitcoinApplications(BaseModel): - - id: str - created_at: int - data: Json - protocol: int - pointer: str - round_id: str - project_id: str - - -class GitcoinIndexingJobs(BaseModel): - - id: Optional[UUID] - created_at: Optional[datetime.datetime] - url: str - network_id: int - is_running: bool = false - skip_rounds: int = 0 - skip_projects: int = 0 - last_updated_at: Optional[datetime.datetime] - is_failed: bool = false - error: Optional[str] - - -class GitcoinProjects(BaseModel): - - id: str - created_at: Optional[datetime.datetime] - data: Json - protocol: int - pointer: str - - -class Logs(BaseModel): - - id: Optional[UUID] - run_id: UUID - created_at: Optional[datetime.datetime] - message: str - - -class Projects(BaseModel): - - id: str - title: Optional[str] - description: Optional[str] - website: Optional[str] - - -class Runs(BaseModel): - - id: Optional[UUID] - worker_id: UUID - created_at: Optional[datetime.datetime] - prompt: str - - -class StrategyEntries(BaseModel): - - id: Optional[UUID] - run_id: UUID - project_id: str - created_at: Optional[datetime.datetime] - reasoning: Optional[str] - impact: Optional[float] - interest: Optional[float] - weight: Optional[float] - - -class Workers(BaseModel): - - id: Optional[UUID] - created_at: Optional[datetime.datetime] diff --git a/web/app/actions/startWorker.ts b/web/app/actions/startWorker.ts index a8f9586..9c102b3 100644 --- a/web/app/actions/startWorker.ts +++ b/web/app/actions/startWorker.ts @@ -17,6 +17,10 @@ export const startWorker = async ( }, }); + if (response.status !== 200) { + throw Error(`Error startring new worker. Status: ${response.status}\nMessage: ${response.statusText}`); + } + const result = await response.json(); if (!result.worker_id || !result.run_id) { throw new Error("Error starting new worker"); diff --git a/web/supabase/dbTypes.ts b/web/supabase/dbTypes.ts index 86d7324..ed0687a 100644 --- a/web/supabase/dbTypes.ts +++ b/web/supabase/dbTypes.ts @@ -182,18 +182,21 @@ export interface Database { description: string | null id: string title: string | null + updated_at: number website: string | null } Insert: { description?: string | null id: string title?: string | null + updated_at: number website?: string | null } Update: { description?: string | null id?: string title?: string | null + updated_at?: number website?: string | null } Relationships: [] diff --git a/web/supabase/migrations/20240118120046_init.sql b/web/supabase/migrations/20240118120046_init.sql index 3897bbb..8017932 100644 --- a/web/supabase/migrations/20240118120046_init.sql +++ b/web/supabase/migrations/20240118120046_init.sql @@ -47,6 +47,7 @@ SELECT CREATE TABLE "public"."projects" ( "id" text NOT NULL, + "updated_at" int NOT NULL, "title" TEXT, "description" TEXT, "website" TEXT, diff --git a/workers/fund_public_goods/db/entities.py b/workers/fund_public_goods/db/entities.py index 9d84442..fe092f7 100644 --- a/workers/fund_public_goods/db/entities.py +++ b/workers/fund_public_goods/db/entities.py @@ -78,6 +78,7 @@ class Logs(BaseModel): class Projects(BaseModel): id: str + updated_at: int = Field(..., alias="updatedAt") title: Optional[str] = None description: Optional[str] = None website: Optional[str] = None diff --git a/workers/fund_public_goods/db/tables/applications.py b/workers/fund_public_goods/db/tables/applications.py index 3d556ad..629fd19 100644 --- a/workers/fund_public_goods/db/tables/applications.py +++ b/workers/fund_public_goods/db/tables/applications.py @@ -5,14 +5,39 @@ def insert( db: Client, row: Applications ): - db.table("applications").insert( - { - "id": id, - "created_at": row.created_at, - "recipient": row.recipient, - "network": row.network, - "round": row.round, - "answers": row.answers, - "project_id": row.project_id - } - ).execute() + db.table("applications").insert({ + "id": row.id, + "created_at": row.created_at, + "recipient": row.recipient, + "network": row.network, + "round": row.round, + "answers": row.answers, + "project_id": row.project_id + }).execute() + +def get_applications( + db: Client, + project_id: str +) -> list[Applications]: + result = (db.table("applications") + .select("id, created_at, recipient, network, round, answers, project_id") + .eq("project_id", project_id) + .execute()) + + if not result.data: + return [] + + applications = [] + + for item in result.data: + applications.append(Applications( + id=item["id"], + created_at=item["created_at"], + recipient=item["recipient"], + network=item["network"], + round=item["round"], + answers=item["answers"], + project_id=item["project_id"] + )) + + return applications diff --git a/workers/fund_public_goods/db/tables/gitcoin.py b/workers/fund_public_goods/db/tables/gitcoin.py index 2437871..2625364 100644 --- a/workers/fund_public_goods/db/tables/gitcoin.py +++ b/workers/fund_public_goods/db/tables/gitcoin.py @@ -4,7 +4,7 @@ from fund_public_goods.db.client import create_admin from fund_public_goods.db import tables, entities -def upsert_project(project: GitcoinProjects): +def upsert_project(project: GitcoinProjects, created_at: int): db = create_admin() db.table("gitcoin_projects").upsert({ @@ -14,18 +14,30 @@ def upsert_project(project: GitcoinProjects): "data": project.data }).execute() - tables.projects.insert(db, entities.Projects( + result = tables.projects.get(db, project.id) + + if result and result.updated_at > created_at: + return + + row = entities.Projects( id=project.id, + updated_at=created_at, title=project.data["title"], description=project.data["description"], website=project.data["website"] - )) + ) + + if result == None: + tables.projects.insert(db, row) + else: + tables.projects.upsert(db, row) def save_application(app: GitcoinApplications, network: int): db = create_admin() db.table("gitcoin_applications").insert({ "id": app.id, + "created_at": app.created_at, "protocol": app.protocol, "pointer": app.pointer, "round_id": app.round_id, @@ -39,7 +51,7 @@ def save_application(app: GitcoinApplications, network: int): recipient=app.data["application"]["recipient"], network=network, round=app.round_id, - answers=app.data["application"]["answers"], + answers=json.dumps(app.data["application"]["answers"]), project_id=app.project_id )) diff --git a/workers/fund_public_goods/db/tables/logs.py b/workers/fund_public_goods/db/tables/logs.py index 7ed9c26..8988c39 100644 --- a/workers/fund_public_goods/db/tables/logs.py +++ b/workers/fund_public_goods/db/tables/logs.py @@ -6,6 +6,6 @@ def insert( row: Logs ): db.table("logs").insert({ - "run_id": row.run_id, + "run_id": str(row.run_id), "message": row.message }).execute() diff --git a/workers/fund_public_goods/db/tables/projects.py b/workers/fund_public_goods/db/tables/projects.py index e7ff33e..d4e15e5 100644 --- a/workers/fund_public_goods/db/tables/projects.py +++ b/workers/fund_public_goods/db/tables/projects.py @@ -1,29 +1,59 @@ from typing import Any, Dict from supabase import Client, PostgrestAPIResponse -import uuid from fund_public_goods.db.entities import Projects def insert( - db: Client, row: Projects + db: Client, + row: Projects ) -> str: - id = str(uuid.uuid4()) - db.table("projects").insert( - { - "id": id, - "title": row.title, - "description": row.description, - "website": row.website, - } - ).execute() - return id - - -def get_projects(db: Client) -> PostgrestAPIResponse[Dict[str, Any]]: + db.table("projects").insert({ + "id": row.id, + "updated_at": row.updated_at, + "title": row.title, + "description": row.description, + "website": row.website, + }).execute() + +def upsert( + db: Client, + row: Projects +) -> str: + db.table("projects").upsert({ + "id": row.id, + "updated_at": row.updated_at, + "title": row.title, + "description": row.description, + "website": row.website, + }).execute() + +def get( + db: Client, + project_id: str +) -> Projects | None: + result = (db.table("projects") + .select("id", "updated_at", "title", "description", "website") + .eq("id", project_id) + .execute()) + + if not result.data: + return None + + data = result.data[0] + + return Projects( + id=data["id"], + updated_at=data["updated_at"], + title=data["title"], + description=data["description"], + website=data["website"] + ) + +def get_projects(db: Client) -> list[Projects]: return ( db.table("projects") .select( - "id, title, description, website, applications(id, recipient, round, answers)" + "id, updated_at, title, description, website, applications(id, recipient, round, answers)" ) .execute() ) diff --git a/workers/fund_public_goods/db/tables/runs.py b/workers/fund_public_goods/db/tables/runs.py index 8c5f3be..c3f0b1e 100644 --- a/workers/fund_public_goods/db/tables/runs.py +++ b/workers/fund_public_goods/db/tables/runs.py @@ -7,7 +7,7 @@ def insert(db: Client, row: Runs) -> str: id = str(uuid.uuid4()) db.table("runs").insert({ "id": id, - "worker_id": row.worker_id, + "worker_id": str(row.worker_id), "prompt": row.prompt }).execute() return id diff --git a/workers/fund_public_goods/db/tables/strategy_entries.py b/workers/fund_public_goods/db/tables/strategy_entries.py index 943621a..28b2131 100644 --- a/workers/fund_public_goods/db/tables/strategy_entries.py +++ b/workers/fund_public_goods/db/tables/strategy_entries.py @@ -8,7 +8,7 @@ def insert( row: StrategyEntries ): db.table("strategy_entries").insert({ - "run_id": row.run_id, + "run_id": str(row.run_id), "project_id": row.project_id, "reasoning": row.reasoning, "impact": row.impact, diff --git a/workers/fund_public_goods/lib/gitcoin/utils.py b/workers/fund_public_goods/lib/gitcoin/utils.py index 5411470..cc5ca1a 100644 --- a/workers/fund_public_goods/lib/gitcoin/utils.py +++ b/workers/fund_public_goods/lib/gitcoin/utils.py @@ -65,8 +65,6 @@ def fetch_project_applications(url: str, round_id: str, skip: int, first: int) - } } - print(f"Fetching projects for round {round_id} ...") - response = requests.post(url, json=data) if response.status_code == 200: diff --git a/workers/fund_public_goods/lib/strategy/utils/evaluate_projects.py b/workers/fund_public_goods/lib/strategy/utils/evaluate_projects.py index 19a3623..c487d80 100644 --- a/workers/fund_public_goods/lib/strategy/utils/evaluate_projects.py +++ b/workers/fund_public_goods/lib/strategy/utils/evaluate_projects.py @@ -41,6 +41,9 @@ def remove_duplicate_projects(projects: list[Project]) -> list[Project]: return unique_projects def get_top_matching_projects(prompt: str, projects: list[Project]) -> list[Project]: + if len(projects) == 0: + return [] + projects_by_id = {project.id: project for project in projects} queries = generate_queries(prompt=prompt, n=3) texts: list[str] = [] @@ -50,7 +53,8 @@ def get_top_matching_projects(prompt: str, projects: list[Project]) -> list[Proj project_text = get_project_text(project=project) texts.append(project_text) metadatas.append({ "id": project["id"] }) - + + db_client = EphemeralClient() collection = Chroma.from_texts( texts=texts, @@ -59,7 +63,7 @@ def get_top_matching_projects(prompt: str, projects: list[Project]) -> list[Proj client=db_client, collection_name="projects" ) - + top_matches: list[Project] = [] for query in queries: diff --git a/workers/fund_public_goods/workflows/index_gitcoin/functions/index_gitcoin_page.py b/workers/fund_public_goods/workflows/index_gitcoin/functions/index_gitcoin_page.py index dbb95e8..1e65f42 100644 --- a/workers/fund_public_goods/workflows/index_gitcoin/functions/index_gitcoin_page.py +++ b/workers/fund_public_goods/workflows/index_gitcoin/functions/index_gitcoin_page.py @@ -1,5 +1,6 @@ from datetime import datetime from typing import cast +import json import inngest from pydantic import parse_obj_as from fund_public_goods.lib.gitcoin.models import ApplicationInfo, RoundInfo @@ -76,7 +77,7 @@ async def index_gitcoin_page( pointer = app.pointer, round_id = app.round_id, project_id = project_id, - data = app_data + data = json.dumps(app_data) ) project_pointer = app_data["application"]["project"]["metaPtr"]["pointer"] @@ -85,10 +86,10 @@ async def index_gitcoin_page( id = app_data["application"]["project"]["id"], protocol = app_data["application"]["project"]["metaPtr"]["protocol"], pointer = project_pointer, - data = project_data, + data = json.dumps(project_data), ) - await step.run("upsert_project_" + str(i), lambda: upsert_project(project)) + await step.run("upsert_project_" + str(i), lambda: upsert_project(project, application.created_at)) await step.run("save_application_" + str(i), lambda: save_application(application, data.network_id)) diff --git a/workers/fund_public_goods/workflows/index_gitcoin/functions/start_index_gitcoin.py b/workers/fund_public_goods/workflows/index_gitcoin/functions/start_index_gitcoin.py index 9ba3466..7db7f32 100644 --- a/workers/fund_public_goods/workflows/index_gitcoin/functions/start_index_gitcoin.py +++ b/workers/fund_public_goods/workflows/index_gitcoin/functions/start_index_gitcoin.py @@ -2,6 +2,7 @@ from fund_public_goods.workflows.index_gitcoin.events import IndexGitcoinPageEvent from fund_public_goods.db.tables.gitcoin import get_non_running_job, is_any_job_running, start_job + @inngest.create_function( fn_id="start_index_gitcoin", # trigger=inngest.TriggerCron(cron="* * * * *"), # every 1 minute @@ -28,18 +29,18 @@ def get_not_running_job_step(): if not job: return "No non-running job found" - await step.run("start_job", lambda: start_job(job.id)) + await step.run("start_job", lambda: start_job(job["id"])) await step.send_event( "index_gitcoin_page", IndexGitcoinPageEvent.Data( - url = job.url, - network_id = job.network_id, + url = job["url"], + network_id = job["networkId"], project_page_size = 100, - skip_rounds = job.skip_rounds, - skip_projects = job.skip_projects, - job_id=job.id + skip_rounds = job["skipRounds"], + skip_projects = job["skipProjects"], + job_id=job["id"] ).to_event() ) - return "Started job: ID=" + job.id + ", URL=" + job.url + return "Started job: ID=" + job["id"] + ", URL=" + job["url"]