From e1421f64443ee6c9395bdc43e0cd29e7fc81e256 Mon Sep 17 00:00:00 2001 From: "0xelessar.eth" <paulo.colombo@pm.me> Date: Wed, 15 Jan 2025 14:18:40 +0100 Subject: [PATCH] feat: add weavevm support (#681) * feat(db): add support for weavevm blob storage * feat(web): add weavevm badge * chore(web): optimize paradex svg image * style(web): format main page source code * feat(api): add weavevm storage * feat(blob-storage-manager): add weavevm blob storage * fix(blob-propagator): skip blob storage queue instantiation for blob storages without a defined worker * feat(blob-storage-manager): add support for specifying Weavem storage API endpoint * feat(docs): add weavem to available storages * chore: add changeset * style: update weavevm name + rename api endpoint env var * fix(web): use dark and light weavevm logo * fix(blob-propagator): fix available blob storages check when creating a new blob propagator --- .changeset/dirty-islands-warn.md | 9 + apps/docs/src/app/docs/environment/page.md | 2 + apps/docs/src/app/docs/storages/page.md | 7 +- .../src/components/Badges/StorageBadge.tsx | 16 +- apps/web/src/components/StorageIcon.tsx | 12 ++ apps/web/src/icons/paradex.svg | 2 +- apps/web/src/icons/weavevm-dark.svg | 3 + apps/web/src/icons/weavevm-light.svg | 3 + apps/web/src/pages/index.tsx | 19 ++- packages/api/src/utils/schemas.ts | 1 + packages/api/src/utils/serializers.ts | 3 + .../blob-propagator/src/BlobPropagator.ts | 41 ++++- packages/blob-propagator/src/constants.ts | 18 +- .../test/BlobPropagator.test.ts | 32 ++++ .../__snapshots__/BlobPropagator.test.ts.snap | 4 + .../test/storage-workers.test.ts | 7 + .../src/storages/WeaveVMStorage.ts | 74 +++++++++ .../src/storages/index.ts | 1 + .../blob-storage-manager/src/utils/storage.ts | 15 ++ .../test/storages/WeaveVMStorage.test.ts | 156 ++++++++++++++++++ .../__snapshots__/WeaveVMStorage.test.ts.snap | 13 ++ .../migration.sql | 2 + packages/db/prisma/schema.prisma | 1 + packages/env/index.ts | 2 + 24 files changed, 406 insertions(+), 37 deletions(-) create mode 100644 .changeset/dirty-islands-warn.md create mode 100644 apps/web/src/icons/weavevm-dark.svg create mode 100644 apps/web/src/icons/weavevm-light.svg create mode 100644 packages/blob-storage-manager/src/storages/WeaveVMStorage.ts create mode 100644 packages/blob-storage-manager/test/storages/WeaveVMStorage.test.ts create mode 100644 packages/blob-storage-manager/test/storages/__snapshots__/WeaveVMStorage.test.ts.snap create mode 100644 packages/db/prisma/migrations/20250109000907_add_weavevm_enum/migration.sql diff --git a/.changeset/dirty-islands-warn.md b/.changeset/dirty-islands-warn.md new file mode 100644 index 000000000..a69df6fbc --- /dev/null +++ b/.changeset/dirty-islands-warn.md @@ -0,0 +1,9 @@ +--- +"@blobscan/blob-storage-manager": minor +"@blobscan/api": minor +"@blobscan/env": minor +"@blobscan/db": minor +"@blobscan/web": minor +--- + +Added Weavevm blob storage support diff --git a/apps/docs/src/app/docs/environment/page.md b/apps/docs/src/app/docs/environment/page.md index b017d12a6..af6922b1e 100644 --- a/apps/docs/src/app/docs/environment/page.md +++ b/apps/docs/src/app/docs/environment/page.md @@ -55,6 +55,8 @@ nextjs: | `BEE_ENDPOINT` | Bee endpoint | No | (empty) | | `FILE_SYSTEM_STORAGE_ENABLED` | Store blobs in filesystem | No | `false` | | `FILE_SYSTEM_STORAGE_PATH` | Store blobs in this path | No | `/tmp/blobscan-blobs` | +| `WEAVEVM_STORAGE_ENABLED` | Weavevm storage usage | No | `false` | +| `WEAVEVM_STORAGE_API_BASE_URL` | Weavevm API base url | No | (empty) | | `STATS_SYNCER_DAILY_CRON_PATTERN` | Cron pattern for the daily stats job | No | `30 0 * * * *` | | `STATS_SYNCER_OVERALL_CRON_PATTERN` | Cron pattern for the overall stats job | No | `*/15 * * * *` | | `SWARM_STAMP_CRON_PATTERN` | Cron pattern for swarm job | No | `*/15 * * * *` | diff --git a/apps/docs/src/app/docs/storages/page.md b/apps/docs/src/app/docs/storages/page.md index f249003c7..0c202c747 100644 --- a/apps/docs/src/app/docs/storages/page.md +++ b/apps/docs/src/app/docs/storages/page.md @@ -10,10 +10,11 @@ nextjs: Blobscan can be configured to use any of the following blob storages: -- PostgreSQL -- Google Cloud Storage -- Ethereum Swarm +- [Ethereum Swarm](https://www.ethswarm.org/) - File system +- Google Cloud Storage +- PostgreSQL +- [Weavevm](https://www.wvm.dev/) (currently supports blob reading only) By default all storages are disabled and you must enable at least one in order to run Blobscan. This is done using [environment variables](/docs/environment). diff --git a/apps/web/src/components/Badges/StorageBadge.tsx b/apps/web/src/components/Badges/StorageBadge.tsx index 64f4faf66..ac7b8843d 100644 --- a/apps/web/src/components/Badges/StorageBadge.tsx +++ b/apps/web/src/components/Badges/StorageBadge.tsx @@ -1,4 +1,4 @@ -import type { FC, HTMLAttributes, ReactNode } from "react"; +import type { FC, HTMLAttributes } from "react"; import React from "react"; import NextLink from "next/link"; @@ -10,32 +10,32 @@ import { Badge } from "./Badge"; type StorageConfig = { name?: string; - icon: ReactNode; style: HTMLAttributes<HTMLDivElement>["className"]; }; const STORAGE_CONFIGS: Record<BlobStorage, StorageConfig> = { file_system: { name: "File System", - icon: <StorageIcon storage="file_system" />, style: "bg-gray-100 hover:bg-gray-200 text-gray-800 hover:text-gray-900 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600 dark:hover:text-gray-200", }, google: { - icon: <StorageIcon storage="google" />, style: "bg-slate-100 hover:bg-slate-200 text-slate-800 hover:text-slate-900 dark:bg-slate-700 dark:text-slate-300 dark:hover:bg-slate-600 dark:hover:text-slate-200", }, swarm: { - icon: <StorageIcon storage="swarm" />, style: "bg-orange-100 hover:bg-orange-200 text-orange-800 hover:text-orange-900 dark:bg-orange-900 dark:text-orange-300 dark:hover:bg-orange-800 dark:hover:text-orange-200", }, postgres: { - icon: <StorageIcon storage="postgres" />, style: "bg-blue-100 dark:bg-blue-900 text-blue-800 dark:text-blue-300 hover:bg-blue-200 dark:hover:bg-blue-800", }, + weavevm: { + name: "WeaveVM", + style: + "bg-gray-100 hover:bg-gray-200 text-gray-800 hover:text-gray-900 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600 dark:hover:text-gray-200", + }, }; type StorageBadgeProps = BadgeProps & { @@ -48,12 +48,12 @@ export const StorageBadge: FC<StorageBadgeProps> = ({ url, ...props }) => { - const { icon, name, style } = STORAGE_CONFIGS[storage]; + const { name, style } = STORAGE_CONFIGS[storage]; return ( <NextLink href={url} target={url !== "#" ? "_blank" : "_self"}> <Badge className={style} {...props}> - {icon} + <StorageIcon storage={storage} /> {name ?? capitalize(storage)} </Badge> </NextLink> diff --git a/apps/web/src/components/StorageIcon.tsx b/apps/web/src/components/StorageIcon.tsx index 9bd986eba..f61c047c9 100644 --- a/apps/web/src/components/StorageIcon.tsx +++ b/apps/web/src/components/StorageIcon.tsx @@ -1,10 +1,13 @@ import NextLink from "next/link"; import { ArchiveBoxIcon } from "@heroicons/react/24/outline"; import cn from "classnames"; +import { useTheme } from "next-themes"; import GoogleIcon from "~/icons/google.svg"; import PostgresIcon from "~/icons/postgres.svg"; import SwarmIcon from "~/icons/swarm.svg"; +import WeaveVMDarkIcon from "~/icons/weavevm-dark.svg"; +import WeaveVMLightIcon from "~/icons/weavevm-light.svg"; import type { BlobStorage, Size } from "~/types"; import { capitalize } from "~/utils"; @@ -19,6 +22,7 @@ export const StorageIcon: React.FC<StorageIconProps> = ({ storage, url = "#", }) => { + const { resolvedTheme } = useTheme(); const commonStyles = cn({ "h-3 w-3": size === "sm", "h-4 w-4": size === "md", @@ -40,6 +44,14 @@ export const StorageIcon: React.FC<StorageIconProps> = ({ case "postgres": storageIcon = <PostgresIcon className={commonStyles} />; break; + case "weavevm": + storageIcon = + resolvedTheme === "light" ? ( + <WeaveVMLightIcon className={commonStyles} /> + ) : ( + <WeaveVMDarkIcon className={commonStyles} /> + ); + break; } return ( diff --git a/apps/web/src/icons/paradex.svg b/apps/web/src/icons/paradex.svg index 40ca2ec7b..efb84f7f9 100644 --- a/apps/web/src/icons/paradex.svg +++ b/apps/web/src/icons/paradex.svg @@ -1,6 +1,6 @@ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"> <g fill="none" fill-rule="evenodd"> <path fill="#04040F" d="M0 0h24v24H0z"/> - <path fill="#CC38FF" d="m16.2 8.888 1.607 2.461h-1.522l-1.537-2.356h-2.086V7.765h2.824c.554 0 1.003-.419 1.003-.936a.9.9 0 0 0-.294-.659 1.03 1.03 0 0 0-.71-.275h-2.823V4.667h2.824c.642 0 1.221.242 1.642.634a2.2 2.2 0 0 1 .314.364 2.024 2.024 0 0 1 .137 2.1c-.273.529-.77.941-1.378 1.123m-5.228-3.224a2.2 2.2 0 0 0-.314-.364c-.42-.392-1-.633-1.641-.633H6.193v1.228h2.825c.278 0 .528.105.709.274a.9.9 0 0 1 .294.66c0 .517-.449.935-1.003.935H6.193v3.585H7.43V8.992h1.588c.922 0 1.718-.501 2.092-1.228a2.03 2.03 0 0 0-.137-2.1m6.834 6.987h-1.513l-1.412 2.176-1.412-2.176h-1.513l2.169 3.341-2.169 3.341h1.513l1.412-2.176 1.412 2.176h1.513l-2.169-3.341zm-6.533 3.341c0 .923-.372 1.758-.974 2.362a3.3 3.3 0 0 1-2.35.979H6.193v-1.23H7.95c1.16 0 2.1-.945 2.1-2.111a2.11 2.11 0 0 0-2.1-2.112H6.193v-1.23H7.95a3.333 3.333 0 0 1 3.324 3.341"/> + <path fill="#CC38FF" d="m16.2 8.888 1.607 2.461h-1.522l-1.537-2.356h-2.086V7.765h2.824c.554 0 1.003-.419 1.003-.936a.9.9 0 0 0-.294-.659 1.03 1.03 0 0 0-.71-.275h-2.823V4.667h2.824c.642 0 1.221.242 1.642.634a2.2 2.2 0 0 1 .314.364 2.02 2.02 0 0 1 .137 2.1c-.273.529-.77.941-1.378 1.123m-5.228-3.224a2.2 2.2 0 0 0-.314-.364c-.42-.392-1-.633-1.641-.633H6.193v1.228h2.825c.278 0 .528.105.709.274a.9.9 0 0 1 .294.66c0 .517-.449.935-1.003.935H6.193v3.585H7.43V8.992h1.588c.922 0 1.718-.501 2.092-1.228a2.03 2.03 0 0 0-.137-2.1m6.834 6.987h-1.513l-1.412 2.176-1.412-2.176h-1.513l2.169 3.341-2.169 3.341h1.513l1.412-2.176 1.412 2.176h1.513l-2.169-3.341zm-6.533 3.341c0 .923-.372 1.758-.974 2.362a3.3 3.3 0 0 1-2.35.979H6.193v-1.23H7.95c1.16 0 2.1-.945 2.1-2.111a2.11 2.11 0 0 0-2.1-2.112H6.193v-1.23H7.95a3.333 3.333 0 0 1 3.324 3.341"/> </g> </svg> diff --git a/apps/web/src/icons/weavevm-dark.svg b/apps/web/src/icons/weavevm-dark.svg new file mode 100644 index 000000000..495b329c9 --- /dev/null +++ b/apps/web/src/icons/weavevm-dark.svg @@ -0,0 +1,3 @@ +<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 668 668"> + <path fill="#fff" d="m248.9 354.7 61.7 61.7-53.4 53.4-.2.2-61.7 61.7h-.1c0 .1-44 44.1-44 44.1-17 17-44.6 17-61.7 0-17-17-17-44.6 0-61.7l159.3-159.3zM573.001 154l-43.8 43.8-61.7 61.7-53.5 53.5-61.7-61.7 115.2-115.2 43.8-43.8c17-17 44.6-17 61.7 0 17 17 17 44.6 0 61.7M310.201 523.1l-61.6 61.6-32.3-32.2 61.6-61.7zM174.598 115.5l-61.6 61.6-18.6-18.6c-17-17-17-44.6 0-61.6s44.6-17 61.6 0zM446.401 387.3l-61.6 61.6-53.4-53.4-61.6-61.6-.3-.3-52.7-52.7 61.6-61.6 52.9 53 61.7 61.7.2.2zM577.7 580.3c-17 17-44.6 17-61.6 0L488 552.2l61.6-61.6 28.1 28.1c17 17 17 44.6 0 61.6M414.476 83.17l-61.66 61.66 32.527 32.527 61.66-61.66zM80.5 251.6l61.7 61.7L393.6 62.2l-33-33S349.1 19 335.4 18.4 314.2 22 314.2 22s-6.3 2.5-10.3 6.4c0 0-1.5 1.2-4 3.7s-9.2 9.1-9.2 9.1zM112.698 448.9l61.6-61.6-114.7-114.8-32.1 32.2s-11.5 10.1-11.4 29.4 11.9 29.9 11.9 29.9l84.8 84.8zM269.5 605.7l251.2-251.2 61.7 61.7-217.9 218.3s-14.9 20.3-43.3 13.8c0 0-10.7-1.8-20.2-11.2s-31.5-31.5-31.5-31.5zM488 279.9l115.2 115.3 32.5-32.1s12-11.5 11.3-29.7-8.3-25.5-8.3-25.5-2.6-3.9-4.4-5l-84.7-84.7z"/> +</svg> diff --git a/apps/web/src/icons/weavevm-light.svg b/apps/web/src/icons/weavevm-light.svg new file mode 100644 index 000000000..db9790c27 --- /dev/null +++ b/apps/web/src/icons/weavevm-light.svg @@ -0,0 +1,3 @@ +<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 668 668"> + <path fill="#000" d="m248.9 354.7 61.7 61.7-53.4 53.4-.2.2-61.7 61.7h-.1c0 .1-44 44.1-44 44.1-17 17-44.6 17-61.7 0-17-17-17-44.6 0-61.7l159.3-159.3zM573.001 154l-43.8 43.8-61.7 61.7-53.5 53.5-61.7-61.7 115.2-115.2 43.8-43.8c17-17 44.6-17 61.7 0 17 17 17 44.6 0 61.7M310.201 523.1l-61.6 61.6-32.3-32.2 61.6-61.7zM174.598 115.5l-61.6 61.6-18.6-18.6c-17-17-17-44.6 0-61.6s44.6-17 61.6 0zM446.401 387.3l-61.6 61.6-53.4-53.4-61.6-61.6-.3-.3-52.7-52.7 61.6-61.6 52.9 53 61.7 61.7.2.2zM577.7 580.3c-17 17-44.6 17-61.6 0L488 552.2l61.6-61.6 28.1 28.1c17 17 17 44.6 0 61.6M414.476 83.17l-61.66 61.66 32.527 32.527 61.66-61.66zM80.5 251.6l61.7 61.7L393.6 62.2l-33-33S349.1 19 335.4 18.4 314.2 22 314.2 22s-6.3 2.5-10.3 6.4c0 0-1.5 1.2-4 3.7s-9.2 9.1-9.2 9.1zM112.698 448.9l61.6-61.6-114.7-114.8-32.1 32.2s-11.5 10.1-11.4 29.4 11.9 29.9 11.9 29.9l84.8 84.8zM269.5 605.7l251.2-251.2 61.7 61.7-217.9 218.3s-14.9 20.3-43.3 13.8c0 0-10.7-1.8-20.2-11.2s-31.5-31.5-31.5-31.5zM488 279.9l115.2 115.3 32.5-32.1s12-11.5 11.3-29.7-8.3-25.5-8.3-25.5-2.6-3.9-4.4-5l-84.7-84.7z"/> +</svg> diff --git a/apps/web/src/pages/index.tsx b/apps/web/src/pages/index.tsx index 741a77470..e602ab75a 100644 --- a/apps/web/src/pages/index.tsx +++ b/apps/web/src/pages/index.tsx @@ -78,7 +78,11 @@ const Home: NextPage = () => { blobs, }; }, [rawBlocksData]); - const overallStats = useMemo(() => rawOverallStats ? deserializeOverallStats(rawOverallStats) : undefined, [rawOverallStats]); + const overallStats = useMemo( + () => + rawOverallStats ? deserializeOverallStats(rawOverallStats) : undefined, + [rawOverallStats] + ); const error = latestBlocksError || @@ -95,7 +99,6 @@ const Home: NextPage = () => { ); } - return ( <div className="flex flex-col items-center justify-center gap-12 sm:gap-20"> <div className=" flex flex-col items-center justify-center gap-8 md:w-8/12"> @@ -127,8 +130,10 @@ const Home: NextPage = () => { <MetricCard name="Total Tx Fees Saved" metric={{ - value: overallStats ? - overallStats.totalBlobAsCalldataFee - overallStats.totalBlobFee : undefined, + value: overallStats + ? overallStats.totalBlobAsCalldataFee - + overallStats.totalBlobFee + : undefined, type: "ethereum", }} compact @@ -176,7 +181,7 @@ const Home: NextPage = () => { <div className="grid grid-cols-1 items-stretch justify-stretch gap-6 lg:grid-cols-3"> <Card header={ - <div className="flex-wrap flex flex-col justify-between gap-3 2xl:flex-row 2xl:items-center"> + <div className="flex flex-col flex-wrap justify-between gap-3 2xl:flex-row 2xl:items-center"> <div>Latest Blocks</div> <Button variant="outline" @@ -209,7 +214,7 @@ const Home: NextPage = () => { </Card> <Card header={ - <div className="flex-wrap flex flex-col justify-between gap-3 2xl:flex-row 2xl:items-center"> + <div className="flex flex-col flex-wrap justify-between gap-3 2xl:flex-row 2xl:items-center"> <div>Latest Blob Transactions</div> <Button variant="outline" @@ -261,7 +266,7 @@ const Home: NextPage = () => { </Card> <Card header={ - <div className="flex-wrap flex flex-col justify-between gap-3 2xl:flex-row 2xl:items-center"> + <div className="flex flex-col flex-wrap justify-between gap-3 2xl:flex-row 2xl:items-center"> <div>Latest Blobs</div> <Button variant="outline" diff --git a/packages/api/src/utils/schemas.ts b/packages/api/src/utils/schemas.ts index 246c6e574..ef543f9d5 100644 --- a/packages/api/src/utils/schemas.ts +++ b/packages/api/src/utils/schemas.ts @@ -6,6 +6,7 @@ const zodBlobStorageEnums = [ "swarm", "postgres", "file_system", + "weavevm", ] as const; const zodRollupEnums = [ diff --git a/packages/api/src/utils/serializers.ts b/packages/api/src/utils/serializers.ts index 2967e87c9..23b73614a 100644 --- a/packages/api/src/utils/serializers.ts +++ b/packages/api/src/utils/serializers.ts @@ -50,6 +50,9 @@ export function buildBlobDataUrl( case "POSTGRES": { return `${env.BLOBSCAN_API_BASE_URL}/blobs/${blobDataUri}/data`; } + case "WEAVEVM": { + return `https://blobscan.shuttleapp.rs/v1/blob/${blobDataUri}`; + } } } diff --git a/packages/blob-propagator/src/BlobPropagator.ts b/packages/blob-propagator/src/BlobPropagator.ts index b922493ad..c68db6986 100644 --- a/packages/blob-propagator/src/BlobPropagator.ts +++ b/packages/blob-propagator/src/BlobPropagator.ts @@ -52,12 +52,13 @@ export type BlobPropagatorConfig = { export const STORAGE_WORKER_PROCESSORS: Record< BlobStorageName, - BlobPropagationWorkerProcessor + BlobPropagationWorkerProcessor | undefined > = { GOOGLE: gcsProcessor, SWARM: swarmProcessor, POSTGRES: postgresProcessor, FILE_SYSTEM: fileSystemProcessor, + WEAVEVM: undefined, }; export class BlobPropagator { @@ -81,25 +82,47 @@ export class BlobPropagator { ...workerOptions, }; + const temporaryBlobStorage = blobStorageManager.getStorage(tmpBlobStorage); + + if (!temporaryBlobStorage) { + throw new BlobPropagatorCreationError("Temporary blob storage not found"); + } + const availableStorageNames = blobStorageManager .getAllStorages() .map((s) => s.name) - .filter((name) => name !== tmpBlobStorage); - const temporaryBlobStorage = blobStorageManager.getStorage(tmpBlobStorage); + .filter((name) => name !== tmpBlobStorage) + .filter((name) => { + const hasWorkerProcessor = !!STORAGE_WORKER_PROCESSORS[name]; - if (!availableStorageNames) { - throw new BlobPropagatorCreationError("No blob storages available"); - } + if (!hasWorkerProcessor) { + logger.warn( + `Worker processor not defined for storage "${name}"; skipping` + ); + } - if (!temporaryBlobStorage) { - throw new BlobPropagatorCreationError("Temporary blob storage not found"); + return hasWorkerProcessor; + }); + + if (!availableStorageNames.length) { + throw new BlobPropagatorCreationError( + "None of the available storages have worker processors defined" + ); } this.storageWorkers = availableStorageNames.map( (storageName: BlobStorageName) => { + const workerProcessor = STORAGE_WORKER_PROCESSORS[storageName]; + + if (!workerProcessor) { + throw new BlobPropagatorCreationError( + `Worker processor not defined for storage "${storageName}"` + ); + } + return this.#createWorker( STORAGE_WORKER_NAMES[storageName], - STORAGE_WORKER_PROCESSORS[storageName]({ + workerProcessor({ prisma, blobStorageManager, }), diff --git a/packages/blob-propagator/src/constants.ts b/packages/blob-propagator/src/constants.ts index 15be77081..384abbce1 100644 --- a/packages/blob-propagator/src/constants.ts +++ b/packages/blob-propagator/src/constants.ts @@ -3,15 +3,15 @@ import type { JobsOptions, WorkerOptions } from "bullmq"; import { BlobStorage as BlobStorageName } from "@blobscan/db/prisma/enums"; import { env } from "@blobscan/env"; -export const STORAGE_WORKER_NAMES = Object.values(BlobStorageName).reduce< - Record<BlobStorageName, string> ->( - (names, storage) => ({ - ...names, - [storage]: `${storage.toLowerCase()}-worker`, - }), - {} as Record<BlobStorageName, string> -); +export const STORAGE_WORKER_NAMES = Object.values(BlobStorageName) + .filter((blobStorageName) => blobStorageName !== "WEAVEVM") + .reduce<Record<BlobStorageName, string>>( + (names, storage) => ({ + ...names, + [storage]: `${storage.toLowerCase()}-worker`, + }), + {} as Record<BlobStorageName, string> + ); export const FINALIZER_WORKER_NAME = "finalizer-worker"; diff --git a/packages/blob-propagator/test/BlobPropagator.test.ts b/packages/blob-propagator/test/BlobPropagator.test.ts index 4d1a8228a..58f7d21b7 100644 --- a/packages/blob-propagator/test/BlobPropagator.test.ts +++ b/packages/blob-propagator/test/BlobPropagator.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { PostgresStorage, + WeaveVMStorage, createStorageFromEnv, getBlobStorageManager, } from "@blobscan/blob-storage-manager"; @@ -37,6 +38,15 @@ export class MockedBlobPropagator extends BlobPropagator { } } +class MockedWeaveVMStorage extends WeaveVMStorage { + constructor() { + super({ + apiBaseUrl: "http://localhost:8000", + chainId: 1, + }); + } +} + describe("BlobPropagator", () => { let blobStorageManager: BlobStorageManager; let tmpBlobStorage: FileSystemStorage; @@ -102,6 +112,28 @@ describe("BlobPropagator", () => { } ); + testValidError( + "should throw a valid error when creating a blob propagator with blob storages without worker processors", + () => { + const weavevmStorage = new MockedWeaveVMStorage(); + + const blobStorageManager = new BlobStorageManager([weavevmStorage]); + + new MockedBlobPropagator({ + blobStorageManager, + prisma, + tmpBlobStorage: env.BLOB_PROPAGATOR_TMP_BLOB_STORAGE, + workerOptions: { + connection, + }, + }); + }, + BlobPropagatorCreationError, + { + checkCause: true, + } + ); + testValidError( "should throw a valid error when creating a blob propagator with no temporary blob storage", async () => { diff --git a/packages/blob-propagator/test/__snapshots__/BlobPropagator.test.ts.snap b/packages/blob-propagator/test/__snapshots__/BlobPropagator.test.ts.snap index 88286809e..e6d16f299 100644 --- a/packages/blob-propagator/test/__snapshots__/BlobPropagator.test.ts.snap +++ b/packages/blob-propagator/test/__snapshots__/BlobPropagator.test.ts.snap @@ -1,5 +1,9 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html +exports[`BlobPropagator > should throw a valid error when creating a blob propagator with blob storages without worker processors 1`] = `"Blob propagator failed: Failed to create blob propagator"`; + +exports[`BlobPropagator > should throw a valid error when creating a blob propagator with blob storages without worker processors 2`] = `"Temporary blob storage not found"`; + exports[`BlobPropagator > should throw a valid error when creating a blob propagator with no blob storages 1`] = `"Blob propagator failed: Failed to create blob propagator"`; exports[`BlobPropagator > should throw a valid error when creating a blob propagator with no blob storages 2`] = `"Temporary blob storage not found"`; diff --git a/packages/blob-propagator/test/storage-workers.test.ts b/packages/blob-propagator/test/storage-workers.test.ts index 4622f72fe..c885fdfba 100644 --- a/packages/blob-propagator/test/storage-workers.test.ts +++ b/packages/blob-propagator/test/storage-workers.test.ts @@ -54,7 +54,14 @@ function runWorkerTests( const storageWorker = STORAGE_WORKER_PROCESSORS[storageName]; + if (!storageWorker) { + throw new Error( + `No worker processor found for the "${storageName}" storage` + ); + } + storageWorkerProcessor = storageWorker(workerParams); + blobStorageManager = workerParams.blobStorageManager; prisma = workerParams.prisma; }); diff --git a/packages/blob-storage-manager/src/storages/WeaveVMStorage.ts b/packages/blob-storage-manager/src/storages/WeaveVMStorage.ts new file mode 100644 index 000000000..3f4ea818d --- /dev/null +++ b/packages/blob-storage-manager/src/storages/WeaveVMStorage.ts @@ -0,0 +1,74 @@ +import { BlobStorage as BlobStorageName } from "@blobscan/db/prisma/enums"; +import { z } from "@blobscan/zod"; + +import type { BlobStorageConfig } from "../BlobStorage"; +import { BlobStorage } from "../BlobStorage"; + +const blobResponseSchema = z.object({ + blob_data: z.string(), +}); + +export interface WeaveVMStorageConfig extends BlobStorageConfig { + apiBaseUrl: string; +} + +export class WeaveVMStorage extends BlobStorage { + apiBaseUrl: string; + + protected constructor({ apiBaseUrl, chainId }: WeaveVMStorageConfig) { + super(BlobStorageName.WEAVEVM, chainId); + + this.apiBaseUrl = apiBaseUrl; + } + + protected async _healthCheck(): Promise<void> { + await fetch(`${this.apiBaseUrl}/v1/stats`); + } + + protected async _getBlob(uri: string): Promise<string> { + const response = await fetch(`${this.apiBaseUrl}/v1/blob/${uri}`); + const jsonRes = await response.json(); + + const res = blobResponseSchema.safeParse(jsonRes); + + if (!res.success) { + throw new Error("Failed to parse blob response", { + cause: res.error.flatten().fieldErrors.blob_data?.join(";"), + }); + } + + return res.data.blob_data; + } + + protected async _storeBlob(_: string, __: string): Promise<string> { + throw new Error( + "Blob storage operation is not allowed for WeaveVM storage" + ); + } + + protected async _removeBlob(_: string): Promise<void> { + throw new Error( + "Blob removal operation is not allowed for WeaveVM storage" + ); + } + + getBlobUri(hash: string) { + return hash; + } + + static async create(config: WeaveVMStorageConfig) { + try { + const storage = new WeaveVMStorage(config); + + await storage.healthCheck(); + + return storage; + } catch (err) { + const err_ = err as Error; + + throw new Error("Failed to create WeaveVM storage", { + cause: err_, + }); + } + } +} diff --git a/packages/blob-storage-manager/src/storages/index.ts b/packages/blob-storage-manager/src/storages/index.ts index 30970e4e3..cf391c82b 100644 --- a/packages/blob-storage-manager/src/storages/index.ts +++ b/packages/blob-storage-manager/src/storages/index.ts @@ -2,3 +2,4 @@ export * from "./FileSystemStorage"; export * from "./GoogleStorage"; export * from "./PostgresStorage"; export * from "./SwarmStorage"; +export * from "./WeaveVMStorage"; diff --git a/packages/blob-storage-manager/src/utils/storage.ts b/packages/blob-storage-manager/src/utils/storage.ts index 14c5eabb7..71bb1c71c 100644 --- a/packages/blob-storage-manager/src/utils/storage.ts +++ b/packages/blob-storage-manager/src/utils/storage.ts @@ -8,6 +8,7 @@ import { GoogleStorage, PostgresStorage, SwarmStorage, + WeaveVMStorage, } from "../storages"; export function removeDuplicatedStorages( @@ -73,5 +74,19 @@ export async function createStorageFromEnv( return fileSystemStorage; } + case BlobStorageName.WEAVEVM: { + if (!env.WEAVEVM_STORAGE_API_BASE_URL) { + throw new Error( + "Missing required env variable for WeavevmStorage: WEAVEVM_STORAGE_API_BASE_URL" + ); + } + + const weavevmStorage = await WeaveVMStorage.create({ + chainId, + apiBaseUrl: env.WEAVEVM_STORAGE_API_BASE_URL, + }); + + return weavevmStorage; + } } } diff --git a/packages/blob-storage-manager/test/storages/WeaveVMStorage.test.ts b/packages/blob-storage-manager/test/storages/WeaveVMStorage.test.ts new file mode 100644 index 000000000..156cd0e6c --- /dev/null +++ b/packages/blob-storage-manager/test/storages/WeaveVMStorage.test.ts @@ -0,0 +1,156 @@ +import { http, HttpResponse } from "msw"; +import type { SetupServerApi } from "msw/node"; +import { setupServer } from "msw/node"; +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; + +import { env } from "@blobscan/env"; +import { testValidError } from "@blobscan/test"; + +import { BlobStorageError } from "../../src/errors"; +import { WeaveVMStorage } from "../../src/storages/WeaveVMStorage"; + +const MOCK_API_BASE_URL = "https://blobscan.weavevm"; + +const MOCK_RESPONSES: Record<string, string> = { + "0x01b4b4b2b62f7a206efcb78e2ed6ef5bde9c1ac33d4d44eccfd43aa951ef1d22": + "0x4fe40fc67f9c3a3ffa2be77d10fe7818", +}; + +class WeaveVMStorageMock extends WeaveVMStorage { + constructor() { + super({ + apiBaseUrl: MOCK_API_BASE_URL, + chainId: env.CHAIN_ID, + }); + } + + healthCheck() { + return super.healthCheck(); + } +} + +describe("WeavevmStorage", () => { + let weavevmServer: SetupServerApi; + let storage: WeaveVMStorageMock; + + beforeAll(() => { + weavevmServer = setupServer( + http.get(`${MOCK_API_BASE_URL}/v1/blob/:uri`, ({ request }) => { + const uri = request.url.split("/").pop(); + + if (!uri) { + return HttpResponse.json({ + code: 400, + message: "invalid path params", + }); + } + + const blobData = MOCK_RESPONSES[uri]; + + if (!blobData) { + return HttpResponse.json({ + code: 404, + message: "blob not found", + }); + } + + return HttpResponse.json({ + blob_data: blobData, + }); + }), + http.get(`${MOCK_API_BASE_URL}/v1/stats`, () => { + return HttpResponse.json({ + blob_versioned_hash: + "0x01b4b4b2b62f7a206efcb78e2ed6ef5bde9c1ac33d4d44eccfd43aa951ef1d22", + last_archived_eth_block: 19559986, + wvm_archive_txid: + "0x01ee8325bc5607a16dd64ff2bcbec7d596b170f31def52615abf6b3f25ceb5a5", + }); + }) + ); + + weavevmServer.listen(); + + return () => { + weavevmServer.close(); + }; + }); + + beforeEach(() => { + storage = new WeaveVMStorageMock(); + }); + + afterEach(() => { + weavevmServer.resetHandlers(); + }); + + it("should create a storage", async () => { + const storage_ = await WeaveVMStorage.create({ + chainId: env.CHAIN_ID, + apiBaseUrl: MOCK_API_BASE_URL, + }); + + expect(storage_.chainId, "Chain ID mismatch").toBe(env.CHAIN_ID); + expect(storage_.apiBaseUrl).toBe(MOCK_API_BASE_URL); + }); + + it("return the correct uri given a blob hash", () => { + const blobHash = "exampleBlobHash"; + const blobUri = storage.getBlobUri(blobHash); + + expect(blobUri).toBe(blobHash); + }); + + it("should return 'OK' if storage is healthy", async () => { + await expect(storage.healthCheck()).resolves.toBe("OK"); + }); + + it("should get a blob given its reference", async () => { + const blobHash = Object.keys(MOCK_RESPONSES)[0] as string; + + const result = await storage.getBlob(blobHash); + + expect(result).toBe(MOCK_RESPONSES[blobHash]); + }); + + testValidError( + "should fail when trying to parse an invalid blob retrieval response", + async () => { + weavevmServer.use( + http.get(`${MOCK_API_BASE_URL}/v1/blob/:uri`, () => { + return HttpResponse.json({ + blob_data: 123, + }); + }) + ); + + await storage.getBlob("invalidBlobHash"); + }, + BlobStorageError, + { + checkCause: true, + } + ); + + testValidError( + "should fail when trying to store a blob", + async () => { + await storage.storeBlob("blobHash", "blobData"); + }, + BlobStorageError, + { + checkCause: true, + } + ); + + testValidError( + "should fail when trying to remove a blob", + async () => { + await storage.removeBlob("blobHash"); + }, + BlobStorageError, + { + checkCause: true, + } + ); +}); diff --git a/packages/blob-storage-manager/test/storages/__snapshots__/WeaveVMStorage.test.ts.snap b/packages/blob-storage-manager/test/storages/__snapshots__/WeaveVMStorage.test.ts.snap new file mode 100644 index 000000000..f73c8f8b0 --- /dev/null +++ b/packages/blob-storage-manager/test/storages/__snapshots__/WeaveVMStorage.test.ts.snap @@ -0,0 +1,13 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`WeavevmStorage > should fail when trying to parse an invalid blob retrieval response 1`] = `"WeaveVMStorageMock failed: Failed to get blob with uri \\"invalidBlobHash\\""`; + +exports[`WeavevmStorage > should fail when trying to parse an invalid blob retrieval response 2`] = `[Error: Failed to parse blob response]`; + +exports[`WeavevmStorage > should fail when trying to remove a blob 1`] = `"WeaveVMStorageMock failed: Failed to remove blob with uri \\"blobHash\\""`; + +exports[`WeavevmStorage > should fail when trying to remove a blob 2`] = `[Error: Blob removal operation is not allowed for WeaveVM storage]`; + +exports[`WeavevmStorage > should fail when trying to store a blob 1`] = `"WeaveVMStorageMock failed: Failed to store blob with hash \\"blobHash\\""`; + +exports[`WeavevmStorage > should fail when trying to store a blob 2`] = `[Error: Blob storage operation is not allowed for WeaveVM storage]`; diff --git a/packages/db/prisma/migrations/20250109000907_add_weavevm_enum/migration.sql b/packages/db/prisma/migrations/20250109000907_add_weavevm_enum/migration.sql new file mode 100644 index 000000000..3278af31f --- /dev/null +++ b/packages/db/prisma/migrations/20250109000907_add_weavevm_enum/migration.sql @@ -0,0 +1,2 @@ +-- AlterEnum +ALTER TYPE "blob_storage" ADD VALUE 'weavevm'; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 8f5a9dc28..d77f11757 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -22,6 +22,7 @@ enum BlobStorage { GOOGLE @map("google") POSTGRES @map("postgres") SWARM @map("swarm") + WEAVEVM @map("weavevm") @@map("blob_storage") } diff --git a/packages/env/index.ts b/packages/env/index.ts index d751e3753..f4c1a6041 100644 --- a/packages/env/index.ts +++ b/packages/env/index.ts @@ -57,6 +57,8 @@ export const env = createEnv({ GOOGLE_STORAGE_ENABLED: booleanSchema.default("false"), GOOGLE_STORAGE_PROJECT_ID: z.string().optional(), GOOGLE_SERVICE_KEY: z.string().optional(), + WEAVEVM_STORAGE_ENABLED: booleanSchema.default("false"), + WEAVEVM_STORAGE_API_BASE_URL: z.string().url().optional(), LOG_LEVEL: z .enum(["debug", "http", "info", "warn", "error"]) .default("http"),