feat: add tx blob gas used (#559)

* feat(db): add tx blob gas used * feat(db): add tx blob gas used backfill migration * feat(db): make tx blob gas used field mandatory * feat: add tx blob gas used * chore: add changeset
Blobscan · Sep 10, 2024 · 3507a88 · 3507a88
1 parent 359df4e
commit 3507a88
Show file tree

Hide file tree

Showing 21 changed files with 137 additions and 31 deletions.
diff --git a/.changeset/wicked-bikes-switch.md b/.changeset/wicked-bikes-switch.md
@@ -0,0 +1,6 @@
+---
+"@blobscan/db": minor
+"@blobscan/api": patch
+---
+
+Added blob gas used to `Transaction` model
diff --git a/packages/api/src/middlewares/withExpands.ts b/packages/api/src/middlewares/withExpands.ts
@@ -27,6 +27,7 @@ export type ZodExpandEnum = (typeof zodExpandEnums)[number];
 
 const expandedTransactionSelect = Prisma.validator<Prisma.TransactionSelect>()({
   blobAsCalldataGasUsed: true,
+  blobGasUsed: true,
   fromId: true,
   toId: true,
   gasPrice: true,
@@ -121,6 +122,7 @@ export const serializedExpandedTransactionSchema = z
     to: z.string().optional(),
     maxFeePerBlobGas: z.string().optional(),
     blobAsCalldataGasUsed: z.string().optional(),
+    blobGasUsed: z.string().optional(),
     category: categorySchema.optional(),
     rollup: rollupSchema.nullable().optional(),
     index: z.number().nonnegative().optional(),
@@ -130,7 +132,6 @@ export const serializedExpandedTransactionSchema = z
       blobAsCalldataGasFee: z.string().optional(),
       blobGasBaseFee: z.string().optional(),
       blobGasMaxFee: z.string().optional(),
-      blobGasUsed: z.string().optional(),
     })
   );
 

diff --git a/packages/api/src/routers/block/getAll.ts b/packages/api/src/routers/block/getAll.ts
@@ -95,14 +95,21 @@ export const getAll = publicProcedure
       blocks = blocks.map((block) => ({
         ...block,
         transactions: block.transactions.map((tx) => {
+          const {
+            maxFeePerBlobGas,
+            blobAsCalldataGasUsed,
+            gasPrice,
+            blobGasUsed,
+          } = tx;
+
           const derivedTxFields =
-            tx.maxFeePerBlobGas && tx.blobAsCalldataGasUsed && tx.gasPrice
+            maxFeePerBlobGas && blobAsCalldataGasUsed && gasPrice && blobGasUsed
               ? calculateDerivedTxBlobGasFields({
-                  blobAsCalldataGasUsed: tx.blobAsCalldataGasUsed,
-                  gasPrice: tx.gasPrice,
+                  blobAsCalldataGasUsed,
+                  blobGasUsed,
+                  gasPrice,
                   blobGasPrice: block.blobGasPrice,
-                  maxFeePerBlobGas: tx.maxFeePerBlobGas,
-                  txBlobsLength: tx.blobs.length,
+                  maxFeePerBlobGas,
                 })
               : {};
 

diff --git a/packages/api/src/routers/block/getByBlockId.ts b/packages/api/src/routers/block/getByBlockId.ts
@@ -90,14 +90,20 @@ export const getByBlockId = publicProcedure
 
       if (expands.transaction) {
         block.transactions = block.transactions.map((tx) => {
+          const {
+            blobAsCalldataGasUsed,
+            blobGasUsed,
+            gasPrice,
+            maxFeePerBlobGas,
+          } = tx;
           const derivedFields =
-            tx.maxFeePerBlobGas && tx.blobAsCalldataGasUsed && tx.gasPrice
+            maxFeePerBlobGas && blobAsCalldataGasUsed && blobGasUsed && gasPrice
               ? calculateDerivedTxBlobGasFields({
-                  blobAsCalldataGasUsed: tx.blobAsCalldataGasUsed,
-                  gasPrice: tx.gasPrice,
+                  blobAsCalldataGasUsed,
+                  blobGasUsed,
+                  gasPrice,
                   blobGasPrice: block.blobGasPrice,
-                  maxFeePerBlobGas: tx.maxFeePerBlobGas,
-                  txBlobsLength: tx.blobs.length,
+                  maxFeePerBlobGas,
                 })
               : {};
 

diff --git a/packages/api/src/routers/indexer/indexData.utils.ts b/packages/api/src/routers/indexer/indexData.utils.ts
@@ -14,6 +14,7 @@ import type { IndexDataFormattedInput } from "./indexData";
 
 const MIN_BLOB_BASE_FEE = BigInt(1);
 const BLOB_BASE_FEE_UPDATE_FRACTION = BigInt(3_338_477);
+const BLOB_GAS_PER_BLOB = BigInt(131_072);
 
 function bigIntToDecimal(bigint: bigint) {
   return new Prisma.Decimal(bigint.toString());
@@ -103,6 +104,9 @@ export function createDBTransactions({
         toId: to,
         index,
         gasPrice: bigIntToDecimal(gasPrice),
+        blobGasUsed: bigIntToDecimal(
+          BigInt(txBlobs.length) * BLOB_GAS_PER_BLOB
+        ),
         blobGasPrice: bigIntToDecimal(blobGasPrice),
         maxFeePerBlobGas: bigIntToDecimal(maxFeePerBlobGas),
         blobAsCalldataGasUsed: bigIntToDecimal(blobGasAsCalldataUsed),

diff --git a/packages/api/src/routers/tx/common/selects.ts b/packages/api/src/routers/tx/common/selects.ts
@@ -24,6 +24,7 @@ export const baseTransactionSelect =
     hash: true,
     fromId: true,
     toId: true,
+    blobGasUsed: true,
     blobAsCalldataGasUsed: true,
     gasPrice: true,
     maxFeePerBlobGas: true,

diff --git a/packages/api/src/routers/tx/common/serializers.ts b/packages/api/src/routers/tx/common/serializers.ts
@@ -29,6 +29,7 @@ const baseSerializedTransactionFieldsSchema = z.object({
   from: z.string(),
   to: z.string(),
   maxFeePerBlobGas: z.string(),
+  blobGasUsed: z.string(),
   blobAsCalldataGasUsed: z.string(),
   category: categorySchema,
   rollup: rollupSchema.nullable(),
@@ -57,16 +58,21 @@ export type SerializedTransaction = z.infer<typeof serializedTransactionSchema>;
 export function addDerivedFieldsToTransaction(
   txQuery: BaseTransaction
 ): FullQueriedTransaction {
-  const { block, blobs, maxFeePerBlobGas, blobAsCalldataGasUsed, gasPrice } =
-    txQuery;
+  const {
+    blobAsCalldataGasUsed,
+    blobGasUsed,
+    block,
+    gasPrice,
+    maxFeePerBlobGas,
+  } = txQuery;
 
   return {
     ...txQuery,
     ...calculateDerivedTxBlobGasFields({
+      blobGasUsed,
       blobAsCalldataGasUsed,
       gasPrice,
       blobGasPrice: block.blobGasPrice,
-      txBlobsLength: blobs.length,
       maxFeePerBlobGas,
     }),
   };
@@ -99,6 +105,7 @@ export function serializeBaseTransactionFields(
     to: toId,
     from: fromId,
     blobAsCalldataGasUsed: serializeDecimal(txQuery.blobAsCalldataGasUsed),
+    blobGasUsed: serializeDecimal(txQuery.blobGasUsed),
     maxFeePerBlobGas: serializeDecimal(txQuery.maxFeePerBlobGas),
     category: serializeCategory(category),
     rollup: serializeRollup(rollup),

diff --git a/packages/api/src/utils/blob.ts b/packages/api/src/utils/blob.ts
@@ -6,7 +6,7 @@ import type {
   BlobReference,
   BlobStorageManager,
 } from "@blobscan/blob-storage-manager";
-import { Prisma } from "@blobscan/db";
+import type { Prisma } from "@blobscan/db";
 import type { BlobDataStorageReference } from "@blobscan/db";
 import { z } from "@blobscan/zod";
 
@@ -16,37 +16,30 @@ export type DerivedTxBlobGasFields = {
   blobAsCalldataGasFee: Prisma.Decimal;
   blobGasBaseFee?: Prisma.Decimal;
   blobGasMaxFee: Prisma.Decimal;
-  blobGasUsed: Prisma.Decimal;
 };
 
-const GAS_PER_BLOB = 2 ** 17; // 131_072
-
 export function calculateDerivedTxBlobGasFields({
+  blobGasUsed,
   blobAsCalldataGasUsed,
   blobGasPrice,
   gasPrice,
-  txBlobsLength,
   maxFeePerBlobGas,
 }: {
+  blobGasUsed: Prisma.Decimal;
   blobAsCalldataGasUsed: Prisma.Decimal;
   blobGasPrice?: Prisma.Decimal;
   gasPrice: Prisma.Decimal;
-  txBlobsLength: number;
   maxFeePerBlobGas: Prisma.Decimal;
 }): DerivedTxBlobGasFields {
-  const blobGasUsed = new Prisma.Decimal(txBlobsLength).mul(GAS_PER_BLOB);
   const blobGasMaxFee = maxFeePerBlobGas.mul(blobGasUsed);
 
   const derivedBlobGasFields: DerivedTxBlobGasFields = {
     blobAsCalldataGasFee: gasPrice.mul(blobAsCalldataGasUsed),
-    blobGasUsed,
     blobGasMaxFee,
   };
 
   if (blobGasPrice) {
-    derivedBlobGasFields.blobGasBaseFee = blobGasPrice.mul(
-      derivedBlobGasFields.blobGasUsed
-    );
+    derivedBlobGasFields.blobGasBaseFee = blobGasPrice.mul(blobGasUsed);
   }
 
   return derivedBlobGasFields;
@@ -56,7 +49,6 @@ export const serializedDerivedTxBlobGasFieldsSchema = z.object({
   blobAsCalldataGasFee: z.string(),
   blobGasBaseFee: z.string().optional(),
   blobGasMaxFee: z.string(),
-  blobGasUsed: z.string(),
 });
 
 export type SerializedDerivedTxBlobGasFields = z.infer<
@@ -67,12 +59,10 @@ export function serializeDerivedTxBlobGasFields({
   blobAsCalldataGasFee,
   blobGasBaseFee,
   blobGasMaxFee,
-  blobGasUsed,
 }: DerivedTxBlobGasFields): SerializedDerivedTxBlobGasFields {
   const serializedFields: SerializedDerivedTxBlobGasFields = {
     blobAsCalldataGasFee: serializeDecimal(blobAsCalldataGasFee),
     blobGasMaxFee: serializeDecimal(blobGasMaxFee),
-    blobGasUsed: serializeDecimal(blobGasUsed),
   };
 
   if (blobGasBaseFee) {

diff --git a/packages/api/test/__snapshots__/blob.test.ts.snap b/packages/api/test/__snapshots__/blob.test.ts.snap
@@ -93,6 +93,7 @@ exports[`Blob router > getAll > when getting expanded blob results > should retu
     "size": 1000,
     "transaction": {
       "blobAsCalldataGasUsed": "1000",
+      "blobGasUsed": "131072",
       "category": "other",
       "from": "address2",
       "index": 0,
@@ -129,6 +130,7 @@ exports[`Blob router > getAll > when getting expanded blob results > should retu
     "size": 1300,
     "transaction": {
       "blobAsCalldataGasUsed": "1000",
+      "blobGasUsed": "393216",
       "category": "rollup",
       "from": "address5",
       "index": 0,
@@ -164,6 +166,7 @@ exports[`Blob router > getAll > when getting expanded blob results > should retu
     "size": 1000,
     "transaction": {
       "blobAsCalldataGasUsed": "1000",
+      "blobGasUsed": "131072",
       "category": "other",
       "from": "address2",
       "index": 0,
@@ -193,6 +196,7 @@ exports[`Blob router > getAll > when getting expanded blob results > should retu
     "size": 1300,
     "transaction": {
       "blobAsCalldataGasUsed": "1000",
+      "blobGasUsed": "393216",
       "category": "rollup",
       "from": "address5",
       "index": 0,

diff --git a/packages/api/test/__snapshots__/tx.test.ts.snap b/packages/api/test/__snapshots__/tx.test.ts.snap
@@ -1039,9 +1039,9 @@ exports[`Transaction router > getAll > when getting filtered transaction results
   {
     "blobAsCalldataGasFee": "1866000",
     "blobAsCalldataGasUsed": "15550",
-    "blobGasBaseFee": "2883584",
-    "blobGasMaxFee": "74186752",
-    "blobGasUsed": "131072",
+    "blobGasBaseFee": "5767168",
+    "blobGasMaxFee": "148373504",
+    "blobGasUsed": "262144",
     "blobs": [
       {
         "index": 0,

diff --git a/packages/api/test/indexer.test.ts b/packages/api/test/indexer.test.ts
@@ -151,6 +151,7 @@ describe("Indexer router", async () => {
           expect(remainingParams).toMatchInlineSnapshot(`
             [
               {
+                "blobGasUsed": "131072",
                 "blockHash": "blockHash2010",
                 "blockNumber": 2010,
                 "blockTimestamp": 2023-09-01T13:50:21.000Z,
@@ -164,6 +165,7 @@ describe("Indexer router", async () => {
                 "toId": "address10",
               },
               {
+                "blobGasUsed": "262144",
                 "blockHash": "blockHash2010",
                 "blockNumber": 2010,
                 "blockTimestamp": 2023-09-01T13:50:21.000Z,

diff --git a/packages/db/prisma/extensions/base.ts b/packages/db/prisma/extensions/base.ts
@@ -270,6 +270,7 @@ export const baseExtension = Prisma.defineExtension((prisma) =>
                 maxFeePerBlobGas,
                 gasPrice,
                 blobAsCalldataGasUsed,
+                blobGasUsed,
                 category,
                 rollup,
               }) => [
@@ -283,6 +284,7 @@ export const baseExtension = Prisma.defineExtension((prisma) =>
                 maxFeePerBlobGas,
                 gasPrice,
                 blobAsCalldataGasUsed,
+                blobGasUsed,
                 Prisma.sql`${category.toLowerCase()}::category`,
                 rollup
                   ? Prisma.sql`${rollup.toLowerCase()}::rollup`
@@ -305,6 +307,7 @@ export const baseExtension = Prisma.defineExtension((prisma) =>
               max_fee_per_blob_gas,
               gas_price,
               blob_as_calldata_gas_used,
+              blob_gas_used,
               category,
               rollup,
               inserted_at,
@@ -320,6 +323,7 @@ export const baseExtension = Prisma.defineExtension((prisma) =>
               max_fee_per_blob_gas = EXCLUDED.max_fee_per_blob_gas,
               gas_price = EXCLUDED.gas_price,
               blob_as_calldata_gas_used = EXCLUDED.blob_as_calldata_gas_used,
+              blob_gas_used = EXCLUDED.blob_gas_used,
               category = EXCLUDED.category,
               rollup = EXCLUDED.rollup,
               updated_at = NOW()

diff --git a/packages/db/prisma/migrations/20240910013656_add_tx_blob_gas_used/migration.sql b/packages/db/prisma/migrations/20240910013656_add_tx_blob_gas_used/migration.sql
@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "transaction" ADD COLUMN     "blob_gas_used" DECIMAL(100,0);
diff --git a/packages/db/prisma/migrations/20240910013741_backfill_tx_blob_gas_used/migration.sql b/packages/db/prisma/migrations/20240910013741_backfill_tx_blob_gas_used/migration.sql
@@ -0,0 +1,36 @@
+DO $$
+DECLARE
+    batch_size INT := 1000; -- Define the size of each batch
+    update_count INT := 0; -- Variable to store the count of updated rows
+BEGIN
+    -- Loop indefinitely until no rows are updated
+    LOOP
+        WITH candidate_rows AS (
+            -- Select rows for update in batches of 1000, using NOWAIT to avoid locking conflicts
+            SELECT hash
+            FROM transaction
+            WHERE blob_gas_used IS NULL
+            LIMIT batch_size
+            FOR UPDATE NOWAIT
+        ), update_rows AS (
+            -- Perform the update for the selected rows
+            UPDATE transaction
+            SET blob_gas_used = (
+              SELECT COUNT(btx.blob_hash) * 131072 AS blob_gas_used
+              FROM blobs_on_transactions btx
+              WHERE btx.tx_hash = transaction.hash
+            )
+            FROM candidate_rows
+            WHERE candidate_rows.hash = transaction.hash
+            RETURNING transaction.hash
+        )
+        -- Count the number of updated rows in this batch
+        SELECT count(1) INTO update_count FROM update_rows;
+
+        -- Exit the loop if no rows were updated in this batch
+        EXIT WHEN update_count = 0;
+
+        -- Optional: Sleep briefly to reduce load on the database
+        PERFORM pg_sleep(0.1);
+    END LOOP;
+END $$;
diff --git a/...s/db/prisma/migrations/20240910015119_make_tx_blob_gas_used_field_mandatory/migration.sql b/...s/db/prisma/migrations/20240910015119_make_tx_blob_gas_used_field_mandatory/migration.sql
@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - Made the column `blob_gas_used` on table `transaction` required. This step will fail if there are existing NULL values in that column.
+
+*/
+-- AlterTable
+ALTER TABLE "transaction" ALTER COLUMN "blob_gas_used" SET NOT NULL;
diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma
@@ -152,6 +152,7 @@ model Transaction {
   blockNumber           Int      @map("block_number")
   blockTimestamp        DateTime @map("block_timestamp")
   index                 Int?
+  blobGasUsed           Decimal  @map("blob_gas_used") @db.Decimal(100, 0)
   maxFeePerBlobGas      Decimal  @map("max_fee_per_blob_gas") @db.Decimal(100, 0)
   gasPrice              Decimal  @map("gas_price") @db.Decimal(100, 0)
   blobAsCalldataGasUsed Decimal  @map("blob_as_calldata_gas_used") @db.Decimal(100, 0)

diff --git a/packages/db/prisma/seed/DataGenerator.ts b/packages/db/prisma/seed/DataGenerator.ts
@@ -266,6 +266,7 @@ export class DataGenerator {
           blockNumber: block.number,
           blockTimestamp: block.timestamp,
           blobAsCalldataGasUsed: new Prisma.Decimal(0),
+          blobGasUsed: new Prisma.Decimal(0),
           gasPrice: bigintToDecimal(gasPrice),
           maxFeePerBlobGas: bigintToDecimal(maxFeePerBlobGas),
           rollup,
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		-- AlterTable
		ALTER TABLE "transaction" ADD COLUMN "blob_gas_used" DECIMAL(100,0);