fix: add blob list and remove infra (#355)

We just need to update `upload-api` dep storacha/w3up#1387 and get `AllocationsStorage` implementation to implement `remove`. All tests run as part of the interface tests imported from https://github.com/web3-storage/w3up/blob/main/packages/upload-api/test/handlers/blob.js Also note that `filecoin-api` was updated (internally to `upload-api` and the options for filecoin storefront skip submit queue were removed, therefore storacha/w3up#1371
storacha · Apr 24, 2024 · af4700c · af4700c
1 parent 24d22ee
commit af4700c
Show file tree

Hide file tree

Showing 6 changed files with 161 additions and 60 deletions.
diff --git a/filecoin/package.json b/filecoin/package.json
@@ -17,7 +17,7 @@
     "@ucanto/principal": "^9.0.1",
     "@ucanto/transport": "^9.1.1",
     "@web3-storage/data-segment": "^5.1.0",
-    "@web3-storage/filecoin-api": "^4.6.1",
+    "@web3-storage/filecoin-api": "^5.0.0",
     "@web3-storage/filecoin-client": "^3.3.1",
     "fr32-sha2-256-trunc254-padded-binary-tree-multihash": "^3.3.0",
     "multiformats": "^13.1.0",

diff --git a/package-lock.json b/package-lock.json
diff --git a/tools/package.json b/tools/package.json
@@ -16,7 +16,7 @@
     "@aws-sdk/client-dynamodb": "^3.226.0",
     "@aws-sdk/util-dynamodb": "^3.226.0",
     "@ipld/dag-ucan": "3.4.0",
-    "@web3-storage/capabilities": "^12.1.0",
+    "@web3-storage/capabilities": "^14.0.0",
     "@web3-storage/data-segment": "5.0.0",
     "sade": "^1.8.1"
   }

diff --git a/upload-api/functions/ucan-invocation-router.js b/upload-api/functions/ucan-invocation-router.js
@@ -262,11 +262,6 @@ export async function ucanInvocationRouter(request) {
         with: serviceSigner.did()
       }
     },
-    options: {
-      // TODO: we compute and put all pieces into the queue on bucket event
-      // We may change this to validate user provided piece
-      skipFilecoinSubmitQueue: false
-    },
     plansStorage,
     requirePaymentPlan,
     usageStorage,

diff --git a/upload-api/package.json b/upload-api/package.json
@@ -23,9 +23,9 @@
     "@ucanto/validator": "^9.0.2",
     "@web-std/fetch": "^4.1.0",
     "@web3-storage/access": "^18.3.1",
-    "@web3-storage/capabilities": "^13.3.1",
+    "@web3-storage/capabilities": "^14.0.0",
     "@web3-storage/did-mailto": "^2.1.0",
-    "@web3-storage/upload-api": "^9.1.5",
+    "@web3-storage/upload-api": "^10.0.0",
     "multiformats": "^13.1.0",
     "nanoid": "^5.0.2",
     "preact": "^10.14.1",

diff --git a/upload-api/stores/allocations.js b/upload-api/stores/allocations.js
@@ -3,9 +3,10 @@ import {
   GetItemCommand,
   PutItemCommand,
   QueryCommand,
+  DeleteItemCommand,
 } from '@aws-sdk/client-dynamodb'
 import { marshall, unmarshall } from '@aws-sdk/util-dynamodb'
-import { RecordKeyConflict, RecordNotFound } from '@web3-storage/upload-api/errors'
+import { RecordKeyConflict, RecordNotFound, StorageOperationFailed } from '@web3-storage/upload-api/errors'
 import { base58btc } from 'multiformats/bases/base58'
 import * as Link from 'multiformats/link'
 
@@ -126,6 +127,44 @@ export function useAllocationsStorage(dynamoDb, tableName) {
       }
       return { ok: { blob } }
     },
+    /**
+     * @param {import('@web3-storage/upload-api').DID} space
+     * @param {Uint8Array} blobMultihash
+     * @returns {ReturnType<AllocationsStorage['remove']>}
+     */
+    async remove(space, blobMultihash) {
+      const key = getKey(space, blobMultihash)
+      const cmd = new DeleteItemCommand({
+        TableName: tableName,
+        Key: key,
+        ConditionExpression: 'attribute_exists(#S) AND attribute_exists(#M)',
+        ExpressionAttributeNames: { '#S': 'space', '#M': 'multihash' },
+        ReturnValues: 'ALL_OLD'
+      })
+
+      try {
+        const res = await dynamoDb.send(cmd)
+        if (!res.Attributes) {
+          throw new Error('missing return values')
+        }
+
+        const raw = unmarshall(res.Attributes)
+        return {
+          ok: {
+            size: Number(raw.size)
+          }
+        }
+      } catch (/** @type {any} */ err) {
+        if (err.name === 'ConditionalCheckFailedException') {
+          return {
+            error: new RecordNotFound()
+          }
+        }
+        return {
+          error: new StorageOperationFailed(err.name)
+        }
+      }
+    },
     /**
      * List all CARs bound to an account
      *
@@ -137,7 +176,7 @@ export function useAllocationsStorage(dynamoDb, tableName) {
       const exclusiveStartKey = options.cursor
         ? marshall({
             space,
-            link: options.cursor,
+            multihash: options.cursor,
           })
         : undefined
 
@@ -150,7 +189,6 @@ export function useAllocationsStorage(dynamoDb, tableName) {
             AttributeValueList: [{ S: space }],
           },
         },
-        ScanIndexForward: !options.pre,
         ExclusiveStartKey: exclusiveStartKey,
         AttributesToGet: ['multihash', 'size', 'insertedAt'],
       })
@@ -165,15 +203,16 @@ export function useAllocationsStorage(dynamoDb, tableName) {
         response.LastEvaluatedKey && unmarshall(response.LastEvaluatedKey)
       const lastLinkCID = lastKey ? lastKey.multihash : undefined
 
-      const before = options.pre ? lastLinkCID : firstLinkCID
-      const after = options.pre ? firstLinkCID : lastLinkCID
+      const before = firstLinkCID
+      const after = lastLinkCID
+
       return {
         ok: {
           size: results.length,
           before,
           after,
           cursor: after,
-          results: options.pre ? results.reverse() : results,
+          results,
         }
       }
     },