feat: cache and store content claims

.env.local

@@ -1,2 +1,3 @@
 TF_WORKSPACE=
-TF_VAR_PRIVATE_KEY=
+TF_VAR_private_key=
+TF_VAR_public_url=

cmd/lambda/getclaim/main.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/aws/aws-lambda-go/lambda"
+	"github.com/awslabs/aws-lambda-go-api-proxy/httpadapter"
+	"github.com/storacha/indexing-service/pkg/aws"
+	"github.com/storacha/indexing-service/pkg/server"
+)
+
+func main() {
+	config := aws.FromEnv(context.Background())
+	service, err := aws.Construct(config)
+	if err != nil {
+		panic(err)
+	}
+	handler := server.GetClaimHandler(service)
+	lambda.Start(httpadapter.NewV2(http.HandlerFunc(handler)).ProxyWithContext)
+}

cmd/lambda/remotesync/main.go

@@ -9,6 +9,7 @@ import (
 	"github.com/ipfs/go-cid"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
 	goredis "github.com/redis/go-redis/v9"
+	"github.com/storacha/go-metadata"
 	"github.com/storacha/indexing-service/pkg/aws"
 	"github.com/storacha/indexing-service/pkg/redis"
 	"github.com/storacha/indexing-service/pkg/service/providerindex"
@@ -47,7 +48,7 @@ func main() {
 	ipniStore := aws.NewS3Store(cfg.Config, cfg.IPNIStoreBucket, cfg.IPNIStorePrefix)
 	chunkLinksTable := aws.NewDynamoProviderContextTable(cfg.Config, cfg.ChunkLinksTableName)
 	metadataTable := aws.NewDynamoProviderContextTable(cfg.Config, cfg.MetadataTableName)
-	publisherStore := store.NewPublisherStore(ipniStore, chunkLinksTable, metadataTable)
+	publisherStore := store.NewPublisherStore(ipniStore, chunkLinksTable, metadataTable, store.WithMetadataContext(metadata.MetadataContext))
 	remoteSyncer := providerindex.NewRemoteSyncer(providerStore, publisherStore)
 	lambda.Start(makeHandler(remoteSyncer))
 }

deploy/lambda.tf

@@ -3,6 +3,9 @@ locals {
     getroot = {
       name = "GETroot"
     }
+    getclaim = {
+      name = "GETclaim"
+    }
     getclaims = {
       name = "GETclaims"
     }
@@ -65,7 +68,9 @@ resource "aws_lambda_function" "lambda" {
         NOTIFIER_HEAD_BUCKET_NAME = aws_s3_bucket.notifier_head_bucket.bucket
         NOTIFIER_SNS_TOPIC_ARN = aws_sns_topic.published_advertisememt_head_change.id
         PRIVATE_KEY = aws_ssm_parameter.private_key.name
+        PUBLIC_URL = var.public_url
         IPNI_STORE_BUCKET_REGIONAL_DOMAIN = aws_s3_bucket.ipni_store_bucket.bucket_regional_domain_name
+        CLAIM_STORE_BUCKET_NAME = aws_s3_bucket.claim_store_bucket.bucket
     }
   }
 
@@ -185,7 +190,8 @@ data "aws_iam_policy_document" "lambda_s3_put_get_document" {
     resources = [
       aws_s3_bucket.caching_bucket.arn,
       aws_s3_bucket.ipni_store_bucket.arn,
-      aws_s3_bucket.notifier_head_bucket.arn
+      aws_s3_bucket.notifier_head_bucket.arn,
+      aws_s3_bucket.claim_store_bucket.arn
     ]
   }
 }

deploy/s3.tf

@@ -47,4 +47,8 @@ resource "aws_s3_bucket_policy" "ipni_store_policy" {
 
 resource "aws_s3_bucket" "notifier_head_bucket" {
   bucket = "${terraform.workspace}-${var.app}-notifier-head-bucket"
-}
+}
+
+resource "aws_s3_bucket" "claim_store_bucket" {
+  bucket = "${terraform.workspace}-${var.app}-claim-store-bucket"
+}

deploy/variables.tf

@@ -15,7 +15,13 @@ variable "allowed_account_ids" {
   type = list(string)
   default = ["505595374361"]
 }
-variable private_key {
+
+variable "private_key" {
   description = "private_key for the peer for this deployment"
   type = string
-}
+}
+
+variable "public_url" {
+  description = "public URL of the peer for this deployment"
+  type = string
+}

go.mod

@@ -1,24 +1,37 @@
 module github.com/storacha/indexing-service
 
-go 1.23
-
-toolchain go1.23.0
+go 1.23.2
 
 require (
+	github.com/aws/aws-lambda-go v1.47.0
 	github.com/aws/aws-sdk-go-v2 v1.32.2
+	github.com/aws/aws-sdk-go-v2/config v1.28.0
+	github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.12
+	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.36.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.66.1
+	github.com/aws/aws-sdk-go-v2/service/sns v1.33.2
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.36.2
+	github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2
+	github.com/awslabs/aws-lambda-go-api-proxy v0.16.2
+	github.com/google/uuid v1.6.0
 	github.com/ipfs/go-cid v0.4.1
+	github.com/ipfs/go-datastore v0.6.0
 	github.com/ipfs/go-ds-flatfs v0.5.1
+	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipld/go-ipld-prime v0.21.1-0.20240917223228-6148356a4c2e
-	github.com/ipni/go-libipni v0.6.13
-	github.com/libp2p/go-libp2p v0.36.5
+	github.com/ipni/go-libipni v0.6.14
+	github.com/libp2p/go-libp2p v0.37.0
 	github.com/multiformats/go-multiaddr v0.13.0
+	github.com/multiformats/go-multibase v0.2.0
 	github.com/multiformats/go-multicodec v0.9.0
 	github.com/multiformats/go-multihash v0.2.3
-	github.com/multiformats/go-varint v0.0.7
 	github.com/redis/go-redis/v9 v9.6.1
-	github.com/storacha/go-ucanto v0.1.1-0.20241022025657-f12c0d06a4ea
-	github.com/storacha/ipni-publisher v0.0.0-20241018055706-032286a2dc3f
+	github.com/storacha/go-capabilities v0.0.0-20241030160329-4cf19ffc732d
+	github.com/storacha/go-metadata v0.0.0-20241031171938-8f3343b7ce5a
+	github.com/storacha/go-ucanto v0.1.1-0.20241028163940-34de8cd912bb
+	github.com/storacha/ipni-publisher v0.0.0-20241031171227-f59f9fc3b0b8
 	github.com/stretchr/testify v1.9.0
+	github.com/urfave/cli/v2 v2.27.4
 )
 
 require (
@@ -40,42 +53,22 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect
 	github.com/aws/smithy-go v1.22.0 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
-	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
-	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/ucan-wg/go-ucan v0.0.0-20240916120445-37f52863156c // indirect
-	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
-	go.opentelemetry.io/otel/metric v1.30.0 // indirect
-)
-
-require (
-	github.com/aws/aws-lambda-go v1.47.0
-	github.com/aws/aws-sdk-go-v2/config v1.27.43
-	github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.15.12
-	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.36.2
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.65.3
-	github.com/aws/aws-sdk-go-v2/service/sns v1.33.2
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.36.2
-	github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2
-	github.com/awslabs/aws-lambda-go-api-proxy v0.16.2
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/google/uuid v1.6.0
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
 	github.com/ipfs/go-block-format v0.2.0 // indirect
 	github.com/ipfs/go-blockservice v0.5.2 // indirect
-	github.com/ipfs/go-datastore v0.6.0
 	github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect
 	github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect
 	github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect
@@ -84,13 +77,14 @@ require (
 	github.com/ipfs/go-ipld-format v0.6.0 // indirect
 	github.com/ipfs/go-ipld-legacy v0.2.1 // indirect
 	github.com/ipfs/go-log v1.0.5 // indirect
-	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipfs/go-merkledag v0.11.0 // indirect
 	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
 	github.com/ipfs/go-verifcid v0.0.3 // indirect
 	github.com/ipld/go-car v0.6.2 // indirect
 	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/libp2p/go-libp2p-pubsub v0.12.0 // indirect
@@ -101,27 +95,29 @@ require (
 	github.com/multiformats/go-base32 v0.1.0 // indirect
 	github.com/multiformats/go-base36 v0.2.0 // indirect
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
-	github.com/multiformats/go-multibase v0.2.0
 	github.com/multiformats/go-multistream v0.5.0 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/onsi/gomega v1.34.1 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/storacha/go-capabilities v0.0.0-20241022031210-04ae6a36f715
-	github.com/storacha/go-metadata v0.0.0-20241021141939-f94d93dcda78
-	github.com/urfave/cli/v2 v2.27.4
+	github.com/ucan-wg/go-ucan v0.0.0-20240916120445-37f52863156c // indirect
 	github.com/whyrusleeping/cbor-gen v0.1.2 // indirect
+	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	go.opentelemetry.io/otel v1.30.0 // indirect
+	go.opentelemetry.io/otel/metric v1.30.0 // indirect
 	go.opentelemetry.io/otel/trace v1.30.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/crypto v0.27.0 // indirect
-	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
-	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect
+	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.3.0 // indirect
 )