From b355b0cd7d6c46bea867419323f490a5062a221c Mon Sep 17 00:00:00 2001 From: Vicente Olmedo Date: Fri, 17 Jan 2025 09:52:40 +0100 Subject: [PATCH 1/3] add infra diagram --- docs/infra.md | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 docs/infra.md diff --git a/docs/infra.md b/docs/infra.md new file mode 100644 index 0000000..6e908a7 --- /dev/null +++ b/docs/infra.md @@ -0,0 +1,64 @@ +```mermaid +graph TB +%% VPC and Networking +VPC[VPC\n10.0.0.0/16] --> PublicSubnets[Public Subnets] +VPC --> PrivateSubnets[Private Subnets] +PublicSubnets --> IGW[Internet Gateway] +PublicSubnets --> NAT[NAT Gateways] +NAT --> PrivateSubnets + + %% API Gateway + APIGW[API Gateway v2\nHTTP API] --> Lambda + APIGW --> CustomDomain[Custom Domain\n*.indexer.storacha.network] + CustomDomain --> Route53[Route53\nDNS Zone] + CustomDomain --> ACM[ACM Certificate] + + %% Lambda Functions + subgraph Lambda[Lambda Functions] + GETroot + GETclaim + GETclaims + POSTclaims + notifier + providercache + remotesync + end + + %% Event Sources + EventBridge[EventBridge\nScheduler] --> notifier + SNSTopic[SNS Topic\nHead Changes] --> remotesync + SQSQueue[SQS Queue\nCaching.fifo] --> providercache + SQSQueue --> SQSDLQueue[Dead Letter Queue] + + %% Storage + Lambda --> DynamoDB + subgraph DynamoDB[DynamoDB Tables] + metadata + chunk_links + legacy_claims + legacy_block_index + end + + Lambda --> S3 + subgraph S3[S3 Buckets] + caching + ipni_store + notifier_head + claim_store + legacy_claims_bucket + end + + %% Cache + Lambda --> ElastiCache + subgraph ElastiCache[Redis Serverless] + providers + indexes + claims + end + + %% Parameters + Lambda --> SSM[SSM Parameter Store\nPrivate Key] + + %% Security + SecurityGroup[Security Group\nLambda] --> CacheSecurityGroup[Security Group\nRedis] +``` From 1ca94477d499833053861ea4324006b450215df4 Mon Sep 17 00:00:00 2001 From: Vicente Olmedo Date: Fri, 17 Jan 2025 09:52:54 +0100 Subject: [PATCH 2/3] add a link to the diagram in the README --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 5af3a52..e9d9f49 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,9 @@ $ go install github.com/storacha/indexing-service/cmd@latest ## Deployment +> [!TIP] +> Take a look at the [infrastructure diagram](docs/infra.md) for an overview of what gets deployed. + Deployment of this service to AWS is managed by terraform which you can invoke with `make`. First, install OpenTofu e.g. From 34acf0af8bfaa43c1aafae844bda58a7913eec62 Mon Sep 17 00:00:00 2001 From: Vicente Olmedo Date: Fri, 17 Jan 2025 10:40:10 +0100 Subject: [PATCH 3/3] fix newlines not rendering properly --- docs/infra.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/infra.md b/docs/infra.md index 6e908a7..f42af94 100644 --- a/docs/infra.md +++ b/docs/infra.md @@ -1,16 +1,16 @@ ```mermaid graph TB -%% VPC and Networking -VPC[VPC\n10.0.0.0/16] --> PublicSubnets[Public Subnets] -VPC --> PrivateSubnets[Private Subnets] -PublicSubnets --> IGW[Internet Gateway] -PublicSubnets --> NAT[NAT Gateways] -NAT --> PrivateSubnets + %% VPC and Networking + VPC[VPC
10.0.0.0/16] --> PublicSubnets[Public Subnets] + VPC --> PrivateSubnets[Private Subnets] + PublicSubnets --> IGW[Internet Gateway] + PublicSubnets --> NAT[NAT Gateways] + NAT --> PrivateSubnets %% API Gateway - APIGW[API Gateway v2\nHTTP API] --> Lambda - APIGW --> CustomDomain[Custom Domain\n*.indexer.storacha.network] - CustomDomain --> Route53[Route53\nDNS Zone] + APIGW[API Gateway v2
HTTP API] --> Lambda + APIGW --> CustomDomain[Custom Domain
*.indexer.storacha.network] + CustomDomain --> Route53[Route53
DNS Zone] CustomDomain --> ACM[ACM Certificate] %% Lambda Functions @@ -25,9 +25,9 @@ NAT --> PrivateSubnets end %% Event Sources - EventBridge[EventBridge\nScheduler] --> notifier - SNSTopic[SNS Topic\nHead Changes] --> remotesync - SQSQueue[SQS Queue\nCaching.fifo] --> providercache + EventBridge[EventBridge
Scheduler] --> notifier + SNSTopic[SNS Topic
Head Changes] --> remotesync + SQSQueue[SQS Queue
Caching.fifo] --> providercache SQSQueue --> SQSDLQueue[Dead Letter Queue] %% Storage @@ -57,8 +57,8 @@ NAT --> PrivateSubnets end %% Parameters - Lambda --> SSM[SSM Parameter Store\nPrivate Key] + Lambda --> SSM[SSM Parameter Store
Private Key] %% Security - SecurityGroup[Security Group\nLambda] --> CacheSecurityGroup[Security Group\nRedis] + SecurityGroup[Security Group
Lambda] --> CacheSecurityGroup[Security Group
Redis] ```