From 3831ed7dfdc66ee9d2e2a5caaeabb1a1c6951045 Mon Sep 17 00:00:00 2001 From: JT Smith Date: Fri, 9 Aug 2024 18:16:41 -0500 Subject: [PATCH] Added a CDK stack for the database and network. --- cdk/bin/cdk.mjs | 22 ++++++++ cdk/cdk.context.json | 10 ++++ cdk/lib/constants.mjs | 10 +++- cdk/lib/database-stack.mjs | 92 ++++++++++++++++++++++++++++++++++ cdk/lib/network-stack.mjs | 23 +++++++++ cdk/lib/upload-stack.mjs | 6 +-- ving/docs/change-log.md | 1 + ving/docs/subsystems/cdk.md | 10 ++-- ving/docs/subsystems/pulumi.md | 58 --------------------- 9 files changed, 166 insertions(+), 66 deletions(-) create mode 100644 cdk/cdk.context.json create mode 100644 cdk/lib/database-stack.mjs create mode 100644 cdk/lib/network-stack.mjs delete mode 100644 ving/docs/subsystems/pulumi.md diff --git a/cdk/bin/cdk.mjs b/cdk/bin/cdk.mjs index 3b00a5f3..5ae6c54c 100644 --- a/cdk/bin/cdk.mjs +++ b/cdk/bin/cdk.mjs @@ -2,6 +2,8 @@ import cdk from 'aws-cdk-lib'; import { UploadStack } from '../lib/upload-stack.mjs'; +import { NetworkStack } from '../lib/network-stack.mjs'; +import { DatabaseStack } from '../lib/database-stack.mjs'; import { generatePrefix, generateSuffix } from '../lib/utils.mjs'; import constants from '../lib/constants.mjs'; @@ -27,3 +29,23 @@ new UploadStack(app, `${prefix}-UploadStack${suffix}`, { env: { account: constants.stages[stage].account, region: constants.stages[stage].region }, }); +if (stage != 'dev') { + + const network = new NetworkStack(app, `${prefix}-NetworkStack${suffix}`, { + stage, + constants, + stageConfig: constants.stages[stage], + formatName: (name) => `${prefix}-${name}${suffix}`, + env: { account: constants.stages[stage].account, region: constants.stages[stage].region }, + }); + + new DatabaseStack(app, `${prefix}-DatabaseStack${suffix}`, { + vpc: network.vpc, + stage, + constants, + stageConfig: constants.stages[stage], + formatName: (name) => `${prefix}-${name}${suffix}`, + env: { account: constants.stages[stage].account, region: constants.stages[stage].region }, + }); + +} \ No newline at end of file diff --git a/cdk/cdk.context.json b/cdk/cdk.context.json new file mode 100644 index 00000000..399e0108 --- /dev/null +++ b/cdk/cdk.context.json @@ -0,0 +1,10 @@ +{ + "availability-zones:account=041977924901:region=us-east-1": [ + "us-east-1a", + "us-east-1b", + "us-east-1c", + "us-east-1d", + "us-east-1e", + "us-east-1f" + ] +} diff --git a/cdk/lib/constants.mjs b/cdk/lib/constants.mjs index 18ab6e43..b4cd57c2 100644 --- a/cdk/lib/constants.mjs +++ b/cdk/lib/constants.mjs @@ -18,8 +18,14 @@ export default { memorySize: 512, // megabytes timeout: 60, // seconds }, - uploadsBucketNameOverride: 'uploads.ving.com', - thumbnailsBucketNameOverride: 'thumbnails.ving.com', + auroraSettings: { + adminUser: 'root', + minCapacity: 0.5, + maxCapacity: 2, + backupRetention: 7, // days + }, + uploadsBucketNameOverride: 'uploads.somedomainthattotallyexists.com', + thumbnailsBucketNameOverride: 'thumbnails.somedomainthattotallyexists.com', }, } }; \ No newline at end of file diff --git a/cdk/lib/database-stack.mjs b/cdk/lib/database-stack.mjs new file mode 100644 index 00000000..0dad67d8 --- /dev/null +++ b/cdk/lib/database-stack.mjs @@ -0,0 +1,92 @@ +import { Stack, Duration } from 'aws-cdk-lib'; +import elasticache from 'aws-cdk-lib/aws-elasticache'; +import cdk from 'aws-cdk-lib'; +import rds from 'aws-cdk-lib/aws-rds'; +import ec2 from 'aws-cdk-lib/aws-ec2'; + +export class DatabaseStack extends Stack { + /** + * + * @param {Construct} scope + * @param {string} id + * @param {StackProps=} props + */ + constructor(scope, id, props) { + super(scope, id, props); + + const vpc = props.vpc; + + const securityGroup = new ec2.SecurityGroup(this, props.formatName('database-sg'), { + description: 'Allow traffic to Aurora and Redis', + vpc, + allowAllOutbound: true, + }); + + // will need to close thse down once we have servers connecting to the database and can use better rules + securityGroup.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.tcp(3306)); + securityGroup.addIngressRule(ec2.Peer.anyIpv4(), ec2.Port.tcp(6379)); + + + /* + * Redis + */ + + const redisSubnetGroup = new elasticache.CfnSubnetGroup(this, props.formatName('redis-sg'), { + description: 'Redis subnet group', + subnetIds: vpc.privateSubnets.map((subnet) => subnet.subnetId), + }); + + const redisCluster = new elasticache.CfnReplicationGroup(this, props.formatName('redis'), { + replicationGroupDescription: 'Redis cluster', + cacheNodeType: 'cache.t3.micro', + engine: 'redis', + engineVersion: '7.0', + numCacheClusters: 1, + automaticFailoverEnabled: false, + cacheSubnetGroupName: redisSubnetGroup.ref, + securityGroupIds: [securityGroup.securityGroupId], + }); + + + /* + * Aurora + */ + + + const clusterParameterGroup = new rds.ParameterGroup(this, props.formatName('mysql8params'), { + engine: rds.DatabaseClusterEngine.auroraMysql({ version: rds.AuroraMysqlEngineVersion.VER_3_07_0 }), + description: 'Custom parameter group for MySLQ 8 on ving', + parameters: { + 'character_set_server': 'utf8', + 'collation_server': 'utf8_unicode_ci', + 'innodb_ft_min_token_size': 2, + 'sql_mode': 'NO_ENGINE_SUBSTITUTION', + }, + }); + + // Create a new Aurora Serverless MySQL cluster + const auroraName = props.formatName('aurora') + const auroraCluster = new rds.DatabaseCluster(this, auroraName, { + engine: rds.DatabaseClusterEngine.auroraMysql({ version: rds.AuroraMysqlEngineVersion.VER_3_07_0 }), + vpc, + credentials: { + username: props.stageConfig.auroraSettings.adminUser || 'root', + }, + clusterIdentifier: auroraName, + serverlessV2MinCapacity: props.stageConfig.auroraSettings.minCapacity || 1, + serverlessV2MaxCapacity: props.stageConfig.auroraSettings.maxCapacity || 2, + backupRetention: Duration.days(props.stageConfig.auroraSettings.backupRetention || 7), + securityGroups: [securityGroup], + parameterGroup: clusterParameterGroup, + deletionProtection: true, + removalPolicy: cdk.RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE, + writer: rds.ClusterInstance.serverlessV2(auroraName + '-writer'), + readers: [ + // will be put in promotion tier 1 and will scale with the writer + rds.ClusterInstance.serverlessV2(auroraName + '-reader1', { scaleWithWriter: true }), + ], + + }); + + } +} \ No newline at end of file diff --git a/cdk/lib/network-stack.mjs b/cdk/lib/network-stack.mjs new file mode 100644 index 00000000..4ec9729c --- /dev/null +++ b/cdk/lib/network-stack.mjs @@ -0,0 +1,23 @@ +import { Stack, Duration } from 'aws-cdk-lib'; +import iam from 'aws-cdk-lib/aws-iam'; +import cdk from 'aws-cdk-lib'; +import rds from 'aws-cdk-lib/aws-rds'; +import ec2 from 'aws-cdk-lib/aws-ec2'; + +export class NetworkStack extends Stack { + /** + * + * @param {Construct} scope + * @param {string} id + * @param {StackProps=} props + */ + constructor(scope, id, props) { + super(scope, id, props); + + this.vpc = new ec2.Vpc(this, props.formatName('vpc'), { + maxAzs: 2, + ipAddresses: ec2.IpAddresses.cidr('10.0.0.0/16'), + }); + + } +} \ No newline at end of file diff --git a/cdk/lib/upload-stack.mjs b/cdk/lib/upload-stack.mjs index f045604f..608d3e46 100644 --- a/cdk/lib/upload-stack.mjs +++ b/cdk/lib/upload-stack.mjs @@ -82,7 +82,7 @@ export class UploadStack extends Stack { // create the bucket const thumbnailsName = props.formatName(props.constants.thumbnailsBucketName); const thumbnailsBucket = new s3.Bucket(this, thumbnailsName, { - bucketName: props.stageConfig.uploadsBucketNameOverride || thumbnailsName, + bucketName: props.stageConfig.thumbnailsBucketNameOverride || thumbnailsName, removalPolicy: cdk.RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE, cors: [ { @@ -149,8 +149,8 @@ export class UploadStack extends Stack { code: lambda.Code.fromAsset('./lib/lambda/func/processUpload'), layers: [nodemodsLayer], role: iamForLambda, - timeout: props.constants.stages[props.stage].uploadsLambdaSettings.timeout, - memorySize: props.constants.stages[props.stage].uploadsLambdaSettings.memorySize, + timeout: Duration.seconds(props.stageConfig.uploadsLambdaSettings.timeout || 60), + memorySize: props.stageConfig.uploadsLambdaSettings.memorySize || 128, environment: { VING_AWS_THUMBNAILS_BUCKET: thumbnailsBucket.bucketName, }, diff --git a/ving/docs/change-log.md b/ving/docs/change-log.md index f68883b0..f1f24023 100644 --- a/ving/docs/change-log.md +++ b/ving/docs/change-log.md @@ -9,6 +9,7 @@ outline: deep * Started replacing Pulumi with CDK. * Added a CDK stack for uploads. * NOTE: Pulumi will be removed in a near future release. Do a `pulumi down; pulumi destroy` to remove the stacks, and then follow the CDK instructions to create recreate the stacks. +* Added a CDK stack for the database and network. ### 2024-08-07 * Switched to using hooks to copy ving.json to the build directory. diff --git a/ving/docs/subsystems/cdk.md b/ving/docs/subsystems/cdk.md index d90dcda7..bf4a186c 100644 --- a/ving/docs/subsystems/cdk.md +++ b/ving/docs/subsystems/cdk.md @@ -74,11 +74,15 @@ Above we always used the `dev` stage. This is what you want to use on your local ## Stacks -Stacks are divisions of AWS resources in CDK. We divide them into areas of responsibility. For example, we have a `uploads` stack that generates the S3 buckets for uploads and thumbnails. We have a `database` stack that generates the Aurora Serverless database and Redis cluster. And we have a `server` stack that generates the EC2 instance that runs the Ving server. +Stacks are divisions of AWS resources in CDK. We divide them into areas of responsibility. For example, we have a `uploads` stack that generates the S3 buckets for uploads and thumbnails. There is also a `network` stack for creating the virtual private cloud that all the hardware will be connected to. We have a `database` stack that generates the Aurora Serverless database and Redis cluster. And in the future we will have a `server` stack that generates the EC2/lambda instances that run the Ving web and jobs servers. You can create your own stacks as well by following the instructions in the [CDK documentation](https://docs.aws.amazon.com/cdk/v2/guide/stacks.html#stacks). ### Special Needs of the Databsae Stack -First by default it will create a database with the root user of `root` and the password `changeMeAfterTheFact`. You'll need to change this password in the AWS console. You can do this by going to the RDS console, clicking on the cluster, and then clicking on the "Modify" button. Then you can change the password. +First, by default it will create a database with the root user of `root` and no password. You'll need to set the password in the AWS console. You can do this by going to the RDS console, clicking on the cluster, and then clicking on the "Modify" button. Then you can change the password. -Second, you may also want to change the minimum and maximum capacity units of the Aurora cluster. You can do this in the `cdk/lib/constants.mjs` file. \ No newline at end of file +Second, you may also want to change the minimum and maximum capacity units of the Aurora cluster. You can do this in the `cdk/lib/constants.mjs` file. + +Third, the stack does not create or deploy your ving database schema. It simply creates the cluster that the database will run on. Follow the [Drizzle](drizzle) documentation to create and deploy your schema. + +*Note:* If you ever want to delete your database, you should change `RETAIN_ON_UPDATE_OR_DELETE` to `DESTROY` in the `DatabaseStack` class first. Then deploy it. And then destroy it. \ No newline at end of file diff --git a/ving/docs/subsystems/pulumi.md b/ving/docs/subsystems/pulumi.md deleted file mode 100644 index 8cb22d67..00000000 --- a/ving/docs/subsystems/pulumi.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -outline: deep ---- -# Pulumi -[Pulumi](https://www.pulumi.com) is an open source Infrastructure as Code (IaC) system. It allows you to automate setting up servers and services. Ving uses it to make configuring AWS faster, rather than giving you a bunch of instructions to perform and hope you get them all right, we've programmed Pulumi to do it for you. - -## Setup - -### Install Pulumi -Follow the [Pulumi install instructions](https://www.pulumi.com/docs/install/). - -### Setup AWS Credentials -If you have not already set up an [AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) file in `~/.aws/credentials` then you'll want to do that now as pulumi will use that to log in to AWS. - -### Edit Config -Then edit the `Pulumi.yaml` file and change the `name` field from `ving` to your project name. This name will be used to prefix all the generated services in AWS so you know what we made. - -There is also a `Pulumi.dev.yaml` file. You'll eventually have one of those for each deployment stage that you create such as `dev`, `qa`, and `prod`. You don't need to modify it now, but it has some variables that are unique to each deployment. - -### Install Pulumi Config -Then type `pulumi up` at your command line. - -The first time you run it, it will ask you to log in to pulumi. You can do that right at the command line by selecting the appropriate option (usually the second one). - -When it asks you about what stack you want, arrow down and hit enter on "create a new stack" and then say `yourusername/dev` unless you already know how to use pulumi and want to use your own stack name. And once it shows you its plan for deployment, use your arrow key to move up to `yes` to deploy it. - - -## Using Pulumi For Your Own Needs -By editing `Pulumi.mjs` you can add your own infrastructure automations. And then use `pulumi up` to roll them out to the server. Just make sure you don't remove or change any of the pulumi code that we have there unless you understand the implications. - - -## Pulumi Prod vs Dev -The documentation above talks about Pulumi for dev deployments. But what about prod deployments? You can switch to prod by typing: - -```bash -pulumi stack select prod -``` - -And then running `pulumi up` again. You can switch back to dev by typing: - -```bash -pulumi stack select dev -``` - -Prod does the following things differently than dev: -- stores its AWS generated variables in .env.prod -- provisions a VPC -- provisions an Aurora Serverless database -- provisions an EC2 instance - -### Deploying Prod (or other stacks) -After deploying prod, or other stacks, there are a few things you'll need to do to set it up: - - -#### Database -First by default it will create a database with the root user of `root` and the password `changeMeAfterTheFact`. You'll need to change this password in the AWS console. You can do this by going to the RDS console, clicking on the cluster, and then clicking on the "Modify" button. Then you can change the password. - -Second, you may also want to change the minimum and maximum capacity units of the Aurora cluster. You can do this by going to the RDS console, clicking on the cluster, and then clicking on the "Modify" button. Then you can change the minimum and maximum capacity units. \ No newline at end of file