From 8cd49227b73a7522a72cb373bf72e721f804b945 Mon Sep 17 00:00:00 2001 From: Kelvin Chappell Date: Mon, 9 Dec 2024 15:28:40 +0000 Subject: [PATCH 01/11] Add config to test Tenable integration --- .../service-catalogue.test.ts.snap | 780 ++++++++++++++++++ packages/cdk/lib/cloudquery/config.ts | 24 + packages/cdk/lib/cloudquery/index.ts | 10 + 3 files changed, 814 insertions(+) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index a849e5a0..a05c716c 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -18821,6 +18821,655 @@ spec: }, "Type": "AWS::IAM::Policy", }, + "CloudquerySourceTenableScheduledEventRule0AC5E644": { + "Properties": { + "ScheduleExpression": "cron(0 3 * * ? *)", + "State": "ENABLED", + "Targets": [ + { + "Arn": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], + }, + "EcsParameters": { + "LaunchType": "FARGATE", + "NetworkConfiguration": { + "AwsVpcConfiguration": { + "AssignPublicIp": "DISABLED", + "SecurityGroups": [ + { + "Fn::GetAtt": [ + "PostgresAccessSecurityGroupServicecatalogue03C78F14", + "GroupId", + ], + }, + ], + "Subnets": { + "Ref": "PrivateSubnets", + }, + }, + }, + "PropagateTags": "TASK_DEFINITION", + "TaskCount": 1, + "TaskDefinitionArn": { + "Ref": "CloudquerySourceTenableTaskDefinition1BA5C9B0", + }, + }, + "Id": "Target0", + "Input": "{}", + "RoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceTenableTaskDefinitionEventsRole9B66EBFC", + "Arn", + ], + }, + }, + ], + }, + "Type": "AWS::Events::Rule", + }, + "CloudquerySourceTenableTaskDefinition1BA5C9B0": { + "Properties": { + "ContainerDefinitions": [ + { + "Command": [ + "/bin/sh", + "-c", + "printf 'kind: source +spec: + name: tenable + registry: cloudquery + path: cloudquery/tenable + version: v2.6.2 + destinations: + - postgresql + tables: + - '*' + spec: + access_key: \${TENABLE_ACCESS_KEY} + secret_key: \${TENABLE_SECRET_KEY} +' > /usr/share/cloudquery/source.yaml;printf 'kind: destination +spec: + name: postgresql + registry: github + path: cloudquery/postgresql + version: v7.2.0 + migrate_mode: forced + spec: + connection_string: >- + user=\${DB_USERNAME} password=\${DB_PASSWORD} host=\${DB_HOST} port=5432 + dbname=postgres sslmode=verify-full +' > /usr/share/cloudquery/destination.yaml;/app/cloudquery sync /usr/share/cloudquery/source.yaml /usr/share/cloudquery/destination.yaml --log-format json --log-console --no-log-file", + ], + "DependsOn": [ + { + "Condition": "HEALTHY", + "ContainerName": "CloudquerySource-TenableAWSOTELCollector", + }, + ], + "DockerLabels": { + "App": "service-catalogue", + "Name": "Tenable", + "Stack": "deploy", + "Stage": "TEST", + }, + "EntryPoint": [ + "", + ], + "Environment": [ + { + "Name": "GOMEMLIMIT", + "Value": "819MiB", + }, + ], + "Essential": true, + "Image": "ghcr.io/guardian/service-catalogue/cloudquery:stable", + "LogConfiguration": { + "LogDriver": "awsfirelens", + "Options": { + "Name": "kinesis_streams", + "region": { + "Ref": "AWS::Region", + }, + "retry_limit": "2", + "stream": { + "Ref": "LoggingStreamName", + }, + }, + }, + "MountPoints": [ + { + "ContainerPath": "/usr/share/cloudquery", + "ReadOnly": false, + "SourceVolume": "config-volume", + }, + { + "ContainerPath": "/app/.cq", + "ReadOnly": false, + "SourceVolume": "cloudquery-volume", + }, + { + "ContainerPath": "/tmp", + "ReadOnly": false, + "SourceVolume": "tmp-volume", + }, + ], + "Name": "CloudquerySource-TenableContainer", + "ReadonlyRootFilesystem": true, + "Secrets": [ + { + "Name": "DB_USERNAME", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":username::", + ], + ], + }, + }, + { + "Name": "DB_HOST", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":host::", + ], + ], + }, + }, + { + "Name": "DB_PASSWORD", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":password::", + ], + ], + }, + }, + { + "Name": "CLOUDQUERY_API_KEY", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "cloudqueryapikeyCCF82F53", + }, + ":api-key::", + ], + ], + }, + }, + ], + }, + { + "Command": [ + "--config=/etc/ecs/ecs-xray.yaml", + ], + "Essential": true, + "HealthCheck": { + "Command": [ + "CMD", + "/healthcheck", + ], + "Interval": 5, + "Retries": 3, + "Timeout": 5, + }, + "Image": "public.ecr.aws/aws-observability/aws-otel-collector:v0.35.0", + "LogConfiguration": { + "LogDriver": "awsfirelens", + "Options": { + "Name": "kinesis_streams", + "region": { + "Ref": "AWS::Region", + }, + "retry_limit": "2", + "stream": { + "Ref": "LoggingStreamName", + }, + }, + }, + "Name": "CloudquerySource-TenableAWSOTELCollector", + "PortMappings": [ + { + "ContainerPort": 4318, + "Protocol": "tcp", + }, + ], + "ReadonlyRootFilesystem": true, + }, + { + "Command": [ + "/bin/sh", + "-c", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + ], + "DockerLabels": { + "App": "service-catalogue", + "Name": "Tenable", + "Stack": "deploy", + "Stage": "TEST", + }, + "EntryPoint": [ + "", + ], + "Essential": false, + "Image": "public.ecr.aws/docker/library/postgres:16-alpine", + "LogConfiguration": { + "LogDriver": "awsfirelens", + "Options": { + "Name": "kinesis_streams", + "region": { + "Ref": "AWS::Region", + }, + "retry_limit": "2", + "stream": { + "Ref": "LoggingStreamName", + }, + }, + }, + "Name": "CloudquerySource-TenablePostgresContainer", + "ReadonlyRootFilesystem": true, + "Secrets": [ + { + "Name": "PGUSER", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":username::", + ], + ], + }, + }, + { + "Name": "PGHOST", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":host::", + ], + ], + }, + }, + { + "Name": "PGPASSWORD", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":password::", + ], + ], + }, + }, + ], + }, + { + "Environment": [ + { + "Name": "STACK", + "Value": "deploy", + }, + { + "Name": "STAGE", + "Value": "TEST", + }, + { + "Name": "APP", + "Value": "service-catalogue", + }, + { + "Name": "GU_REPO", + "Value": "guardian/service-catalogue", + }, + ], + "Essential": true, + "FirelensConfiguration": { + "Type": "fluentbit", + }, + "Image": "ghcr.io/guardian/devx-logs:2", + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "CloudquerySourceTenableTaskDefinitionCloudquerySourceTenableFirelensLogGroup04061777", + }, + "awslogs-region": { + "Ref": "AWS::Region", + }, + "awslogs-stream-prefix": "deploy/TEST/service-catalogue", + }, + }, + "MountPoints": [ + { + "ContainerPath": "/init", + "ReadOnly": false, + "SourceVolume": "firelens-volume", + }, + ], + "Name": "CloudquerySource-TenableFirelens", + "ReadonlyRootFilesystem": true, + }, + ], + "Cpu": "256", + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "CloudquerySourceTenableTaskDefinitionExecutionRoleA128084D", + "Arn", + ], + }, + "Family": "ServiceCatalogueCloudquerySourceTenableTaskDefinition01DF6BF3", + "Memory": "1024", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": [ + "FARGATE", + ], + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "Tenable", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + "TaskRoleArn": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskTenable93E15E95", + "Arn", + ], + }, + "Volumes": [ + { + "Name": "config-volume", + }, + { + "Name": "cloudquery-volume", + }, + { + "Name": "tmp-volume", + }, + { + "Name": "firelens-volume", + }, + ], + }, + "Type": "AWS::ECS::TaskDefinition", + }, + "CloudquerySourceTenableTaskDefinitionCloudquerySourceTenableFirelensLogGroup04061777": { + "DeletionPolicy": "Retain", + "Properties": { + "RetentionInDays": 1, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "Tenable", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + }, + "CloudquerySourceTenableTaskDefinitionEventsRole9B66EBFC": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "events.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "Tenable", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceTenableTaskDefinitionEventsRoleDefaultPolicy5D979357": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "ecs:RunTask", + "Condition": { + "ArnEquals": { + "ecs:cluster": { + "Fn::GetAtt": [ + "servicecatalogueCluster5FC34DC5", + "Arn", + ], + }, + }, + }, + "Effect": "Allow", + "Resource": { + "Ref": "CloudquerySourceTenableTaskDefinition1BA5C9B0", + }, + }, + { + "Action": "ecs:TagResource", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":ecs:", + { + "Ref": "AWS::Region", + }, + ":*:task/", + { + "Ref": "servicecatalogueCluster5FC34DC5", + }, + "/*", + ], + ], + }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceTenableTaskDefinitionExecutionRoleA128084D", + "Arn", + ], + }, + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "servicecatalogueTESTtaskTenable93E15E95", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceTenableTaskDefinitionEventsRoleDefaultPolicy5D979357", + "Roles": [ + { + "Ref": "CloudquerySourceTenableTaskDefinitionEventsRole9B66EBFC", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "CloudquerySourceTenableTaskDefinitionExecutionRoleA128084D": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Name", + "Value": "Tenable", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "CloudquerySourceTenableTaskDefinitionExecutionRoleDefaultPolicyA338312D": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + }, + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "cloudqueryapikeyCCF82F53", + }, + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "CloudquerySourceTenableTaskDefinitionCloudquerySourceTenableFirelensLogGroup04061777", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "CloudquerySourceTenableTaskDefinitionExecutionRoleDefaultPolicyA338312D", + "Roles": [ + { + "Ref": "CloudquerySourceTenableTaskDefinitionExecutionRoleA128084D", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, "DataAudit2FEB3068": { "DependsOn": [ "DataAuditRoleDefaultPolicyD0BF34E5", @@ -28240,6 +28889,137 @@ spec: }, "Type": "AWS::IAM::Role", }, + "servicecatalogueTESTtaskTenable93E15E95": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/AWSXrayWriteOnlyAccess", + ], + ], + }, + ], + "RoleName": "service-catalogue-TEST-task-Tenable", + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "servicecatalogueTESTtaskTenableDefaultPolicy47B334CC": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "kinesis:Describe*", + "kinesis:Put*", + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":kinesis:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":stream/", + { + "Ref": "LoggingStreamName", + }, + ], + ], + }, + }, + { + "Action": "rds-db:connect", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":rds-db:", + { + "Ref": "AWS::Region", + }, + ":", + { + "Ref": "AWS::AccountId", + }, + ":dbuser:", + { + "Fn::GetAtt": [ + "PostgresInstance16DE4286E", + "DbiResourceId", + ], + }, + "/{{resolve:secretsmanager:", + { + "Ref": "PostgresInstance1SecretAttachmentBA0D257D", + }, + ":SecretString:username::}}", + ], + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "servicecatalogueTESTtaskTenableDefaultPolicy47B334CC", + "Roles": [ + { + "Ref": "servicecatalogueTESTtaskTenable93E15E95", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, "snykcredentials4D951A18": { "DeletionPolicy": "Delete", "Properties": { diff --git a/packages/cdk/lib/cloudquery/config.ts b/packages/cdk/lib/cloudquery/config.ts index 364b42bf..f5c338a6 100644 --- a/packages/cdk/lib/cloudquery/config.ts +++ b/packages/cdk/lib/cloudquery/config.ts @@ -326,6 +326,30 @@ export function amigoBakePackagesConfig( }; } +export function TenableConfig(): CloudqueryConfig { + return { + kind: 'source', + spec: { + name: 'tenable', + registry: 'cloudquery', + path: 'cloudquery/tenable', + version: 'v2.6.2', + destinations: ['postgresql'], + tables: [ + 'tenable_tvm_vulnerabilities', + 'tenable_tvm_scans', + 'tenable_tvm_scans_details', + 'tenable_was2_vulnerabilities', + 'tenable_tvm_assets', + ], + spec: { + access_key: '${TENABLE_ACCESS_KEY}', + secret_key: '${TENABLE_SECRET_KEY}', + }, + }, + }; +} + // Tables we are skipping because they are slow and or uninteresting to us. export const skipTables = [ 'aws_ec2_vpc_endpoint_services', // this resource includes services that are available from AWS as well as other AWS Accounts diff --git a/packages/cdk/lib/cloudquery/index.ts b/packages/cdk/lib/cloudquery/index.ts index e58d36d7..a4523dbc 100644 --- a/packages/cdk/lib/cloudquery/index.ts +++ b/packages/cdk/lib/cloudquery/index.ts @@ -26,6 +26,7 @@ import { serviceCatalogueConfigDirectory, skipTables, snykSourceConfig, + TenableConfig, } from './config'; import { Images } from './images'; import { @@ -646,6 +647,14 @@ export function addCloudqueryEcsCluster( ], }; + const tenableSource: CloudquerySource = { + name: 'Tenable', + description: 'Tenable data.', + schedule: nonProdSchedule ?? Schedule.cron({ minute: '0', hour: '3' }), + config: TenableConfig(), + memoryLimitMiB: 1024, + }; + return new CloudqueryCluster(scope, `${app}Cluster`, { app, vpc, @@ -664,6 +673,7 @@ export function addCloudqueryEcsCluster( githubLanguagesSource, ns1Source, amigoBakePackagesSource, + tenableSource, ], }); } From cc1390a834b1c1590d98ff11d1fb169308af2328 Mon Sep 17 00:00:00 2001 From: Kelvin Chappell Date: Mon, 9 Dec 2024 15:40:38 +0000 Subject: [PATCH 02/11] Include credentials --- packages/cdk/lib/cloudquery/index.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/cdk/lib/cloudquery/index.ts b/packages/cdk/lib/cloudquery/index.ts index a4523dbc..7c172cb8 100644 --- a/packages/cdk/lib/cloudquery/index.ts +++ b/packages/cdk/lib/cloudquery/index.ts @@ -647,12 +647,26 @@ export function addCloudqueryEcsCluster( ], }; + const tenableCredentials = new SecretsManager(scope, 'tenable-credentials', { + secretName: `/${stage}/${stack}/${app}/tenable-credentials`, + }); + const tenableSource: CloudquerySource = { name: 'Tenable', description: 'Tenable data.', schedule: nonProdSchedule ?? Schedule.cron({ minute: '0', hour: '3' }), config: TenableConfig(), memoryLimitMiB: 1024, + secrets: { + TENABLE_ACCESS_KEY: Secret.fromSecretsManager( + tenableCredentials, + 'access_key', + ), + TENABLE_SECRET_KEY: Secret.fromSecretsManager( + tenableCredentials, + 'secret_key', + ), + }, }; return new CloudqueryCluster(scope, `${app}Cluster`, { From 3895b13d61274f87c9180055cc9d1c4b92b99e5e Mon Sep 17 00:00:00 2001 From: Kelvin Chappell Date: Mon, 9 Dec 2024 15:42:03 +0000 Subject: [PATCH 03/11] Update test snapshot --- .../service-catalogue.test.ts.snap | 73 ++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index a05c716c..de538db1 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -18886,7 +18886,11 @@ spec: destinations: - postgresql tables: - - '*' + - tenable_tvm_vulnerabilities + - tenable_tvm_scans + - tenable_tvm_scans_details + - tenable_was2_vulnerabilities + - tenable_tvm_assets spec: access_key: \${TENABLE_ACCESS_KEY} secret_key: \${TENABLE_SECRET_KEY} @@ -18959,6 +18963,34 @@ spec: "Name": "CloudquerySource-TenableContainer", "ReadonlyRootFilesystem": true, "Secrets": [ + { + "Name": "TENABLE_ACCESS_KEY", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "tenablecredentialsF2FF9557", + }, + ":access_key::", + ], + ], + }, + }, + { + "Name": "TENABLE_SECRET_KEY", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "tenablecredentialsF2FF9557", + }, + ":secret_key::", + ], + ], + }, + }, { "Name": "DB_USERNAME", "ValueFrom": { @@ -19058,7 +19090,7 @@ spec: "Command": [ "/bin/sh", "-c", - "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('%', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", + "psql -c "INSERT INTO cloudquery_table_frequency VALUES ('tenable_tvm_vulnerabilities', 'DAILY'),('tenable_tvm_scans', 'DAILY'),('tenable_tvm_scans_details', 'DAILY'),('tenable_was2_vulnerabilities', 'DAILY'),('tenable_tvm_assets', 'DAILY') ON CONFLICT (table_name) DO UPDATE SET frequency = 'DAILY'"", ], "DockerLabels": { "App": "service-catalogue", @@ -19425,6 +19457,16 @@ spec: "Properties": { "PolicyDocument": { "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + ], + "Effect": "Allow", + "Resource": { + "Ref": "tenablecredentialsF2FF9557", + }, + }, { "Action": [ "secretsmanager:GetSecretValue", @@ -29047,6 +29089,33 @@ spec: "Type": "AWS::SecretsManager::Secret", "UpdateReplacePolicy": "Delete", }, + "tenablecredentialsF2FF9557": { + "DeletionPolicy": "Delete", + "Properties": { + "GenerateSecretString": {}, + "Name": "/TEST/deploy/service-catalogue/tenable-credentials", + "Tags": [ + { + "Key": "gu:cdk:version", + "Value": "TEST", + }, + { + "Key": "gu:repo", + "Value": "guardian/service-catalogue", + }, + { + "Key": "Stack", + "Value": "deploy", + }, + { + "Key": "Stage", + "Value": "TEST", + }, + ], + }, + "Type": "AWS::SecretsManager::Secret", + "UpdateReplacePolicy": "Delete", + }, }, } `; From 265013d3437c5519cd136ab07888c4cafde61f65 Mon Sep 17 00:00:00 2001 From: Kelvin Chappell Date: Mon, 9 Dec 2024 15:56:29 +0000 Subject: [PATCH 04/11] Rename Tenable secrets Because there's another secret scheduled for deletion with the same name. --- .../lib/__snapshots__/service-catalogue.test.ts.snap | 10 +++++----- packages/cdk/lib/cloudquery/index.ts | 10 +++++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index de538db1..b329c8e0 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -18970,7 +18970,7 @@ spec: "", [ { - "Ref": "tenablecredentialsF2FF9557", + "Ref": "tenableaccesscredentials362C8D64", }, ":access_key::", ], @@ -18984,7 +18984,7 @@ spec: "", [ { - "Ref": "tenablecredentialsF2FF9557", + "Ref": "tenableaccesscredentials362C8D64", }, ":secret_key::", ], @@ -19464,7 +19464,7 @@ spec: ], "Effect": "Allow", "Resource": { - "Ref": "tenablecredentialsF2FF9557", + "Ref": "tenableaccesscredentials362C8D64", }, }, { @@ -29089,11 +29089,11 @@ spec: "Type": "AWS::SecretsManager::Secret", "UpdateReplacePolicy": "Delete", }, - "tenablecredentialsF2FF9557": { + "tenableaccesscredentials362C8D64": { "DeletionPolicy": "Delete", "Properties": { "GenerateSecretString": {}, - "Name": "/TEST/deploy/service-catalogue/tenable-credentials", + "Name": "/TEST/deploy/service-catalogue/tenable-access-credentials", "Tags": [ { "Key": "gu:cdk:version", diff --git a/packages/cdk/lib/cloudquery/index.ts b/packages/cdk/lib/cloudquery/index.ts index 7c172cb8..739261f7 100644 --- a/packages/cdk/lib/cloudquery/index.ts +++ b/packages/cdk/lib/cloudquery/index.ts @@ -647,9 +647,13 @@ export function addCloudqueryEcsCluster( ], }; - const tenableCredentials = new SecretsManager(scope, 'tenable-credentials', { - secretName: `/${stage}/${stack}/${app}/tenable-credentials`, - }); + const tenableCredentials = new SecretsManager( + scope, + 'tenable-access-credentials', + { + secretName: `/${stage}/${stack}/${app}/tenable-access-credentials`, + }, + ); const tenableSource: CloudquerySource = { name: 'Tenable', From 2d7e01293113fff5e3d29cac2d06204524c8bf0a Mon Sep 17 00:00:00 2001 From: Kelvin Chappell Date: Thu, 12 Dec 2024 09:27:12 +0000 Subject: [PATCH 05/11] Update plugin version to latest --- .env | 4 ++++ packages/cdk/lib/cloudquery/config.ts | 2 +- packages/cdk/lib/cloudquery/versions.ts | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 3f735215..8ad5ecdb 100644 --- a/.env +++ b/.env @@ -35,6 +35,10 @@ CQ_NS1=0.1.3 # See https://github.com/guardian/cq-image-packages CQ_IMAGE_PACKAGES=1.0.0 +# See https://hub.cloudquery.io/plugins/source/cloudquery/tenable/latest/versions +CQ_TENABLE=2.6.3 + + # --- FOR LOCAL DEVELOPMENT ONLY --- STAGE=DEV DATABASE_USER=postgres diff --git a/packages/cdk/lib/cloudquery/config.ts b/packages/cdk/lib/cloudquery/config.ts index f5c338a6..c2fa409f 100644 --- a/packages/cdk/lib/cloudquery/config.ts +++ b/packages/cdk/lib/cloudquery/config.ts @@ -333,7 +333,7 @@ export function TenableConfig(): CloudqueryConfig { name: 'tenable', registry: 'cloudquery', path: 'cloudquery/tenable', - version: 'v2.6.2', + version: `v${Versions.CloudqueryTenable}`, destinations: ['postgresql'], tables: [ 'tenable_tvm_vulnerabilities', diff --git a/packages/cdk/lib/cloudquery/versions.ts b/packages/cdk/lib/cloudquery/versions.ts index be544e3a..405d7d80 100644 --- a/packages/cdk/lib/cloudquery/versions.ts +++ b/packages/cdk/lib/cloudquery/versions.ts @@ -28,4 +28,5 @@ export const Versions = { CloudqueryGithubLanguages: envOrError('CQ_GITHUB_LANGUAGES'), CloudqueryNs1: envOrError('CQ_NS1'), CloudqueryImagePackages: envOrError('CQ_IMAGE_PACKAGES'), + CloudqueryTenable: envOrError('CQ_TENABLE'), }; From 89c1bb3b7ee0022a2b70dac436d41a66d12c1bca Mon Sep 17 00:00:00 2001 From: Kelvin Chappell Date: Thu, 12 Dec 2024 14:33:23 +0000 Subject: [PATCH 06/11] Update test snapshot --- packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index b329c8e0..0e582511 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -18882,7 +18882,7 @@ spec: name: tenable registry: cloudquery path: cloudquery/tenable - version: v2.6.2 + version: v2.6.3 destinations: - postgresql tables: From e3672b5ddaf096c35f8c63ecb9606f2b202c12e8 Mon Sep 17 00:00:00 2001 From: Kelvin Chappell Date: Thu, 12 Dec 2024 15:18:23 +0000 Subject: [PATCH 07/11] Give job more memory --- packages/cdk/lib/cloudquery/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cdk/lib/cloudquery/index.ts b/packages/cdk/lib/cloudquery/index.ts index 739261f7..e695b7ea 100644 --- a/packages/cdk/lib/cloudquery/index.ts +++ b/packages/cdk/lib/cloudquery/index.ts @@ -660,7 +660,7 @@ export function addCloudqueryEcsCluster( description: 'Tenable data.', schedule: nonProdSchedule ?? Schedule.cron({ minute: '0', hour: '3' }), config: TenableConfig(), - memoryLimitMiB: 1024, + memoryLimitMiB: 2048, secrets: { TENABLE_ACCESS_KEY: Secret.fromSecretsManager( tenableCredentials, From a3278a0c0b4e052d25e2467fc122988a00a306d7 Mon Sep 17 00:00:00 2001 From: Kelvin Chappell Date: Thu, 12 Dec 2024 15:27:16 +0000 Subject: [PATCH 08/11] Don't schedule yet --- .../cdk/lib/__snapshots__/service-catalogue.test.ts.snap | 6 +++--- packages/cdk/lib/cloudquery/index.ts | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index 0e582511..f0fc8bbe 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -18823,7 +18823,7 @@ spec: }, "CloudquerySourceTenableScheduledEventRule0AC5E644": { "Properties": { - "ScheduleExpression": "cron(0 3 * * ? *)", + "ScheduleExpression": "cron(0 3 * * ? 2026)", "State": "ENABLED", "Targets": [ { @@ -18925,7 +18925,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "819MiB", + "Value": "1638MiB", }, ], "Essential": true, @@ -19218,7 +19218,7 @@ spec: ], }, "Family": "ServiceCatalogueCloudquerySourceTenableTaskDefinition01DF6BF3", - "Memory": "1024", + "Memory": "2048", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", diff --git a/packages/cdk/lib/cloudquery/index.ts b/packages/cdk/lib/cloudquery/index.ts index e695b7ea..8549dd28 100644 --- a/packages/cdk/lib/cloudquery/index.ts +++ b/packages/cdk/lib/cloudquery/index.ts @@ -658,7 +658,9 @@ export function addCloudqueryEcsCluster( const tenableSource: CloudquerySource = { name: 'Tenable', description: 'Tenable data.', - schedule: nonProdSchedule ?? Schedule.cron({ minute: '0', hour: '3' }), + schedule: + nonProdSchedule ?? + Schedule.cron({ minute: '0', hour: '3', year: '2026' }), // TODO: sync more often if this data is useful config: TenableConfig(), memoryLimitMiB: 2048, secrets: { From 9a7cdb95139085a3562007d2903e049ba548e96f Mon Sep 17 00:00:00 2001 From: Kelvin Chappell Date: Thu, 12 Dec 2024 15:55:41 +0000 Subject: [PATCH 09/11] Increase memory for task --- packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap | 4 ++-- packages/cdk/lib/cloudquery/index.ts | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index f0fc8bbe..e45e81d1 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -18925,7 +18925,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "1638MiB", + "Value": "6553MiB", }, ], "Essential": true, @@ -19218,7 +19218,7 @@ spec: ], }, "Family": "ServiceCatalogueCloudquerySourceTenableTaskDefinition01DF6BF3", - "Memory": "2048", + "Memory": "8192", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", diff --git a/packages/cdk/lib/cloudquery/index.ts b/packages/cdk/lib/cloudquery/index.ts index 8549dd28..ca4a6378 100644 --- a/packages/cdk/lib/cloudquery/index.ts +++ b/packages/cdk/lib/cloudquery/index.ts @@ -662,7 +662,8 @@ export function addCloudqueryEcsCluster( nonProdSchedule ?? Schedule.cron({ minute: '0', hour: '3', year: '2026' }), // TODO: sync more often if this data is useful config: TenableConfig(), - memoryLimitMiB: 2048, + // memoryLimitMiB: 2048, too low + memoryLimitMiB: 8192, secrets: { TENABLE_ACCESS_KEY: Secret.fromSecretsManager( tenableCredentials, From fb5bc4304e97f97a82d0cb435598c82a46f51a84 Mon Sep 17 00:00:00 2001 From: Kelvin Chappell Date: Thu, 12 Dec 2024 16:04:58 +0000 Subject: [PATCH 10/11] Reduce memory for task --- packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap | 4 ++-- packages/cdk/lib/cloudquery/index.ts | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index e45e81d1..8f1ad1d7 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -18925,7 +18925,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "6553MiB", + "Value": "3276MiB", }, ], "Essential": true, @@ -19218,7 +19218,7 @@ spec: ], }, "Family": "ServiceCatalogueCloudquerySourceTenableTaskDefinition01DF6BF3", - "Memory": "8192", + "Memory": "4096", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", diff --git a/packages/cdk/lib/cloudquery/index.ts b/packages/cdk/lib/cloudquery/index.ts index ca4a6378..f5056879 100644 --- a/packages/cdk/lib/cloudquery/index.ts +++ b/packages/cdk/lib/cloudquery/index.ts @@ -663,7 +663,8 @@ export function addCloudqueryEcsCluster( Schedule.cron({ minute: '0', hour: '3', year: '2026' }), // TODO: sync more often if this data is useful config: TenableConfig(), // memoryLimitMiB: 2048, too low - memoryLimitMiB: 8192, + memoryLimitMiB: 4096, + // memoryLimitMiB: 8192, // No Fargate configuration exists for given values: 256 CPU, 8192 memory secrets: { TENABLE_ACCESS_KEY: Secret.fromSecretsManager( tenableCredentials, From e89e0eab15cd9484b190837c68c5f56b7ae74eb7 Mon Sep 17 00:00:00 2001 From: Kelvin Chappell Date: Thu, 12 Dec 2024 16:19:51 +0000 Subject: [PATCH 11/11] Increase memory for task --- .../cdk/lib/__snapshots__/service-catalogue.test.ts.snap | 6 +++--- packages/cdk/lib/cloudquery/index.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index 8f1ad1d7..e7c88263 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -18925,7 +18925,7 @@ spec: "Environment": [ { "Name": "GOMEMLIMIT", - "Value": "3276MiB", + "Value": "6553MiB", }, ], "Essential": true, @@ -19210,7 +19210,7 @@ spec: "ReadonlyRootFilesystem": true, }, ], - "Cpu": "256", + "Cpu": "1024", "ExecutionRoleArn": { "Fn::GetAtt": [ "CloudquerySourceTenableTaskDefinitionExecutionRoleA128084D", @@ -19218,7 +19218,7 @@ spec: ], }, "Family": "ServiceCatalogueCloudquerySourceTenableTaskDefinition01DF6BF3", - "Memory": "4096", + "Memory": "8192", "NetworkMode": "awsvpc", "RequiresCompatibilities": [ "FARGATE", diff --git a/packages/cdk/lib/cloudquery/index.ts b/packages/cdk/lib/cloudquery/index.ts index f5056879..9e0a215f 100644 --- a/packages/cdk/lib/cloudquery/index.ts +++ b/packages/cdk/lib/cloudquery/index.ts @@ -663,8 +663,8 @@ export function addCloudqueryEcsCluster( Schedule.cron({ minute: '0', hour: '3', year: '2026' }), // TODO: sync more often if this data is useful config: TenableConfig(), // memoryLimitMiB: 2048, too low - memoryLimitMiB: 4096, - // memoryLimitMiB: 8192, // No Fargate configuration exists for given values: 256 CPU, 8192 memory + memoryLimitMiB: 8192, + cpu: 1024, // have to increase to get extra memory - see https://docs.aws.amazon.com/en_us/AmazonECS/latest/developerguide/task_definition_parameters.html#task_size secrets: { TENABLE_ACCESS_KEY: Secret.fromSecretsManager( tenableCredentials,