warehouse.yaml

AWSTemplateFormatVersion: 2010-09-09
Description : "Hosting an NCCID warehouse instance"

Parameters:
  BucketNameParameter:
    Type: String
    Default: nccid-data-warehouse
    Description: The base name of the storage bucket

Resources:

  # Main bucket
  WarehouseBucket:
    Type: AWS::S3::Bucket
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      AccessControl: Private
      BucketName: !Ref "BucketNameParameter"
      VersioningConfiguration:
        Status: Enabled
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: AES256
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        RestrictPublicBuckets: true
        IgnorePublicAcls: true
      LifecycleConfiguration:
        Rules:
          - Id: "clear-incomplete-uploads"
            AbortIncompleteMultipartUpload:
              DaysAfterInitiation: 7
            Status: Enabled
      InventoryConfigurations:
        - Destination:
            BucketArn: !GetAtt [WarehouseInventoryBucket, Arn]
            Format: CSV
          OptionalFields:
            - Size
            - LastModifiedDate
          Enabled: true
          Id: "daily-full-inventory"
          IncludedObjectVersions: Current
          ScheduleFrequency: Daily
      ReplicationConfiguration:
        Role: !GetAtt [ReplicaRole, Arn]
        Rules:
        - Destination:
            Bucket: !GetAtt [WarehouseBackupBucket, Arn]
            StorageClass: STANDARD_IA
          Id: Backup
          Prefix: 'raw-'
          Status: Enabled

  # Backup role
  ReplicaRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Action: ['sts:AssumeRole']
          Effect: Allow
          Principal:
            Service: [s3.amazonaws.com]

  ReplicaPolicy:
    Type: AWS::IAM::Policy
    Properties:
      PolicyDocument:
        Statement:
        - Action:
          - s3:Get*
          - s3:ListBucket
          Effect: 'Allow'
          Resource:
            - !GetAtt [WarehouseBucket, Arn]
            - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/*"]]
        - Action:
          - s3:ReplicateObject
          - s3:ReplicateDelete
          - s3:ReplicateTags
          - s3:GetObjectVersionTagging
          Effect: 'Allow'
          Resource: !Join ["", [!GetAtt [WarehouseBackupBucket, Arn], "/*"]]
      PolicyName: ReplicaPolicy
      Roles: [!Ref 'ReplicaRole']

  # Backup bucket
  WarehouseBackupBucket:
    Type: AWS::S3::Bucket
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      AccessControl: Private
      BucketName: !Sub "${BucketNameParameter}-backup"
      VersioningConfiguration:
        Status: Enabled
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: AES256
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        RestrictPublicBuckets: true
        IgnorePublicAcls: true

  # Inventory bucket
  WarehouseInventoryBucket:
    Type: AWS::S3::Bucket
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      AccessControl: Private
      BucketName: !Join ["", [!Ref "BucketNameParameter", "-inventory"]]
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: AES256
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        RestrictPublicBuckets: true
        IgnorePublicAcls: true
      LifecycleConfiguration:
        Rules:
          - Id: "BucketRecycling"
            ExpirationInDays: 7
            Status: Enabled

  InventoryBucketPolicy:
    Type: AWS::S3::BucketPolicy
    Properties:
      Bucket: !Ref WarehouseInventoryBucket
      PolicyDocument:
        Version: 2012-10-17
        Id: "ReceiveInventoryPolicy"
        Statement:
          - Effect: Allow
            Principal:
              Service: s3.amazonaws.com
            Action: s3:PutObject
            Resource: !Join ["", [!GetAtt [WarehouseInventoryBucket, Arn], "/*"]]
            Condition:
              StringEquals:
                  s3:x-amz-acl: bucket-owner-full-control
                  aws:SourceAccount: !Ref "AWS::AccountId"
              ArnLike:
                  aws:SourceArn: !GetAtt [WarehouseBucket, Arn]

  # Processed data bucket
  WarehouseProcessedBucket:
    Type: AWS::S3::Bucket
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      AccessControl: Private
      BucketName: !Join ["", [!Ref "BucketNameParameter", "-processed"]]
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: AES256
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        RestrictPublicBuckets: true
        IgnorePublicAcls: true
      LifecycleConfiguration:
        Rules:
          - Id: "BucketRecycling"
            ExpirationInDays: 90
            Status: Enabled

  # IAM
  RotateOwnCredentials:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      ManagedPolicyName: !Join ["", ["rotate-credentials-", !Ref "BucketNameParameter", "-users"]]
      Description: Allow rotating keys for programmatic users
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - "iam:*AccessKey*"
              - "iam:GetUser"
            Resource:
              - "arn:aws:iam::*:user/${aws:username}"

  UploadToRawPolicy:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      ManagedPolicyName: !Join ["", ["upload-to-", !Ref "BucketNameParameter", "-raw"]]
      Description: Allow uploading to the raw prefix in the warehouse bucket
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - "s3:PutObject"
            Resource:
              !Join ["", [!GetAtt [WarehouseBucket, Arn], "/raw-${aws:username}/*"]]
  DataUploaderGroup:
    Type: AWS::IAM::Group
    Properties:
      ManagedPolicyArns:
        - !Ref UploadToRawPolicy
        - !Ref RotateOwnCredentials

  ETLPolicy:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      ManagedPolicyName: !Join ["", ["etl-", !Ref "BucketNameParameter"]]
      Description: Allow ETL operations on the non-raw prefixes in the warehouse bucket
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - "s3:ListBucket"
              - "s3:GetBucketLocation"
            Resource:
              - !GetAtt [WarehouseBucket, Arn]
              - !GetAtt [WarehouseInventoryBucket, Arn]
          - Effect: Allow
            Action:
              - "s3:GetObject"
            Resource:
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/*"]]
              - !Join ["", [!GetAtt [WarehouseInventoryBucket, Arn], "/*"]]
          - Effect: Allow
            Action:
              - "s3:PutObject"
            Resource:
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/training/*"]]
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/validation/*"]]
          - Effect: Allow
            Action:
              - "sns:Publish"
            Resource:
              - !Ref Notifications
  ETLGroup:
    Type: AWS::IAM::Group
    Properties:
      ManagedPolicyArns:
        - !Ref ETLPolicy
        - !Ref RotateOwnCredentials

  ProcessingPolicy:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      ManagedPolicyName: !Join ["", ["processing-", !Ref "BucketNameParameter"]]
      Description: Allow data processing operations on the warehouse content
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - "s3:ListBucket"
              - "s3:GetBucketLocation"
            Resource:
              - !GetAtt [WarehouseBucket, Arn]
              - !GetAtt [WarehouseInventoryBucket, Arn]
              - !GetAtt [WarehouseProcessedBucket, Arn]
          - Effect: Allow
            Action:
              - "s3:GetObject"
            Resource:
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/training/*"]]
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/validation/*"]]
              - !Join ["", [!GetAtt [WarehouseInventoryBucket, Arn], "/*"]]
              - !Join ["", [!GetAtt [WarehouseProcessedBucket, Arn], "/*"]]
          - Effect: Allow
            Action:
              - "s3:PutObject"
            Resource:
              - !Join ["", [!GetAtt [WarehouseProcessedBucket, Arn], "/*"]]
          - Effect: Allow
            Action:
              - "sns:Publish"
            Resource:
              - !Ref Notifications
  ProcessingGroup:
    Type: AWS::IAM::Group
    Properties:
      ManagedPolicyArns:
        - !Ref ProcessingPolicy
        - !Ref RotateOwnCredentials

  ProcessedDataAccessPolicy:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      ManagedPolicyName: !Join ["", ["processed-data-", !Ref "BucketNameParameter"]]
      Description: Allow reading processed warehouse data
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - "s3:ListBucket"
              - "s3:GetBucketLocation"
            Resource:
              - !GetAtt [WarehouseProcessedBucket, Arn]
          - Effect: Allow
            Action:
              - "s3:GetObject"
            Resource:
              - !Join ["", [!GetAtt [WarehouseProcessedBucket, Arn], "/*"]]
  ProcessedDataAccessGroup:
    Type: AWS::IAM::Group
    Properties:
      ManagedPolicyArns:
        - !Ref ProcessedDataAccessPolicy

  ReadOnlyAccessWholeBucketPolicy:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      ManagedPolicyName: !Join ["", ["read-only-", !Ref "BucketNameParameter"]]
      Description: Allow read-only view of the warehouse bucket
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - "s3:ListBucket"
            Resource:
              !GetAtt [WarehouseBucket, Arn]
          - Effect: Allow
            Action:
              - "s3:GetObject"
            Resource:
              !Join ["", [!GetAtt [WarehouseBucket, Arn], "/*"]]
  DataVerifierGroup:
    Type: AWS::IAM::Group
    Properties:
      ManagedPolicyArns:
        - !Ref ReadOnlyAccessWholeBucketPolicy
        - !Ref RotateOwnCredentials

  TrainingCTPolicy:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      ManagedPolicyName: !Join ["", ["training-ct-", !Ref "BucketNameParameter"]]
      Description: Allow reading the CT and patient data sections of the training set
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - "s3:ListBucket"
              - "s3:HeadBucket"
            Resource:
              !GetAtt [WarehouseBucket, Arn]
            Condition:
              StringLike:
                s3:prefix: ["training/ct/*", "training/ct-metadata/*", "training/data/*"]
          - Effect: Allow
            Action:
              - "s3:GetObject"
            Resource:
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/training/ct/*"]]
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/training/ct-metadata/*"]]
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/training/data/*"]]
  TrainingCTGroup:
    Type: AWS::IAM::Group
    Properties:
      ManagedPolicyArns:
        - !Ref TrainingCTPolicy
        - !Ref RotateOwnCredentials

  TrainingMRIPolicy:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      ManagedPolicyName: !Join ["", ["training-mri-", !Ref "BucketNameParameter"]]
      Description: Allow reading the MRI and patient data sections of the training set
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - "s3:ListBucket"
              - "s3:HeadBucket"
            Resource:
              !GetAtt [WarehouseBucket, Arn]
            Condition:
              StringLike:
                s3:prefix: ["training/mri/*", "training/mri-metadata/*", "training/data/*"]
          - Effect: Allow
            Action:
              - "s3:GetObject"
            Resource:
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/training/mri/*"]]
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/training/mri-metadata/*"]]
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/training/data/*"]]
  TrainingMRIGroup:
    Type: AWS::IAM::Group
    Properties:
      ManagedPolicyArns:
        - !Ref TrainingMRIPolicy
        - !Ref RotateOwnCredentials

  TrainingXrayPolicy:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      ManagedPolicyName: !Join ["", ["training-xray-", !Ref "BucketNameParameter"]]
      Description: Allow reading the X-ray and patient data sections of the training set
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - "s3:ListBucket"
              - "s3:HeadBucket"
            Resource:
              !GetAtt [WarehouseBucket, Arn]
            Condition:
              StringLike:
                s3:prefix: ["training/xray/*", "training/xray-metadata/*", "training/data/*"]
          - Effect: Allow
            Action:
              - "s3:GetObject"
            Resource:
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/training/xray/*"]]
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/training/xray-metadata/*"]]
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/training/data/*"]]
  TrainingXrayGroup:
    Type: AWS::IAM::Group
    Properties:
      ManagedPolicyArns:
        - !Ref TrainingXrayPolicy
        - !Ref RotateOwnCredentials

  TrainingAllPolicy:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      ManagedPolicyName: !Join ["", ["training-all-", !Ref "BucketNameParameter"]]
      Description: Allow reading all data in the training set
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - "s3:ListBucket"
              - "s3:HeadBucket"
            Resource:
              !GetAtt [WarehouseBucket, Arn]
            Condition:
              StringLike:
                s3:prefix: ["training/*"]
          - Effect: Allow
            Action:
              - "s3:GetObject"
            Resource:
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/training/*"]]
  TrainingAllGroup:
    Type: AWS::IAM::Group
    Properties:
      ManagedPolicyArns:
        - !Ref TrainingAllPolicy
        - !Ref RotateOwnCredentials

  ValidationAllPolicy:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      ManagedPolicyName: !Join ["", ["validation-all-", !Ref "BucketNameParameter"]]
      Description: Allow reading all data in the validation set
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - "s3:ListBucket"
              - "s3:HeadBucket"
            Resource:
              !GetAtt [WarehouseBucket, Arn]
            Condition:
              StringLike:
                s3:prefix: ["validation/*"]
          - Effect: Allow
            Action:
              - "s3:GetObject"
            Resource:
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/validation/*"]]
  ValidationAllGroup:
    Type: AWS::IAM::Group
    Properties:
      ManagedPolicyArns:
        - !Ref ValidationAllPolicy
        - !Ref RotateOwnCredentials

  # CloudTrail
  CloudTrailLogsBucket:
    Type: AWS::S3::Bucket
    DeletionPolicy: Retain
    UpdateReplacePolicy: Retain
    Properties:
      BucketName: !Join ["", [!Ref "BucketNameParameter", "-object-logs"]]
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: AES256
      PublicAccessBlockConfiguration:
        BlockPublicAcls: true
        BlockPublicPolicy: true
        RestrictPublicBuckets: true
        IgnorePublicAcls: true
  CloudTrailLogsBucketPolicy:
    Type: AWS::S3::BucketPolicy
    Properties:
      Bucket: !Ref CloudTrailLogsBucket
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service: cloudtrail.amazonaws.com
            Action: s3:GetBucketAcl
            Resource: !GetAtt [CloudTrailLogsBucket, Arn]
          - Effect: Allow
            Principal:
              Service: cloudtrail.amazonaws.com
            Action: s3:PutObject
            Resource: !Join ["", [!GetAtt [CloudTrailLogsBucket, Arn], "/AWSLogs/",!Ref "AWS::AccountId", "/*"]]
            Condition:
              StringEquals:
                s3:x-amz-acl: bucket-owner-full-control
  CloudTrailLogs:
    Type: AWS::CloudTrail::Trail
    DependsOn:
      # Have to explicitly pass the dependency on the bucket policy, otherwise
      # they both get created at the same time and the trail creation fails
      - CloudTrailLogsBucketPolicy
    Properties:
      IsLogging: true
      S3BucketName: !Ref CloudTrailLogsBucket
      EventSelectors:
        - DataResources:
            - Type: AWS::S3::Object
              Values:
                - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/"]]
                - !Join ["", [!GetAtt [WarehouseBackupBucket, Arn], "/"]]

  AuditorReadOnlyPolicy:
    Type: AWS::IAM::ManagedPolicy
    Properties:
      ManagedPolicyName: !Join ["", ["auditor-", !Ref "BucketNameParameter"]]
      Description: Allow read-only access to the object logs
      PolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Action:
              - "s3:ListBucket"
              - "s3:HeadBucket"
            Resource:
              !GetAtt CloudTrailLogsBucket.Arn
          - Effect: Allow
            Action:
              - "s3:GetObject"
            Resource:
              - !Sub "${CloudTrailLogsBucket.Arn}/AWSLogs/${AWS::AccountId}/*"
  AuditorGroup:
    Type: AWS::IAM::Group
    Properties:
      ManagedPolicyArns:
        - !Ref AuditorReadOnlyPolicy
        - !Ref RotateOwnCredentials

  # Compute
  VPC:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: 10.40.0.0/16
      EnableDnsSupport: true
      EnableDnsHostnames: true
      InstanceTenancy: default
      Tags:
        - Key: Name
          Value: !Join ["", [!Ref "BucketNameParameter", "-VPC"]]

  InternetGateway:
    Type: AWS::EC2::InternetGateway
    DependsOn: VPC
  AttachGateway:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      VpcId: !Ref VPC
      InternetGatewayId: !Ref InternetGateway

  PublicSubnetA:
      Type: AWS::EC2::Subnet
      Properties:
        VpcId: !Ref VPC
        CidrBlock: 10.40.10.0/24
        AvailabilityZone: !Select [ 0, !GetAZs ]    # Get the first AZ in the list
        Tags:
        - Key: Name
          Value: !Join ["", [!Ref "BucketNameParameter", "-publicA"]]

  PublicSubnetB:
      Type: AWS::EC2::Subnet
      Properties:
        VpcId: !Ref VPC
        CidrBlock: 10.40.20.0/24
        AvailabilityZone: !Select [ 1, !GetAZs ]    # Get the second AZ in the list
        Tags:
        - Key: Name
          Value: !Join ["", [!Ref "BucketNameParameter", "-publicB"]]

  PublicRouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref VPC
      Tags:
      - Key: Name
        Value: !Join ["", [!Ref "BucketNameParameter", "-public"]]

  PublicRoute1:
    Type: AWS::EC2::Route
    DependsOn: AttachGateway
    Properties:
      RouteTableId: !Ref PublicRouteTable
      DestinationCidrBlock: 0.0.0.0/0
      GatewayId: !Ref InternetGateway

  PublicSubnetARouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref PublicSubnetA
      RouteTableId: !Ref PublicRouteTable
  PublicSubnetBRouteTableAssociation:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref PublicSubnetB
      RouteTableId: !Ref PublicRouteTable

  S3Endpoint:
    Type: AWS::EC2::VPCEndpoint
    Properties:
      PolicyDocument:
        Version: 2012-10-17
        Id: "S3EndpointAccessPolicy"
        Statement:
          - Effect: Allow
            Principal: "*"
            Action: "s3:*"
            Resource:
              # Access to the relevant warehoues buckets only
              - !GetAtt [WarehouseBucket, Arn]
              - !Join ["", [!GetAtt [WarehouseBucket, Arn], "/*"]]
              - !GetAtt [WarehouseInventoryBucket, Arn]
              - !Join ["", [!GetAtt [WarehouseInventoryBucket, Arn], "/*"]]
              - !GetAtt [WarehouseProcessedBucket, Arn]
              - !Join ["", [!GetAtt [WarehouseProcessedBucket, Arn], "/*"]]
              # Required to pull from ECR through this endpoint
              - !Sub arn:aws:s3:::prod-${AWS::Region}-starport-layer-bucket/*
      RouteTableIds:
        - !Ref PublicRouteTable
      ServiceName: !Sub com.amazonaws.${AWS::Region}.s3
      VpcId: !Ref VPC

  FargateSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: HTTP access from anywhere for container pull
      SecurityGroupIngress:
        - IpProtocol: "tcp"
          CidrIp: 0.0.0.0/0
          FromPort: 80
          ToPort: 80
          Description: "HTTP access from anywhere"
      VpcId:
        Ref: VPC
      Tags:
        - Key: Name
          Value: !Sub "${AWS::StackName}-http-anywhere"

  DockerRepository:
    Type: AWS::ECR::Repository
    Properties:
      RepositoryName: !Sub "${AWS::StackName}-pipeline"
      LifecyclePolicy:
        LifecyclePolicyText: |
          {
            "rules": [
            {
              "rulePriority": 1,
              "description": "Only keep 5 untagged images",
              "selection": {
                "tagStatus": "untagged",
                "countType": "imageCountMoreThan",
                "countNumber": 5
              },
              "action": { "type": "expire" }
            }]
          }

  ECSCluster:
    Type: 'AWS::ECS::Cluster'
    Properties:
      ClusterName: !Sub "${AWS::StackName}-cluster"
      ClusterSettings:
        - Name: containerInsights
          Value: enabled

  # This is a role which is used by the ECS tasks themselves.
  # From https://github.com/awslabs/aws-cloudformation-templates/blob/7671e34887dec84bcaebab33bf0fa8e46e77137b/aws/services/ECS/FargateLaunchType/clusters/public-vpc.yml  # This is a role which is used by the ECS tasks themselves.
  ECSTaskExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Principal:
            Service: [ecs-tasks.amazonaws.com]
          Action: ['sts:AssumeRole']
      Path: /
      Policies:
        - PolicyName: AmazonECSTaskExecutionRolePolicy
          PolicyDocument:
            Statement:
            - Effect: Allow
              Action:
                # Allow the ECS Tasks to download images from ECR
                - 'ecr:GetAuthorizationToken'
                - 'ecr:BatchCheckLayerAvailability'
                - 'ecr:GetDownloadUrlForLayer'
                - 'ecr:BatchGetImage'

                # Allow the ECS tasks to upload logs to CloudWatch
                - 'logs:CreateLogStream'
                - 'logs:PutLogEvents'
              Resource: '*'

  # The submitting centres extracting task
  SubmittingcentresTaskdefinition:
    Type: AWS::ECS::TaskDefinition
    Properties:
      Family:  !Sub "${AWS::StackName}-SubmittingCentres"
      RequiresCompatibilities:
        - "FARGATE"
      Cpu: "512"  # 0.5 vCPU
      Memory: "1GB"
      NetworkMode: "awsvpc"
      ExecutionRoleArn: !GetAtt [ECSTaskExecutionRole, Arn]
      TaskRoleArn: !GetAtt [PipelineRole, Arn]
      ContainerDefinitions:
        -
          Name: "pipeline"
          Image: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}-pipeline:prod"
          Essential: true
          Command:
            - submittingcentres
          Environment:
            - Name: COMMIT
              Value: master
            - Name: WAREHOUSE_BUCKET
              Value: !Ref WarehouseBucket
            - Name: TOPIC_ARN
              Value: !Ref Notifications
          # Send logs to CloudWatch Logs
          LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-region: !Ref AWS::Region
              awslogs-group: !Ref TaskLogGroup
              awslogs-stream-prefix: ecs

  # The the actual pipeline task
  PipelineTaskdefinition:
    Type: AWS::ECS::TaskDefinition
    Properties:
      Family:  !Sub "${AWS::StackName}-Pipeline"
      RequiresCompatibilities:
        - "FARGATE"
      Cpu: "1024"  # 1 vCPU
      Memory: "8GB"
      NetworkMode: "awsvpc"
      ExecutionRoleArn: !GetAtt [ECSTaskExecutionRole, Arn]
      TaskRoleArn: !GetAtt [PipelineRole, Arn]
      ContainerDefinitions:
        -
          Name: "pipeline"
          Image: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}-pipeline:prod"
          Essential: true
          Command:
            - warehouseloader
          Environment:
            - Name: COMMIT
              Value: master
            - Name: WAREHOUSE_BUCKET
              Value: !Ref WarehouseBucket
            - Name: TOPIC_ARN
              Value: !Ref Notifications
          # Send logs to CloudWatch Logs
          LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-region: !Ref AWS::Region
              awslogs-group: !Ref TaskLogGroup
              awslogs-stream-prefix: ecs

  # The data processing pipeline task for the dashboards
  ProcessingTaskdefinition:
    Type: AWS::ECS::TaskDefinition
    Properties:
      Family:  !Sub "${AWS::StackName}-DataProcessing"
      RequiresCompatibilities:
        - "FARGATE"
      Cpu: "1024"  # 1 vCPU
      Memory: "2GB"
      NetworkMode: "awsvpc"
      ExecutionRoleArn: !GetAtt [ECSTaskExecutionRole, Arn]
      TaskRoleArn: !GetAtt [ProcessingRole, Arn]
      ContainerDefinitions:
        -
          Name: "pipeline"
          Image: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}-pipeline:prod"
          Essential: true
          Command:
            - dataprocess
          Environment:
            - Name: COMMIT
              Value: master
            - Name: WAREHOUSE_BUCKET
              Value: !Ref WarehouseBucket
            - Name: TOPIC_ARN
              Value: !Ref Notifications
          # Send logs to CloudWatch Logs
          LogConfiguration:
            LogDriver: awslogs
            Options:
              awslogs-region: !Ref AWS::Region
              awslogs-group: !Ref TaskLogGroup
              awslogs-stream-prefix: ecs

  TaskLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub /ecs/${AWS::StackName}-pipeline
      # 2 years of pipeline run log retention
      RetentionInDays: 731

  PipelineRole:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Principal:
            Service: [ecs-tasks.amazonaws.com]
          Action: ['sts:AssumeRole']
      Path: /
      ManagedPolicyArns:
        - !Ref ETLPolicy

  ProcessingRole:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Statement:
        - Effect: Allow
          Principal:
            Service: [ecs-tasks.amazonaws.com]
          Action: ['sts:AssumeRole']
      Path: /
      ManagedPolicyArns:
        - !Ref ProcessingPolicy

  Notifications:
    Type: AWS::SNS::Topic
    Properties:
      DisplayName: !Sub ${AWS::StackName}-notifications