diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..486225a --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,86 @@ +name: deploy-probot-terraform + +on: + pull_request: + workflow_dispatch: + push: + branches: + - "pull-request/[0-9]+" + - "main" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +permissions: + id-token: write + contents: read + +jobs: + deploy: + name: Deploy Probot Application + runs-on: ubuntu-latest + + steps: + - name: Get AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.SERVERLESS_AWS_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION }} + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Install npm dependencies + run: npm ci + + - name: Test Probot + run: npm run test + + - name: Build Probot + run: npm run build + + - name: Copy release draft template + run: cp src/plugins/ReleaseDrafter/draft_template.njk dist/plugins/ReleaseDrafter + + - name: Package Lambda functions + run: | + zip -r probot.zip . + zip -r authorizer.zip . -x "probot.zip" + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.9.2" + + - name: Terraform Format Check + working-directory: terraform + run: terraform fmt -check + + - name: Terraform Init + working-directory: terraform + run: terraform init + + - name: Terraform Validate + working-directory: terraform + run: terraform validate + + - name: Terraform Plan + id: plan + working-directory: terraform + run: terraform plan -out tfplan + env: + TF_VAR_app_id: ${{ secrets.APP_ID }} + TF_VAR_webhook_secret: ${{ secrets.WEBHOOK_SECRET }} + TF_VAR_private_key: ${{ secrets.PRIVATE_KEY }} + TF_VAR_gputester_pat: ${{ secrets.GPUTESTER_PAT }} + + - name: Terraform Apply + if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' + working-directory: terraform + run: terraform apply -auto-approve tfplan diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml deleted file mode 100644 index 570a479..0000000 --- a/.github/workflows/deploy.yaml +++ /dev/null @@ -1,50 +0,0 @@ -name: deploy-probot - -on: - workflow_dispatch: - push: - branches: main - paths: - - "src/**" - - "package*.json" - - "serverless.yaml" - - ".github/workflows/deploy.yaml" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: - id-token: write - contents: read - -jobs: - deploy: - name: Deploy Probot Application - runs-on: ubuntu-latest - container: node:18 - steps: - - name: Get AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - role-to-assume: ${{ vars.SERVERLESS_AWS_ROLE_ARN }} - aws-region: ${{ vars.AWS_REGION }} - - name: Checkout code - uses: actions/checkout@v3 - - name: Install npm dependencies - run: npm ci - - name: Install serverless framework - run: npm install -g serverless@3 - - name: Test Probot - run: npm run test - - name: Build Probot - run: npm run build - - name: Copy release draft template - run: cp src/plugins/ReleaseDrafter/draft_template.njk dist/plugins/ReleaseDrafter - - name: Deploy Probot - run: npm run deploy - env: - APP_ID: ${{ secrets.APP_ID }} - WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} - PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} - GPUTESTER_PAT: ${{ secrets.GPUTESTER_PAT }} diff --git a/.github/workflows/prs.yaml b/.github/workflows/prs.yaml deleted file mode 100644 index 8d7aeec..0000000 --- a/.github/workflows/prs.yaml +++ /dev/null @@ -1,21 +0,0 @@ -name: unit-tests -on: - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - probot: - runs-on: ubuntu-latest - container: node:18 - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Install packages - run: npm ci - - name: Run build - run: npm run build - - name: Run unit tests - run: npm run test diff --git a/.gitignore b/.gitignore index bbcb5c5..9df9656 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,9 @@ npm-debug.log coverage dist .serverless +.terraform/ +*.tfstate +*.tfstate.* +*.tfplan +.terraform.lock.hcl +*.tfvars diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ec3d201 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,33 @@ +# Contributing + +Any new functionality should be introduced as a new plugin in the [src/plugins](./src/plugins) directory. New plugins should make use of the shared `featureIsDisabled` function so that repositories can disable the feature if they desire. New plugins should also have an entry added in [config.ts](./src/config.ts) + +## Making Infrastructure Changes + +The project uses Terraform to manage AWS infrastructure. The configuration files are located in the `terraform/` directory. + +### Structure + +- `main.tf`: Provider configuration and backend setup +- `lambda.tf`: Lambda function definitions +- `iam.tf`: IAM roles and policies +- `api_gateway.tf`: API Gateway configuration +- `cloudwatch.tf`: CloudWatch log groups +- `variables.tf`: Input variables +- `outputs.tf`: Output values + +### Testing Changes + +1. Make your changes to the Terraform files +2. Run `terraform fmt` to ensure consistent formatting +3. Run `terraform validate` to check for configuration errors +4. Create a PR - the GitHub Actions workflow will automatically: + - Check formatting + - Validate configuration + - Generate and post a plan to the PR + +### Deployment + +Infrastructure changes are automatically deployed when merged to `main`. The deployment: +- Packages and uploads Lambda functions to S3 +- Applies Terraform changes with the new configuration diff --git a/README.md b/README.md index 6d8ce66..178ddda 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,55 @@ The plugins are listed in the [src/plugins](./src/plugins) folder. - **Branch Checker** - Set a status on PRs that checks whether they are targeting either the repo's _default branch_ or _default branch + 1_ - **Recently Updated** - Sets a status on PRs based on whether a PR is `X` commits out-of-date compared to the based branch. `X` defaults to `5`, but is configurable via the `recently_updated_threshold` option in the `.github/ops-bot.yaml` configuration file. -## Deployment +## Infrastructure -The _Serverless_ framework is used to deploy the Probot application to an AWS Lambda instance. The deployment configuration can be seen in the [serverless.yaml](./serverless.yaml) file. A deployment will happen automatically anytime a change is merged to the `main` branch affecting any of the following files: source code files, `package.json` file, or `serverless.yaml` file. See the [deploy.yaml](/.github/workflows/deploy.yaml) GitHub Action for more details. +The project's infrastructure is managed using Terraform. Key components include: + +- AWS Lambda functions for the Probot handler and authorizer +- API Gateway with custom authorizer +- IAM roles and policies +- CloudWatch log groups +- S3 bucket for deployment artifacts + +### Prerequisites + +- Terraform v1.9.2 or later +- AWS CLI configured with appropriate credentials +- Node.js 18.x + +### Deployment + +The deployment is automated via GitHub Actions. For manual deployment: + +1. Build the application: +```bash +npm install +npm run build +``` +2. Package Lambda functions: +```bash +zip -r probot-{version}.zip dist +zip -r authorizer-{version}.zip dist/authorizer.js +``` +3. Upload to S3: +```bash +aws s3 cp probot-{version}.zip s3://rapidsai-serverless-deployments/serverless/ops-bot/prod/ +aws s3 cp authorizer-{version}.zip s3://rapidsai-serverless-deployments/serverless/ops-bot/prod/ +``` +4. Deploy infrastructure: +```bash +cd terraform +terraform init +terraform plan +terraform apply +``` + +### Required Environment Variables + +- `APP_ID`: GitHub App ID +- `WEBHOOK_SECRET`: GitHub Webhook Secret +- `PRIVATE_KEY`: GitHub App Private Key +- `GPUTESTER_PAT`: GPU Tester Personal Access Token ## npm Scripts @@ -30,7 +76,3 @@ npm run test # Deploy npm run deploy ``` - -## Contributing - -Any new functionality should be introduced as a new plugin in the [src/plugins](./src/plugins) directory. New plugins should make use of the shared `featureIsDisabled` function so that repositories can disable the feature if they desire. New plugins should also have an entry added in [config.ts](./src/config.ts) diff --git a/serverless.yaml b/serverless.yaml deleted file mode 100644 index c4d7e96..0000000 --- a/serverless.yaml +++ /dev/null @@ -1,49 +0,0 @@ -service: ops-bot - -provider: - name: aws - region: us-east-2 - memorySize: 1024 - stage: dev - runtime: nodejs18.x - logRetentionInDays: 60 - apiGateway: - shouldStartNameWithService: true - deploymentBucket: - name: rapidsai-serverless-deployments - environment: - NODE_ENV: production - LOG_FORMAT: json - LOG_LEVEL: debug - APP_ID: ${env:APP_ID} - WEBHOOK_SECRET: ${env:WEBHOOK_SECRET} - PRIVATE_KEY: ${env:PRIVATE_KEY} - GPUTESTER_PAT: ${env:GPUTESTER_PAT} - iam: - role: - statements: - - Effect: Allow - Action: - - "lambda:InvokeFunction" - Resource: - Fn::Sub: - - arn:aws:lambda:${aws:region}:${aws:accountId}:function:${AWS::StackName}-${fnName} - # A YAML anchor is used since `Ref` results in circular dependency issues between - # the Lambda function and this policy. - - fnName: &probotFn handleProbot - -functions: - *probotFn : - handler: dist/probot.handler - timeout: 900 - authorizerFn: - handler: dist/authorizer.handler - environment: - probotFnName: - Fn::Sub: - - ${AWS::StackName}-${fnName} - - fnName: *probotFn - events: - - http: - path: / - method: POST diff --git a/terraform/api_gateway.tf b/terraform/api_gateway.tf new file mode 100644 index 0000000..ab12b31 --- /dev/null +++ b/terraform/api_gateway.tf @@ -0,0 +1,43 @@ +resource "aws_api_gateway_rest_api" "ops_bot" { + name = "ops-bot" +} + +resource "aws_api_gateway_method" "root_post" { + rest_api_id = aws_api_gateway_rest_api.ops_bot.id + # resource_id = aws_api_gateway_resource.proxy.id + resource_id = aws_api_gateway_rest_api.ops_bot.root_resource_id + http_method = "POST" + authorization = "NONE" +} + +resource "aws_api_gateway_integration" "lambda" { + rest_api_id = aws_api_gateway_rest_api.ops_bot.id + resource_id = aws_api_gateway_rest_api.ops_bot.root_resource_id + http_method = aws_api_gateway_method.root_post.http_method + credentials = aws_iam_role.api_gateway_authorizer.arn + + integration_http_method = "POST" + type = "AWS_PROXY" + uri = aws_lambda_function.authorizer.invoke_arn +} + +resource "aws_api_gateway_deployment" "ops_bot" { + rest_api_id = aws_api_gateway_rest_api.ops_bot.id + triggers = { + redeployment = sha1(jsonencode([ + aws_api_gateway_rest_api.ops_bot.root_resource_id, + aws_api_gateway_method.root_post.id, + aws_api_gateway_integration.lambda.id, + ])) + } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_api_gateway_stage" "ops_bot" { + deployment_id = aws_api_gateway_deployment.ops_bot.id + rest_api_id = aws_api_gateway_rest_api.ops_bot.id + stage_name = "prod" +} diff --git a/terraform/cloudwatch.tf b/terraform/cloudwatch.tf new file mode 100644 index 0000000..c625789 --- /dev/null +++ b/terraform/cloudwatch.tf @@ -0,0 +1,9 @@ +resource "aws_cloudwatch_log_group" "probot_handler" { + name = "/aws/lambda/ops-bot-handleProbot" + retention_in_days = 60 +} + +resource "aws_cloudwatch_log_group" "authorizer" { + name = "/aws/lambda/ops-bot-authorizerFn" + retention_in_days = 60 +} diff --git a/terraform/iam.tf b/terraform/iam.tf new file mode 100644 index 0000000..a3380e6 --- /dev/null +++ b/terraform/iam.tf @@ -0,0 +1,73 @@ +resource "aws_iam_role" "lambda_role" { + name = "ops-bot-lambda-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Action = ["sts:AssumeRole"] + Effect = "Allow" + Principal = { + Service = ["lambda.amazonaws.com"] + } + }] + }) +} + +resource "aws_iam_role_policy" "lambda_policy" { + name = "ops-bot-lambda-policy" + role = aws_iam_role.lambda_role.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = ["lambda:InvokeFunction"] + Resource = [ + aws_lambda_function.probot_handler.arn + ] + }, + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = ["arn:aws:logs:*:*:*"] + } + ] + }) +} + +resource "aws_iam_role" "api_gateway_authorizer" { + name = "ops-bot-api-gateway-authorizer" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Action = ["sts:AssumeRole"] + Effect = "Allow" + Principal = { + Service = ["apigateway.amazonaws.com"] + } + }] + }) +} + +resource "aws_iam_role_policy" "api_gateway_authorizer" { + name = "ops-bot-api-gateway-authorizer" + role = aws_iam_role.api_gateway_authorizer.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = ["lambda:InvokeFunction"] + Resource = [ + aws_lambda_function.authorizer.arn, + aws_lambda_function.probot_handler.arn + ] + }] + }) +} diff --git a/terraform/lambda.tf b/terraform/lambda.tf new file mode 100644 index 0000000..12af5fe --- /dev/null +++ b/terraform/lambda.tf @@ -0,0 +1,64 @@ +resource "aws_lambda_function" "probot_handler" { + depends_on = [aws_cloudwatch_log_group.probot_handler] + filename = "../probot.zip" + source_code_hash = filebase64sha256("../probot.zip") + function_name = "ops-bot-handleProbot" + role = aws_iam_role.lambda_role.arn + handler = "dist/probot.handler" + runtime = "nodejs18.x" + timeout = 900 + memory_size = 1024 + + environment { + variables = { + NODE_ENV = "production" + LOG_FORMAT = "json" + LOG_LEVEL = "debug" + APP_ID = var.app_id + WEBHOOK_SECRET = var.webhook_secret + PRIVATE_KEY = var.private_key + GPUTESTER_PAT = var.gputester_pat + } + } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_lambda_function" "authorizer" { + depends_on = [aws_cloudwatch_log_group.authorizer] + filename = "../authorizer.zip" + source_code_hash = filebase64sha256("../authorizer.zip") + function_name = "ops-bot-authorizerFn" + role = aws_iam_role.lambda_role.arn + handler = "dist/authorizer.handler" + runtime = "nodejs18.x" + memory_size = 1024 + + environment { + variables = { + probotFnName = aws_lambda_function.probot_handler.function_name + } + } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_lambda_permission" "api_gw_resource_policy" { + statement_id = "AllowAPIGatewayInvoke" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.probot_handler.function_name + principal = "apigateway.amazonaws.com" + source_arn = "${aws_api_gateway_rest_api.ops_bot.execution_arn}/*/*" +} + +resource "aws_lambda_permission" "api_gw_authorizer_policy" { + statement_id = "AllowAPIGatewayInvoke" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.authorizer.function_name + principal = "apigateway.amazonaws.com" + source_arn = "${aws_api_gateway_rest_api.ops_bot.execution_arn}/*/*" +} diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..1ce9077 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,18 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } + + backend "s3" { + bucket = "rapidsai-serverless-deployments" + key = "ops-bot/terraform.tfstate" + region = "us-east-2" + } +} + +provider "aws" { + region = "us-east-2" +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000..940058c --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,4 @@ +output "api_gateway_url" { + description = "Base URL for API Gateway stage" + value = "${aws_api_gateway_stage.ops_bot.invoke_url}/" +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..bf42257 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,29 @@ +variable "aws_region" { + description = "AWS region" + type = string + default = "us-east-2" +} + +variable "app_id" { + description = "GitHub App ID" + type = string + sensitive = true +} + +variable "webhook_secret" { + description = "GitHub Webhook Secret" + type = string + sensitive = true +} + +variable "private_key" { + description = "GitHub App Private Key" + type = string + sensitive = true +} + +variable "gputester_pat" { + description = "GPU Tester PAT" + type = string + sensitive = true +}