From 5237e57521fa51e3c4343cd1d146a04476e5f21c Mon Sep 17 00:00:00 2001 From: Shrey Bana Date: Tue, 31 Dec 2024 15:15:14 +0530 Subject: [PATCH] feat: ci nix setup --- .github/workflows/nix.yaml | 5 ++ docker-compose.yaml | 20 ++++-- flake.nix | 6 +- makefile | 122 +++++++++++++++++-------------------- 4 files changed, 82 insertions(+), 71 deletions(-) diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml index 8ae73b98..5129b222 100644 --- a/.github/workflows/nix.yaml +++ b/.github/workflows/nix.yaml @@ -15,3 +15,8 @@ jobs: steps: - uses: actions/checkout@v4 - run: om ci --extra-access-tokens ${{ secrets.GITHUB_TOKEN }} run --systems "${{ matrix.system }}" + - name: tests + shell: bash + run: | + # ls -la + nix develop --impure --command bash -c 'make run &; make test' diff --git a/docker-compose.yaml b/docker-compose.yaml index 0e775938..f93f768f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,9 +1,9 @@ -version: "3.4" - services: postgres: - build: ./docker-compose/postgres/ + image: public.ecr.aws/docker/library/postgres:12-alpine container_name: superposition_postgres + volumes: + - ./docker-compose/postgres/db_init.sql:/docker-entrypoint-initdb.d/db_init.sql ports: - "5432:5432" environment: @@ -11,9 +11,14 @@ services: POSTGRES_DB: "config" restart: on-failure network_mode: bridge + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 localstack: - build: ./docker-compose/localstack/ + image: public.ecr.aws/localstack/localstack:1.3.0 container_name: superposition_localstack ports: - "4510-4559:4510-4559" # external service port range @@ -23,7 +28,14 @@ services: environment: LOCALSTACK_SERVICES: s3, sns, sqs, logs, cloudwatch, kms AWS_DEFAULT_REGION: ap-south-1 + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test EDGE_PORT: 4566 + healthcheck: + test: ["CMD-SHELL", "aws --endpoint-url=http://localhost:4566 --region=ap-south-1 sts get-caller-identity"] + interval: 10s + timeout: 5s + retries: 5 # redis: # image: redis:7 diff --git a/flake.nix b/flake.nix index 0cc6a38b..24ee8503 100644 --- a/flake.nix +++ b/flake.nix @@ -36,6 +36,7 @@ }: { formatter = pkgs.nixpkgs-fmt; + packages.default = self'.packages.superposition; packages.static-assets = pkgs.buildNpmPackage { name = "static-assets"; version = "1.0.0"; @@ -60,7 +61,7 @@ ''; installPhase = "true"; }; - packages.image = pkgs.dockerTools.buildImage { + packages.container-image = pkgs.dockerTools.buildImage { name = "superposition"; tag = "latest"; @@ -93,7 +94,6 @@ ]; # Add your devShell tools here packages = with pkgs; [ - docker-compose gnumake # Why do we need this? stdenv.cc @@ -107,6 +107,8 @@ leptosfmt wasm-pack tailwindcss + podman + podman-compose ## For inspecting OCI(docker) images. dive # go client diff --git a/makefile b/makefile index 106beaac..d472f077 100644 --- a/makefile +++ b/makefile @@ -3,15 +3,20 @@ DOCKER_DNS ?= localhost TENANT ?= dev SHELL := /usr/bin/env bash FEATURES ?= ssr +COMPOSE_CMD = podman-compose +COMPOSE_UP = $(COMPOSE_CMD) up -d +COMPOSE_DOWN = $(COMPOSE_CMD) down +DB_UP := $(shell podman container ls | grep -q 'superposition_postgres' && echo 1 || echo 0) +LOCALSTACK_UP := $(shell podman container ls | grep -q 'superposition_localstack' && echo 1 || echo 0) +TIMEOUT ?= 30 .PHONY: db-init setup kill run - ci-test - validate-aws-connection - validate-psql-connection + ci-setup + test cac db-init: @@ -19,28 +24,18 @@ db-init: -diesel migration run --locked-schema --config-file=crates/superposition_types/src/experimentation/diesel.toml cleanup: - -docker rm -f $$(docker container ls --filter name=^context-aware-config -a -q) - -docker rmi -f $$(docker images | grep context-aware-config-postgres | cut -f 10 -d " ") + -podman rm -f $$(podman container ls --filter name=^context-aware-config -a -q) + -podman rmi -f $$(podman images | grep context-aware-config-postgres | cut -f 10 -d " ") -cac-migration: cleanup - docker-compose up -d postgres +cac-migration: cleanup db cp .env.example .env - while ! make validate-psql-connection; \ - do echo "waiting for postgres bootup"; \ - sleep 0.5; \ - done diesel migration run --locked-schema --config-file=crates/superposition_types/src/cac/diesel.toml - docker-compose down + $(COMPOSE_DOWN) -exp-migration: cleanup - docker-compose up -d postgres + exp-migration: cleanup db cp .env.example .env - while ! make validate-psql-connection; \ - do echo "waiting for postgres bootup"; \ - sleep 0.5; \ - done --diesel migration run --locked-schema --config-file=crates/superposition_types/src/experimentation/diesel.toml - docker-compose down + $(COMPOSE_DOWN) migration: cac-migration exp-migration @@ -50,38 +45,37 @@ legacy_db_setup: tenant: grep 'DATABASE_URL=' .env | sed -e 's/DATABASE_URL=//' | xargs ./scripts/create-tenant.sh -t $(TENANT) -d -validate-aws-connection: - aws --no-cli-pager --endpoint-url=http://$(DOCKER_DNS):4566 --region=ap-south-1 sts get-caller-identity +db: +ifeq ($(DB_UP),0) + @echo "Booting postgres..." +## Filtering some output to remove spurious error logs. + @$(COMPOSE_UP) postgres 2>&1 | grep -v "already exists" +endif + @podman wait --condition=healthy "superposition_postgres" > /dev/null + @echo "DB Up!" + +localstack: +ifeq ($(LOCALSTACK_UP),0) + @echo "Booting localstack..." +## Filtering some output to remove spurious error logs. + @$(COMPOSE_UP) localstack 2>&1 | grep -v "already exists" +endif + @podman wait --condition=healthy "superposition_localstack" > /dev/null + @echo "Localstack Up!" + +infra: db localstack + +env-setup: infra + npm ci + cp .env.example .env -validate-psql-connection: - pg_isready -h $(DOCKER_DNS) -p 5432 +test-tenant: TENANT = 'test' +test-tenant: tenant +dev-tenant: TENANT = 'dev' +dev-tenant: tenant -env-setup: - npm ci - -docker-compose up -d postgres localstack - cp .env.example .env - while ! make validate-psql-connection validate-aws-connection; \ - do echo "waiting for postgres, localstack bootup"; \ - sleep 0.5; \ - done - -test-tenant: - make tenant TENANT='test' - -dev-tenant: - make tenant TENANT='dev' - -ci-setup: env-setup test-tenant - npm ci --loglevel=error - make run -e DOCKER_DNS=$(DOCKER_DNS) 2>&1 | tee test_logs & - while ! grep -q "starting in Actix" test_logs; \ - do echo "ci-test: waiting for bootup..." && sleep 4; \ - done - # NOTE: `make db-init` finally starts a postgres container and runs all the migrations with locked-schema option - # to prevent update of schema.rs for both cac and experimentation. - # NOTE: The container spinned-up here is the actual container being used in development - echo setup completed successfully!!! +ci-setup: env-setup test-tenant run setup: env-setup @@ -89,7 +83,7 @@ kill: -pkill -f target/debug/superposition & get-password: - export DB_PASSWORD=`./docker-compose/localstack/get_db_password.sh` && echo $$DB_PASSWORD + export DB_PASSWORD=`./podman-compose/localstack/get_db_password.sh` && echo $$DB_PASSWORD superposition: cargo run --color always --bin superposition --no-default-features --features=$(FEATURES) @@ -101,7 +95,7 @@ superposition_legacy: cargo run --color always --bin superposition --no-default-features --features='ssr superposition_types/disable_db_data_validation context_aware_config/disable_db_data_validation experimentation_platform/disable_db_data_validation' superposition_dev: - # export DB_PASSWORD=`./docker-compose/localstack/get_db_password.sh` +# export DB_PASSWORD=`./podman-compose/localstack/get_db_password.sh` cargo watch -x 'run --color always --bin superposition --no-default-features --features=$(FEATURES)' @@ -123,22 +117,20 @@ backend: build: frontend backend -run: kill frontend - while ! make validate-psql-connection validate-aws-connection; \ - do echo "waiting for postgres, localstack bootup"; \ - sleep 0.5; \ - done - make superposition -e DOCKER_DNS=$(DOCKER_DNS) - -run_legacy: kill build - while ! make validate-psql-connection validate-aws-connection; \ - do echo "waiting for postgres, localstack bootup"; \ - sleep 0.5; \ - done - make superposition_legacy -e DOCKER_DNS=$(DOCKER_DNS) - -ci-test: ci-setup +run: kill infra frontend superposition + +run_legacy: kill infra build superposition_legacy + +test: cargo test + @timeout=$(TIMEOUT); \ + while ! nc -z localhost 8080 && [ $$timeout -gt 0 ]; do \ + sleep 1; \ + ((timeout--)); \ + done; \ + if [ $$timeout -eq 0 ]; then \ + echo "Timedout waiting for service to recieve connections, is superposition running?" && exit 1; \ + fi npm run test tailwind: