From a431858ba7798aa8206ee8af02ef7e4af869f6eb Mon Sep 17 00:00:00 2001
From: Mike Perrone <mike@gable.ai>
Date: Fri, 26 Jan 2024 10:37:53 -0800
Subject: [PATCH] Add support for integration tests with Docker

@alexdemeo gave this to me to help me run the integration tests locally
when implementing https://github.com/recap-build/recap/pull/414.

I assume he grabbed the docker config from here:

https://github.com/mjperrone/recap/blob/a0cbb2a47d2e3a789d76b5ba0f222b9ea8c25f8a/.github/workflows/ci.yaml#L61-L88

I grabbed the `RECAP_URLS` value from here:

https://github.com/mjperrone/recap/blob/a0cbb2a47d2e3a789d76b5ba0f222b9ea8c25f8a/.github/workflows/ci.yaml#L165

Admittedly I did not run all the integration tests as that was too long (just
`RECAP_URLS='["postgresql://postgres:password@localhost:5432/testdb"]' pdm run
pytest tests/integration/clients/test_postgresql.py` but this should be closer
to describing how to all the int tests.
---
 DEVELOPER.md             |  6 ++-
 tests/docker-compose.yml | 90 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+), 1 deletion(-)
 create mode 100644 tests/docker-compose.yml

diff --git a/DEVELOPER.md b/DEVELOPER.md
index f96d5f0..ab16cc3 100644
--- a/DEVELOPER.md
+++ b/DEVELOPER.md
@@ -30,9 +30,13 @@ These tests require internet access because they download the spec from Recap's
 
 Recap has integration tests that validate some of Recap's readers against real systems. These tests are located in the `tests/integration` directory.
 
+These tests require a postgres and mysql database to be running to work. To spin them up, you can use docker-compose:
+
+    docker-compose --project-directory tests/ up
+
 Run the integration tests:
 
-    pdm run integration
+    RECAP_URLS='["postgresql://postgres:password@localhost:5432/testdb"]' pdm run integration
 
 These test require various systems to be running as defined in the .github/workflows/ci.yml file.
 
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
new file mode 100644
index 0000000..230f734
--- /dev/null
+++ b/tests/docker-compose.yml
@@ -0,0 +1,90 @@
+version: '3.8'
+
+services:
+  postgres:
+    image: postgres:14
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: testdb
+    ports:
+      - "5432:5432"
+    healthcheck:
+      test: ["CMD", "pg_isready"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  mysql:
+    image: mysql:8.0
+    environment:
+      MYSQL_ROOT_PASSWORD: password
+      MYSQL_DATABASE: testdb
+      MYSQL_USER: mysql
+      MYSQL_PASSWORD: password
+    ports:
+      - "3306:3306"
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "--silent"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  bigquery:
+    image: ghcr.io/criccomini/bigquery-emulator:0.4.3-envvar
+    environment:
+      BIGQUERY_EMULATOR_PROJECT: test_project
+      BIGQUERY_EMULATOR_DATASET: test_dataset
+    ports:
+      - "9050:9050"
+
+  zookeeper:
+    image: confluentinc/cp-zookeeper:latest
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+    healthcheck:
+      test: ["CMD", "echo", "ruok", "|", "nc", "localhost", "2181"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  kafka:
+    image: confluentinc/cp-kafka:latest
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+    depends_on:
+      - zookeeper
+    healthcheck:
+      test: ["CMD", "kafka-topics", "--bootstrap-server", "localhost:9092", "--list"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  schema-registry:
+    image: confluentinc/cp-schema-registry:latest
+    ports:
+      - "8081:8081"
+    environment:
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry
+      SCHEMA_REGISTRY_LISTENERS: http://schema-registry:8081
+      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
+      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9092
+    depends_on:
+      - kafka
+      - zookeeper
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://schema-registry:8081/"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  hive-metastore:
+    image: ghcr.io/criccomini/hive-metastore-standalone:latest
+    ports:
+      - "9083:9083"