From a7a56fa7eaf06d5d4cfdce1bf3f8b8db70d58aba Mon Sep 17 00:00:00 2001
From: Chandrasekharan M <chandrasekharan@zipstack.com>
Date: Tue, 28 Jan 2025 19:38:15 +0530
Subject: [PATCH] feat: Added compose override support and docs to use it

---
 .gitignore                          |  1 +
 docker/README.md                    | 30 +++++++++++++++++++++++++----
 docker/sample.compose.override.yaml | 14 ++++++++++++++
 3 files changed, 41 insertions(+), 4 deletions(-)
 create mode 100644 docker/sample.compose.override.yaml

diff --git a/.gitignore b/.gitignore
index 0f029490b..a7588844a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -636,6 +636,7 @@ docker/*.env
 !docker/sample*.env
 docker/public_tools.json
 docker/proxy_overrides.yaml
+docker/compose.override.yaml
 docker/workflow_data/
 
 # Tool development
diff --git a/docker/README.md b/docker/README.md
index 8529196db..d895f069c 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -1,4 +1,6 @@
-# Docker Build
+# Docker Commands
+
+## Docker Build
 
 ```bash
 # Build all services
@@ -8,7 +10,7 @@ VERSION=dev docker compose -f docker-compose.build.yaml build
 VERSION=dev docker compose -f docker-compose.build.yaml build frontend
 ```
 
-# Docker Run
+## Docker Run
 
 **NOTE**: First copy `sample.*.env` files to `*.env` and update as required.
 
@@ -22,7 +24,7 @@ VERSION=dev docker compose -f docker-compose.yaml up -d frontend
 
 Now access frontend at http://frontend.unstract.localhost
 
-# Docker Build and Run Optional Services
+## Docker Build and Run Optional Services
 
 Some services are kept optional and will not be built or started by default. Run them as follows.
 
@@ -33,7 +35,27 @@ VERSION=dev docker compose -f docker-compose.build.yaml --profile optional build
 VERSION=dev docker compose -f docker-compose.yaml --profile optional up -d
 ```
 
-# `src` Folder Layout and `gunicorn`
+## Overriding a service's config
+
+By making use of the [merge compose files](https://docs.docker.com/compose/how-tos/multiple-compose-files/merge/) feature its
+possible to override some configuration that's used by the services.
+
+Copy and rename the `sample.compose.override.yaml` to `compose.override.yaml` and update it as necessary.
+
+```bash
+cp sample.compose.override.yaml compose.override.yaml
+
+# Configuration in docker-compose.yaml gets overridden
+VERSION=dev docker compose -f docker-compose.yaml -f compose.override.yaml up -d
+```
+
+This can be useful during development to
+
+- not run some memory intensive services
+- use commands with different arguments to save resources
+- mount additional volumes or define additional env to configure behaviour
+
+## `src` Folder Layout and `gunicorn`
 
 For the following project structure:
 
diff --git a/docker/sample.compose.override.yaml b/docker/sample.compose.override.yaml
new file mode 100644
index 000000000..f4ac352b2
--- /dev/null
+++ b/docker/sample.compose.override.yaml
@@ -0,0 +1,14 @@
+# Refer https://docs.docker.com/compose/how-tos/multiple-compose-files/merge/
+# This is an example configuration which aims to minimize the number of celery workers
+# It helps reduce memory usage during development
+services:
+  worker:
+    command: "-A backend worker --loglevel=info -Q celery,celery_periodic_logs,celery_log_task_queue,celery_api_deployments --autoscale=${WORKER_AUTOSCALE}"
+
+  worker-logging:
+    profiles:
+      - high_memory
+
+  worker-api-deployment:
+    profiles:
+      - high_memory