generated from pierrz/papel
-
Notifications
You must be signed in to change notification settings - Fork 1
299 lines (260 loc) · 12.1 KB
/
ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
# CI setup, building the main images and running the tests against them
# - Can be self-hosted for debugging purposes
# - TODO: fix the Github action about PR comments from coverage report (very end of this file)
name: CI pipeline
on:
workflow_dispatch: # manual trigger
pull_request:
branches:
- main
# - "feature/**"
push:
branches: [ "main" ]
env:
REGISTRY: ghcr.io
RELEASE: 0.5.8 # /!\ increase this to avoid overwriting older image
jobs:
build-and-push-api:
name: API - build/push image
runs-on: ubuntu-latest
# runs-on: self-hosted
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/[email protected]
- name: Run 'build-push' local action
uses: ./.github/actions/build-push
with:
name: API
src: api
registry: ${{ env.REGISTRY }}
actor: ${{ github.actor }}
token: ${{ secrets.GITHUB_TOKEN }}
image: pierrz/biggie_api_img
tag: ${{ env.RELEASE }}
labels: ${{ steps.meta.outputs.labels }}
build-and-push-orchestrator:
name: Orchestrator - build/push image
runs-on: ubuntu-latest
# runs-on: self-hosted
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/[email protected]
- name: Run 'build-push' local action
uses: ./.github/actions/build-push
with:
name: Orchestrator
src: orchestrator
registry: ${{ env.REGISTRY }}
actor: ${{ github.actor }}
token: ${{ secrets.GITHUB_TOKEN }}
image: pierrz/biggie_orchestrator_img
tag: ${{ env.RELEASE }}
labels: ${{ steps.meta.outputs.labels }}
test:
name: Tests
needs: [build-and-push-api, build-and-push-orchestrator]
runs-on: ubuntu-latest
# runs-on: self-hosted
permissions:
packages: write
# specific to py-cov-action/python-coverage-comment-action (WIP)
pull-requests: write
contents: write
env:
NETWORK: ci_network
SPARK_MASTER_URL: "spark://spark-master:7077"
POSTGRESDB_HOST: "biggie_postgres:5432"
MONGODB_URI: "mongodb://${{ secrets.MONGO_USERNAME }}:${{ secrets.MONGO_PASSWORD }}@biggie_mongo:27017/?authSource=${{ secrets.DB_NAME }}"
steps:
- name: Checkout repository
uses: actions/[email protected]
- name: Log in to the Container registry
uses: docker/[email protected]
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Create network
run: docker network create -d bridge ${{ env.NETWORK }}
- name: Create Mongo container
run: |
docker build db/mongo --tag biggie_mongo:latest
chmod +x ./db/mongo/mongo-init.sh
docker run -d --name biggie_mongo \
--env DB_NAME=${{ secrets.DB_NAME }} \
--env MONGO_INITDB_ROOT_USERNAME=${{ secrets.MONGO_INITDB_ROOT_USERNAME }} \
--env MONGO_INITDB_ROOT_PASSWORD=${{ secrets.MONGO_INITDB_ROOT_PASSWORD }} \
--env MONGO_USERNAME=${{ secrets.MONGO_USERNAME }} \
--env MONGO_PASSWORD=${{ secrets.MONGO_PASSWORD }} \
--env ME_CONFIG_MONGODB_ADMINUSERNAME=${{ secrets.ME_CONFIG_MONGODB_ADMINUSERNAME }} \
--env ME_CONFIG_MONGODB_ADMINPASSWORD=${{ secrets.ME_CONFIG_MONGODB_ADMINPASSWORD }} \
--network=${{ env.NETWORK }} \
--volume ./db/mongo/mongo-init.sh:/docker-entrypoint-initdb.d/mongo-init.sh \
--volume ./db/mongo/mongod.conf:/etc/mongod.conf \
biggie_mongo:latest
- name: Create Postgres container
run: |
docker build db/postgres --tag biggie_postgres:latest
chmod +x ./db/postgres/init-postgres.sh
docker run -d --name biggie_postgres \
--env DB_NAME=${{ secrets.DB_NAME }} \
--env POSTGRES_DB=${{ secrets.POSTGRES_DB }} \
--env POSTGRES_USER=${{ secrets.POSTGRES_USER }} \
--env POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }} \
--env POSTGRES_APP_USER=${{ secrets.POSTGRES_APP_USER }} \
--env POSTGRES_APP_PASSWORD=${{ secrets.POSTGRES_APP_PASSWORD }} \
--network=${{ env.NETWORK }} \
--volume ./db/postgres/init-postgres.sh:/docker-entrypoint-initdb.d/init-postgres.sh \
biggie_postgres:latest
# /!\ The flag --network-alias spark-master is CRITICAL for the master /!\
- name: Create Spark containers (master and workers)
run: |
docker build spark_cluster --tag biggie_spark:latest
docker run -d --name biggie_spark-master \
--env SPARK_MODE=master \
--publish 7077:7077 \
--network=${{ env.NETWORK }} \
--network-alias spark-master \
biggie_spark:latest \
&& docker run -d --name biggie_spark-worker-1 \
--env SPARK_MODE=worker \
--env SPARK_MASTER_URL=${{ env.SPARK_MASTER_URL }} \
--network=${{ env.NETWORK }} \
biggie_spark:latest \
&& docker run -d --name biggie_spark-worker-2 \
--env SPARK_MODE=worker \
--env SPARK_MASTER_URL=${{ env.SPARK_MASTER_URL }} \
--network=${{ env.NETWORK }} \
biggie_spark:latest
# This can be useful to check the DBs after changes in the init scripts or authentication methods
# - name: Checks
# run: |
# docker ps -a
# docker ps --format "table {{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Status}}"
# docker logs biggie_mongo
# docker logs biggie_postgres
- name: API tests
run: |
docker run --name biggie_api-test \
--env DOCKER_SUBNET_BASE=${{ secrets.DOCKER_SUBNET_BASE }} \
--env API_PORT=${{ secrets.API_PORT }} \
--env MONGODB_URI=${{ env.MONGODB_URI }} \
--env DB_NAME=${{ secrets.DB_NAME }} \
--env POSTGRESDB_HOST=${{ env.POSTGRESDB_HOST }} \
--env POSTGRES_APP_USER=${{ secrets.POSTGRES_APP_USER }} \
--env POSTGRES_APP_PASSWORD=${{ secrets.POSTGRES_APP_PASSWORD }} \
--env POSTGRES_APP_PASSWORD=${{ secrets.POSTGRES_APP_PASSWORD }} \
--network=${{ env.NETWORK }} \
--volume ${{ secrets.DATA_DIR }}:/opt/data \
ghcr.io/pierrz/biggie_api_img:latest \
sh -c "uv sync --only-group test && \
uv run pytest -vv --cov-config=pyproject.toml --cov=. --cov-report=html && \
mkdir /opt/data/api_coverage && cp .coverage /opt/data/api_coverage/.coverage"
- name: Orchestrator tests
run: |
docker run --name biggie_orchestrator-test \
--env CELERY_BROKER_URL=${{ secrets.CELERY_BROKER_URL }} \
--env CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND }} \
--env MONGODB_URI=${{ env.MONGODB_URI }} \
--env DB_NAME=${{ secrets.DB_NAME }} \
--env POSTGRESDB_HOST=${{ env.POSTGRESDB_HOST }} \
--env POSTGRES_APP_USER=${{ secrets.POSTGRES_APP_USER }} \
--env POSTGRES_APP_PASSWORD=${{ secrets.POSTGRES_APP_PASSWORD }} \
--env TOKEN_GITHUB_API=${{ secrets.TOKEN_GITHUB_API }} \
--env TEST_MODE="True" \
--network=${{ env.NETWORK }} \
--volume ${{ secrets.DATA_DIR }}:/opt/data \
ghcr.io/pierrz/biggie_orchestrator_img:latest \
sh -c "uv sync --only-group test && \
uv run pytest -vv --asyncio-mode=strict --cov-config=pyproject.toml --cov=. --cov-report=html && \
mkdir /opt/data/orchestrator_coverage && cp .coverage /opt/data/orchestrator_coverage/.coverage"
- name: Upload coverage reports
uses: actions/[email protected]
with:
name: html-coverage-report
path: |
${{ secrets.DATA_DIR }}/orchestrator_coverage_html_report
${{ secrets.DATA_DIR }}/api_coverage_html_report
# /!\ Step triggered only when the pipeline is running in the self-hosted runner /!\
- name: Delete Docker elements in self-hosted runner
if: runner.labels == 'self-hosted' && always()
run: |
# Get names of all containers starting with biggie_
echo "Getting containers name ..."
CONTAINER_NAMES=$(docker ps -a --format '{{.Names}}' | grep '^biggie_' || true)
if [ ! -z "$CONTAINER_NAMES" ]; then
# Get volume names associated with the containers
echo "Retrieving created volumes ..."
VOLUME_NAMES=$(docker container inspect $CONTAINER_NAMES \
-f '{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}} {{end}}{{end}}' | tr ' ' '\n' | sort -u | tr '\n' ' ') || echo "Failed to inspect containers"
# Remove containers
echo "Removing containers ..."
docker container stop -f $CONTAINER_NAMES || echo "Failed to remove some containers"
docker container rm -f $CONTAINER_NAMES || echo "Failed to remove some containers"
# Remove volumes
if [ ! -z "$VOLUME_NAMES" ]; then
echo "Removing volumes ..."
docker volume rm $VOLUME_NAMES || echo "Failed to remove some volumes"
else
echo "No volumes to remove"
fi
else
echo "No biggie_* containers found"
fi
# Remove network
echo "Removing created network ..."
docker network rm ${{ env.NETWORK }} || true
# TODO: fix py-cov-action/python-coverage-comment-action errors (permissions, absolute/relative files)
# Cf. https://github.com/py-cov-action/python-coverage-comment-action?tab=readme-ov-file#merging-multiple-coverage-reports
# Error: Cannot read .coverage files because files are absolute. You need to configure coverage to write relative paths by adding the following option to your coverage configuration file:
# [run]
# relative_files = true
# Note that the specific format can be slightly different if you're using setup.cfg or pyproject.toml. See details in: https://coverage.readthedocs.io/en/latest/config.html#config-run-relative-files
# Error: Critical error. This error possibly occurred because the permissions of the workflow are set incorrectly. You can see the correct setting of permissions here: https://github.com/py-cov-action/python-coverage-comment-action#basic-usage
# for refined merging approach
# - name: Merge coverage reports
# run: |
# ls -al ${{ github.workspace }}/data
# ls -al data
# pip install coverage
# coverage combine --keep ${{ github.workspace }}/data/.coverage.api ${{ github.workspace }}/data/.coverage.orchestrator
# coverage html -d ${{ github.workspace }}/data
# cp ${{ github.workspace }}/data/.coverage ${{ github.workspace }}/data/merged_coverage/.coverage
# - name: Comment coverage
# uses: py-cov-action/[email protected]
# with:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# COVERAGE_PATH: data/orchestrator_coverage
# MINIMUM_GREEN: 80
# MINIMUM_ORANGE: 50
# COMMENT_TEMPLATE: |
# ### CELERY Coverage Report
# {coverage_report}
# - name: Store Pull Request comment to be posted
# uses: actions/upload-artifact@v4
# if: steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true'
# with:
# name: python-coverage-comment-action-orchestrator
# path: python-coverage-comment-action.txt
# - name: Comment coverage
# uses: py-cov-action/[email protected]
# with:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# COVERAGE_PATH: ${{ github.workspace }}/data/api_coverage
# MINIMUM_GREEN: 80
# MINIMUM_ORANGE: 50
# COMMENT_TEMPLATE: |
# ### API Coverage Report
# {coverage_report}
# - name: Store Pull Request comment to be posted
# uses: actions/upload-artifact@v4
# if: steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true'
# with:
# name: python-coverage-comment-action-api
# path: python-coverage-comment-action.txt