Skip to content

Commit

Permalink
Switch gateway to node/tsx
Browse files Browse the repository at this point in the history
Bun just has too many issues with individual packages:

- vitest (use bun test?)
- prisma
- react-router (as of latest testing)
- inquirer
- probably others that I'm forgetting
  • Loading branch information
knpwrs committed Jan 24, 2025
1 parent 0b71a46 commit 99b2cb3
Show file tree
Hide file tree
Showing 9 changed files with 15,937 additions and 2,739 deletions.
24 changes: 14 additions & 10 deletions .github/workflows/check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,25 @@ jobs:
runs-on: depot-ubuntu-22.04
steps:
- uses: actions/checkout@v3
- uses: oven-sh/setup-bun@v2
- uses: actions/setup-node@v4
with:
bun-version: "1.1.34"
node-version: "23.6.0"
cache: "npm"
cache-dependency-path: ./services/gateway/package-lock.json
- name: Check gateway
working-directory: ./services/gateway
run: |
bun install
bun run check
npm install
npm run check
web:
runs-on: depot-ubuntu-22.04
steps:
- uses: actions/checkout@v3
- uses: actions/setup-node@v4
with:
node-version: "20.18.0"
node-version: "23.6.0"
cache: "npm"
cache-dependency-path: ./apps/web/package-lock.json
cache-dependency-path: ./services/gateway/package-lock.json
- name: Check web
working-directory: ./apps/web
run: |
Expand Down Expand Up @@ -57,11 +59,13 @@ jobs:
runs-on: depot-ubuntu-22.04
steps:
- uses: actions/checkout@v3
- uses: oven-sh/setup-bun@v2
- uses: actions/setup-node@v4
with:
bun-version: "1.1.34"
node-version: "23.6.0"
cache: "npm"
cache-dependency-path: ./services/gateway/package-lock.json
- name: Test gateway
working-directory: ./services/gateway
run: |
bun install
bun run check
npm ci
npm test
25 changes: 10 additions & 15 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,18 @@ stages:
# \____|_| |_|_____\____|_|\_\

check-gateway:
image: oven/bun:1.2.0-slim
image: node:23.6.0-slim
stage: check
script:
- cd $CI_PROJECT_DIR/services/gateway
- bun install
- bun run check
- npm ci --cache $CI_PROJECT_DIR/.npm --prefer-offline
- npm run check
cache:
key:
files:
- $CI_PROJECT_DIR/services/gateway/bun.lock
- $CI_PROJECT_DIR/services/gateway/package-lock.json
paths:
- $HOME/.bun/install/cache
- $CI_PROJECT_DIR/.npm/

check-web:
image: node:23.6.1-bookworm-slim
Expand Down Expand Up @@ -72,23 +72,18 @@ check-scripts:
- $HOME/.bun/install/cache

test-gateway:
image: oven/bun:1.2.0-slim
image: node:23.6.0-slim
stage: check
before_script:
# Install node (required for vitest)
- apt-get update && apt-get install -y curl
- curl -sS https://webi.sh/node | sh
- source $HOME/.config/envman/PATH.env
script:
- cd services/gateway
- bun install
- cd $CI_PROJECT_DIR/services/gateway
- npm ci --cache .npm --prefer-offline
- npm test
cache:
key:
files:
- $CI_PROJECT_DIR/services/gateway/bun.lock
- $CI_PROJECT_DIR/services/gateway/package-lock.json
paths:
- $HOME/.bun/install/cache
- $CI_PROJECT_DIR/.npm/

# ____ _ _ ___ _ ____
# | __ )| | | |_ _| | | _ \
Expand Down
36 changes: 18 additions & 18 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ services:
build:
context: ./services/gateway
target: server
command: bun run dev:server
command: npm run dev:server
restart: unless-stopped
ports:
- 127.0.0.1:${HOST_GATEWAY_PORT}:3000
Expand All @@ -155,16 +155,16 @@ services:
ZXCVBN_MINIMUM_SCORE: ${ZXCVBN_MINIMUM_SCORE}
<<: *s3-env
volumes:
- ./services/gateway:/home/bun/app
- gateway-node-modules:/home/bun/app/node_modules
- ./services/gateway:/home/node/app
- gateway-node-modules:/home/node/app/node_modules
depends_on:
- temporal

background-worker:
build:
context: ./services/gateway
target: background-worker
command: bun run dev:background-worker
command: npm run dev:background-worker
restart: unless-stopped
environment:
NODE_ENV: development
Expand All @@ -179,8 +179,8 @@ services:
<<: *s3-env
SMTP_URL: smtp://mailpit:1025?pool=true
volumes:
- ./services/gateway:/home/bun/app
- background-worker-node-modules:/home/bun/app/node_modules
- ./services/gateway:/home/node/app
- background-worker-node-modules:/home/node/app/node_modules

background-worker-go:
build:
Expand All @@ -201,7 +201,7 @@ services:
build:
context: ./services/gateway
target: probe-worker
command: bun run dev:probe-worker
command: npm run dev:probe-worker
restart: unless-stopped
environment:
NODE_ENV: development
Expand All @@ -213,14 +213,14 @@ services:
MAX_CONCURRENT_ACTIVITY_TASK_EXECUTIONS: 5
<<: *s3-env
volumes:
- ./services/gateway:/home/bun/app
- transcode-worker-node-modules:/home/bun/app/node_modules
- ./services/gateway:/home/node/app
- transcode-worker-node-modules:/home/node/app/node_modules

transcode-worker:
build:
context: ./services/gateway
target: transcode-worker
command: bun run dev:transcode-worker
command: npm run dev:transcode-worker
restart: unless-stopped
environment:
NODE_ENV: development
Expand All @@ -232,14 +232,14 @@ services:
MAX_CONCURRENT_ACTIVITY_TASK_EXECUTIONS: 1
<<: *s3-env
volumes:
- ./services/gateway:/home/bun/app
- transcode-worker-node-modules:/home/bun/app/node_modules
- ./services/gateway:/home/node/app
- transcode-worker-node-modules:/home/node/app/node_modules

transcribe-worker:
build:
context: ./services/gateway
target: transcribe-worker
command: bun run dev:transcribe-worker
command: npm run dev:transcribe-worker
restart: unless-stopped
environment:
NODE_ENV: development
Expand All @@ -252,14 +252,14 @@ services:
WHISPER_MODEL: tiny.en
<<: *s3-env
volumes:
- ./services/gateway:/home/bun/app
- transcribe-worker-node-modules:/home/bun/app/node_modules
- ./services/gateway:/home/node/app
- transcribe-worker-node-modules:/home/node/app/node_modules

import-worker:
build:
context: ./services/gateway
target: import-worker
command: bun run dev:import-worker
command: npm run dev:import-worker
restart: unless-stopped
environment:
NODE_ENV: development
Expand All @@ -270,8 +270,8 @@ services:
TEMPORAL_SHUTDOWN_GRACE_TIME: 1m
<<: *s3-env
volumes:
- ./services/gateway:/home/bun/app
- import-worker-node-modules:/home/bun/app/node_modules
- ./services/gateway:/home/node/app
- import-worker-node-modules:/home/node/app/node_modules

#
# Vendor Services
Expand Down
14 changes: 7 additions & 7 deletions justfile
Original file line number Diff line number Diff line change
Expand Up @@ -41,20 +41,20 @@ temporal *args:
docker compose exec temporal-admin-tools temporal {{args}}

gateway-db-push:
docker compose exec gateway bun run prisma:db:push
docker compose exec gateway npm run prisma:db:push

gateway-db-reset:
docker compose exec gateway bun run prisma:migrate:reset
docker compose exec gateway npm run prisma:migrate:reset
docker compose restart postgres

gateway-prisma-generate:
docker compose exec gateway bun run prisma:migrate:dev
docker compose exec gateway npm run prisma:migrate:dev

gateway-es-push-mappings:
docker compose exec gateway bun run es:push-mappings
docker compose exec gateway npm run es:push-mappings

gateway-migrate-dev:
docker compose exec gateway bun run prisma:migrate:dev
docker compose exec gateway npm run prisma:migrate:dev
cd services/gateway; npm run prisma:generate

gateway-schedule:
Expand Down Expand Up @@ -109,7 +109,7 @@ npmci-web-next:
npmci: npmci-gateway npmci-web npmci-web-next npmci-scripts

seed-db:
docker compose exec gateway bun run prisma:db:seed
docker compose exec gateway npm run prisma:db:seed
seed-s3-ingest:
rclone sync --fast-list --checksum -P ./seed-data/lcdevs3/letschurch-dev-ingest lcdevs3:letschurch-dev-ingest
seed-s3-public:
Expand All @@ -118,7 +118,7 @@ seed-s3: seed-s3-ingest seed-s3-public
seed: seed-s3 seed-db

truncate:
docker compose exec gateway bun run prisma:db:truncate
docker compose exec gateway npm run prisma:db:truncate

check-gateway:
cd services/gateway; npm run check
Expand Down
48 changes: 19 additions & 29 deletions services/gateway/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,51 +11,41 @@ RUN mkdir build
WORKDIR /home/build/audiowaveform/build
RUN cmake -D BUILD_STATIC=1 .. && make

# Prisma generate currently requires node
# Temporal workers require node
FROM oven/bun:1.1.42-slim AS bun-with-node
RUN apt-get update && \
apt-get install -y curl ffmpeg git python3 python3-pip && \
mkdir -p /etc/apt/keyrings && \
curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \
apt-get update && \
apt-get install -y nodejs && \
rm -rf /var/lib/apt/lists/* && \
apt-get clean

# Debian Base Image currently required for Prisma and M1 Macs: https://github.com/prisma/prisma/issues/8478
# Temporal does not support alpine: https://github.com/temporalio/sdk-typescript/issues/850
FROM bun-with-node AS base
FROM node:23.6.0-slim AS base
# Prisma requires lbssl-dev and ca-certificates
RUN apt-get update && apt-get install -y curl libssl-dev ca-certificates neovim
COPY package.json bun.lock ./
RUN bun install
WORKDIR /home/node/app
COPY package.json package-lock.json ./
RUN npm install
ENV NODE_ENV=production EDITOR=nvim
COPY . .

FROM base AS server
RUN bun run prisma:generate
CMD bun run start:server
RUN npm run prisma:generate
CMD npm run start:server

FROM base AS background-worker
RUN apt-get install -y curl imagemagick jpegoptim
COPY --from=videah/oxipng:7.0.0 /usr/local/bin/oxipng /usr/local/bin/oxipng
RUN bun run prisma:generate
CMD bun run start:background-worker
RUN npm run prisma:generate
CMD npm run start:background-worker

FROM base AS probe-worker
RUN apt-get install -y ffmpeg
CMD bun run start:probe-worker
CMD npm run start:probe-worker

FROM base AS transcode-worker
RUN apt-get install -y imagemagick jpegoptim ffmpeg
COPY --from=build-audiowaveform /home/build/audiowaveform/build/audiowaveform /usr/bin/
CMD bun run start:transcode-worker
CMD npm run start:transcode-worker

FROM nvidia/cuda:12.6.2-cudnn-runtime-ubuntu22.04 AS transcribe-worker
ARG WHISPER_MODEL=tiny.en
COPY --from=oven/bun:1.1.36-slim /usr/local/bin/bun /usr/local/bin/bun
COPY --from=node:23.6.0-slim /usr/local/bin/node /usr/local/bin/
COPY --from=node:23.6.0-slim /usr/local/lib/node_modules /usr/local/lib/node_modules
RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm
RUN apt-get update && \
apt-get install -y ca-certificates curl gnupg python3 python3-pip git ffmpeg && \
mkdir -p /opt/whisper/models && \
Expand All @@ -68,15 +58,15 @@ RUN apt-get update && \
rm -rf /var/lib/apt/lists/* && \
apt-get clean && \
pip3 install git+https://github.com/Softcatala/[email protected] && mkdir -p /opt/whisper/models
WORKDIR /home/bun/app
COPY package.json bun.lock ./
RUN bun install
WORKDIR /home/node/app
COPY package.json package-lock.json ./
RUN npm install
COPY . .
ENV NODE_ENV=production
CMD bun run start:transcribe-worker
CMD npm run start:transcribe-worker

FROM base AS import-worker
RUN apt-get install -y python3 ffmpeg
COPY --from=jauderho/yt-dlp:2025.01.15 /usr/local/bin/yt-dlp /usr/local/bin/yt-dlp
RUN bun run prisma:generate
CMD bun run start:import-worker
RUN npm run prisma:generate
CMD npm run start:import-worker
Loading

0 comments on commit 99b2cb3

Please sign in to comment.