Merge pull request #11551 from wellcomecollection/rk/everything-every…

…where-all-at-once-sausage-fingers

Add configuration to run against local APIs

README.md

@@ -55,18 +55,35 @@ yarn install
 # yarn {appName = content|identity}
 # e.g.
 yarn content
-# you may also run all of them concurrently.
-# this may add a prefix to the URL such as `/catalogue/`
-# and is only for local cross projects development
-yarn run-concurrently
 ```
-### Port
+### Running the app with local APIs
 
-By default webapps will run on port `3000`.
+To run the app using a local copy of the concept, content, and/or catalogue APIs you can run:
 
-You can specify a port by setting the `PORT` in your `.env.development`.
+```bash
+yarn config-local-apis
+```
+
+Configure the content app to use the local APIs under https:///api-dev.wellcomecollection.org/ by adding the following to the `.env` file in `./content`:
+
+```
+API_ENV_OVERRIDE="dev"
+```
+
+This will configure local version of nginx to proxy requests to the local APIs, 
+see [scripts/configure-local-apis](./scripts/configure-local-apis) for more information.
+
+### Running the app with local Identity
 
-This is useful if you want to run webapps simultaneously, or you may just use `yarn run-concurrently` as explained above.
+Currently the identity app is not included in the local APIs configuration, so you will need to run it separately. In order to run `content` & `identity` together, you will need to run the following commands in different terminal windows while **not** using the local APIs configuration:
+
+```bash
+yarn content
+```
+
+```bash
+yarn identity
+```
 
 ### Running CI steps locally
 

common/next/next.config.js

@@ -18,7 +18,7 @@ const createConfig =
     const prodSubdomain = process.env.PROD_SUBDOMAIN || '';
     const buildHash = process.env.BUILD_HASH || 'test';
     const isProd = process.env.NODE_ENV === 'production';
-    const identityHost = process.env.IDENTITY_HOST || 'http://localhost:3000';
+    const identityHost = process.env.IDENTITY_HOST || 'http://localhost:3003';
     const shouldAnalyzeBundle = !!process.env.BUNDLE_ANALYZE;
 
     const rewriteEntries = options.rewriteEntries || [];

content/webapp/.env.example

@@ -16,4 +16,7 @@ NEXT_PUBLIC_CIVICUK_API_KEY=
 # Items api keys
 # secret values are stored in the AWS experience account
 items_api_key_stage=
-items_api_key_prod=
+items_api_key_prod=
+
+# Used in local development
+NEXT_PUBLIC_API_ENV_OVERRIDE=

content/webapp/package.json

@@ -4,7 +4,7 @@
   "license": "MIT",
   "scripts": {
     "build": "next build",
-    "dev": "NODE_ENV=development nodemon --watch server.ts --watch app.ts --exec ts-node --project tsconfig.server.json server",
+    "dev": "NODE_EXTRA_CA_CERTS=\"$(mkcert -CAROOT)/rootCA.pem\" NODE_ENV=development nodemon --watch server.ts --watch app.ts --exec ts-node --project tsconfig.server.json server",
     "start": "NODE_ENV=production ts-node-transpile-only --project tsconfig.server.json server",
     "test": "NODE_ENV=test jest --no-cache",
     "test:watch": "yarn test -- --watchAll"

content/webapp/services/wellcome/catalogue/index.ts

@@ -15,6 +15,7 @@ import {
 export const rootUris = {
   prod: 'https://api.wellcomecollection.org/catalogue',
   stage: 'https://api-stage.wellcomecollection.org/catalogue',
+  dev: 'https://api-dev.wellcomecollection.org/catalogue',
 };
 
 export const notFound = (): WellcomeApiError => ({

content/webapp/services/wellcome/content/index.ts

@@ -12,6 +12,7 @@ import { ContentResultsList, ResultType } from './types/api';
 const rootUris = {
   prod: 'https://api.wellcomecollection.org/content',
   stage: 'https://api-stage.wellcomecollection.org/content',
+  dev: 'https://api-dev.wellcomecollection.org/content',
 };
 
 export async function contentListQuery<Params, Result extends ResultType>(

content/webapp/services/wellcome/index.ts

@@ -3,13 +3,17 @@ import fetch, { Response } from 'node-fetch';
 
 import { Toggles } from '@weco/toggles';
 
+type envOptions = 'prod' | 'stage' | 'dev';
+
+const API_ENV_OVERRIDE = process.env.NEXT_PUBLIC_API_ENV_OVERRIDE as envOptions;
+
 export type GlobalApiOptions = {
-  env: 'prod' | 'stage';
+  env: envOptions;
   index?: string;
 };
 
 export const globalApiOptions = (toggles?: Toggles): GlobalApiOptions => ({
-  env: toggles?.stagingApi?.value ? 'stage' : 'prod',
+  env: API_ENV_OVERRIDE ?? (toggles?.stagingApi?.value ? 'stage' : 'prod'),
 });
 
 // Used as a helper to return a typesafe empty results list

identity/README.md

@@ -31,5 +31,5 @@ Then run:
 
 ```console
 $ docker-compose build identity
-$ docker-compose run -p 3000:3000 identity yarn start:dev
+$ docker-compose run -p 3003:3003 identity yarn start:dev
 ```

identity/webapp/package.json

@@ -10,7 +10,7 @@
   "license": "MIT",
   "scripts": {
     "start": "NODE_ENV=production ts-node-transpile-only -r tsconfig-paths/register src/index.ts",
-    "start:dev": "CREDENTIALS=$(AWS_PROFILE=identity-developer aws secretsmanager get-secret-value --secret-id=identity/stage/local_dev_client/credentials) NODE_ENV=development nodemon --ignore './src/frontend/' src/index.ts",
+    "start:dev": "PORT=3003 CREDENTIALS=$(AWS_PROFILE=identity-developer aws secretsmanager get-secret-value --secret-id=identity/stage/local_dev_client/credentials) NODE_ENV=development nodemon --ignore './src/frontend/' src/index.ts",
     "watch:client": "yarn build:frontend --watch",
     "build": "NODE_ENV=production && yarn build:next",
     "build:next": "next build",

identity/webapp/src/index.ts

@@ -26,7 +26,7 @@ async function main() {
     process.env.AUTH0_DOMAIN = 'stage.account.wellcomecollection.org';
     process.env.IDENTITY_API_HOST =
       'https://v1-api.stage.account.wellcomecollection.org';
-    process.env.SITE_BASE_URL = 'http://localhost:3000';
+    process.env.SITE_BASE_URL = `http://localhost:${port}`;
     process.env.AUTH0_CLIENT_SECRET = credentials.client_secret;
     process.env.IDENTITY_API_KEY = credentials.api_key;
     process.env.SESSION_KEYS = 'correct-horse-battery-staple'; // https://xkcd.com/936/

package.json

@@ -18,8 +18,7 @@
     "test:playwright:mobile": "echo \"⭕️ Running playwright mobile tests...\" && cd playwright && PLAYWRIGHT_BASE_URL=https://www-stage.wellcomecollection.org NODE_ENV=test platform=mobile yarn test && echo \"✅️ playwright mobile tests finished\"",
     "test:all:unit": "yarn test:common && yarn test:content && yarn test:identity",
     "e2e-stage": "yarn test:playwright && yarn test:playwright:mobile",
-    "run-concurrently": "./scripts/run-concurrently.sh",
-    "run-concurrently:clean": "./scripts/run-concurrently.sh -c",
+    "config-local-apis": "./scripts/configure-local-apis.sh",
     "lint": "eslint \"**/*.{js,ts,tsx}\"",
     "lint:css": "stylelint \"**/*.{js,ts,tsx}\"",
     "tsc": "yarn workspaces run tsc --noEmit --skipLibCheck"

scripts/configure-local-apis.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SCRIPTS_DIR=${DIR}/configure-local-apis
+
+# colours
+YELLOW='\033[1;33m'
+MAGENTA='\033[1;35m'
+GREEN='\033[1;32m'
+NC='\033[0m' # no colour - reset console colour
+
+setup_deps() {( set -e
+  echo -e "${MAGENTA}Setting up OSX dependencies ...${NC}"
+  if ! command -v brew &> /dev/null
+  then
+    echo "Please install brew.sh: https://brew.sh/"
+    exit 1
+  fi
+  brew install mkcert nginx
+)}
+
+install_certs() {( set -e
+  echo -e "${MAGENTA}Setting up local certificates for wellcomecollection.org${NC} ..."
+  $SCRIPTS_DIR/setup-mkcert.sh www-dev.wellcomecollection.org
+  $SCRIPTS_DIR/setup-mkcert.sh api-dev.wellcomecollection.org
+)}
+
+add_to_etc_hosts() {( set -e
+  echo -e "${MAGENTA}Adding entries to /etc/hosts ...${NC}"
+  $SCRIPTS_DIR/add-to-etc-hosts.sh www-dev.wellcomecollection.org
+  $SCRIPTS_DIR/add-to-etc-hosts.sh api-dev.wellcomecollection.org
+)}
+
+configure_nginx() {( set -e
+  echo -e "${MAGENTA}Configuring nginx ...${NC}"
+  $SCRIPTS_DIR/update-nginx-config.sh
+)}
+
+setup_deps
+install_certs
+add_to_etc_hosts
+configure_nginx
+
+# echo sparkles then "All done! in green
+
+echo -e "\n${GREEN} ✨ All done!${NC}"
+echo -e "\n${YELLOW}Please restart nginx: \"sudo bash -c 'nginx -s stop && nginx'\"${NC}"

scripts/configure-local-apis/README.md

@@ -0,0 +1,20 @@
+# Developer NGINX configuration scripts
+
+The scripts in this folder assist with the configuration of a local nginx installation for development purposes.
+
+These scripts have been shamelessly adapted (mostly copied) from https://github.com/guardian/dev-nginx. Credit to
+the developers of that project!
+
+## Usage
+
+Run the `configure-local-apis.sh` script in the parent directory.
+
+You can view the nginx configuration files in [weco-local.conf](./weco-local.conf), 
+it depends on thelocal APIs to be started successfully, and use the specified ports.
+
+- Catalogue Search, Works & Images API: http://localhost:8080
+- Catalogue Items API: http://localhost:8081
+- Concept API: http://localhost:3001
+- Content API: http://localhost:3002
+
+Configuration for Identity, Accounts and Requests is not included in this script.

scripts/configure-local-apis/add-to-etc-hosts.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -e
+
+# colours
+YELLOW='\033[1;33m'
+NC='\033[0m' # no colour - reset console colour
+
+if [[ $# -lt 1 ]]
+then
+  echo "add-to-hosts-file <DOMAIN>"
+  echo "Add an entry to hosts file to resolve to 127.0.0.1"
+	echo "Example: add-to-hosts-file foo.local"
+	exit 1
+fi
+
+DOMAIN=$1
+
+if grep '127.0.0.1' /etc/hosts | grep ${DOMAIN} ; then
+  echo -e "✅ /etc/hosts entry already exists for ${DOMAIN}"
+else
+  echo -e "🔧 ${YELLOW}adding /etc/hosts entry for ${DOMAIN}. Requires sudo - enter password when prompted.${NC}"
+  sudo sh -c "echo '127.0.0.1		${DOMAIN}' >> /etc/hosts"
+fi

scripts/configure-local-apis/locate-nginx.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -e
+
+nginx -V 2>&1 | grep "configure arguments:" | sed 's/[^*]*conf-path=\([^ ]*\)\/nginx\.conf.*/\1/g'

scripts/configure-local-apis/setup-mkcert.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Create a certificate using mkcert. Assumes you have installed mkcert previously.
+# Will add the CA to the truststore for macOS, Firefox and Java.
+
+# colours
+YELLOW='\033[1;33m'
+NC='\033[0m' # no colour - reset console colour
+
+if [[ $# -lt 1 ]]
+then
+  echo -e "Create a certificate for ${YELLOW}development use only${NC} using mkcert."
+  echo -e "See https://github.com/FiloSottile/mkcert for more information."
+  echo
+	echo "Example usage: $0 foo.local"
+	exit 1
+fi
+
+if type -p java > /dev/null ; then
+  # ensure JAVA_HOME is set for mkcert to install local root CA in the java trust store
+  # see https://github.com/FiloSottile/mkcert#supported-root-stores
+  if test -z ${JAVA_HOME} ; then
+    if [[ $(uname -s) == Darwin ]] ; then
+      echo -e "☕️ Running macOS and JAVA_HOME is not set. Attempting to set it..."
+      export JAVA_HOME=$(/usr/libexec/java_home)
+      echo -e "✅ JAVA_HOME now set to ${JAVA_HOME}"
+    else
+      echo -e "☕️ Java is installed but JAVA_HOME is not set. Set it before running this script."
+      exit 1
+    fi
+  fi
+else
+  echo -e "☕️ Did not detect an installation of Java."
+fi
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+NGINX_HOME=$("${DIR}/locate-nginx.sh")
+CERT_DIRECTORY=$HOME/.weco/mkcert
+
+DOMAIN=$1
+
+KEY_FILE=${CERT_DIRECTORY}/${DOMAIN}.key
+CERT_FILE=${CERT_DIRECTORY}/${DOMAIN}.crt
+
+mkcert -install
+
+echo -e "🔐 Creating certificate for: ${YELLOW}${DOMAIN}${NC}"
+mkdir -p ${CERT_DIRECTORY}
+mkcert -key-file=${KEY_FILE} -cert-file=${CERT_FILE} ${DOMAIN}
+
+echo -e "Symlinking the certificate for nginx at ${NGINX_HOME}"
+ln -sf ${KEY_FILE} ${NGINX_HOME}/${DOMAIN}.key
+ln -sf ${CERT_FILE} ${NGINX_HOME}/${DOMAIN}.crt
+

scripts/configure-local-apis/update-nginx-config.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# colours
+YELLOW='\033[1;33m'
+NC='\033[0m' # no colour - reset console colour
+
+NGINX_HOME=$("${DIR}/locate-nginx.sh")
+
+# ensure $NGINX_HOME/servers exists
+mkdir -p $NGINX_HOME/servers
+
+# check if $NGINX_HOME/servers/weco-local.conf exists and has the same contents as $DIR/weco-local.conf
+if [ -f $NGINX_HOME/servers/weco-local.conf ]; then
+  if cmp -s $DIR/weco-local.conf $NGINX_HOME/servers/weco-local.conf; then
+    echo -e "✅ ${NGINX_HOME}/servers/weco-local.conf is up to date"
+    exit 0
+  else
+    echo -e "🔧 ${YELLOW}updating ${NGINX_HOME}/servers/weco-local.conf${NC}"
+  fi
+else
+  echo -e "🔧 ${YELLOW}creating ${NGINX_HOME}/servers/weco-local.conf${NC}"
+fi
+
+cp $DIR/weco-local.conf $NGINX_HOME/servers/