Merge pull request #80 from caktus/develop

Production Deploy 04/20/2021

.circleci/config.yml

@@ -58,7 +58,7 @@ jobs:
           name: Test
           command: |
             . env/bin/activate
-            ./run_tests.sh
+            pytest
       - slack/status:
           success_message: ":caktus: ${CIRCLE_BRANCH} tests passed. <${CIRCLE_BUILD_URL}|Build> :successful:"
           failure_message: ":sadcactus: ${CIRCLE_BRANCH} tests failed. <${CIRCLE_BUILD_URL}|Build> :failed:"

.dockerignore

@@ -6,8 +6,13 @@ docker-compose.yml
 jgdc.yml
 Dockerfile
 setup.cfg
-node_modules
 *.dump
 *.env
 .git
 ncdata
+frontend/build
+frontend/node_modules
+.direnv
+deploy/
+.env
+.envrc

.gitignore

@@ -44,8 +44,10 @@ npm-debug.log
 env-local.sh
 *.dump
 .envrc
+.direnv
 .vscode
 jgdc.yml
 aws_ec2.yml
 deploy/roles
 docker-compose.override.yml
+htmlcov

Dockerfile

@@ -1,10 +1,10 @@
-FROM node:10-alpine as static_files
+FROM node:12.16-alpine as static_files
 
 WORKDIR /code
 ENV PATH /code/node_modules/.bin:$PATH
-COPY package.json package-lock.json gulpfile.js /code/
+COPY frontend/package.json frontend/package-lock.json /code/
 RUN npm install --silent
-COPY . /code/
+COPY frontend/ /code/
 RUN npm run build
 
 FROM python:3.8-slim as base
@@ -56,8 +56,8 @@ RUN mkdir /code/
 WORKDIR /code/
 ADD . /code/
 
-COPY --from=static_files /code/traffic_stops/static /code/traffic_stops/static
-COPY --from=static_files /code/node_modules/bootstrap /code/node_modules/bootstrap
+# Copy React SPA build into final image
+COPY --from=static_files /code/build /code/build
 
 # uWSGI will listen on this port
 EXPOSE 8000

Makefile

@@ -22,12 +22,6 @@ setup:
 	@echo 'Setting up the environment...'
 	make install_requirements
 
-run-dev:
-	@echo 'Running local development'
-	docker-compose up -d --remove-orphans
-	npm run dev &
-	python manage.py runserver
-
 run-tests:
 	@echo 'Checking for migrations'
 	python manage.py makemigrations --dry-run | grep 'No changes detected' || (echo 'There are changes which require migrations.' && exit 1)

deploy/deploy-hosting-services.yml

@@ -0,0 +1,8 @@
+- name: caktus hosting services
+  hosts: production
+  vars:
+    ansible_connection: local
+    ansible_python_interpreter: "{{ ansible_playbook_python }}"
+  gather_facts: false
+  roles:
+    - role: caktus.k8s-hosting-services

deploy/host_vars/production.yml

@@ -113,3 +113,17 @@ k8s_environment_variables:
   NC_FTP_HOST: "{{ env_nc_ftp_host }}"
   NC_FTP_USER: "{{ env_nc_ftp_user }}"
   NC_FTP_PASSWORD: "{{ env_nc_ftp_password }}"
+
+
+# ** TS_BACKUPS
+k8s_hosting_services_project_name: "{{ app_name }}"
+k8s_hosting_services_aws_secret_access_key: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  66346166663139656536333430393532616530633234626234356435303239323530313937306430
+  6436646430376262346230623339616538346538656432630a623531306337383064333763613532
+  35383664646437386662326665373130323031316133353562313932346536316130613566326664
+  3533623661316635340a663162636630343066376235383138396436343038613666383061393035
+  30653766333761613335383633353034623737323030306266373966643831323663323661393261
+  6338393036373961333736646565306137666636623365616636
+k8s_hosting_services_database_url: "{{ env_database_url }}"
+k8s_hosting_services_healthcheck_url: https://hc-ping.com/c80a410a-f739-4613-9aa6-bda801e7cfdb

deploy/host_vars/staging.yml

@@ -67,15 +67,15 @@ database_password: !vault |
   3263343333363130630a373033353533613064653033623138313334623537383037356262383662
   36613231353732663637316637383061376566663466373865356539626539376161
 
-# k8s_container_htpasswd: !vault |
-#           $ANSIBLE_VAULT;1.1;AES256
-#           31356461656536343532333632356338616462346436386566643438376237333935373531633762
-#           6436356465366162666366326530353936373461393932370a386665613137343137333262356235
-#           38353965393664313666393138643930323161343234393630616164363732633330373436306235
-#           3837386533393162370a663536653965383137346133333732353634323932306339626465623237
-#           38326131626331653931396137613861623630373162626637396637643135626531663538653332
-#           38326363333664626262343631666233663733313562333035663233333530383432616137323630
-#           333037613330396430643061653438313766
+k8s_container_htpasswd: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          31356461656536343532333632356338616462346436386566643438376237333935373531633762
+          6436356465366162666366326530353936373461393932370a386665613137343137333262356235
+          38353965393664313666393138643930323161343234393630616164363732633330373436306235
+          3837386533393162370a663536653965383137346133333732353634323932306339626465623237
+          38326131626331653931396137613861623630373162626637396637643135626531663538653332
+          38326363333664626262343631666233663733313562333035663233333530383432616137323630
+          333037613330396430643061653438313766
 
 k8s_environment_variables:
   DATABASE_URL: "{{ env_database_url }}"

deploy/requirements.yml

@@ -2,7 +2,7 @@
 
 - src: https://github.com/caktus/ansible-role-django-k8s
   name: caktus.django-k8s
-  version: v0.0.8
+  version: v0.0.11
 
 - src: https://github.com/caktus/ansible-role-aws-web-stacks
   name: caktus.aws-web-stacks
@@ -11,3 +11,7 @@
 - src: https://github.com/caktus/ansible-role-k8s-web-cluster
   name: caktus.k8s-web-cluster
   version: v0.0.7
+
+- src: https://github.com/caktus/ansible-role-k8s-hosting-services
+  name: caktus.k8s-hosting-services
+  version: v0.0.1

docs/dev-setup.rst

@@ -6,10 +6,8 @@ Stops project. To begin you should have the following applications installed on
 your local development system:
 
 - Python 3.8
-- NodeJS >= 10.21.0
+- NodeJS >= 12.6.0
 - `pip >= 8 or so <http://www.pip-installer.org/>`_
-- `virtualenv >= 1.10 <http://www.virtualenv.org/>`_
-- `virtualenvwrapper >= 3.0 <http://pypi.python.org/pypi/virtualenvwrapper>`_
 - Postgres >= 12
 
 
@@ -41,10 +39,6 @@ necessary requirements::
     $ mkvirtualenv --python=`which python3.8` traffic-stops
     (traffic-stops)$ pip install -U pip
     (traffic-stops)$ make setup
-    (traffic-stops)$ npm install
-
-If ``npm install`` fails, make sure you're using ``npm`` from a reasonable version
-of NodeJS, as documented at the top of this document.
 
 Next, we'll set up our local environment variables. We use `django-dotenv
 <https://github.com/jpadilla/django-dotenv>`_ to help with this. It reads environment variables
@@ -64,19 +58,29 @@ Migrate the project databases::
     (traffic-stops)$ ./migrate_all_dbs.sh
 
 
+Frontend setup::
+
+  (traffic-stops)$ cd frontend
+  (traffic-stops)$ npm i 
+
+If ``npm install`` fails, make sure you're using ``npm`` from a reasonable version
+of NodeJS, as documented at the top of this document.
+
+
 Development
 -----------
 
-You should be able to run the development server via the configured ``dev`` script::
+Start API dev server::
 
-    (traffic-stops)$ npm run dev
+    (traffic-stops)$ ./manage.py runserver
 
-Or, on a custom port and address::
+In another terminal window, start the frontend dev server::
 
-    (traffic-stops)$ npm run dev -- --address=0.0.0.0 --port=8020
+    (traffic-stops)$ cd frontend
+    (traffic-stops)$ npm run start
 
 Any changes made to Python, Javascript or Less files will be detected and rebuilt transparently as
-long as the development server is running.
+long as both dev servers are running.
 
 
 When running migrations
@@ -86,83 +90,3 @@ This is a multi-database project.  Whenever you have unapplied migrations,
 either added locally or via an update from the source repository, the
 migrations need to be applied to all databases by running the
 ``./migrate_all_dbs.sh`` command.
-
-
-Docker
-======
-
-You can use the provided ``docker-compose`` environment to create a local development environment.
-See previous section for more detailed instructions about how this project is configured.
-
-Setup your local development settings::
-
-  cp traffic_stops/settings/local.example.py traffic_stops/settings/local.py
-  # uncomment lines below "UNCOMMENT BELOW IF USING DOCKER SETUP" in local.py
-  echo "DJANGO_SETTINGS_MODULE=traffic_stops.settings.local" > .env
-
-For basic setup, run the following commands::
-
-  docker-compose up -d db  # start the PostgreSQL container in the background
-  docker-compose build web  # build the container (can take a while)
-  docker-compose run --rm web createdb -E UTF-8 traffic_stops
-  docker-compose run --rm web createdb -E UTF-8 traffic_stops_nc
-  docker-compose run --rm web ./migrate_all_dbs.sh
-
-Run ``npm install`` inside the ``web`` container (you'll need to do this for any update to
-``package.json``)::
-
-  rm -rf ./node_modules  # if needed
-  docker-compose run --rm web bash -lc "npm install"
-
-You can now run the web container and tail the logs::
-
-  # start up the dev server, and watch the logs:
-  docker-compose up -d web && docker-compose logs -f web
-
-These are other useful docker-compose commands::
-
-  # explicitly execute runserver in the foreground (for breakpoints):
-  docker-compose stop web
-  docker-compose run --rm --service-ports web python manage.py runserver 0.0.0.0:8000
-
-Visit http://localhost:8003/ in your browser.
-
-
-Restore Production Data
------------------------
-
-The data import process for each state can take a long time. You can load the production data using
-the following steps:
-
-First download a dump (in this case, NC) of the database::
-
-  ssh opendatapolicing.com 'sudo -u postgres pg_dump -Fc -Ox traffic_stops_nc_production' > traffic_stops_nc_production.pgdump
-
-Now run ``pg_restore`` within the ``web`` container::
-
-  docker-compose stop web  # free up connections to the DB
-  docker-compose run --rm web dropdb traffic_stops_nc
-  docker-compose run --rm web createdb -E UTF-8 traffic_stops_nc
-  docker-compose run --rm web pg_restore -Ox -d traffic_stops_nc traffic_stops_nc_production.pgdump
-  rm traffic_stops_nc_production.pgdump  # so it doesn't get built into the container
-
-You can also load the primary DB with user accounts and state statistics::
-
-  ssh opendatapolicing.com 'sudo -u postgres pg_dump -Fc -Ox traffic_stops_production' > traffic_stops_production.pgdump
-  docker-compose stop web  # free up connections to the DB
-  docker-compose run --rm web dropdb traffic_stops
-  docker-compose run --rm web createdb -E UTF-8 traffic_stops
-  docker-compose run --rm web pg_restore -Ox -d traffic_stops traffic_stops_production.pgdump
-  rm traffic_stops_production.pgdump  # so it doesn't get built into the container
-
-
-Deployment
-----------
-
-You can run a deployment from within a docker container using the following commands::
-
-  docker-compose run --rm web /bin/bash
-  eval $(ssh-agent)
-  ssh-add ~/.ssh/YOUR_KEY
-
-  fab -u YOUR_USER staging salt:"test.ping"