Merge pull request #87 from caktus/develop

Production Deploy 12/10/2021

.circleci/config.yml

@@ -8,15 +8,15 @@ commands:
     steps:
       - restore_cache:
           keys:
-              - v7-pip-dependencies-{{ checksum "requirements/test/test.txt" }}-{{ checksum "requirements/base/base.txt" }}
+              - v10-pip-dependencies-{{ checksum "requirements/test/test.txt" }}-{{ checksum "requirements/base/base.txt" }}
               # fallback to using the latest cache if no exact match is found
-              - v7-pip-dependencies-
+              - v10-pip-dependencies-
   save_cache_cmd:
     steps:
       - save_cache:
           paths:
             - "env"
-          key: v7-pip-dependencies-{{ checksum "requirements/test/test.txt" }}-{{ checksum "requirements/base/base.txt" }}
+          key: v10-pip-dependencies-{{ checksum "requirements/test/test.txt" }}-{{ checksum "requirements/base/base.txt" }}
 orbs:
   slack: circleci/[email protected]
 

.gitignore

@@ -43,6 +43,7 @@ npm-debug.log
 .python-version
 env-local.sh
 *.dump
+*.pgdump
 .envrc
 .direnv
 .vscode

Dockerfile

@@ -7,7 +7,7 @@ RUN npm install --silent
 COPY frontend/ /code/
 RUN npm run build
 
-FROM python:3.8-slim as base
+FROM python:3.8-slim-bullseye as base
 
 # Create a group and user to run our app
 ARG APP_USER=appuser
@@ -43,6 +43,7 @@ RUN set -ex \
     build-essential \
     libpcre3-dev \
     libpq-dev \
+    git-core \
     " \
     && apt-get update && apt-get install -y --no-install-recommends $BUILD_DEPS \
     && pip install -U -q pip-tools \

README.rst

@@ -1,25 +1,15 @@
-NC Traffic Stops
+NC CopWatch
 ================
 
-.. image:: https://badge.waffle.io/OpenDataPolicingNC/Traffic-Stops.svg?label=ready&title=Ready 
- :target: https://waffle.io/OpenDataPolicingNC/Traffic-Stops 
- :alt: 'Stories in Ready'
+.. image:: https://circleci.com/gh/caktus/Traffic-Stops.svg?style=svg
+    :target: https://circleci.com/gh/caktus/Traffic-Stops
 
-.. image:: https://readthedocs.org/projects/nc-traffic-stops/badge/?version=latest
-  :target: http://nc-traffic-stops.readthedocs.org/en/latest/
-  :alt: Documentation Status
-
-.. image:: https://travis-ci.org/OpenDataPolicingNC/Traffic-Stops.svg?branch=master
-    :target: https://travis-ci.org/OpenDataPolicingNC/Traffic-Stops
-
-NC Traffic Stops is a website to monitor and identify racial profiling
+NC CopWatch is a website to monitor and identify racial profiling
 practices by North Carolina law enforcement agencies. This project is lead by
 `Forward Justice`_, a nonpartisan law, policy, and strategy center dedicated to advancing racial, 
 social, and economic justice in the U.S. South.
 
 Please see the `production documentation`_ and `development documentation`_
 for more information.
 
-.. _production documentation: http://nc-traffic-stops.readthedocs.org/en/latest/
-.. _development documentation: http://nc-traffic-stops.readthedocs.org/en/dev/
 .. _Forward Justice: https://forwardjustice.org/

deploy/deploy-hosting-services.yml

@@ -6,3 +6,7 @@
   gather_facts: false
   roles:
     - role: caktus.k8s-hosting-services
+  tasks:
+    - import_role:
+        name: caktus.k8s-hosting-services
+        tasks_from: monitoring

deploy/group_vars/all.yml

@@ -42,6 +42,7 @@ cloudformation_stack:
     UseAES256Encryption: "true"
     CustomerManagedCmkArn: ""
     ContainerInstanceType: t3a.medium
+    ContainerVolumeSize: 40
     DatabaseAllocatedStorage: 100
     DatabaseClass: db.t3.large
     DatabaseEngineVersion: "12"
@@ -71,17 +72,33 @@ k8s_ci_repository_arn: arn:aws:ecr:us-east-2:606178775542:repository/traff-appli
 k8s_ci_vault_password_arn: arn:aws:secretsmanager:us-east-2:606178775542:secret:trafficstops-ansible-vault-password-XKpR8f
 
 k8s_letsencrypt_email: [email protected]
-# New Relic Infrastructure: Caktus Paid Account
-k8s_newrelic_license_key: !vault |
-  $ANSIBLE_VAULT;1.1;AES256
-  31623963653434303137323231656263643235616539316537346331646133313732316465623865
-  3438623336353035323437653033313434646366383236390a656531636336663530373462323331
-  32643434333833363433663932316534373565663035383334336231313366373763303263393836
-  3035363662323335630a306331303761303434633235616564386362353766336462656535663033
-  31333537343865616436623063386539303339653165636664633736666365623337326363646437
-  6565393035313438666364363231353562613334376135663031
+k8s_iam_users: [copelco]
+
+# Pin ingress-nginx and cert-manager to current versions so future upgrades of this
+# role will not upgrade these charts without your intervention:
+# https://github.com/kubernetes/ingress-nginx/releases
+k8s_ingress_nginx_chart_version: "3.39.0"
+# https://github.com/jetstack/cert-manager/releases
+k8s_cert_manager_chart_version: "v1.6.1"
+# AWS only:
+# Use the newer load balancer type (NLB). DO NOT edit k8s_aws_load_balancer_type after
+# creating your Service.
+k8s_aws_load_balancer_type: nlb
+
+# ----------------------------------------------------------------------------
+# caktus.k8s-hosting-services: Logging and monitoring configuration
+# ----------------------------------------------------------------------------
 
 k8s_papertrail_logspout_destination: "syslog+tls://logs2.papertrailapp.com:20851"
 k8s_papertrail_logspout_memory_limit: 128Mi
 
-k8s_iam_users: [copelco]
+# New Relic Infrastructure: [email protected]
+k8s_newrelic_chart_version: "3.2.4"
+k8s_newrelic_license_key: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  37656631623333346263383231386165666531333961373931383661366338343634333362356430
+  3963613833663637313632373465613730383365626461630a383432346335386632303935356532
+  61643737636132336339336332396262623362333663333130393031376338363266363430326136
+  6131396135646236360a323766623330313365306539316263393533623063346166653433316631
+  39356263623363653934333064376364363562303236646238666234356136663539343064383463
+  3161356339656137373935623562366134393765346466643365

deploy/requirements.yml

@@ -2,16 +2,16 @@
 
 - src: https://github.com/caktus/ansible-role-django-k8s
   name: caktus.django-k8s
-  version: v0.0.11
+  version: v1.3.0
 
 - src: https://github.com/caktus/ansible-role-aws-web-stacks
   name: caktus.aws-web-stacks
   version: ''
 
 - src: https://github.com/caktus/ansible-role-k8s-web-cluster
   name: caktus.k8s-web-cluster
-  version: v0.0.7
+  version: v1.1.0
 
 - src: https://github.com/caktus/ansible-role-k8s-hosting-services
   name: caktus.k8s-hosting-services
-  version: v0.0.1
+  version: v0.3.0

docs/deploy.rst

@@ -14,7 +14,7 @@ Caktus AWS Access
     .. code-block::
 
         [trafficstops]
-        role_arn = arn:aws:iam::000000000000:role/CaktusAccessRole
+        role_arn = arn:aws:iam::000000000000:role/CaktusAccountAccessRole-Admins
         source_profile = caktus
 
     See LastPass entry *Traffic Stops AWS Profile role_arn* for the AWS account
@@ -111,4 +111,3 @@ Deploy application
 4. Deploy::
 
     inv staging deploy --tag=...
-=======

docs/hosting-services.md

@@ -0,0 +1,34 @@
+# Hosting Services
+
+The services configured for this project are:
+* PostgreSQL database backups to S3 (within Caktus AWS account)
+    * Currently, this is only `traffic_stops`, which contains users, census data, etc.
+    * `traffic_stops_nc` is not backed up since the entire dataset is re-imported daily.
+* Papertrail logging (to Caktus account)
+* New Relic Infrastructure monitoring (Account: `[email protected]`)
+
+
+## Production database disaster recovery
+
+In the event a restore from a historical backup is needed, access to the [Caktus
+AssumeRole is
+required](https://github.com/caktus/caktus-hosting-services/blob/main/docs/aws-assumerole.md#aws-accounts).
+Once you have that access, you can use invoke tools to pull historical backups.
+
+To download the latest `daily` backup:
+
+```sh
+inv utils.get-db-backup
+```
+
+
+## Production backup configuration
+
+[caktus.k8s-hosting-services](https://github.com/caktus/ansible-role-k8s-hosting-services)
+manages database backups.
+
+Run this command to set up database backups and monitoring services:
+
+```sh
+inv deploy.playbook -n deploy-hosting-services.yml
+```

frontend/src/Components/Charts/ChartPrimitives/Bar.js

@@ -2,10 +2,11 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import { VictoryAxis, VictoryBar, VictoryChart, VictoryContainer } from 'victory';
 import { AXIS_STYLE } from './chartConstants';
+import ChartLoading from 'Components/Charts/ChartPrimitives/ChartLoading';
 import BarSkeleton from 'Components/Elements/Skeletons/BarSkeleton';
 
 function Bar({ data, chartProps, xAxisProps, yAxisProps, barProps }) {
-  if (!data) return <BarSkeleton />;
+  if (!data) return <ChartLoading skeleton={BarSkeleton} />;
   return (
     <VictoryChart
       {...chartProps}

frontend/src/Components/Charts/ChartPrimitives/ChartBase.js

@@ -2,9 +2,9 @@ import React, { useState, useEffect } from 'react';
 import { ChartBaseStyled } from './ChartBase.styled';
 
 // Children
-import PieSkeleton from 'Components/Elements/Skeletons/PieSkeleton';
 import ResponsiveChartContainer from 'Components/Charts/ResponsiveChartContainer.styled';
 import Legend from 'Components/Charts/ChartPrimitives/Legend/Legend';
+import DataLoading from 'Components/Charts/ChartPrimitives/DataLoading';
 
 function ChartBase({
   children,
@@ -60,7 +60,7 @@ function ChartBase({
       {...props}
     >
       {hasError && <p>some chart error message</p>}
-      {isLoading && <PieSkeleton />}
+      {isLoading && <DataLoading />}
       <h2>{chartTitle}</h2>
       {!hideLegend && (
         <Legend

frontend/src/Components/Charts/ChartPrimitives/ChartLoading.js

@@ -1,16 +1,18 @@
 import React from 'react';
-import { ChartLoadingStyled } from './ChartLoading.styled';
+import * as S from './ChartLoading.styled';
 
-function ChartLoading({ pastDelay }) {
-  if (pastDelay) {
-    return (
-      <ChartLoadingStyled>
-        <p>ChartLoading</p>
-      </ChartLoadingStyled>
-    );
-  } else {
-    return null;
-  }
+// Hooks
+import useOfficerId from 'Hooks/useOfficerId';
+
+function ChartLoading({ skeleton: Skeleton }) {
+  const officerId = useOfficerId();
+
+  return (
+    <S.ChartLoading>
+      <h3>Loading {officerId ? "Officer" : "Agency"} data...</h3>
+      <Skeleton scale={2} />
+    </S.ChartLoading>
+  );
 }
 
 export default ChartLoading;

frontend/src/Components/Charts/ChartPrimitives/ChartLoading.styled.js

@@ -1,3 +1,11 @@
 import styled from 'styled-components';
 
-export const ChartLoadingStyled = styled.div``;
+export const ChartLoading = styled.div`
+  padding: 2rem 0;
+  h3 {
+    text-align: center;
+    font-size: 28px;
+    font-weight: 200;
+    color: ${props => props.theme.colors.grey};
+  }
+`;

frontend/src/Components/Charts/ChartPrimitives/DataLoading.js

@@ -0,0 +1,21 @@
+import React from 'react';
+import { DataLoadingStyled } from './DataLoading.styled';
+
+// Hooks
+import useOfficerId from 'Hooks/useOfficerId';
+
+// Children
+import PieSkeleton from 'Components/Elements/Skeletons/PieSkeleton';
+
+function DataLoading() {
+  const officerId = useOfficerId();
+
+  return (
+    <DataLoadingStyled>
+      <h3>Loading {officerId ? "Officer" : "Agency"} data...</h3>
+      <PieSkeleton scale={2} />
+    </DataLoadingStyled>
+  );
+}
+
+export default DataLoading;

frontend/src/Components/Charts/ChartPrimitives/DataLoading.styled.js

@@ -0,0 +1,11 @@
+import styled from 'styled-components';
+
+export const DataLoadingStyled = styled.div`
+  padding: 2rem 0;
+  h3 {
+    text-align: center;
+    font-size: 28px;
+    font-weight: 200;
+    color: ${props => props.theme.colors.grey};
+  }
+`;

frontend/src/Components/Charts/ChartPrimitives/GroupedBar.js

@@ -3,6 +3,7 @@ import PropTypes from 'prop-types';
 
 import { VictoryChart, VictoryGroup, VictoryBar, VictoryAxis, VictoryContainer } from 'victory';
 import { AXIS_STYLE } from './chartConstants';
+import ChartLoading from 'Components/Charts/ChartPrimitives/ChartLoading';
 import BarSkeleton from 'Components/Elements/Skeletons/BarSkeleton';
 
 function GroupedBar({
@@ -16,7 +17,7 @@ function GroupedBar({
   iAxisProps,
   barProps,
 }) {
-  if (loading) return <BarSkeleton />;
+  if (loading) return <ChartLoading skeleton={BarSkeleton} />
 
   return (
     <VictoryChart

frontend/src/Components/Charts/ChartPrimitives/Line.js

@@ -5,6 +5,7 @@ import { AXIS_STYLE } from './chartConstants';
 
 // Deps
 import { VictoryChart, VictoryLine, VictoryAxis, VictoryContainer } from 'victory';
+import ChartLoading from 'Components/Charts/ChartPrimitives/ChartLoading';
 import BarSkeleton from 'Components/Elements/Skeletons/BarSkeleton';
 
 function Line({
@@ -17,7 +18,7 @@ function Line({
   dAxisProps = {},
   iAxisProps = {},
 }) {
-  if (loading) return <BarSkeleton />;
+  if (loading) return <ChartLoading skeleton={BarSkeleton} />
 
   return (
     <VictoryChart containerComponent={<VictoryContainer style={{ touchAction: 'auto' }} />}>

frontend/src/Components/Charts/ChartPrimitives/Pie.js

@@ -3,6 +3,7 @@ import styled from 'styled-components';
 import { useTheme } from 'styled-components';
 
 // Elements
+import ChartLoading from 'Components/Charts/ChartPrimitives/ChartLoading'
 import PieSkeleton from 'Components/Elements/Skeletons/PieSkeleton';
 import { VictoryPie, VictoryLabel, VictoryTooltip } from 'victory';
 import { P, WEIGHTS } from 'styles/StyledComponents/Typography';
@@ -23,7 +24,7 @@ function Pie({ data, loading }) {
     return d.length === 0 || d.every((dt) => dt.y === 0);
   };
 
-  if (loading) return <PieSkeleton />;
+  if (loading) return <ChartLoading skeleton={PieSkeleton} />;
 
   if (_dataIsZeros(data)) {
     return (

frontend/src/Components/Charts/ChartPrimitives/StackedBar.js

@@ -4,11 +4,12 @@ import PropTypes from 'prop-types';
 import { VictoryChart, VictoryStack, VictoryBar, VictoryAxis, VictoryContainer } from 'victory';
 import { AXIS_STYLE } from './chartConstants';
 
-// Childre
+// Children
+import ChartLoading from 'Components/Charts/ChartPrimitives/ChartLoading';
 import BarSkeleton from 'Components/Elements/Skeletons/BarSkeleton';
 
 function StackedBar({ data, loading, tickValues }) {
-  if (loading) return <BarSkeleton />;
+  if (loading) return <ChartLoading skeleton={BarSkeleton} />
 
   return (
     <VictoryChart