diff --git a/.circleci/config.yml b/.circleci/config.yml index 51d0e13b..68f940b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,15 +8,15 @@ commands: steps: - restore_cache: keys: - - v7-pip-dependencies-{{ checksum "requirements/test/test.txt" }}-{{ checksum "requirements/base/base.txt" }} + - v10-pip-dependencies-{{ checksum "requirements/test/test.txt" }}-{{ checksum "requirements/base/base.txt" }} # fallback to using the latest cache if no exact match is found - - v7-pip-dependencies- + - v10-pip-dependencies- save_cache_cmd: steps: - save_cache: paths: - "env" - key: v7-pip-dependencies-{{ checksum "requirements/test/test.txt" }}-{{ checksum "requirements/base/base.txt" }} + key: v10-pip-dependencies-{{ checksum "requirements/test/test.txt" }}-{{ checksum "requirements/base/base.txt" }} orbs: slack: circleci/slack@3.4.2 diff --git a/.gitignore b/.gitignore index 719d7d18..5c6dec07 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ npm-debug.log .python-version env-local.sh *.dump +*.pgdump .envrc .direnv .vscode diff --git a/Dockerfile b/Dockerfile index 1e9f8748..fbd86ae3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN npm install --silent COPY frontend/ /code/ RUN npm run build -FROM python:3.8-slim as base +FROM python:3.8-slim-bullseye as base # Create a group and user to run our app ARG APP_USER=appuser @@ -43,6 +43,7 @@ RUN set -ex \ build-essential \ libpcre3-dev \ libpq-dev \ + git-core \ " \ && apt-get update && apt-get install -y --no-install-recommends $BUILD_DEPS \ && pip install -U -q pip-tools \ diff --git a/README.rst b/README.rst index 621c9afe..46b938a5 100644 --- a/README.rst +++ b/README.rst @@ -1,18 +1,10 @@ -NC Traffic Stops +NC CopWatch ================ -.. image:: https://badge.waffle.io/OpenDataPolicingNC/Traffic-Stops.svg?label=ready&title=Ready - :target: https://waffle.io/OpenDataPolicingNC/Traffic-Stops - :alt: 'Stories in Ready' +.. image:: https://circleci.com/gh/caktus/Traffic-Stops.svg?style=svg + :target: https://circleci.com/gh/caktus/Traffic-Stops -.. image:: https://readthedocs.org/projects/nc-traffic-stops/badge/?version=latest - :target: http://nc-traffic-stops.readthedocs.org/en/latest/ - :alt: Documentation Status - -.. image:: https://travis-ci.org/OpenDataPolicingNC/Traffic-Stops.svg?branch=master - :target: https://travis-ci.org/OpenDataPolicingNC/Traffic-Stops - -NC Traffic Stops is a website to monitor and identify racial profiling +NC CopWatch is a website to monitor and identify racial profiling practices by North Carolina law enforcement agencies. This project is lead by `Forward Justice`_, a nonpartisan law, policy, and strategy center dedicated to advancing racial, social, and economic justice in the U.S. South. @@ -20,6 +12,4 @@ social, and economic justice in the U.S. South. Please see the `production documentation`_ and `development documentation`_ for more information. -.. _production documentation: http://nc-traffic-stops.readthedocs.org/en/latest/ -.. _development documentation: http://nc-traffic-stops.readthedocs.org/en/dev/ .. _Forward Justice: https://forwardjustice.org/ diff --git a/deploy/deploy-hosting-services.yml b/deploy/deploy-hosting-services.yml index 9c1008f0..574946d4 100644 --- a/deploy/deploy-hosting-services.yml +++ b/deploy/deploy-hosting-services.yml @@ -6,3 +6,7 @@ gather_facts: false roles: - role: caktus.k8s-hosting-services + tasks: + - import_role: + name: caktus.k8s-hosting-services + tasks_from: monitoring diff --git a/deploy/group_vars/all.yml b/deploy/group_vars/all.yml index 2c458a62..4a5cc886 100644 --- a/deploy/group_vars/all.yml +++ b/deploy/group_vars/all.yml @@ -42,6 +42,7 @@ cloudformation_stack: UseAES256Encryption: "true" CustomerManagedCmkArn: "" ContainerInstanceType: t3a.medium + ContainerVolumeSize: 40 DatabaseAllocatedStorage: 100 DatabaseClass: db.t3.large DatabaseEngineVersion: "12" @@ -71,17 +72,33 @@ k8s_ci_repository_arn: arn:aws:ecr:us-east-2:606178775542:repository/traff-appli k8s_ci_vault_password_arn: arn:aws:secretsmanager:us-east-2:606178775542:secret:trafficstops-ansible-vault-password-XKpR8f k8s_letsencrypt_email: admin@caktusgroup.com -# New Relic Infrastructure: Caktus Paid Account -k8s_newrelic_license_key: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 31623963653434303137323231656263643235616539316537346331646133313732316465623865 - 3438623336353035323437653033313434646366383236390a656531636336663530373462323331 - 32643434333833363433663932316534373565663035383334336231313366373763303263393836 - 3035363662323335630a306331303761303434633235616564386362353766336462656535663033 - 31333537343865616436623063386539303339653165636664633736666365623337326363646437 - 6565393035313438666364363231353562613334376135663031 +k8s_iam_users: [copelco] + +# Pin ingress-nginx and cert-manager to current versions so future upgrades of this +# role will not upgrade these charts without your intervention: +# https://github.com/kubernetes/ingress-nginx/releases +k8s_ingress_nginx_chart_version: "3.39.0" +# https://github.com/jetstack/cert-manager/releases +k8s_cert_manager_chart_version: "v1.6.1" +# AWS only: +# Use the newer load balancer type (NLB). DO NOT edit k8s_aws_load_balancer_type after +# creating your Service. +k8s_aws_load_balancer_type: nlb + +# ---------------------------------------------------------------------------- +# caktus.k8s-hosting-services: Logging and monitoring configuration +# ---------------------------------------------------------------------------- k8s_papertrail_logspout_destination: "syslog+tls://logs2.papertrailapp.com:20851" k8s_papertrail_logspout_memory_limit: 128Mi -k8s_iam_users: [copelco] +# New Relic Infrastructure: admin+newrelic@caktusgroup.com +k8s_newrelic_chart_version: "3.2.4" +k8s_newrelic_license_key: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 37656631623333346263383231386165666531333961373931383661366338343634333362356430 + 3963613833663637313632373465613730383365626461630a383432346335386632303935356532 + 61643737636132336339336332396262623362333663333130393031376338363266363430326136 + 6131396135646236360a323766623330313365306539316263393533623063346166653433316631 + 39356263623363653934333064376364363562303236646238666234356136663539343064383463 + 3161356339656137373935623562366134393765346466643365 diff --git a/deploy/requirements.yml b/deploy/requirements.yml index fdd60ed0..9cf8e56b 100644 --- a/deploy/requirements.yml +++ b/deploy/requirements.yml @@ -2,7 +2,7 @@ - src: https://github.com/caktus/ansible-role-django-k8s name: caktus.django-k8s - version: v0.0.11 + version: v1.3.0 - src: https://github.com/caktus/ansible-role-aws-web-stacks name: caktus.aws-web-stacks @@ -10,8 +10,8 @@ - src: https://github.com/caktus/ansible-role-k8s-web-cluster name: caktus.k8s-web-cluster - version: v0.0.7 + version: v1.1.0 - src: https://github.com/caktus/ansible-role-k8s-hosting-services name: caktus.k8s-hosting-services - version: v0.0.1 + version: v0.3.0 diff --git a/docs/deploy.rst b/docs/deploy.rst index 4cc32e89..41d86ac5 100644 --- a/docs/deploy.rst +++ b/docs/deploy.rst @@ -14,7 +14,7 @@ Caktus AWS Access .. code-block:: [trafficstops] - role_arn = arn:aws:iam::000000000000:role/CaktusAccessRole + role_arn = arn:aws:iam::000000000000:role/CaktusAccountAccessRole-Admins source_profile = caktus See LastPass entry *Traffic Stops AWS Profile role_arn* for the AWS account @@ -111,4 +111,3 @@ Deploy application 4. Deploy:: inv staging deploy --tag=... -======= diff --git a/docs/hosting-services.md b/docs/hosting-services.md new file mode 100644 index 00000000..32bfe2ec --- /dev/null +++ b/docs/hosting-services.md @@ -0,0 +1,34 @@ +# Hosting Services + +The services configured for this project are: +* PostgreSQL database backups to S3 (within Caktus AWS account) + * Currently, this is only `traffic_stops`, which contains users, census data, etc. + * `traffic_stops_nc` is not backed up since the entire dataset is re-imported daily. +* Papertrail logging (to Caktus account) +* New Relic Infrastructure monitoring (Account: `admin+newrelic@caktusgroup.com`) + + +## Production database disaster recovery + +In the event a restore from a historical backup is needed, access to the [Caktus +AssumeRole is +required](https://github.com/caktus/caktus-hosting-services/blob/main/docs/aws-assumerole.md#aws-accounts). +Once you have that access, you can use invoke tools to pull historical backups. + +To download the latest `daily` backup: + +```sh +inv utils.get-db-backup +``` + + +## Production backup configuration + +[caktus.k8s-hosting-services](https://github.com/caktus/ansible-role-k8s-hosting-services) +manages database backups. + +Run this command to set up database backups and monitoring services: + +```sh +inv deploy.playbook -n deploy-hosting-services.yml +``` diff --git a/frontend/src/Components/Charts/ChartPrimitives/Bar.js b/frontend/src/Components/Charts/ChartPrimitives/Bar.js index 947581b7..be1bed30 100644 --- a/frontend/src/Components/Charts/ChartPrimitives/Bar.js +++ b/frontend/src/Components/Charts/ChartPrimitives/Bar.js @@ -2,10 +2,11 @@ import React from 'react'; import PropTypes from 'prop-types'; import { VictoryAxis, VictoryBar, VictoryChart, VictoryContainer } from 'victory'; import { AXIS_STYLE } from './chartConstants'; +import ChartLoading from 'Components/Charts/ChartPrimitives/ChartLoading'; import BarSkeleton from 'Components/Elements/Skeletons/BarSkeleton'; function Bar({ data, chartProps, xAxisProps, yAxisProps, barProps }) { - if (!data) return ; + if (!data) return ; return ( {hasError &&

some chart error message

} - {isLoading && } + {isLoading && }

{chartTitle}

{!hideLegend && ( -

ChartLoading

- - ); - } else { - return null; - } +// Hooks +import useOfficerId from 'Hooks/useOfficerId'; + +function ChartLoading({ skeleton: Skeleton }) { + const officerId = useOfficerId(); + + return ( + +

Loading {officerId ? "Officer" : "Agency"} data...

+ + + ); } export default ChartLoading; diff --git a/frontend/src/Components/Charts/ChartPrimitives/ChartLoading.styled.js b/frontend/src/Components/Charts/ChartPrimitives/ChartLoading.styled.js index ceb68d76..84b525e5 100644 --- a/frontend/src/Components/Charts/ChartPrimitives/ChartLoading.styled.js +++ b/frontend/src/Components/Charts/ChartPrimitives/ChartLoading.styled.js @@ -1,3 +1,11 @@ import styled from 'styled-components'; -export const ChartLoadingStyled = styled.div``; +export const ChartLoading = styled.div` + padding: 2rem 0; + h3 { + text-align: center; + font-size: 28px; + font-weight: 200; + color: ${props => props.theme.colors.grey}; + } +`; diff --git a/frontend/src/Components/Charts/ChartPrimitives/DataLoading.js b/frontend/src/Components/Charts/ChartPrimitives/DataLoading.js new file mode 100644 index 00000000..789ba94f --- /dev/null +++ b/frontend/src/Components/Charts/ChartPrimitives/DataLoading.js @@ -0,0 +1,21 @@ +import React from 'react'; +import { DataLoadingStyled } from './DataLoading.styled'; + +// Hooks +import useOfficerId from 'Hooks/useOfficerId'; + +// Children +import PieSkeleton from 'Components/Elements/Skeletons/PieSkeleton'; + +function DataLoading() { + const officerId = useOfficerId(); + + return ( + +

Loading {officerId ? "Officer" : "Agency"} data...

+ + + ); +} + +export default DataLoading; diff --git a/frontend/src/Components/Charts/ChartPrimitives/DataLoading.styled.js b/frontend/src/Components/Charts/ChartPrimitives/DataLoading.styled.js new file mode 100644 index 00000000..2e13471a --- /dev/null +++ b/frontend/src/Components/Charts/ChartPrimitives/DataLoading.styled.js @@ -0,0 +1,11 @@ +import styled from 'styled-components'; + +export const DataLoadingStyled = styled.div` + padding: 2rem 0; + h3 { + text-align: center; + font-size: 28px; + font-weight: 200; + color: ${props => props.theme.colors.grey}; + } +`; diff --git a/frontend/src/Components/Charts/ChartPrimitives/GroupedBar.js b/frontend/src/Components/Charts/ChartPrimitives/GroupedBar.js index 31f238d8..691596ed 100644 --- a/frontend/src/Components/Charts/ChartPrimitives/GroupedBar.js +++ b/frontend/src/Components/Charts/ChartPrimitives/GroupedBar.js @@ -3,6 +3,7 @@ import PropTypes from 'prop-types'; import { VictoryChart, VictoryGroup, VictoryBar, VictoryAxis, VictoryContainer } from 'victory'; import { AXIS_STYLE } from './chartConstants'; +import ChartLoading from 'Components/Charts/ChartPrimitives/ChartLoading'; import BarSkeleton from 'Components/Elements/Skeletons/BarSkeleton'; function GroupedBar({ @@ -16,7 +17,7 @@ function GroupedBar({ iAxisProps, barProps, }) { - if (loading) return ; + if (loading) return return ( ; + if (loading) return return ( }> diff --git a/frontend/src/Components/Charts/ChartPrimitives/Pie.js b/frontend/src/Components/Charts/ChartPrimitives/Pie.js index e4888e7a..83aa173c 100644 --- a/frontend/src/Components/Charts/ChartPrimitives/Pie.js +++ b/frontend/src/Components/Charts/ChartPrimitives/Pie.js @@ -3,6 +3,7 @@ import styled from 'styled-components'; import { useTheme } from 'styled-components'; // Elements +import ChartLoading from 'Components/Charts/ChartPrimitives/ChartLoading' import PieSkeleton from 'Components/Elements/Skeletons/PieSkeleton'; import { VictoryPie, VictoryLabel, VictoryTooltip } from 'victory'; import { P, WEIGHTS } from 'styles/StyledComponents/Typography'; @@ -23,7 +24,7 @@ function Pie({ data, loading }) { return d.length === 0 || d.every((dt) => dt.y === 0); }; - if (loading) return ; + if (loading) return ; if (_dataIsZeros(data)) { return ( diff --git a/frontend/src/Components/Charts/ChartPrimitives/StackedBar.js b/frontend/src/Components/Charts/ChartPrimitives/StackedBar.js index f0e21096..c978d579 100644 --- a/frontend/src/Components/Charts/ChartPrimitives/StackedBar.js +++ b/frontend/src/Components/Charts/ChartPrimitives/StackedBar.js @@ -4,11 +4,12 @@ import PropTypes from 'prop-types'; import { VictoryChart, VictoryStack, VictoryBar, VictoryAxis, VictoryContainer } from 'victory'; import { AXIS_STYLE } from './chartConstants'; -// Childre +// Children +import ChartLoading from 'Components/Charts/ChartPrimitives/ChartLoading'; import BarSkeleton from 'Components/Elements/Skeletons/BarSkeleton'; function StackedBar({ data, loading, tickValues }) { - if (loading) return ; + if (loading) return return ( import(/* webpackChunkName: 'Overview' */ 'Components/Charts/Overview/Overview') } - renderLoading={() => } + renderLoading={() => } renderError={() => } /> } + renderLoading={() => } renderError={() => } /> import(/* webpackChunkName: 'Searches' */ 'Components/Charts/Searches/Searches') } - renderLoading={() => } + renderLoading={() => } renderError={() => } /> import(/* webpackChunkName: 'SearchRate' */ 'Components/Charts/SearchRate/SearchRate') } - renderLoading={() => } + renderLoading={() => } renderError={() => } /> import(/* webpackChunkName: 'Contraband' */ 'Components/Charts/Contraband/Contraband') } - renderLoading={() => } + renderLoading={() => } renderError={() => } /> @@ -68,7 +69,7 @@ function Charts() { importComponent={() => import(/* webpackChunkName: 'UseOfForce' */ 'Components/Charts/UseOfForce/UseOfForce') } - renderLoading={() => } + renderLoading={() => } renderError={() => } /> diff --git a/frontend/src/Components/Charts/ChartSections/ChartPageBase.styled.js b/frontend/src/Components/Charts/ChartSections/ChartPageBase.styled.js index ec025d49..8a8beb5f 100644 --- a/frontend/src/Components/Charts/ChartSections/ChartPageBase.styled.js +++ b/frontend/src/Components/Charts/ChartSections/ChartPageBase.styled.js @@ -5,6 +5,8 @@ import { phoneOnly, smallerThanDesktop } from 'styles/breakpoints'; export const ChartPageBase = styled(motion.article)` flex: 1; overflow-y: scroll; + align-self: flex-start; + height: 100%; `; export const ChartPageContent = styled.div` diff --git a/frontend/src/Components/Elements/Skeletons/BarSkeleton.js b/frontend/src/Components/Elements/Skeletons/BarSkeleton.js index 937ff911..1c0ee606 100644 --- a/frontend/src/Components/Elements/Skeletons/BarSkeleton.js +++ b/frontend/src/Components/Elements/Skeletons/BarSkeleton.js @@ -4,7 +4,7 @@ import React from 'react'; import ContentLoader from 'react-content-loader'; import LoaderBase from 'Components/Elements/Skeletons/LoaderBase'; -function PieSkeleton() { +function BarSkeleton() { return ( parseInt(val, 10) * scale return ( - - - - - - - - - - + + + + + + + + + + ); diff --git a/nc/admin.py b/nc/admin.py index df2996bf..b76f116b 100644 --- a/nc/admin.py +++ b/nc/admin.py @@ -1,9 +1,45 @@ from django.contrib import admin -from nc.models import Agency +from nc.models import Agency, StopSummary class AgencyAdmin(admin.ModelAdmin): list_display = ("name", "census_profile_id") +class StopSummaryAdmin(admin.ModelAdmin): + list_display = ( + "id", + "agency_name", + "year", + "stop_purpose", + "engage_force", + "search_type", + "contraband_found", + "officer_id", + "driver_race", + "driver_ethnicity", + "count", + ) + list_filter = ( + "stop_purpose", + "engage_force", + "search_type", + "contraband_found", + "year", + "agency", + ) + list_select_related = ("agency",) + search_fields = ( + "id", + "count", + "officer_id", + "agency__name", + ) + ordering = ("id",) + + def agency_name(self, obj): + return obj.agency.name + + admin.site.register(Agency, AgencyAdmin) +admin.site.register(StopSummary, StopSummaryAdmin) diff --git a/nc/data/importer.py b/nc/data/importer.py index a73a690a..3256e9d9 100644 --- a/nc/data/importer.py +++ b/nc/data/importer.py @@ -11,7 +11,7 @@ from django.core.mail import EmailMessage from django.db import connections, transaction from nc.data import copy_nc -from nc.models import Agency, Search, Stop +from nc.models import Agency, Search, Stop, StopSummary from nc.prime_cache import run as prime_cache_run from tsdata.dataset_facts import compute_dataset_facts from tsdata.sql import drop_constraints_and_indexes @@ -24,7 +24,9 @@ MAGIC_NC_FTP_URL = "ftp://nc.us/" -def run(url, destination=None, zip_path=None, min_stop_id=None, max_stop_id=None, prime_cache=True): +def run( + url, destination=None, zip_path=None, min_stop_id=None, max_stop_id=None, prime_cache=False +): """ Download NC data, extract, convert to CSV, and load into PostgreSQL @@ -92,6 +94,11 @@ def run(url, destination=None, zip_path=None, min_stop_id=None, max_stop_id=None ) logger.info("NC dataset facts: %r", facts) + # update materialized view + logger.info("Updating materialized view") + StopSummary.refresh() + logger.info("Materialized view updated") + # prime the query cache for large NC agencies if prime_cache: prime_cache_run() diff --git a/nc/management/commands/prime_cache.py b/nc/management/commands/prime_cache.py index 56414de9..e9aed342 100644 --- a/nc/management/commands/prime_cache.py +++ b/nc/management/commands/prime_cache.py @@ -11,7 +11,19 @@ def add_arguments(self, parser): dest="cutoff", help="Stop priming cache for agencies once it takes less than this", ) + parser.add_argument("--clear-cache", "-c", action="store_true", default=False) + parser.add_argument("--skip-agencies", action="store_true", default=False) + parser.add_argument("--skip-officers", action="store_true", default=False) + parser.add_argument( + "--officer-cutoff-count", type=int, default=None, + ) def handle(self, *args, **options): cutoff = float(options["cutoff"]) if options["cutoff"] else None - prime_cache.run(cutoff_duration_secs=cutoff) + prime_cache.run( + cutoff_duration_secs=cutoff, + clear_cache=options["clear_cache"], + skip_agencies=options["skip_agencies"], + skip_officers=options["skip_officers"], + officer_cutoff_count=options["officer_cutoff_count"], + ) diff --git a/nc/migrations/0005_stopsummary.py b/nc/migrations/0005_stopsummary.py new file mode 100644 index 00000000..7479ae78 --- /dev/null +++ b/nc/migrations/0005_stopsummary.py @@ -0,0 +1,31 @@ +# Generated by Django 2.2.17 on 2021-11-13 15:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('nc', '0004_agency_last_reported_stop'), + ] + + operations = [ + migrations.CreateModel( + name='StopSummary', + fields=[ + ('id', models.PositiveIntegerField(primary_key=True, serialize=False)), + ('year', models.IntegerField()), + ('stop_purpose', models.PositiveSmallIntegerField(choices=[(1, 'Speed Limit Violation'), (2, 'Stop Light/Sign Violation'), (3, 'Driving While Impaired'), (4, 'Safe Movement Violation'), (5, 'Vehicle Equipment Violation'), (6, 'Vehicle Regulatory Violation'), (7, 'Seat Belt Violation'), (8, 'Investigation'), (9, 'Other Motor Vehicle Violation'), (10, 'Checkpoint')])), + ('engage_force', models.BooleanField()), + ('search_type', models.PositiveSmallIntegerField(choices=[(1, 'Consent'), (2, 'Search Warrant'), (3, 'Probable Cause'), (4, 'Search Incident to Arrest'), (5, 'Protective Frisk')])), + ('contraband_found', models.BooleanField()), + ('officer_id', models.CharField(max_length=15)), + ('driver_race', models.CharField(choices=[('A', 'Asian'), ('B', 'Black'), ('I', 'Native American'), ('U', 'Other'), ('W', 'White')], max_length=2)), + ('driver_ethnicity', models.CharField(choices=[('H', 'Hispanic'), ('N', 'Non-Hispanic')], max_length=2)), + ('count', models.IntegerField()), + ], + options={ + 'managed': False, + }, + ), + ] diff --git a/nc/models.py b/nc/models.py index 40c934e1..cb2d046c 100755 --- a/nc/models.py +++ b/nc/models.py @@ -1,5 +1,6 @@ from caching.base import CachingManager, CachingMixin from django.db import models +from django_pgviews import view as pg from tsdata.models import CensusProfile PURPOSE_CHOICES = ( @@ -160,3 +161,60 @@ def census_profile(self): return profile.get_census_dict() else: return dict() + + +STOP_SUMMARY_VIEW_SQL = """ + SELECT + ROW_NUMBER() OVER () AS id + , "nc_stop"."agency_id" + , DATE_PART('year', DATE_TRUNC('year', date AT TIME ZONE 'America/New_York'))::integer AS "year" + , "nc_stop"."purpose" AS "stop_purpose" + , "nc_stop"."engage_force" + , "nc_search"."type" AS "search_type" + , (CASE + WHEN nc_contraband.contraband_id IS NULL THEN false + ELSE true + END) AS contraband_found + , "nc_stop"."officer_id" + , "nc_person"."race" AS "driver_race" + , "nc_person"."ethnicity" AS "driver_ethnicity" + , COUNT("nc_stop"."date")::integer AS "count" + FROM "nc_stop" + INNER JOIN "nc_person" + ON ("nc_stop"."stop_id" = "nc_person"."stop_id" AND "nc_person"."type" = 'D') + LEFT OUTER JOIN "nc_search" + ON ("nc_stop"."stop_id" = "nc_search"."stop_id") + LEFT OUTER JOIN "nc_contraband" + ON ("nc_stop"."stop_id" = "nc_contraband"."stop_id") + GROUP BY + 2, 3, 4, 5, 6, 7, 8, 9, 10 + ORDER BY "agency_id", "year" ASC; +""" # noqa + + +class StopSummary(pg.ReadOnlyMaterializedView): + sql = STOP_SUMMARY_VIEW_SQL + # Don't create view with data, this will be manually managed + # and refreshed by the data import process + # https://github.com/mikicz/django-pgviews#with-no-data + with_data = False + + id = models.PositiveIntegerField(primary_key=True) + year = models.IntegerField() + agency = models.ForeignKey("Agency", on_delete=models.DO_NOTHING) + stop_purpose = models.PositiveSmallIntegerField(choices=PURPOSE_CHOICES) + engage_force = models.BooleanField() + search_type = models.PositiveSmallIntegerField(choices=SEARCH_TYPE_CHOICES) + contraband_found = models.BooleanField() + officer_id = models.CharField(max_length=15) + driver_race = models.CharField(max_length=2, choices=RACE_CHOICES) + driver_ethnicity = models.CharField(max_length=2, choices=ETHNICITY_CHOICES) + count = models.IntegerField() + + class Meta: + managed = False + indexes = [ + models.Index(fields=["agency", "officer_id", "search_type"]), + models.Index(fields=["engage_force"]), + models.Index(fields=["contraband_found"]), + ] diff --git a/nc/prime_cache.py b/nc/prime_cache.py index c0d3e9a0..c95da01d 100644 --- a/nc/prime_cache.py +++ b/nc/prime_cache.py @@ -1,20 +1,21 @@ import logging -import time +from time import perf_counter from django.conf import settings -from django.db.models import Count +from django.core.cache import cache +from django.db.models import Count, F, Sum from django.test.client import Client from django.urls import reverse -from nc.models import Agency +from nc.models import Stop, StopSummary logger = logging.getLogger(__name__) -ENDPOINTS = ( - "stops", - "stops_by_reason", - "use_of_force", - "searches", - "searches_by_type", - "contraband_hit_rate", +API_ENDPOINT_NAMES = ( + "nc:agency-api-stops", + "nc:agency-api-stops-by-reason", + "nc:agency-api-searches", + "nc:agency-api-searches-by-type", + "nc:agency-api-contraband-hit-rate", + "nc:agency-api-use-of-force", ) DEFAULT_CUTOFF_SECS = 4 @@ -51,7 +52,120 @@ def avoid_newrelic_bug(): pass -def run(cutoff_duration_secs=None): +class Timer: + def __init__(self, cutoff): + self.cutoff = cutoff + + def __enter__(self): + self.start = perf_counter() + return self + + def __exit__(self, type, value, traceback): + self.elapsed = perf_counter() - self.start + self.stop = self.elapsed < self.cutoff + self.readout = f"{self.elapsed} < {self.cutoff} = {self.stop}" + + +class CachePrimer: + def __init__(self, cutoff_secs=0, cutoff_count=None): + self.cutoff_secs = cutoff_secs + self.cutoff_count = cutoff_count + self.count = 0 + + def request(self, uri, payload=None): + c = Client() + if settings.ALLOWED_HOSTS and settings.ALLOWED_HOSTS[0] != "*": + host = settings.ALLOWED_HOSTS[0] + else: + host = "127.0.0.1" + logger.debug(f"Querying {uri}") + response = c.get(uri, data=payload, HTTP_HOST=host) + if response.status_code != 200: + logger.warning("Status not OK: {} ({})".format(uri, response.status_code)) + raise Exception("Request to %s failed: %s", uri, response.status_code) + + def get_endpoints(self): + for idx, row in enumerate(self.get_queryset()): + with Timer(self.cutoff_secs) as timer: + yield self.get_urls(row) + officer_id = row.get("officer_id", "") + logger.info( + ( + "Primed cache for agency %s:%s " + "[officer_id=%s] with " + "%s stops in %.2f secs (%s of %s)" + ), + row["agency_id"], + row["agency_name"], + officer_id, + "{:,}".format(row["num_stops"]), + timer.elapsed, + idx, + self.count, + ) + if timer.stop or (self.cutoff_count and idx == self.cutoff_count): + logger.info("Cutoff reached, stopping...") + break + + def prime(self): + logger.info(f"{self} starting") + self.count = self.get_queryset().count() + logger.info(f"{self} priming {self.count:,} objects") + for endpoints in self.get_endpoints(): + for endpoint in endpoints: + self.request(endpoint) + + def __repr__(self): + options = [] + if self.cutoff_secs: + options.append(f"cutoff_secs={self.cutoff_secs}") + if self.cutoff_count: + options.append(f"cutoff_count={self.cutoff_count}") + return f"<{self.__class__.__name__} {' '.join(options)}>" + + +class AgencyStopsPrimer(CachePrimer): + def get_queryset(self): + return ( + Stop.objects.no_cache() + .annotate(agency_name=F("agency_description")) + .values("agency_name", "agency_id") + .annotate(num_stops=Count("stop_id")) + .order_by("-num_stops") + ) + + def get_urls(self, row): + urls = [] + for endpoint_name in API_ENDPOINT_NAMES: + urls.append(reverse(endpoint_name, args=[row["agency_id"]])) + return urls + + +class OfficerStopsPrimer(CachePrimer): + def get_queryset(self): + return ( + StopSummary.objects.all() + .annotate(agency_name=F("agency__name")) + .values("agency_name", "agency_id", "officer_id") + .annotate(num_stops=Sum("count")) + .order_by("-num_stops") + ) + + def get_urls(self, row): + urls = [] + for endpoint_name in API_ENDPOINT_NAMES: + agency_url = reverse(endpoint_name, args=[row["agency_id"]]) + urls.append(f"{agency_url}?officer={row['officer_id']}") + return urls + + +def run( + cutoff_duration_secs=None, + clear_cache=False, + skip_agencies=False, + skip_officers=False, + officer_cutoff_count=None, +): """ Prime query cache for "big" NC agencies. @@ -78,51 +192,16 @@ def run(cutoff_duration_secs=None): avoid_newrelic_bug() - logger.info("NC prime_cache starting") - logger.info("Finding largest agencies by stop counts") - agencies = [ - (a.id, a.name, a.num_stops) - for a in Agency.objects.annotate(num_stops=Count("stops")).order_by("-num_stops") - ] - api = reverse("nc:agency-api-list") - agencies_processed = 0 - for agency_id, agency_name, num_stops in agencies: - elapsed = [] # collect times for each request - # prime each API endpoint - for endpoint in ENDPOINTS: - uri = "{}/{}/{}/".format(api.rstrip("/"), agency_id, endpoint) - start_time = time.time() - req(uri) - elapsed.append(time.time() - start_time) - # # prime first search page - # payload = {"agency": agency_name} - # search_uri = reverse("nc:stops-search") - # start_time = time.time() - # req(search_uri, payload) - elapsed.append(time.time() - start_time) - elapsed = sum(elapsed) - logger.info( - "Primed cache for agency %s:%s with %s stops in %.2f secs", - agency_id, - agency_name, - "{:,}".format(num_stops), - elapsed, - ) - agencies_processed += 1 - num_remaining_agencies = len(agencies) - agencies_processed - if elapsed < cutoff_duration_secs and num_remaining_agencies > 0: - logger.info("Not priming cache for %s remaining agencies", num_remaining_agencies) - break - - -def req(uri, payload=None): - c = Client() - if settings.ALLOWED_HOSTS and settings.ALLOWED_HOSTS[0] != "*": - host = settings.ALLOWED_HOSTS[0] - else: - host = "127.0.0.1" - logger.debug(f"Querying {uri}") - response = c.get(uri, data=payload, HTTP_HOST=host) - if response.status_code != 200: - logger.warning("Status not OK: {} ({})".format(uri, response.status_code)) - raise Exception("Request to %s failed: %s", uri, response.status_code) + if clear_cache: + logger.info("Clearing cache") + cache.clear() + + if not skip_agencies: + AgencyStopsPrimer(cutoff_secs=cutoff_duration_secs).prime() + + if not skip_officers: + OfficerStopsPrimer( + cutoff_secs=0, cutoff_count=officer_cutoff_count + ).prime() # cache all officer endpoins for now + + logger.info("Complete") diff --git a/nc/tests/factories.py b/nc/tests/factories.py index 30af5e83..b2a775ce 100644 --- a/nc/tests/factories.py +++ b/nc/tests/factories.py @@ -5,14 +5,25 @@ from nc import models -class AgencyFactory(factory.django.DjangoModelFactory): +class ViewRefreshFactory(factory.django.DjangoModelFactory): + """ + Refresh materialized view after object creation so tests don't have to + manually invoke this functionality. + """ + + @factory.post_generation + def refresh_view(obj, create, extracted, **kwargs): + models.StopSummary.refresh() + + +class AgencyFactory(ViewRefreshFactory): class Meta(object): model = models.Agency name = factory.Sequence(lambda n: "Agency %03d" % n) -class PersonFactory(factory.django.DjangoModelFactory): +class PersonFactory(ViewRefreshFactory): class Meta(object): model = models.Person @@ -24,7 +35,7 @@ class Meta(object): type = "D" -class StopFactory(factory.django.DjangoModelFactory): +class StopFactory(ViewRefreshFactory): class Meta(object): model = models.Stop @@ -47,7 +58,7 @@ def year(self, create, extracted, **kwargs): self.date = self.date.replace(year=extracted, day=day) -class SearchFactory(factory.django.DjangoModelFactory): +class SearchFactory(ViewRefreshFactory): class Meta(object): model = models.Search @@ -57,7 +68,7 @@ class Meta(object): type = factory.fuzzy.FuzzyChoice(x[0] for x in models.SEARCH_TYPE_CHOICES) -class ContrabandFactory(factory.django.DjangoModelFactory): +class ContrabandFactory(ViewRefreshFactory): class Meta(object): model = models.Contraband diff --git a/nc/views.py b/nc/views.py index 79b5f0c5..c33dba32 100644 --- a/nc/views.py +++ b/nc/views.py @@ -1,11 +1,9 @@ -from django.conf import settings -from django.db import connections -from django.db.models import Count, Q +from django.db.models import Q, Sum from django_filters.rest_framework import DjangoFilterBackend from nc import serializers from nc.filters import DriverStopsFilter from nc.models import SEARCH_TYPE_CHOICES as SEARCH_TYPE_CHOICES_TUPLES -from nc.models import Agency, Person, Stop +from nc.models import Agency, Person, StopSummary from nc.pagination import NoCountPagination from rest_framework import viewsets from rest_framework.decorators import action @@ -62,13 +60,8 @@ class AgencyViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = serializers.AgencySerializer def query(self, results, group_by, filter_=None): - # date trunc on year, respecting NC time zone - year_sql = connections[Stop.objects.db].ops.datetime_trunc_sql( - "year", "date", settings.NC_TIME_ZONE, - ) - qs = Stop.objects.extra(select={"year": year_sql}) - # filter down stops by agency only those who were drivers - qs = qs.filter(agency=self.get_object(), person__type="D") + # filter down stops by agency + qs = StopSummary.objects.filter(agency=self.get_object()) # filter down by officer if supplied officer = self.request.query_params.get("officer", None) if officer: @@ -77,26 +70,27 @@ def query(self, results, group_by, filter_=None): qs = qs.filter(filter_) # group by specified fields and order by year qs = qs.values(*group_by).order_by("year") - for stop in qs.annotate(count=Count("date")): + qs = qs.annotate(count=Sum("count")) + for stop in qs: data = {} if "year" in group_by: - data["year"] = stop["year"].year - if "purpose" in group_by: - purpose = PURPOSE_CHOICES.get(stop["purpose"], stop["purpose"]) + data["year"] = stop["year"] + if "stop_purpose" in group_by: + purpose = PURPOSE_CHOICES.get(stop["stop_purpose"], stop["stop_purpose"]) data["purpose"] = purpose - if "search__type" in group_by: + if "search_type" in group_by: data["search_type"] = SEARCH_TYPE_CHOICES.get( - stop["search__type"], stop["search__type"], + stop["search_type"], stop["search_type"], ) - if "person__race" in group_by: + if "driver_race" in group_by: # The 'Hispanic' ethnicity option is now being aggreggated into its # own race category, and its count excluded from the other counts. - if stop["person__ethnicity"] == "H": + if stop["driver_ethnicity"] == "H": race = GROUPS.get("H", "H") else: - race = GROUPS.get(stop["person__race"], stop["person__race"]) + race = GROUPS.get(stop["driver_race"], stop["driver_race"]) data.setdefault(race, 0) data[race] += stop["count"] @@ -107,7 +101,7 @@ def query(self, results, group_by, filter_=None): @cache_response(key_func=query_cache_key_func) def stops(self, request, pk=None): results = GroupedData(by="year", defaults=GROUP_DEFAULTS) - self.query(results, group_by=("year", "person__race", "person__ethnicity")) + self.query(results, group_by=("year", "driver_race", "driver_ethnicity")) return Response(results.flatten()) @action(detail=True, methods=["get"]) @@ -116,14 +110,14 @@ def stops_by_reason(self, request, pk=None): response = {} # stops results = GroupedData(by=("purpose", "year"), defaults=GROUP_DEFAULTS) - self.query(results, group_by=("purpose", "year", "person__race", "person__ethnicity")) + self.query(results, group_by=("stop_purpose", "year", "driver_race", "driver_ethnicity")) response["stops"] = results.flatten() # searches results = GroupedData(by=("purpose", "year"), defaults=GROUP_DEFAULTS) self.query( results, - group_by=("purpose", "year", "person__race", "person__ethnicity"), - filter_=Q(search__isnull=False), + group_by=("stop_purpose", "year", "driver_race", "driver_ethnicity"), + filter_=Q(search_type__isnull=False), ) response["searches"] = results.flatten() return Response(response) @@ -132,26 +126,26 @@ def stops_by_reason(self, request, pk=None): @cache_response(key_func=query_cache_key_func) def use_of_force(self, request, pk=None): results = GroupedData(by="year", defaults=GROUP_DEFAULTS) - q = Q(search__isnull=False) & Q(engage_force="t") - self.query(results, group_by=("year", "person__race", "person__ethnicity"), filter_=q) + q = Q(search_type__isnull=False) & Q(engage_force="t") + self.query(results, group_by=("year", "driver_race", "driver_ethnicity"), filter_=q) return Response(results.flatten()) @action(detail=True, methods=["get"]) @cache_response(key_func=query_cache_key_func) def searches(self, request, pk=None): results = GroupedData(by="year", defaults=GROUP_DEFAULTS) - q = Q(search__isnull=False) - self.query(results, group_by=("year", "person__race", "person__ethnicity"), filter_=q) + q = Q(search_type__isnull=False) + self.query(results, group_by=("year", "driver_race", "driver_ethnicity"), filter_=q) return Response(results.flatten()) @action(detail=True, methods=["get"]) - @cache_response(key_func=query_cache_key_func) + # @cache_response(key_func=query_cache_key_func) def searches_by_type(self, request, pk=None): results = GroupedData(by=("search_type", "year"), defaults=GROUP_DEFAULTS) - q = Q(search__isnull=False) + q = Q(search_type__isnull=False) self.query( results, - group_by=("search__type", "year", "person__race", "person__ethnicity",), + group_by=("search_type", "year", "driver_race", "driver_ethnicity",), filter_=q, ) return Response(results.flatten()) @@ -162,13 +156,13 @@ def contraband_hit_rate(self, request, pk=None): response = {} # searches results = GroupedData(by="year", defaults=GROUP_DEFAULTS) - q = Q(search__isnull=False) - self.query(results, group_by=("year", "person__race", "person__ethnicity"), filter_=q) + q = Q(search_type__isnull=False) + self.query(results, group_by=("year", "driver_race", "driver_ethnicity"), filter_=q) response["searches"] = results.flatten() # searches results = GroupedData(by="year", defaults=GROUP_DEFAULTS) - q = Q(search__contraband__isnull=False) - self.query(results, group_by=("year", "person__race", "person__ethnicity"), filter_=q) + q = Q(contraband_found=True) + self.query(results, group_by=("year", "driver_race", "driver_ethnicity"), filter_=q) response["contraband"] = results.flatten() return Response(response) diff --git a/requirements/base/base.in b/requirements/base/base.in index 525b479a..1f643795 100644 --- a/requirements/base/base.in +++ b/requirements/base/base.in @@ -5,12 +5,13 @@ census us dealer django-cache-machine -django-click +django-click==2.2.0 django-crispy-forms django-dotenv django-extensions django-filter django-memoize +django-pgviews-redux==0.8.0 django-redis djangorestframework dj-database-url diff --git a/requirements/base/base.txt b/requirements/base/base.txt index 99bb1c1e..0916f212 100644 --- a/requirements/base/base.txt +++ b/requirements/base/base.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile --output-file=requirements/base/base.txt requirements/base/base.in @@ -16,15 +16,22 @@ certifi==2020.6.20 # via requests chardet==3.0.4 # via requests -click==7.0 +click==7.1.2 # via django-click dealer==2.1.0 # via -r requirements/base/base.in dj-database-url==0.5.0 # via -r requirements/base/base.in +django==2.2.17 + # via + # -r requirements/base/base.in + # django-filter + # django-memoize + # django-redis + # djangorestframework django-cache-machine==1.1.0 # via -r requirements/base/base.in -django-click==2.1.1 +django-click==2.2.0 # via -r requirements/base/base.in django-crispy-forms==1.10.0 # via -r requirements/base/base.in @@ -36,15 +43,10 @@ django-filter==2.4.0 # via -r requirements/base/base.in django-memoize==2.3.1 # via -r requirements/base/base.in +django-pgviews-redux==0.8.0 + # via -r requirements/base/base.in django-redis==4.12.1 # via -r requirements/base/base.in -django==2.2.17 - # via - # -r requirements/base/base.in - # django-filter - # django-memoize - # django-redis - # djangorestframework djangorestframework==3.11.0 # via # -r requirements/base/base.in @@ -83,7 +85,6 @@ requests==2.24.0 six==1.15.0 # via # -r requirements/base/base.in - # django-click # django-extensions # python-dateutil sqlparse==0.3.1 diff --git a/requirements/deploy/deploy.txt b/requirements/deploy/deploy.txt index 28180d5f..ca5cbb35 100644 --- a/requirements/deploy/deploy.txt +++ b/requirements/deploy/deploy.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile --output-file=requirements/deploy/deploy.txt requirements/deploy/deploy.in diff --git a/requirements/dev/dev.in b/requirements/dev/dev.in index c602debf..6983a8f7 100644 --- a/requirements/dev/dev.in +++ b/requirements/dev/dev.in @@ -3,7 +3,8 @@ -c ../test/test.txt # deploy --e git+https://github.com/caktus/invoke-kubesae@0.0.14#egg=invoke-kubesae +invoke-kubesae==0.0.16 +ansible==4.8.0 troposphere # docs diff --git a/requirements/dev/dev.txt b/requirements/dev/dev.txt index 313badd5..d95569bd 100644 --- a/requirements/dev/dev.txt +++ b/requirements/dev/dev.txt @@ -1,17 +1,21 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile --output-file=requirements/dev/dev.txt requirements/dev/dev.in # --e git+https://github.com/caktus/invoke-kubesae@0.0.14#egg=invoke-kubesae - # via -r requirements/dev/dev.in alabaster==0.7.12 # via sphinx -ansible==2.9.6 - # via invoke-kubesae +ansible==4.8.0 + # via + # -r requirements/dev/dev.in + # invoke-kubesae +ansible-core==2.11.6 + # via ansible appnope==0.1.0 - # via -r requirements/dev/dev.in + # via + # -r requirements/dev/dev.in + # ipython argh==0.26.2 # via sphinx-autobuild attrs==19.3.0 @@ -24,12 +28,12 @@ babel==2.8.0 # via sphinx backcall==0.1.0 # via ipython +boto==2.49.0 + # via -r requirements/dev/dev.in boto3==1.17.35 # via # -r requirements/dev/dev.in # invoke-kubesae -boto==2.49.0 - # via -r requirements/dev/dev.in botocore==1.20.35 # via # awscli @@ -50,7 +54,7 @@ chardet==3.0.4 # via # -c requirements/dev/../base/base.txt # requests -click==7.0 +click==7.1.2 # via # -c requirements/dev/../base/base.txt # -c requirements/dev/../test/test.txt @@ -60,19 +64,19 @@ colorama==0.4.3 # awscli # invoke-kubesae cryptography==2.9 - # via ansible + # via ansible-core decorator==4.4.2 # via # ipython # traitlets dictdiffer==0.8.1 # via openshift -django-debug-toolbar==2.2 - # via -r requirements/dev/dev.in django==2.2.17 # via # -c requirements/dev/../base/base.txt # django-debug-toolbar +django-debug-toolbar==2.2 + # via -r requirements/dev/dev.in docutils==0.15.2 # via # awscli @@ -88,15 +92,17 @@ imagesize==1.2.0 # via sphinx invoke==1.4.1 # via invoke-kubesae -ipython-genutils==0.2.0 - # via traitlets +invoke-kubesae==0.0.16 + # via -r requirements/dev/dev.in ipython==7.14.0 # via -r requirements/dev/dev.in +ipython-genutils==0.2.0 + # via traitlets jedi==0.17.0 # via ipython jinja2==2.11.2 # via - # ansible + # ansible-core # openshift # sphinx jmespath==0.10.0 @@ -105,10 +111,10 @@ jmespath==0.10.0 # botocore jsonschema==3.2.0 # via kubernetes-validate -kubernetes-validate==1.18.0 - # via -r requirements/dev/dev.in kubernetes==11.0.0 # via openshift +kubernetes-validate==1.18.0 + # via -r requirements/dev/dev.in livereload==2.6.2 # via sphinx-autobuild markupsafe==1.1.1 @@ -120,6 +126,7 @@ openshift==0.11.0 packaging==20.3 # via # -c requirements/dev/../test/test.txt + # ansible-core # sphinx parso==0.7.0 # via jedi @@ -139,12 +146,12 @@ ptyprocess==0.6.0 # via pexpect pudb==2019.2 # via -r requirements/dev/dev.in -pyasn1-modules==0.2.8 - # via google-auth pyasn1==0.4.8 # via # pyasn1-modules # rsa +pyasn1-modules==0.2.8 + # via google-auth pycparser==2.20 # via cffi pygments==2.6.1 @@ -174,30 +181,32 @@ pytz==2019.3 pyyaml==5.3.1 # via # -c requirements/dev/../test/test.txt - # ansible + # ansible-core # awscli # cfn-flip # kubernetes # kubernetes-validate # sphinx-autobuild -requests-oauthlib==1.3.0 - # via kubernetes requests==2.24.0 # via # -c requirements/dev/../base/base.txt # kubernetes # requests-oauthlib # sphinx +requests-oauthlib==1.3.0 + # via kubernetes +resolvelib==0.5.4 + # via ansible-core rsa==3.4.2 # via # awscli # google-auth rstcheck==3.3.1 # via -r requirements/dev/dev.in -ruamel.yaml.clib==0.2.0 - # via ruamel.yaml ruamel.yaml==0.16.10 # via openshift +ruamel.yaml.clib==0.2.6 + # via ruamel.yaml s3transfer==0.3.3 # via # awscli @@ -220,10 +229,10 @@ six==1.15.0 # websocket-client snowballstemmer==2.0.0 # via sphinx -sphinx-autobuild==0.7.1 - # via -r requirements/dev/dev.in sphinx==3.1.1 # via -r requirements/dev/dev.in +sphinx-autobuild==0.7.1 + # via -r requirements/dev/dev.in sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 @@ -247,7 +256,7 @@ tornado==6.0.4 # sphinx-autobuild traitlets==4.3.3 # via ipython -troposphere==2.6.1 +troposphere==3.1.1 # via -r requirements/dev/dev.in urllib3==1.25.9 # via diff --git a/requirements/test/test.in b/requirements/test/test.in index e1a443c1..2087258e 100644 --- a/requirements/test/test.in +++ b/requirements/test/test.in @@ -1,6 +1,5 @@ # test.in -c ../base/base.txt --c ../dev/dev.txt pyyaml isort @@ -12,5 +11,5 @@ coverage # Linting flake8 # Pinned due to bug introduced in release 20.8b0 -black==19.10b0 +black==21.10b0 pre-commit diff --git a/requirements/test/test.txt b/requirements/test/test.txt index a9f004c2..47035701 100644 --- a/requirements/test/test.txt +++ b/requirements/test/test.txt @@ -1,26 +1,20 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.8 # To update, run: # # pip-compile --output-file=requirements/test/test.txt requirements/test/test.in # appdirs==1.4.3 - # via - # black - # virtualenv + # via virtualenv attrs==19.3.0 - # via - # -c requirements/test/../dev/dev.txt - # black - # pytest -black==19.10b0 + # via pytest +black==21.10b0 # via -r requirements/test/test.in cfgv==3.1.0 # via pre-commit -click==7.0 +click==7.1.2 # via # -c requirements/test/../base/base.txt - # -c requirements/test/../dev/dev.txt # black coverage==5.1 # via @@ -44,13 +38,15 @@ mccabe==0.6.1 # via flake8 more-itertools==8.2.0 # via pytest +mypy-extensions==0.4.3 + # via black nodeenv==1.3.5 # via pre-commit packaging==20.3 - # via - # -c requirements/test/../dev/dev.txt - # pytest -pathspec==0.8.0 + # via pytest +pathspec==0.9.0 + # via black +platformdirs==2.4.0 # via black pluggy==0.13.1 # via pytest @@ -63,26 +59,22 @@ pycodestyle==2.6.0 pyflakes==2.2.0 # via flake8 pyparsing==2.4.7 - # via - # -c requirements/test/../dev/dev.txt - # packaging -pytest-cov==2.8.1 - # via -r requirements/test/test.in -pytest-django==3.9.0 - # via -r requirements/test/test.in + # via packaging pytest==5.4.2 # via # -r requirements/test/test.in # pytest-cov # pytest-django +pytest-cov==2.8.1 + # via -r requirements/test/test.in +pytest-django==3.9.0 + # via -r requirements/test/test.in python-dateutil==2.8.1 # via # -c requirements/test/../base/base.txt - # -c requirements/test/../dev/dev.txt # faker pyyaml==5.3.1 # via - # -c requirements/test/../dev/dev.txt # -r requirements/test/test.in # pre-commit regex==2020.5.7 @@ -90,21 +82,18 @@ regex==2020.5.7 six==1.15.0 # via # -c requirements/test/../base/base.txt - # -c requirements/test/../dev/dev.txt # packaging # python-dateutil # virtualenv text-unidecode==1.3 # via faker toml==0.10.0 - # via - # black - # pre-commit -typed-ast==1.4.1 + # via pre-commit +tomli==1.2.2 + # via black +typing-extensions==4.0.0 # via black virtualenv==20.0.20 # via pre-commit wcwidth==0.1.9 - # via - # -c requirements/test/../dev/dev.txt - # pytest + # via pytest diff --git a/tasks.py b/tasks.py index bfb3918a..5248b444 100644 --- a/tasks.py +++ b/tasks.py @@ -43,6 +43,7 @@ def ansible_playbook(c, name, extra="", verbosity=1): ns.add_collection(kubesae.deploy) ns.add_collection(kubesae.pod) ns.add_collection(kubesae.info) +ns.add_collection(kubesae.utils) ns.add_task(staging) ns.add_task(production) @@ -55,6 +56,7 @@ def ansible_playbook(c, name, extra="", verbosity=1): "aws": {"region": "us-east-2",}, "cluster": "trafficstops-stack-cluster", "container_name": "app", + "hosting_services_backup_folder": "trafficstops", "repository": "606178775542.dkr.ecr.us-east-2.amazonaws.com/traff-appli-gvyudgfsjhrz", "run": { "echo": True, diff --git a/traffic_stops/settings/base.py b/traffic_stops/settings/base.py index 35d2d10e..d08153ce 100644 --- a/traffic_stops/settings/base.py +++ b/traffic_stops/settings/base.py @@ -53,6 +53,10 @@ DATABASE_ROUTERS = ["traffic_stops.routers.StateDatasetRouter"] DATABASE_ETL_USER = "" +# Only recreate materialized views when view SQL schema changed +# https://github.com/mikicz/django-pgviews#conditional-materialized-views-recreate +MATERIALIZED_VIEWS_CHECK_SQL_CHANGED = True + # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name # although not all choices may be available on all operating systems. @@ -172,6 +176,7 @@ def __init__(self, tz_name=None): "crispy_forms", "django_filters", "rest_framework", + "django_pgviews", # Custom apps "tsdata", "nc",