Merge pull request #163 from caktus/develop

Production release v1.8.0

Author: copelco

.devcontainer/devcontainer.json

@@ -0,0 +1,51 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.245.0/containers/docker-existing-docker-compose
+// If you want to run as a non-root user in the container, see .devcontainer/docker-compose.yml.
+{
+	"name": "CopWatch",
+
+	// Update the 'dockerComposeFile' list if you have more compose files or use different names.
+	// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
+	"dockerComposeFile": [
+		"../docker-compose.yml",
+		"docker-compose.yml"
+	],
+
+	// The 'service' property is the name of the service for the container that VS Code should
+	// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
+	"service": "django",
+
+	// The optional 'workspaceFolder' property is the path VS Code should open by default when
+	// connected. This is typically a file mount in .devcontainer/docker-compose.yml
+	"workspaceFolder": "/code",
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Uncomment the next line if you want start specific services in your Docker Compose config.
+	// "runServices": [],
+
+	// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
+	"shutdownAction": "stopCompose",
+
+	// Uncomment the next line to run commands after the container is created - for example installing curl.
+	"postCreateCommand": "sudo chown appuser /home/appuser/.cache /code/frontend/node_modules /code/venv /code/public /var/run/docker.sock",
+
+	// Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "appuser",
+
+	// A command to run each time the container is successfully started.
+	"postStartCommand": "git config --global --add safe.directory ${containerWorkspaceFolder}",
+
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.python",
+				"ms-python.vscode-pylance",
+				"batisteo.vscode-django",
+				"ms-vsliveshare.vsliveshare-pack",
+				"yzhang.markdown-all-in-one"
+			]
+		}
+	}
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,37 @@
+version: '3.4'
+services:
+  # Update this to the name of the service you want to work with in your docker-compose.yml file
+  django:
+    # If you want add a non-root user to your Dockerfile, you can use the "remoteUser"
+    # property in devcontainer.json to cause VS Code its sub-processes (terminals, tasks,
+    # debugging) to execute as the user. Uncomment the next line if you want the entire
+    # container to run as this user instead. Note that, on Linux, you may need to
+    # ensure the UID and GID of the container user you create matches your local user.
+    # See https://aka.ms/vscode-remote/containers/non-root for details.
+    #
+    # user: vscode
+
+    # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer
+    # folder. Note that the path of the Dockerfile and context is relative to the *primary*
+    # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile"
+    # array). The sample below assumes your primary file is in the root of your project.
+    #
+    # build:
+    #   context: .
+    #   dockerfile: .devcontainer/Dockerfile
+
+    volumes:
+      # Update this to wherever you want VS Code to mount the folder of your project
+      - .:/code:cached
+
+      # Uncomment the next line to use Docker from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker-compose for details.
+      - /var/run/docker.sock:/var/run/docker.sock
+
+    # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.
+    # cap_add:
+    #   - SYS_PTRACE
+    # security_opt:
+    #   - seccomp:unconfined
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: /bin/sh -c "while sleep 1000; do :; done"

.dockerignore

@@ -16,3 +16,7 @@ frontend/node_modules
 deploy/
 .env
 .envrc
+.devcontainer
+.github
+nc/notebooks
+venv

.github/workflows/deploy.yaml

@@ -19,7 +19,7 @@ jobs:
           fetch-depth: 0
       - name: Set env vars (production)
         if: endsWith(github.ref, '/main')
-        run: | 
+        run: |
           echo "ENV=production" >> $GITHUB_ENV
           echo "ENV_URL=https://nccopwatch.org/" >> $GITHUB_ENV
       - uses: actions/setup-python@v2

.github/workflows/test.yaml

@@ -34,7 +34,7 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt update
-          sudo apt install -y --no-install-recommends postgresql-client-12
+          sudo apt install -y --no-install-recommends postgresql-client-14
           python -m pip install pip-tools
           pip-sync requirements/base/base.txt requirements/test/test.txt
       - name: Create NC database

.gitignore

@@ -15,6 +15,7 @@ traffic_stops.log
 traffic_stops.log.1
 reports
 env
+venv
 jmeter.log
 npm-debug.log
 .transifexrc
@@ -52,3 +53,4 @@ aws_ec2.yml
 deploy/roles
 docker-compose.override.yml
 htmlcov
+acs*.json

.pre-commit-config.yaml

@@ -5,25 +5,27 @@ repos:
       - id: black
         language_version: python3.8
         exclude: migrations
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v4.3.21
+  - repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        exclude: settings|migrations|tests
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
     hooks:
       - id: isort
-        # Resolves conflict with black
-        args: [--multi-line=3, --trailing-comma, --force-grid-wrap=0, --use-parentheses, --line-width=100]
+        args: ["--profile", "black", "--filter-files"]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.4.0
+    rev: v4.1.0
     hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+        exclude: '^(deployment/|deploy)'
+      - id: check-added-large-files
+      - id: check-json
+        exclude: .devcontainer/devcontainer.json|.vscode/launch.json|frontend/.eslintrc.json
       - id: check-docstring-first
       - id: check-merge-conflict
-      - id: check-json
-        exclude: docs/dev
-      - id: check-yaml
-        exclude: (deploy/host_vars|deploy/stack|deploy)
-      - id: check-xml
       - id: debug-statements
-      - id: detect-aws-credentials
-        args: [--allow-missing-credentials]
       - id: detect-private-key
-      - id: flake8
-        exclude: settings|migrations|test

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16.16-alpine3.16 as static_files
+FROM node:16-bullseye-slim as static_files
 
 WORKDIR /code
 ENV PATH /code/node_modules/.bin:$PATH
@@ -24,7 +24,6 @@ RUN set -ex \
     mime-support \
     postgresql-client \
     vim \
-    gdal-bin \
     " \
     && seq 1 8 | xargs -I{} mkdir -p /usr/share/man/man{} \
     && apt-get update && apt-get install -y --no-install-recommends $RUN_DEPS \
@@ -48,10 +47,11 @@ RUN set -ex \
     && apt-get update && apt-get install -y --no-install-recommends $BUILD_DEPS \
     && pip install -U -q pip-tools \
     && pip-sync requirements/base/base.txt requirements/deploy/deploy.txt \
-    \
     && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false $BUILD_DEPS \
     && rm -rf /var/lib/apt/lists/*
 
+FROM base AS deploy
+
 # Copy your application code to the container (make sure you create a .dockerignore file if any large files or directories should be excluded)
 RUN mkdir /code/
 WORKDIR /code/
@@ -67,7 +67,6 @@ EXPOSE 8000
 ENV DJANGO_SETTINGS_MODULE=traffic_stops.settings.deploy
 
 # Call collectstatic (customize the following line with the minimal environment variables needed for manage.py to run):
-RUN touch /code/.env
 RUN DATABASE_URL='' ENVIRONMENT='' DJANGO_SECRET_KEY='dummy' DOMAIN='' python manage.py collectstatic --noinput
 
 # Tell uWSGI where to find your wsgi file (change this):
@@ -82,6 +81,8 @@ ENV UWSGI_WORKERS=2 UWSGI_THREADS=4
 # uWSGI static file serving configuration (customize or comment out if not needed):
 ENV UWSGI_STATIC_MAP="/static/=/code/static/" UWSGI_STATIC_EXPIRES_URI="/static/.*\.[a-f0-9]{12,}\.(css|js|png|jpg|jpeg|gif|ico|woff|ttf|otf|svg|scss|map|txt) 315360000"
 
+RUN touch /code/.env
+
 # Change to a non-root user
 USER ${APP_USER}:${APP_USER}
 
@@ -90,3 +91,86 @@ ENTRYPOINT ["/code/docker-entrypoint.sh"]
 
 # Start uWSGI
 CMD ["newrelic-admin", "run-program", "uwsgi", "--single-interpreter", "--enable-threads", "--show-config"]
+
+FROM python:3.8-slim-bullseye AS dev
+
+ARG USERNAME=appuser
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+# Create non-root user
+RUN groupadd --gid $USER_GID $USERNAME \
+    && useradd --uid $USER_UID --gid $USER_GID --create-home --shell /bin/bash $USERNAME
+
+# Install packages for Dev Container development
+#   build-essential -- for gcc to compile non-wheel packages with C dependencies
+#   docker-ce-cli -- docker CLI
+#   docker-compose-plugin -- docker compose CLI
+#   git-core -- to pull, commit, and push from dev container
+#   gnupg2 -- GNU privacy guard - a free PGP replacement
+#   libpq-dev -- header files for PostgreSQL
+#   openssh-client -- for git over SSH
+#   sudo -- to run commands as superuser
+#   vim -- enhanced vi editor for commits
+ENV KUBE_CLIENT_VERSION="v1.22.15"
+ENV HELM_VERSION="3.8.2"
+RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
+    --mount=type=cache,mode=0755,target=/root/.cache/pip \
+    set -ex \
+    && RUN_DEPS=" \
+    build-essential \
+    docker-ce-cli \
+    docker-compose-plugin \
+    git-core \
+    gnupg2 \
+    libpcre3 \
+    libpq-dev \
+    mime-support \
+    nodejs \
+    openssh-client \
+    postgresql-client-12 \
+    sudo \
+    vim \
+    " \
+    && apt-get update && apt-get -y install curl wget gnupg2 lsb-release \
+    # starship.rs prompt
+    && curl -sS https://starship.rs/install.sh | sh -s -- -y \
+    # kubectl
+    && curl --silent -L https://dl.k8s.io/release/$KUBE_CLIENT_VERSION/bin/linux/$(dpkg --print-architecture)/kubectl -o /usr/local/bin/kubectl \
+    && chmod +x /usr/local/bin/kubectl \
+    # helm
+    && curl --silent -L https://get.helm.sh/helm-v$HELM_VERSION-linux-$(dpkg --print-architecture).tar.gz --output - | tar -xzC /tmp \
+    && mv /tmp/linux-$(dpkg --print-architecture)/helm /usr/local/bin/helm \
+    && chmod +x /usr/local/bin/helm \
+    # docker
+    && curl https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor | tee /etc/apt/trusted.gpg.d/docker.gpg >/dev/null \
+    && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/trusted.gpg.d/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null \
+    # nodejs
+    && sh -c 'echo "deb https://deb.nodesource.com/node_16.x $(lsb_release -cs) main" > /etc/apt/sources.list.d/nodesource.list' \
+    && wget --quiet -O- https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
+    # PostgreSQL
+    && sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \
+    && curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null \
+    # dev packages
+    && apt-get update \
+    && apt-get install -y --no-install-recommends $RUN_DEPS \
+    # sudo
+    && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
+    && chmod 0440 /etc/sudoers.d/$USERNAME
+
+# Install Docker Buildx component for Docker v23.0.0+
+COPY --from=docker/buildx-bin:latest /buildx /usr/libexec/docker/cli-plugins/docker-buildx
+
+COPY --chown=$USER_UID:$USER_GID . /code/
+
+USER $USERNAME
+RUN set -ex \
+    && touch /code/.env \
+    && echo 'eval "$(starship init bash)"' >> ~/.bashrc
+
+ENV DJANGO_SETTINGS_MODULE=traffic_stops.settings.dev
+ENV PATH=/code/venv/bin:$PATH
+
+WORKDIR /code
+
+CMD ["python", "/code/manage.py", "runserver", "0.0.0.0:8000"]

Makefile

@@ -26,3 +26,10 @@ run-tests:
 	@echo 'Checking for migrations'
 	python manage.py makemigrations --dry-run | grep 'No changes detected' || (echo 'There are changes which require migrations.' && exit 1)
 	python manage.py test
+
+
+dl-db-dump:
+	@echo 'Downloading new db dump from s3'
+	export AWS_PROFILE=copwatch
+	inv aws.configure-eks-kubeconfig
+	inv staging pod.get-db-dump --db-var=DATABASE_URL_NC

README.rst

@@ -3,7 +3,7 @@ NC CopWatch
 
 NC CopWatch is a website to monitor and identify racial profiling
 practices by North Carolina law enforcement agencies. This project is lead by
-`Forward Justice`_, a nonpartisan law, policy, and strategy center dedicated to advancing racial, 
+`Forward Justice`_, a nonpartisan law, policy, and strategy center dedicated to advancing racial,
 social, and economic justice in the U.S. South.
 
 Please see the `production documentation`_ and `development documentation`_

deploy/group_vars/all.yml

@@ -57,6 +57,21 @@ cloudformation_stack:
   tags:
     Environment: "{{ app_name }}"
 
+# Install Descheduler to attempt to spread out pods again after node failures
+k8s_install_descheduler: yes
+# You must set the k8s_descheduler_chart_version to match the Kubernetes
+# node version (0.23.x -> K8s 1.23.x); see:
+# https://github.com/kubernetes-sigs/descheduler#compatibility-matrix
+k8s_descheduler_chart_version: v0.22.1
+# See values.yaml for options:
+# https://github.com/kubernetes-sigs/descheduler/blob/master/charts/descheduler/values.yaml#L63
+k8s_descheduler_release_values:
+  deschedulerPolicy:
+    strategies:
+      # During upgrades or reboots, don't pre-emptively drain a node.
+      RemovePodsViolatingNodeTaints:
+        enabled: false
+
 # ----------------------------------------------------------------------------
 # caktus.k8s-web-cluster: An Ansible role to help configure Kubernetes
 #                         clusters for web apps.
@@ -77,7 +92,7 @@ k8s_iam_users: [copelco]
 # Pin ingress-nginx and cert-manager to current versions so future upgrades of this
 # role will not upgrade these charts without your intervention:
 # https://github.com/kubernetes/ingress-nginx/releases
-k8s_ingress_nginx_chart_version: "4.0.19"
+k8s_ingress_nginx_chart_version: "4.4.2"
 # https://github.com/jetstack/cert-manager/releases
 k8s_cert_manager_chart_version: "v1.7.2"
 # AWS only:
@@ -92,13 +107,14 @@ k8s_aws_load_balancer_type: nlb
 k8s_papertrail_logspout_destination: "syslog+tls://logs2.papertrailapp.com:20851"
 k8s_papertrail_logspout_memory_limit: 128Mi
 
-# New Relic Infrastructure: admin+newrelic@caktusgroup.com
-k8s_newrelic_chart_version: "3.5.1"
+# New Relic Account: forwardjustice-team@caktusgroup.com
+k8s_newrelic_chart_version: "5.0.4"
+k8s_newrelic_logging_enabled: true
 k8s_newrelic_license_key: !vault |
   $ANSIBLE_VAULT;1.1;AES256
-  37656631623333346263383231386165666531333961373931383661366338343634333362356430
-  3963613833663637313632373465613730383365626461630a383432346335386632303935356532
-  61643737636132336339336332396262623362333663333130393031376338363266363430326136
-  6131396135646236360a323766623330313365306539316263393533623063346166653433316631
-  39356263623363653934333064376364363562303236646238666234356136663539343064383463
-  3161356339656137373935623562366134393765346466643365
+  66633739306132643365363862313730386239643136363637336638323066343033653466663234
+  6663633261383536383936343939383634636530613633370a363032623332656465303231653535
+  33303336323831303939646432313835386261646632343165373963616132343039333432386562
+  3163313364666132340a636330353366613061306361303737303332383431336263323135393232
+  65303530343134383464616561383139643263326661636133316534303934346438643366666663
+  3136353834393937356364356235393236643835663965643532