Merge pull request #191 from caktus/develop

Prepare production release v1.9.0

Author: copelco

.github/workflows/deploy.yaml

@@ -14,15 +14,15 @@ jobs:
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Set env vars (production)
         if: endsWith(github.ref, '/main')
         run: |
           echo "ENV=production" >> $GITHUB_ENV
           echo "ENV_URL=https://nccopwatch.org/" >> $GITHUB_ENV
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: '3.9'
           cache: 'pip'

.github/workflows/test.yaml

@@ -25,8 +25,8 @@ jobs:
         ports:
           - 5432:5432
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: '3.9'
           cache: 'pip'
@@ -36,7 +36,7 @@ jobs:
           sudo apt update
           sudo apt install -y --no-install-recommends postgresql-client-14
           python -m pip install pip-tools
-          pip-sync requirements/base/base.txt requirements/test/test.txt
+          pip-sync requirements/base/base.txt requirements/dev/dev.txt requirements/test/test.txt
       - name: Create NC database
         run: |
           psql $DATABASE_URL -c 'CREATE DATABASE traffic_stops_nc;'
@@ -49,3 +49,6 @@ jobs:
           DJANGO_SETTINGS_MODULE: traffic_stops.settings.dev
           DATABASE_URL: postgres://postgres:postgres@localhost:5432/traffic_stops
           DATABASE_URL_NC: postgres://postgres:postgres@localhost:5432/traffic_stops_nc
+      - name: Test deploy image build
+        run: |
+          inv image.build

.gitignore

@@ -53,4 +53,5 @@ aws_ec2.yml
 deploy/roles
 docker-compose.override.yml
 htmlcov
+nc/notebooks/**/**.html
 acs*.json

.pre-commit-config.yaml

@@ -23,6 +23,7 @@ repos:
       - id: check-yaml
         exclude: '^(deployment/|deploy)'
       - id: check-added-large-files
+        args: ['--maxkb=8000']
       - id: check-json
         exclude: .devcontainer/devcontainer.json|.vscode/launch.json|frontend/.eslintrc.json
       - id: check-docstring-first

Dockerfile

@@ -3,6 +3,7 @@ FROM node:16-bullseye-slim as static_files
 WORKDIR /code
 ENV PATH /code/node_modules/.bin:$PATH
 COPY frontend/package.json frontend/package-lock.json /code/
+RUN npm install -g npm@latest
 RUN npm install --silent
 COPY frontend/ /code/
 RUN npm run build

Makefile

@@ -33,3 +33,9 @@ dl-db-dump:
 	export AWS_PROFILE=copwatch
 	inv aws.configure-eks-kubeconfig
 	inv staging pod.get-db-dump --db-var=DATABASE_URL_NC
+
+refresh-view:
+	@echo 'Migrating all databases'
+	./migrate_all_dbs.sh
+	@echo 'Refreshing the StopSummary materialized view'
+	python manage.py refresh_pgviews --database=traffic_stops_nc

deploy/group_vars/all.yml

@@ -51,9 +51,11 @@ cloudformation_stack:
     DatabaseUser: "{{ app_name }}_admin"
     DatabasePassword: "{{ admin_database_password }}"
     DatabaseName: "{{ app_name }}"
-    DomainName: "{{ app_name }}.caktus-built.com"
+    DomainName: nccopwatch.org
     DomainNameAlternates: ""
-    AssetsUseCloudFront: "false"
+    AssetsCloudFrontDomain: files.nccopwatch.org
+    AssetsCloudFrontCertArn: arn:aws:acm:us-east-1:606178775542:certificate/379950bb-4b29-4308-8418-122674fe1076
+    AssetsUseCloudFront: "true"
   tags:
     Environment: "{{ app_name }}"
 

deploy/group_vars/k8s.yml

@@ -31,9 +31,8 @@ k8s_container_resources:
   requests:
     memory: 256Mi
     cpu: 50m
-  limits:
-    cpu: 250m
 k8s_migrations_enabled: true
+k8s_migrations_timeout: 300  # 5 minutes
 k8s_migration_command:
   - bash
   - migrate_all_dbs.sh
@@ -62,10 +61,12 @@ env_database_url_nc: "postgres://{{ app_name }}_{{ env_name }}:{{ database_passw
 env_django_settings: "traffic_stops.settings.deploy"
 env_broker_url: redis://redis:6379/0
 env_cache_host: redis://redis:6379/1
-env_default_file_storage: "{{ app_name }}.storages.MediaBoto3Storage"
-env_media_storage_bucket_name: "{{ app_name }}-{{ env_name }}-assets"
+# Uploaded media files
+env_default_file_storage: "traffic_stops.storages.MediaBoto3Storage"
 env_aws_default_acl: public-read
-env_media_location: media/
+env_media_storage_bucket_name: trafficstops-stack-assetsbucket-y0dbtam3g132
+env_media_storage_custom_domain: files.nccopwatch.org
+env_media_location: ""
 # New Relic Account: forwardjustice-team@caktusgroup.com
 env_new_relic_app_name: "{{ k8s_namespace }}"
 env_new_relic_license_key: "{{ k8s_newrelic_license_key }}"
@@ -109,6 +110,7 @@ k8s_environment_variables:
   # *** Uploaded media
   DEFAULT_FILE_STORAGE: "{{ env_default_file_storage }}"
   MEDIA_STORAGE_BUCKET_NAME: "{{ env_media_storage_bucket_name }}"
+  MEDIA_S3_CUSTOM_DOMAIN: "{{ env_media_storage_custom_domain }}"
   AWS_DEFAULT_ACL: "{{ env_aws_default_acl }}"
   AWS_DEFAULT_REGION: "{{ aws_region }}"
   MEDIA_LOCATION: "{{ env_media_location }}"

deploy/host_vars/staging.yml

@@ -6,6 +6,8 @@ k8s_domain_names:
 env_contact_us_emails:
   - forwardjustice-team@caktusgroup.com
 
+env_media_location: "staging/"
+
 database_password: !vault |
   $ANSIBLE_VAULT;1.1;AES256
   66636130383863366430366534663963316536393136316164336535613635353339646430306536