Alpine, Gunicorn, and Helm

.dockerignore

@@ -0,0 +1,14 @@
+# Ignore all
+**
+
+# ...except these directories
+!ccdb5_api
+!complaint_search
+
+# ...and these files
+!docker-entrypoint.sh
+!manage.py
+!initial-data.sh
+!refresh-data.sh
+!setup.py
+!gunicorn.conf.py

Dockerfile

@@ -0,0 +1,48 @@
+FROM python:3.8-alpine as base
+
+# Hard labels
+LABEL maintainer="[email protected]"
+
+# Ensure that the environment uses UTF-8 encoding by default
+ENV LANG en_US.UTF-8
+ENV ENV /etc/profile
+ENV PIP_NO_CACHE_DIR true
+# Stops Python default buffering to stdout, improving logging to the console.
+ENV PYTHONUNBUFFERED 1
+ENV APP_HOME /src/app
+ENV ALLOWED_HOSTS '["*"]'
+
+WORKDIR ${APP_HOME}
+
+RUN apk update --no-cache && apk upgrade --no-cache && \
+    pip install --upgrade pip setuptools
+
+FROM base as builder
+
+RUN apk add --no-cache --virtual .build-deps gcc git libffi-dev musl-dev postgresql-dev
+
+COPY setup.py setup.py
+RUN mkdir /build && pip install --prefix=/build .
+
+FROM base as dev
+
+RUN apk add --no-cahce --virtual .backend-deps curl postgresql
+
+COPY --from=builder /build /usr/local
+COPY setup.py setup.py
+RUN pip install '.[testing]'
+
+EXPOSE 8000
+
+ENTRYPOINT ["./docker-entrypoint.sh"]
+CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"]
+
+FROM base as final
+
+COPY --from=builder /build /usr/local
+RUN pip install gunicorn
+RUN apk add --no-cache --virtual .deps postgresql-client
+
+COPY . .
+
+CMD ["gunicorn", "-c", "gunicorn.conf.py"]

ccdb5_api/settings.py

@@ -29,7 +29,7 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True
 
-ALLOWED_HOSTS = []
+ALLOWED_HOSTS = os.getenv("ALLOWED_HOSTS", [])
 
 
 # Application definition
@@ -98,12 +98,24 @@
 # Database
 # https://docs.djangoproject.com/en/stable/ref/settings/#databases
 
-DATABASES = {
-    "default": {
-        "ENGINE": "django.db.backends.sqlite3",
-        "NAME": os.path.join(BASE_DIR, "db.sqlite3"),
+if os.getenv("PGUSER"):
+    DATABASES = {
+        "default": {
+            "ENGINE": "django.db.backends.postgresql",
+            "NAME": os.getenv("PGDATABASE", "ccdb"),
+            "USER": os.getenv("PGUSER", "cfpb"),
+            "PASSWORD": os.getenv("PGPASSWORD", "cfpb"),
+            "HOST": os.getenv("PGHOST", "localhost"),
+            "PORT": os.getenv("PGPORT", "5432"),
+        }
+    }
+else:
+    DATABASES = {
+        "default": {
+            "ENGINE": "django.db.backends.sqlite3",
+            "NAME": os.path.join(BASE_DIR, "db.sqlite3"),
+        }
     }
-}
 
 
 # Internationalization

docker-entrypoint.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+
+set -e
+
+if [ -f '.env' ]; then
+  . '.env'
+fi
+
+echo "Using $(python3 --version 2>&1) located at $(which python3)"
+
+# Do first-time set up of the database if necessary
+if [ ! -z "$RUN_MIGRATIONS" ]; then
+  # Wait for the database to be ready for initialization tasks
+  if [ ! -z "$PGHOST" ]; then
+    until pg_isready --host="${PGHOST:-localhost}" --port="${PGPORT:-5432}"
+    do
+      echo "Waiting for postgres at: ${PGHOST:-localhost}:${PGPORT:-5432}"
+      sleep 1;
+    done
+
+    # Check the DB if it needs refresh or migrations (initial-data.sh)
+    if ! psql "postgres://${PGUSER:-cfpb}:${PGPASSWORD:-cfpb}@${PGHOST:-localhost}:${PGPORT:-5432}/${PGDATABASE:-ccdb}" -c 'SELECT COUNT(*) FROM auth_user' >/dev/null 2>&1 || [ ! -z $FORCE_DB_REBUILD ]; then
+      echo "Doing first-time database and search index setup..."
+      if [ -n "$DB_DUMP_FILE" ] || [ -n "$DB_DUMP_URL" ]; then
+        echo "Running refresh-data.sh... $DB_DUMP_FILE"
+        ./refresh-data.sh "$DB_DUMP_FILE"
+        echo "Create the cache table..."
+        ./manage.py createcachetable
+
+        # refresh-data.sh runs migrations,
+        # unset vars to prevent further action
+        unset RUN_MIGRATIONS
+      else
+        # Detected the database is empty, or force rebuild was requested,
+        # but we have no valid data sources to load data.
+        echo "WARNING: Database rebuild request detected, but missing DB_DUMP_FILE/DB_DUMP_URL variable. Unable to load data!!"
+      fi
+    else
+      echo "Data detected, FORCE_DB_REBUILD not requested. Skipping data load!"
+    fi
+  fi
+
+  # Check if we still need to run migrations, if so, run them
+  if [ ! -z $RUN_MIGRATIONS ]; then
+    echo "Running initial-data.sh (migrations)..."
+    ./initial-data.sh
+  fi
+fi
+
+# Execute the Docker CMD
+exec "$@"

gunicorn.conf.py

@@ -0,0 +1,48 @@
+import json
+import multiprocessing
+import os
+
+from __future__ import print_function  # noqa: F404
+
+
+# Django WSGI application path in pattern MODULE_NAME:VARIABLE_NAME
+wsgi_app = "ccdb5_api.wsgi:application"
+
+workers_per_core_str = os.getenv("WORKERS_PER_CORE", "2")
+web_concurrency_str = os.getenv("WEB_CONCURRENCY", None)
+host = os.getenv("HOST", "0.0.0.0")
+port = os.getenv("PORT", "8000")
+bind_env = os.getenv("BIND", None)
+use_loglevel = os.getenv("LOG_LEVEL", "info").lower()
+if bind_env:
+    use_bind = bind_env
+else:
+    use_bind = "{host}:{port}".format(host=host, port=port)
+
+cores = multiprocessing.cpu_count()
+workers_per_core = float(workers_per_core_str)
+default_web_concurrency = workers_per_core * cores
+if web_concurrency_str:
+    web_concurrency = int(web_concurrency_str)
+    assert web_concurrency > 0
+else:
+    web_concurrency = int(default_web_concurrency)
+
+# Gunicorn config variables
+loglevel = use_loglevel
+workers = web_concurrency
+bind = use_bind
+keepalive = 120
+errorlog = "-"
+
+# For debugging and testing
+log_data = {
+    "loglevel": loglevel,
+    "workers": workers,
+    "bind": bind,
+    # Additional, non-gunicorn variables
+    "workers_per_core": workers_per_core,
+    "host": host,
+    "port": port,
+}
+print(json.dumps(log_data))

helm/.gitignore

@@ -0,0 +1 @@
+charts/

helm/ccdb5-api/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

helm/ccdb5-api/Chart.lock

@@ -0,0 +1,12 @@
+dependencies:
+- name: postgresql
+  repository: https://charts.bitnami.com/bitnami
+  version: 11.6.19
+- name: opensearch
+  repository: https://opensearch-project.github.io/helm-charts/
+  version: 2.0.0
+- name: opensearch-dashboards
+  repository: https://opensearch-project.github.io/helm-charts/
+  version: 2.0.0
+digest: sha256:7726e430c78cab54ca25b6032b9267d6b2e4a3d5576822d44697a80d81d2073a
+generated: "2022-09-24T05:19:15.252965-06:00"

helm/ccdb5-api/Chart.yaml

@@ -0,0 +1,39 @@
+apiVersion: v2
+name: ccdb5-api
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"
+
+# Dependency Charts
+dependencies:
+  - name: postgresql
+    version: "11.6.19"
+    repository: "https://charts.bitnami.com/bitnami"
+    condition: postgresql.enabled
+  - name: opensearch
+    version: "1.14.1"
+    repository: "https://opensearch-project.github.io/helm-charts/"
+    condition: opensearch.enabled
+  - name: opensearch-dashboards
+    version: "1.8.3"
+    repository: "https://opensearch-project.github.io/helm-charts/"
+    condition: opensearch-dashboards.enabled

helm/ccdb5-api/templates/NOTES.txt

@@ -0,0 +1,22 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "ccdb5-api.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "ccdb5-api.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "ccdb5-api.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "ccdb5-api.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+{{- end }}