diff --git a/showCrime/Dockerfile b/showCrime/Dockerfile new file mode 100644 index 0000000..ae879d9 --- /dev/null +++ b/showCrime/Dockerfile @@ -0,0 +1,39 @@ +FROM python:3.6-jessie + +WORKDIR /app/showCrime + +ENV DJANGO_SETTINGS_MODULE showCrime.settings +ENV MATPLOTLIBRC /app/showCrime +ENV PUBLIC_ROOT /public/showCrime + +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + build-essential \ + gettext \ + libffi-dev \ + libgdal-dev \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + +COPY Makefile /app/showCrime +COPY requirements.txt /app/showCrime +COPY requirements.dev.txt /app/showCrime +COPY conf/wait-for-it.sh /app/showCrime + +RUN make requirements + +COPY showCrime /app/showCrime + +RUN mkdir -p /logs \ + && touch /logs/app.log \ + && touch /logs/gunicorn.log + +ENV PUBLIC_ROOT /public +ENV LOG_FILE_PATH /logs +ENV ENABLE_LOGGING_TO_FILE true + +VOLUME /public/media + +EXPOSE 8000 + +ENTRYPOINT ["/app/showCrime/docker-entrypoint.sh"] diff --git a/showCrime/Makefile b/showCrime/Makefile new file mode 100644 index 0000000..f70b761 --- /dev/null +++ b/showCrime/Makefile @@ -0,0 +1,59 @@ +.DEFAULT_GOAL := test + +.PHONY: clean detect_missing_migrations help requirements test validate quality production-requirements migrate static clean_static + +# Generates a help message. Borrowed from https://github.com/pydanny/cookiecutter-djangopackage. +help: ## Display this help message + @echo "Please use \`make \` where is one of" + @perl -nle'print $& if m{^[\.a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m %-25s\033[0m %s\n", $$1, $$2}' + +docker.build: ## Build the Docker containers + docker-compose build + +docker.pull: ## Pull the Docker containers + docker-compose pull + +%.down: ## Stop the (local|production) Docker containers + docker-compose -f docker-compose.yml -f docker-compose.$*.yml down + +%.restart: ## Restart the (local|production) Docker containers + docker-compose -f docker-compose.yml -f docker-compose.$*.yml restart + +%.shell: ## Open a shell into the (local|production) app Docker container + docker-compose -f docker-compose.yml -f docker-compose.$*.yml exec app /bin/bash + +%.up: ## Start the (local|production) Docker containers + docker-compose -f docker-compose.yml -f docker-compose.$*.yml up -d + +clean: ## Delete generated byte code and coverage reports + find . -name '*.pyc' -delete + coverage erase + +requirements: ## Install requirements for local development + pip install -r requirements.dev.txt + +production-requirements: ## Install requirements for production + pip install -r requirements.txt + +test: clean ## Run tests and generate coverage report + coverage run -m pytest --durations=25 -v + coverage report -m + +quality: ## Run pep8 and Pylint + isort --check-only --recursive . + pycodestyle . *.py + pylint --rcfile=pylintrc . *.py + +validate: quality test ## Run tests and quality checks + +detect_missing_migrations: ## Determine if any apps are missing generated migrations + python manage.py makemigrations --check --dry-run || (echo "Migration files are missing. Please run the "makemigrations" management command, and commit the migrations." && false) + +migrate: ## Apply database migrations + python manage.py migrate --noinput + +static: ## Gather all static assets for production + python manage.py collectstatic --noinput + +clean_static: ## Remove all generated static files + rm -rf ./public/ diff --git a/showCrime/conf/nginx.conf b/showCrime/conf/nginx.conf new file mode 100644 index 0000000..998beb0 --- /dev/null +++ b/showCrime/conf/nginx.conf @@ -0,0 +1,29 @@ +upstream app { + server app:8000 fail_timeout=0; +} + +server { + listen 80; + server_name 0.0.0.0; + + location = /favicon.ico { + access_log off; + log_not_found off; + } + + location /media/ { + alias /public/media/; + } + + location /static/ { + alias /public/static/; + } + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://app/; + } +} diff --git a/showCrime/conf/wait-for-it.sh b/showCrime/conf/wait-for-it.sh new file mode 100755 index 0000000..8b6092d --- /dev/null +++ b/showCrime/conf/wait-for-it.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# SOURCE: https://github.com/vishnubob/wait-for-it +# +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + WAITFORIT_BUSYTIMEFLAG="-t" + +else + WAITFORIT_ISBUSY=0 + WAITFORIT_BUSYTIMEFLAG="" +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/showCrime/docker-compose.local.yml b/showCrime/docker-compose.local.yml new file mode 100644 index 0000000..8e3b24f --- /dev/null +++ b/showCrime/docker-compose.local.yml @@ -0,0 +1,37 @@ +version: "3" +services: + db: + container_name: oakcrime.db + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=oakcrime + image: mdillon/postgis:11 + volumes: + - db-data:/var/lib/postgresql/data + # NOTE: If you need to access the database from an external tool, + # expose this port and restart the container. + # ports: + # - 5432:5432 + restart: always + app: + # Instruct Gunicorn to reload code when it is changed locally. + command: --reload + depends_on: + - db + environment: + - DEBUG=true + - SECRET_KEY=replace-me + - DATABASE_URL=postgis://postgres:postgres@oakcrime.db:5432/oakcrime?connect_timeout=60 + - DEFAULT_FILE_STORAGE=django.core.files.storage.FileSystemStorage + - SERVER_EMAIL=root@localhost + - EMAIL_URL=smtp://user@:password@localhost:25 + links: + - db + ports: + # This port is primarily exposed for debugging. Use the web service's port to properly access the service. + - 8000:8000 + volumes: + - .:/app/showCrime:cached +volumes: + db-data: diff --git a/showCrime/docker-compose.yml b/showCrime/docker-compose.yml new file mode 100644 index 0000000..671fb75 --- /dev/null +++ b/showCrime/docker-compose.yml @@ -0,0 +1,32 @@ +version: "3" +services: + app: + build: + context: . + dockerfile: Dockerfile + container_name: oakcrime.app + image: openoakland/oakcrime:latest + restart: always + stdin_open: true + tty: true + volumes: + - logs:/logs + - media:/public/media/ + - static:/public/static/ + web: + container_name: oakcrime.web + image: nginx:1.15-alpine + links: + - app + ports: + - 8080:80 + restart: always + volumes: + - ./conf/nginx.conf:/etc/nginx/conf.d/default.conf:ro + - logs:/var/log/nginx + - media:/public/media:ro + - static:/public/static:ro +volumes: + logs: + media: + static: diff --git a/showCrime/docker-entrypoint.sh b/showCrime/docker-entrypoint.sh new file mode 100755 index 0000000..597dbea --- /dev/null +++ b/showCrime/docker-entrypoint.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# Exit if any of the lines below fail! This is especially important if, for example, migrations fail. +# If we do not exit early, we end up running the application without the migrations or, if security groups +# haven't been properly setup, without a database! +set -e + +# Prepare log files and start outputting logs to stdout +touch /logs/app.log +touch /logs/gunicorn.log +tail -n 0 -f /logs/*.log & + +mkdir -p /public/static +make migrate +make static + +echo Starting Gunicorn... +gunicorn showCrime.wsgi \ + --workers=2 \ + --worker-class=gevent \ + --bind=0.0.0.0:8000 \ + --log-file=/logs/gunicorn.log \ + "$@" diff --git a/showCrime/requirements.dev.txt b/showCrime/requirements.dev.txt index f8134d6..28a73ff 100644 --- a/showCrime/requirements.dev.txt +++ b/showCrime/requirements.dev.txt @@ -1,2 +1,4 @@ +-r requirements.txt + django-debug-toolbar==1.11 pylint==2.2.2 diff --git a/showCrime/requirements.txt b/showCrime/requirements.txt index 4369dab..2c9aa55 100644 --- a/showCrime/requirements.txt +++ b/showCrime/requirements.txt @@ -4,7 +4,9 @@ django-environ==0.4.5 djangorestframework==3.9.0 editdistance==0.4 geojson==2.4.1 +gevent==1.4.0 googlemaps==3.0.2 +gunicorn==19.9.0 mapbox==0.17.2 matplotlib==2.2.3 numpy==1.15.4