Merge pull request #805 from sdss/docker

Adds docker setup for marvin

docker/Dockerfile

@@ -0,0 +1,54 @@
+# Use an official Python runtime as a parent image
+FROM python:3.10-slim
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Install library files
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git libpq-dev gcc build-essential mime-support && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Define an argument for the Git tag, defaulting to "main"
+ARG MARVIN_TAG=main
+
+# Clone the repository and checkout the specific tag
+RUN git clone https://github.com/sdss/marvin.git /app && \
+    git checkout ${MARVIN_TAG}
+
+# Install any needed packages
+RUN pip install --no-cache-dir ".[web,db]"
+
+# Resolve some dependency issues
+RUN pip install gunicorn
+RUN pip install "jinja2<3.1"
+RUN pip install "packaging<21"
+RUN pip install "itsdangerous==2.0.1"
+RUN pip install "pillow<10"
+
+# Create the log directory and ensure it's writable
+RUN mkdir -p /tmp/marvin/logs && \
+    chmod -R 755 /tmp/marvin
+
+# Make port 8000 available to the world outside this container
+EXPOSE 8000
+
+# Copy the gunicorn config file into the container
+COPY ./gunicorn_config.py /app/python/gunicorn_config.py
+
+# Create the marvin config file
+RUN mkdir -p /root/.marvin && chmod -R 755 /root/.marvin
+RUN echo "use_sentry: False\nadd_github_message: False\ncheck_access: False" > /root/.marvin/marvin.yml
+
+# Update permissions
+RUN chmod -R 755 /app/python/marvin/web
+
+# Set environment variables
+ENV FLASK_APP="marvin.web.uwsgi_conf_files.app:app"
+
+# Change to python dir
+WORKDIR /app/python
+
+# Run the application with uWSGI
+CMD ["gunicorn", "-c", "gunicorn_config.py", "marvin.web.uwsgi_conf_files.app:app"]

docker/README.md

@@ -0,0 +1,53 @@
+
+
+# Marvin Docker
+
+Describes the setup of a containerized system hosting the Marvin
+web application.
+
+
+## Initial Setup
+
+There are two bind mounts:
+
+- The SDSS SAS filesystem
+- The host machine pgpass config file
+
+1. Set a `SAS_BASE_DIR` environment variable that points to the
+top level directory of the SDSS SAS filesystem.
+
+2. Create or check that a pgpass config file exists at `$HOME/.pgpass`, which contains the manga database connection string info `host.docker.internal:5432:manga:marvin:(password)`.  Replace the `(password)` with the local database password.
+
+The marvin docker setup attempts to connect to the local host machine postgres database directly using `host.docker.internal`
+
+
+## Run docker compose
+
+All commands are relative to within the docker folder.  From the top-level repo, run `cd docker`
+
+To build and run the docker compose system:
+```bash
+docker compose up
+```
+To force a build, you can do: `docker compose up --build`
+
+Navigate to `http://localhost:8080/marvin/`.
+
+To bring the system down:
+```bash
+docker compose down
+```
+
+The docker compose system starts three services:
+- Marvin Webapp - the backend web application, mounted to port 8000
+- Nginx - the nginx web service, mounted to port 8080
+- Redis - a redis database for caching
+
+The final web application is available at `localhost:8080/marvin`
+
+## Build and Tag Version
+
+By default the marvin docker build will checkout the latest `main` branch of marvin on github and tag the image as `marvin:latest`.  To
+change this to a specific branch or tag of marvin, e.g. `2.8.0` or `branch_name`, set the `MARVIN_TAG` environment variable.  Then
+rebuild the system with `docker compose up --build`.  This will checkout that branch or tag of the marvin repository and create
+a new marvin docker image with that tag name.

docker/docker-compose.yml

@@ -0,0 +1,55 @@
+version: '3.9'
+name: marvin
+services:
+  nginx:
+    container_name: nginx
+    image: nginx:latest
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf
+      - socket_logs:/tmp/marvin
+      - web_assets:/usr/share/nginx/html
+    ports:
+      - 8080:80
+    depends_on:
+      - marvin
+    networks:
+      - frontend
+
+  redis:
+    container_name: redis
+    image: redis:latest
+    networks:
+     - backend
+
+  marvin:
+    container_name: marvin
+    build:
+      context: .
+      args:
+        MARVIN_TAG: ${MARVIN_TAG:-main}
+    image: marvin:${MARVIN_TAG:-latest}
+    ports:
+      - 8000:8000
+    volumes:
+      - ${SAS_BASE_DIR}:/root/sas/
+      - $HOME/.pgpass:/root/.pgpass
+      - socket_logs:/tmp/marvin
+      - web_assets:/app/python/marvin/web
+    environment:
+      - SESSION_REDIS=redis://redis:6379
+      - MARVIN_BASE=marvin
+      - PUBLIC_SERVER=True
+      - MANGADB_CONFIG=docker
+    networks:
+      - backend
+      - frontend
+    depends_on:
+      - redis
+
+networks:
+  backend:
+  frontend:
+
+volumes:
+  socket_logs:
+  web_assets:

docker/gunicorn_config.py

@@ -0,0 +1,8 @@
+import os
+
+socket_dir = os.getenv("MARVIN_SOCKET_DIR", '/tmp/marvin')
+bind = [f"unix:{socket_dir}/marvin.sock", "0.0.0.0:8000"]
+workers = 1
+daemon = False
+errorlog = os.path.join(os.getenv("MARVIN_LOGS_DIR", '/tmp/marvin/logs'), 'marvin_app_error.log')
+accesslog = os.path.join(os.getenv("MARVIN_LOGS_DIR", '/tmp/marvin/logs'), 'marvin_app_access.log')

docker/nginx.conf

@@ -0,0 +1,58 @@
+events {
+    use epoll;
+    worker_connections 51200;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    gzip on;
+
+    error_log /tmp/marvin/nginx-error.log warn;
+    access_log /tmp/marvin/nginx-access.log;
+
+    client_max_body_size 20m;
+
+    server {
+        listen 80 default_server;
+        server_name localhost;
+
+        location / {
+            proxy_pass http://marvin:8000/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_redirect off;
+            proxy_buffering off;
+        }
+
+        location /api/ {
+            proxy_pass http://marvin:8000/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_redirect off;
+            proxy_buffering off;
+
+            error_log  /tmp/marvin/logs/marvin_api_error.log  error;
+            access_log  /tmp/marvin/logs/marvin_api_access.log;
+        }
+
+        location /marvin/static/ {
+            alias /usr/share/nginx/html/static/;
+            autoindex off;
+        }
+
+        location /marvin/lib/ {
+            alias /usr/share/nginx/html/lib/;
+            autoindex off;
+        }
+    }
+}
+

python/marvin/__init__.py

@@ -402,7 +402,7 @@ def dapall(self, value):
 
     def _setDbConfig(self):
         ''' Set the db configuration '''
-        self.db = getDbMachine()
+        self.db = getDbMachine() or os.getenv("MANGADB_CONFIG")
 
     def _update_releases(self):
         ''' Update the allowed releases based on access '''

python/marvin/db/dbconfig.ini

@@ -36,3 +36,10 @@ jhu:
     port: 5432
     database: manga
 
+docker:
+    name: docker
+    user: marvin
+    host: host.docker.internal
+    port: 5432
+    database: manga
+

python/marvin/web/__init__.py

@@ -71,18 +71,21 @@ def create_app(debug=False, local=False, object_config=None):
     config._inapp = True
     url_prefix = '/marvin' if local else '/{0}'.format(marvin_base)
 
+    # get env
+    env = os.getenv('FLASK_ENV', 'production')
+
     # ----------------------------------
     # Load the appropriate Flask configuration object for debug or production
     if not object_config:
-        if app.debug or local:
+        if app.debug or local or (env in ('dev', 'development')):
             app.logger.info('Loading Development Config!')
-            object_config = type('Config', (DevConfig, CustomConfig), dict())
-        elif config.db and config.db == 'jhu':
+            object_config = type('Config', (DevConfig, CustomConfig), {})
+        elif (env == 'docker') or (config.db and config.db == 'jhu'):
             app.logger.info('Loading Docker Config!')
-            object_config = type('Config', (DockerConfig, CustomConfig), dict())
+            object_config = type('Config', (DockerConfig, CustomConfig), {})
         else:
             app.logger.info('Loading Production Config!')
-            object_config = type('Config', (ProdConfig, CustomConfig), dict())
+            object_config = type('Config', (ProdConfig, CustomConfig), {})
     app.config.from_object(object_config)
 
     # ------------------------------------------

setup.cfg

@@ -92,9 +92,9 @@ dev =
 	matplotlib>=3.1.1
 	flake8>=3.7.9
 	doc8>=0.8.0
-	pytest>=6.2.2
+	pytest>=7.2.2
 	pytest-cov>=2.8.1
-	pytest-sugar>=0.9.4
+	pytest-sugar<1.0.0
 	pytest-remotedata>=0.3.2
     pytest-flask>=0.10.0
     pytest-xdist>=1.18.1