Merge pull request #697 from nsidc/prefix-envvars

Prefix all envvars our code/deployment cares about with `QGREENLAND_`

doc/contributor/how-to/run-qgreenland.md

@@ -1,13 +1,10 @@
 # How to run QGreenland Core
 
-This project uses Docker and `docker-compose` to run each of its components as
-services.  See Docker's [Getting started guide](https://docs.docker.com/get-started/).
+This project uses Docker and `docker-compose` to run each of its components as services.
+See Docker's [Getting started guide](https://docs.docker.com/get-started/).
 
-The docker-compose stack runs Luigi (with visualizer at port 8082) as a service
-for running tasks, as well as NGINX (port 80, 443) for hosting outputs.
-
-
-## How to start the service stack
+The `docker-compose` stack runs Luigi (with visualizer at port 8082) as a service for
+running tasks, as well as NGINX (port 80, 443) for hosting outputs.
 
 ```{caution}
 Docker Desktop for OSX has some "gotchas". Running with "Use gRPC FUSE for file sharing"
@@ -18,33 +15,70 @@ https://docs.docker.com/desktop/mac/
 ```
 
 
+## How to configure the service stack
+
+### Development overrides
+
+Development overrides enable:
+
+* Build the Docker image from local source instead of using a versioned Docker image
+* Mount the source code into the Docker container, so the container doesn't need to be
+  re-built on each change
+
+To set up development overrides on your machine:
+
+```
+ln -s docker-compose.dev.yml docker-compose.override.yml
+```
+
+
 ### Envvars
 
+Some envvars are used by the source code, others are used by the `docker-compose`
+config.
+
+
+#### Mandatory envvars
+
 In order to download data behind Earthdata Login, you must `export` the
-following environment variables on the docker host before starting the stack:
+following environment variables:
 
-* `EARTHDATA_USERNAME`
-* `EARTHDATA_PASSWORD`
+* `QGREENLAND_EARTHDATA_USERNAME`
+* `QGREENLAND_EARTHDATA_PASSWORD`
 
 Developers at NSIDC may use the values stored in Vault at the following path:
 `nsidc/apps/qgreenland`. Those outside of NSIDC must use their personal
 Earthdata Login credentials. New users to Earthdata can register here:
 https://urs.earthdata.nasa.gov/users/new
 
 
-### Directory set up
+##### Optional envvars
 
-If you prefer not to store data alongside the source code, set up symlinks from the
-following locations to your desired location:
+The source code looks at these envvars, if set:
 
-* `./data/private-archive`
-* `./data/working-storage`
+* `QGREENLAND_ENVIRONMENT`: defaults to `dev`
+* `QGREENLAND_ENV_MANAGER`: defaults to `conda`
+
+
+#### Optional Docker Compose envvars
+
+Our source code expects to run in a container and has hard-coded path constants. We
+should move these envvars and defaults into the source code, but for now they're for
+configuring the compose stack to route directories on the host to the hard-coded
+container locations.
+
+* `QGREENLAND_VERSION`: The `nsidc/qgreenland` docker image tag to use. Defaults to
+  `latest`.
+* `QGREENLAND_DATA_WORKING_STORAGE`: defaults to `./data/working-storage`
+* `QGREENLAND_DATA_PRIVATE_ARCHIVE`: defaults to `./data/private-archive`
+* `QGREENLAND_DATA_LOGS`: defaults to `./data/logs`
 
 Visit our [storage architecture reference
-documentation](../reference/architecture/storage.md) to learn more.
+documentation](../reference/architecture/storage.md) to learn more about storage
+locations.
 
 
-### Go!
+## How to start the service stack
 
 Start the stack with docker-compose:
 

docker-compose.yml

@@ -5,7 +5,7 @@ services:
   # Luigi runs as a service and must have jobs submitted to it
   # (`scripts/run.sh`)
   luigi:
-    image: "nsidc/qgreenland:${QGREENLAND_VERSION:?QGREENLAND_VERSION must be set}"
+    image: "nsidc/qgreenland:${QGREENLAND_VERSION:-latest}"
     container_name: "luigi"
     volumes:
       # Code
@@ -15,16 +15,16 @@ services:
       - "./luigi/state:/luigi/state:rw"
 
       # Input (private) storage
-      - "${DATA_PRIVATE_ARCHIVE:-./data/private-archive}:/private-archive:ro"
+      - "${QGREENLAND_DATA_PRIVATE_ARCHIVE:-./data/private-archive}:/private-archive:ro"
       # Read-write storage
-      # HACK: Should use ${DATA_WORKING_STORAGE}, but we were having issues
-      # with our NFS appliance some time ago, started using two working storage
-      # locations temporarily, and we haven't re-tested yet.
-      - "${DATA_WORKING_STORAGE_TMP:-./data/working-storage}:/working-storage:rw"
+      # HACK: Should use ${QGREENLAND_DATA_WORKING_STORAGE}, but we were having
+      # issues with our storage appliance some time ago, started using two
+      # working storage locations temporarily, and we haven't re-tested yet.
+      - "${QGREENLAND_DATA_WORKING_STORAGE_TMP:-./data/working-storage}:/working-storage:rw"
     environment:
-      - "ENVIRONMENT"
-      - "EARTHDATA_USERNAME"
-      - "EARTHDATA_PASSWORD"
+      - "QGREENLAND_EARTHDATA_USERNAME=${QGREENLAND_EARTHDATA_USERNAME:?QGREENLAND_EARTHDATA_USERNAME must be set}"
+      - "QGREENLAND_EARTHDATA_PASSWORD=${QGREENLAND_EARTHDATA_PASSWORD:?QGREENLAND_EARTHDATA_PASSWORD must be set}"
+      - "QGREENLAND_ENVIRONMENT"
       - "QGREENLAND_ENV_MANAGER=micromamba"
       # Configure Luigi to find its config in luigi/conf/luigi.toml
       - "LUIGI_CONFIG_PARSER=toml"
@@ -50,9 +50,9 @@ services:
     volumes:
       - "./nginx/default.conf:/etc/nginx/conf.d/default.conf:ro"
       - "./nginx/nginx.conf:/etc/nginx/nginx.conf:ro"
-      - "${DATA_WORKING_STORAGE:-./data/working-storage}/release-layers:/usr/share/nginx/html/layers:ro"
-      - "${DATA_WORKING_STORAGE:-./data/working-storage}/release-packages:/usr/share/nginx/html/packages:ro"
-      - "${DATA_LOGS:-./data/logs}:/logs:rw"
+      - "${QGREENLAND_DATA_WORKING_STORAGE:-./data/working-storage}/release-layers:/usr/share/nginx/html/layers:ro"
+      - "${QGREENLAND_DATA_WORKING_STORAGE:-./data/working-storage}/release-packages:/usr/share/nginx/html/packages:ro"
+      - "${QGREENLAND_DATA_LOGS:-./data/logs}:/logs:rw"
     ports:
       - "80:80"
       - "443:443"

qgreenland/constants/project.py

@@ -1,7 +1,7 @@
 import os
 
 PROJECT = "QGreenland"
-ENVIRONMENT = os.environ.get("ENVIRONMENT", "dev")
+ENVIRONMENT = os.environ.get("QGREENLAND_ENVIRONMENT", "dev")
 ENV_MANAGER = os.environ.get("QGREENLAND_ENV_MANAGER", "conda")
 
 # In seconds. See

qgreenland/util/edl.py

@@ -49,9 +49,16 @@ def create_earthdata_authenticated_session(s=None, *, hosts, verify):
 
 
 def _get_earthdata_creds():
-    if not os.environ.get("EARTHDATA_USERNAME"):
-        raise RuntimeError("Environment variable EARTHDATA_USERNAME must be defined.")
-    if not os.environ.get("EARTHDATA_PASSWORD"):
-        raise RuntimeError("Environment variable EARTHDATA_PASSWORD must be defined.")
+    if not os.environ.get("QGREENLAND_EARTHDATA_USERNAME"):
+        raise RuntimeError(
+            "Environment variable QGREENLAND_EARTHDATA_USERNAME must be defined."
+        )
+    if not os.environ.get("QGREENLAND_EARTHDATA_PASSWORD"):
+        raise RuntimeError(
+            "Environment variable QGREENLAND_EARTHDATA_PASSWORD must be defined."
+        )
 
-    return (os.environ["EARTHDATA_USERNAME"], os.environ["EARTHDATA_PASSWORD"])
+    return (
+        os.environ["QGREENLAND_EARTHDATA_USERNAME"],
+        os.environ["QGREENLAND_EARTHDATA_PASSWORD"],
+    )