From 8fc013e459bdbe0aec003a88e89d79ac747e0d78 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Tue, 25 Jul 2023 19:10:09 -0600 Subject: [PATCH 1/4] Prefix all envvars our code/deployment cares about with `QGREENLAND_` This is a good practice to avoid namespace conflicts and to make configration easier to debug. --- doc/contributor/how-to/run-qgreenland.md | 35 +++++++++++++++++------- docker-compose.yml | 22 +++++++-------- qgreenland/constants/project.py | 2 +- qgreenland/util/edl.py | 17 ++++++++---- 4 files changed, 49 insertions(+), 27 deletions(-) diff --git a/doc/contributor/how-to/run-qgreenland.md b/doc/contributor/how-to/run-qgreenland.md index e64234a4..f7930c33 100644 --- a/doc/contributor/how-to/run-qgreenland.md +++ b/doc/contributor/how-to/run-qgreenland.md @@ -20,11 +20,13 @@ https://docs.docker.com/desktop/mac/ ### Envvars +### Earthdata Login + In order to download data behind Earthdata Login, you must `export` the -following environment variables on the docker host before starting the stack: +following environment variables: -* `EARTHDATA_USERNAME` -* `EARTHDATA_PASSWORD` +* `QGREENLAND_EARTHDATA_USERNAME` +* `QGREENLAND_EARTHDATA_PASSWORD` Developers at NSIDC may use the values stored in Vault at the following path: `nsidc/apps/qgreenland`. Those outside of NSIDC must use their personal @@ -32,16 +34,29 @@ Earthdata Login credentials. New users to Earthdata can register here: https://urs.earthdata.nasa.gov/users/new -### Directory set up +#### Optional envvars + +The source code looks at these envvars, if set: + +* `QGREENLAND_ENVIRONMENT`: defaults to `dev` +* `QGREENLAND_ENV_MANAGER`: defaults to `conda` + + +### Mandatory Docker Compose envvars + +* `QGREENLAND_VERSION`: The `nsidc/qgreenland` docker image tag to use. + -If you prefer not to store data alongside the source code, set up symlinks from the -following locations to your desired location: +### Optional Docker Compose envvars -* `./data/private-archive` -* `./data/working-storage` +Our source code expects to run in a container and has hard-coded path constants. We +should move these envvars and defaults into the source code, but for now they're for +configuring the compose stack to route directories on the host to the hard-coded +container locations. -Visit our [storage architecture reference -documentation](../reference/architecture/storage.md) to learn more. +* `QGREENLAND_DATA_WORKING_STORAGE`: defaults to `./data/working-storage` +* `QGREENLAND_DATA_PRIVATE_ARCHIVE`: defaults to `./data/private-archive` +* `QGREENLAND_DATA_LOGS`: defaults to `./data/logs` ### Go! diff --git a/docker-compose.yml b/docker-compose.yml index f86e047c..89b0a237 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,16 +15,16 @@ services: - "./luigi/state:/luigi/state:rw" # Input (private) storage - - "${DATA_PRIVATE_ARCHIVE:-./data/private-archive}:/private-archive:ro" + - "${QGREENLAND_DATA_PRIVATE_ARCHIVE:-./data/private-archive}:/private-archive:ro" # Read-write storage - # HACK: Should use ${DATA_WORKING_STORAGE}, but we were having issues - # with our NFS appliance some time ago, started using two working storage - # locations temporarily, and we haven't re-tested yet. - - "${DATA_WORKING_STORAGE_TMP:-./data/working-storage}:/working-storage:rw" + # HACK: Should use ${QGREENLAND_DATA_WORKING_STORAGE}, but we were having + # issues with our storage appliance some time ago, started using two + # working storage locations temporarily, and we haven't re-tested yet. + - "${QGREENLAND_DATA_WORKING_STORAGE_TMP:-./data/working-storage}:/working-storage:rw" environment: - - "ENVIRONMENT" - - "EARTHDATA_USERNAME" - - "EARTHDATA_PASSWORD" + - "QGREENLAND_EARTHDATA_USERNAME=${QGREENLAND_EARTHDATA_USERNAME:?QGREENLAND_EARTHDATA_USERNAME must be set}" + - "QGREENLAND_EARTHDATA_PASSWORD=${QGREENLAND_EARTHDATA_PASSWORD:?QGREENLAND_EARTHDATA_PASSWORD must be set}" + - "QGREENLAND_ENVIRONMENT" - "QGREENLAND_ENV_MANAGER=micromamba" # Configure Luigi to find its config in luigi/conf/luigi.toml - "LUIGI_CONFIG_PARSER=toml" @@ -50,9 +50,9 @@ services: volumes: - "./nginx/default.conf:/etc/nginx/conf.d/default.conf:ro" - "./nginx/nginx.conf:/etc/nginx/nginx.conf:ro" - - "${DATA_WORKING_STORAGE:-./data/working-storage}/release-layers:/usr/share/nginx/html/layers:ro" - - "${DATA_WORKING_STORAGE:-./data/working-storage}/release-packages:/usr/share/nginx/html/packages:ro" - - "${DATA_LOGS:-./data/logs}:/logs:rw" + - "${QGREENLAND_DATA_WORKING_STORAGE:-./data/working-storage}/release-layers:/usr/share/nginx/html/layers:ro" + - "${QGREENLAND_DATA_WORKING_STORAGE:-./data/working-storage}/release-packages:/usr/share/nginx/html/packages:ro" + - "${QGREENLAND_DATA_LOGS:-./data/logs}:/logs:rw" ports: - "80:80" - "443:443" diff --git a/qgreenland/constants/project.py b/qgreenland/constants/project.py index 00643f80..527d8e36 100644 --- a/qgreenland/constants/project.py +++ b/qgreenland/constants/project.py @@ -1,7 +1,7 @@ import os PROJECT = "QGreenland" -ENVIRONMENT = os.environ.get("ENVIRONMENT", "dev") +ENVIRONMENT = os.environ.get("QGREENLAND_ENVIRONMENT", "dev") ENV_MANAGER = os.environ.get("QGREENLAND_ENV_MANAGER", "conda") # In seconds. See diff --git a/qgreenland/util/edl.py b/qgreenland/util/edl.py index d482f977..dc146602 100644 --- a/qgreenland/util/edl.py +++ b/qgreenland/util/edl.py @@ -49,9 +49,16 @@ def create_earthdata_authenticated_session(s=None, *, hosts, verify): def _get_earthdata_creds(): - if not os.environ.get("EARTHDATA_USERNAME"): - raise RuntimeError("Environment variable EARTHDATA_USERNAME must be defined.") - if not os.environ.get("EARTHDATA_PASSWORD"): - raise RuntimeError("Environment variable EARTHDATA_PASSWORD must be defined.") + if not os.environ.get("QGREENLAND_EARTHDATA_USERNAME"): + raise RuntimeError( + "Environment variable QGREENLAND_EARTHDATA_USERNAME must be defined." + ) + if not os.environ.get("QGREENLAND_EARTHDATA_PASSWORD"): + raise RuntimeError( + "Environment variable QGREENLAND_EARTHDATA_PASSWORD must be defined." + ) - return (os.environ["EARTHDATA_USERNAME"], os.environ["EARTHDATA_PASSWORD"]) + return ( + os.environ["QGREENLAND_EARTHDATA_USERNAME"], + os.environ["QGREENLAND_EARTHDATA_PASSWORD"], + ) From 8c8c1a7fa1c6ec07f6b07b954291c62566cb41e8 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Tue, 25 Jul 2023 19:18:59 -0600 Subject: [PATCH 2/4] Re-org docs and add more info on development configuration --- doc/contributor/how-to/run-qgreenland.md | 42 +++++++++++++++++------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/doc/contributor/how-to/run-qgreenland.md b/doc/contributor/how-to/run-qgreenland.md index f7930c33..00c48d75 100644 --- a/doc/contributor/how-to/run-qgreenland.md +++ b/doc/contributor/how-to/run-qgreenland.md @@ -1,13 +1,10 @@ # How to run QGreenland Core -This project uses Docker and `docker-compose` to run each of its components as -services. See Docker's [Getting started guide](https://docs.docker.com/get-started/). +This project uses Docker and `docker-compose` to run each of its components as services. +See Docker's [Getting started guide](https://docs.docker.com/get-started/). -The docker-compose stack runs Luigi (with visualizer at port 8082) as a service -for running tasks, as well as NGINX (port 80, 443) for hosting outputs. - - -## How to start the service stack +The `docker-compose` stack runs Luigi (with visualizer at port 8082) as a service for +running tasks, as well as NGINX (port 80, 443) for hosting outputs. ```{caution} Docker Desktop for OSX has some "gotchas". Running with "Use gRPC FUSE for file sharing" @@ -18,9 +15,30 @@ https://docs.docker.com/desktop/mac/ ``` +## How to configure the service stack + +### Development overrides + +Development overrides enable: + +* Build the Docker image from local source instead of using a versioned Docker image +* Mount the source code into the Docker container, so the container doesn't need to be + re-built on each change + +To set up development overrides on your machine: + +``` +ln -s docker-compose.dev.yml docker-compose.override.yml +``` + + ### Envvars -### Earthdata Login +Some envvars are used by the source code, others are used by the `docker-compose` +config. + + +#### Mandatory envvars In order to download data behind Earthdata Login, you must `export` the following environment variables: @@ -34,7 +52,7 @@ Earthdata Login credentials. New users to Earthdata can register here: https://urs.earthdata.nasa.gov/users/new -#### Optional envvars +##### Optional envvars The source code looks at these envvars, if set: @@ -42,12 +60,12 @@ The source code looks at these envvars, if set: * `QGREENLAND_ENV_MANAGER`: defaults to `conda` -### Mandatory Docker Compose envvars +#### Mandatory Docker Compose envvars * `QGREENLAND_VERSION`: The `nsidc/qgreenland` docker image tag to use. -### Optional Docker Compose envvars +#### Optional Docker Compose envvars Our source code expects to run in a container and has hard-coded path constants. We should move these envvars and defaults into the source code, but for now they're for @@ -59,7 +77,7 @@ container locations. * `QGREENLAND_DATA_LOGS`: defaults to `./data/logs` -### Go! +## How to start the service stack Start the stack with docker-compose: From bccb93b0834948d04e250012ee694ee5d9cc59b1 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Tue, 25 Jul 2023 19:34:18 -0600 Subject: [PATCH 3/4] Make version envvar optional -- breaks dev override The mandatory variable check runs even if the string with the mandatory variable check is overridden by another compose file. This seems like a flaw in docker-compose's behavior -- overrides should come first, then evaluation, right? I'm sure there's a good reason things are the way they are. --- doc/contributor/how-to/run-qgreenland.md | 9 ++++++--- docker-compose.yml | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/contributor/how-to/run-qgreenland.md b/doc/contributor/how-to/run-qgreenland.md index 00c48d75..87817cfe 100644 --- a/doc/contributor/how-to/run-qgreenland.md +++ b/doc/contributor/how-to/run-qgreenland.md @@ -62,9 +62,6 @@ The source code looks at these envvars, if set: #### Mandatory Docker Compose envvars -* `QGREENLAND_VERSION`: The `nsidc/qgreenland` docker image tag to use. - - #### Optional Docker Compose envvars Our source code expects to run in a container and has hard-coded path constants. We @@ -72,10 +69,16 @@ should move these envvars and defaults into the source code, but for now they're configuring the compose stack to route directories on the host to the hard-coded container locations. +* `QGREENLAND_VERSION`: The `nsidc/qgreenland` docker image tag to use. Defaults to + `latest`. * `QGREENLAND_DATA_WORKING_STORAGE`: defaults to `./data/working-storage` * `QGREENLAND_DATA_PRIVATE_ARCHIVE`: defaults to `./data/private-archive` * `QGREENLAND_DATA_LOGS`: defaults to `./data/logs` +Visit our [storage architecture reference +documentation](../reference/architecture/storage.md) to learn more about storage +locations. + ## How to start the service stack diff --git a/docker-compose.yml b/docker-compose.yml index 89b0a237..458e769e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: # Luigi runs as a service and must have jobs submitted to it # (`scripts/run.sh`) luigi: - image: "nsidc/qgreenland:${QGREENLAND_VERSION:?QGREENLAND_VERSION must be set}" + image: "nsidc/qgreenland:${QGREENLAND_VERSION:-latest}" container_name: "luigi" volumes: # Code From 2c4ee06cd68f9c9159287e4f547a20c7bb4792ae Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Tue, 25 Jul 2023 19:36:07 -0600 Subject: [PATCH 4/4] Remove empty doc subsection --- doc/contributor/how-to/run-qgreenland.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/contributor/how-to/run-qgreenland.md b/doc/contributor/how-to/run-qgreenland.md index 87817cfe..b523b1da 100644 --- a/doc/contributor/how-to/run-qgreenland.md +++ b/doc/contributor/how-to/run-qgreenland.md @@ -60,8 +60,6 @@ The source code looks at these envvars, if set: * `QGREENLAND_ENV_MANAGER`: defaults to `conda` -#### Mandatory Docker Compose envvars - #### Optional Docker Compose envvars Our source code expects to run in a container and has hard-coded path constants. We