diff --git a/doc/contributor/how-to/run-qgreenland.md b/doc/contributor/how-to/run-qgreenland.md index e64234a4..b523b1da 100644 --- a/doc/contributor/how-to/run-qgreenland.md +++ b/doc/contributor/how-to/run-qgreenland.md @@ -1,13 +1,10 @@ # How to run QGreenland Core -This project uses Docker and `docker-compose` to run each of its components as -services. See Docker's [Getting started guide](https://docs.docker.com/get-started/). +This project uses Docker and `docker-compose` to run each of its components as services. +See Docker's [Getting started guide](https://docs.docker.com/get-started/). -The docker-compose stack runs Luigi (with visualizer at port 8082) as a service -for running tasks, as well as NGINX (port 80, 443) for hosting outputs. - - -## How to start the service stack +The `docker-compose` stack runs Luigi (with visualizer at port 8082) as a service for +running tasks, as well as NGINX (port 80, 443) for hosting outputs. ```{caution} Docker Desktop for OSX has some "gotchas". Running with "Use gRPC FUSE for file sharing" @@ -18,13 +15,36 @@ https://docs.docker.com/desktop/mac/ ``` +## How to configure the service stack + +### Development overrides + +Development overrides enable: + +* Build the Docker image from local source instead of using a versioned Docker image +* Mount the source code into the Docker container, so the container doesn't need to be + re-built on each change + +To set up development overrides on your machine: + +``` +ln -s docker-compose.dev.yml docker-compose.override.yml +``` + + ### Envvars +Some envvars are used by the source code, others are used by the `docker-compose` +config. + + +#### Mandatory envvars + In order to download data behind Earthdata Login, you must `export` the -following environment variables on the docker host before starting the stack: +following environment variables: -* `EARTHDATA_USERNAME` -* `EARTHDATA_PASSWORD` +* `QGREENLAND_EARTHDATA_USERNAME` +* `QGREENLAND_EARTHDATA_PASSWORD` Developers at NSIDC may use the values stored in Vault at the following path: `nsidc/apps/qgreenland`. Those outside of NSIDC must use their personal @@ -32,19 +52,33 @@ Earthdata Login credentials. New users to Earthdata can register here: https://urs.earthdata.nasa.gov/users/new -### Directory set up +##### Optional envvars -If you prefer not to store data alongside the source code, set up symlinks from the -following locations to your desired location: +The source code looks at these envvars, if set: -* `./data/private-archive` -* `./data/working-storage` +* `QGREENLAND_ENVIRONMENT`: defaults to `dev` +* `QGREENLAND_ENV_MANAGER`: defaults to `conda` + + +#### Optional Docker Compose envvars + +Our source code expects to run in a container and has hard-coded path constants. We +should move these envvars and defaults into the source code, but for now they're for +configuring the compose stack to route directories on the host to the hard-coded +container locations. + +* `QGREENLAND_VERSION`: The `nsidc/qgreenland` docker image tag to use. Defaults to + `latest`. +* `QGREENLAND_DATA_WORKING_STORAGE`: defaults to `./data/working-storage` +* `QGREENLAND_DATA_PRIVATE_ARCHIVE`: defaults to `./data/private-archive` +* `QGREENLAND_DATA_LOGS`: defaults to `./data/logs` Visit our [storage architecture reference -documentation](../reference/architecture/storage.md) to learn more. +documentation](../reference/architecture/storage.md) to learn more about storage +locations. -### Go! +## How to start the service stack Start the stack with docker-compose: diff --git a/docker-compose.yml b/docker-compose.yml index f86e047c..458e769e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: # Luigi runs as a service and must have jobs submitted to it # (`scripts/run.sh`) luigi: - image: "nsidc/qgreenland:${QGREENLAND_VERSION:?QGREENLAND_VERSION must be set}" + image: "nsidc/qgreenland:${QGREENLAND_VERSION:-latest}" container_name: "luigi" volumes: # Code @@ -15,16 +15,16 @@ services: - "./luigi/state:/luigi/state:rw" # Input (private) storage - - "${DATA_PRIVATE_ARCHIVE:-./data/private-archive}:/private-archive:ro" + - "${QGREENLAND_DATA_PRIVATE_ARCHIVE:-./data/private-archive}:/private-archive:ro" # Read-write storage - # HACK: Should use ${DATA_WORKING_STORAGE}, but we were having issues - # with our NFS appliance some time ago, started using two working storage - # locations temporarily, and we haven't re-tested yet. - - "${DATA_WORKING_STORAGE_TMP:-./data/working-storage}:/working-storage:rw" + # HACK: Should use ${QGREENLAND_DATA_WORKING_STORAGE}, but we were having + # issues with our storage appliance some time ago, started using two + # working storage locations temporarily, and we haven't re-tested yet. + - "${QGREENLAND_DATA_WORKING_STORAGE_TMP:-./data/working-storage}:/working-storage:rw" environment: - - "ENVIRONMENT" - - "EARTHDATA_USERNAME" - - "EARTHDATA_PASSWORD" + - "QGREENLAND_EARTHDATA_USERNAME=${QGREENLAND_EARTHDATA_USERNAME:?QGREENLAND_EARTHDATA_USERNAME must be set}" + - "QGREENLAND_EARTHDATA_PASSWORD=${QGREENLAND_EARTHDATA_PASSWORD:?QGREENLAND_EARTHDATA_PASSWORD must be set}" + - "QGREENLAND_ENVIRONMENT" - "QGREENLAND_ENV_MANAGER=micromamba" # Configure Luigi to find its config in luigi/conf/luigi.toml - "LUIGI_CONFIG_PARSER=toml" @@ -50,9 +50,9 @@ services: volumes: - "./nginx/default.conf:/etc/nginx/conf.d/default.conf:ro" - "./nginx/nginx.conf:/etc/nginx/nginx.conf:ro" - - "${DATA_WORKING_STORAGE:-./data/working-storage}/release-layers:/usr/share/nginx/html/layers:ro" - - "${DATA_WORKING_STORAGE:-./data/working-storage}/release-packages:/usr/share/nginx/html/packages:ro" - - "${DATA_LOGS:-./data/logs}:/logs:rw" + - "${QGREENLAND_DATA_WORKING_STORAGE:-./data/working-storage}/release-layers:/usr/share/nginx/html/layers:ro" + - "${QGREENLAND_DATA_WORKING_STORAGE:-./data/working-storage}/release-packages:/usr/share/nginx/html/packages:ro" + - "${QGREENLAND_DATA_LOGS:-./data/logs}:/logs:rw" ports: - "80:80" - "443:443" diff --git a/qgreenland/constants/project.py b/qgreenland/constants/project.py index 00643f80..527d8e36 100644 --- a/qgreenland/constants/project.py +++ b/qgreenland/constants/project.py @@ -1,7 +1,7 @@ import os PROJECT = "QGreenland" -ENVIRONMENT = os.environ.get("ENVIRONMENT", "dev") +ENVIRONMENT = os.environ.get("QGREENLAND_ENVIRONMENT", "dev") ENV_MANAGER = os.environ.get("QGREENLAND_ENV_MANAGER", "conda") # In seconds. See diff --git a/qgreenland/util/edl.py b/qgreenland/util/edl.py index d482f977..dc146602 100644 --- a/qgreenland/util/edl.py +++ b/qgreenland/util/edl.py @@ -49,9 +49,16 @@ def create_earthdata_authenticated_session(s=None, *, hosts, verify): def _get_earthdata_creds(): - if not os.environ.get("EARTHDATA_USERNAME"): - raise RuntimeError("Environment variable EARTHDATA_USERNAME must be defined.") - if not os.environ.get("EARTHDATA_PASSWORD"): - raise RuntimeError("Environment variable EARTHDATA_PASSWORD must be defined.") + if not os.environ.get("QGREENLAND_EARTHDATA_USERNAME"): + raise RuntimeError( + "Environment variable QGREENLAND_EARTHDATA_USERNAME must be defined." + ) + if not os.environ.get("QGREENLAND_EARTHDATA_PASSWORD"): + raise RuntimeError( + "Environment variable QGREENLAND_EARTHDATA_PASSWORD must be defined." + ) - return (os.environ["EARTHDATA_USERNAME"], os.environ["EARTHDATA_PASSWORD"]) + return ( + os.environ["QGREENLAND_EARTHDATA_USERNAME"], + os.environ["QGREENLAND_EARTHDATA_PASSWORD"], + )