diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 04791f53..83ed2ef5 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -1,4 +1,4 @@ -name: "Test and (if tag) build" +name: "Test, release, and build" on: # TODO: We currently get double workflows when pushing to pull request @@ -6,7 +6,7 @@ on: # instead? push: branches: - - "*" + - "main" tags: - "v[0-9]+.[0-9]+.[0-9]+*" pull_request: @@ -21,6 +21,7 @@ defaults: jobs: test: + name: "Run tests" runs-on: "ubuntu-latest" steps: - name: "Check out repository" @@ -36,7 +37,6 @@ jobs: - name: "Install Conda environment" uses: "mamba-org/setup-micromamba@v1" with: - micromamba-version: "1.4.2-2" environment-file: "conda-lock.yml" # When using a lock-file, we have to set an environment name. environment-name: "qgreenland-ci" @@ -63,9 +63,56 @@ jobs: QT_QPA_PLATFORM: offscreen - build: + # TODO: Consider extracting everything below this line to a separate workflow + # which is triggered by pushes to the default branch and GitHub releases. + build-and-release-image: + name: "Build and (if `main` or tag) release container image" runs-on: "ubuntu-latest" needs: ["test"] + env: + IMAGE_NAME: "nsidc/qgreenland" + # GitHub Actions expressions don't have great conditional support, so + # writing a ternary expression looks a lot like bash. In Python, this + # would read as: + # 'latest' if github.ref_type == 'branch' else github.ref_name + # https://docs.github.com/en/actions/learn-github-actions/expressions + IMAGE_TAG: "${{ github.ref_type == 'branch' && 'latest' || github.ref_name }}" + steps: + - name: "Check out repository" + uses: "actions/checkout@v3" + + - name: "Build Docker image" + run: | + docker build -t "${IMAGE_NAME}:${IMAGE_TAG}" . + + - name: "DockerHub login (if `main` or tag)" + if: "github.event_name == 'tag' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch)" + uses: "docker/login-action@v2" + with: + username: "${{secrets.DOCKER_USER}}" + password: "${{secrets.DOCKER_PASS}}" + + - name: "GHCR login (if `main` or tag)" + if: "github.event_name == 'tag' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch)" + uses: "docker/login-action@v2" + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: "Release to DockerHub and GHCR (if `main` or tag)" + if: "github.event_name == 'tag' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch)" + run: | + docker push "${IMAGE_NAME}:${IMAGE_TAG}" + + docker tag "${IMAGE_NAME}:${IMAGE_TAG}" "ghcr.io/${IMAGE_NAME}:${IMAGE_TAG}" + docker push "ghcr.io/${IMAGE_NAME}:${IMAGE_TAG}" + + + build-package: + name: "Build QGreenland project (if tag)" + runs-on: "ubuntu-latest" + needs: ["build-and-release-image"] if: "github.ref_type == 'tag'" steps: - name: "Trigger Jenkins to build QGreenland Core" diff --git a/Dockerfile b/Dockerfile index 6ef0c7be..551d96a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,41 +1,40 @@ # This docker image simply runs luigi's centralized scheduler with the needed -# dependencies installed into a conda environment. It's expected that task code -# will always be mounted in using volumes! +# dependencies installed into a conda environment. All QGreenland tasks are +# available at `TASKS_DIR`. FROM axiom/docker-luigi:3.0.3-alpine AS luigi # This build stage only exists to grab the luigi run script. Luigi dependency # itself is specified in `environment.yml` +# TODO: Why is this necessary? Does `luigid` not come along with the conda package? FROM mambaorg/micromamba:1.4.2 AS micromamba COPY --from=luigi /bin/run /usr/local/bin/luigid USER root -ENV TASKS_MOUNT_DIR=/luigi/tasks/qgreenland - # `libgl1-mesa-glx` is required for pyqgis # `git` is required for analyzing the current version # `make` is required for building sphinx docs # `texlive-latex-extra` is required for pdf doc builds +# TODO: Remove `make` RUN apt-get update && apt-get install -y \ git \ make \ libgl1-mesa-glx \ texlive-latex-extra -# Enable our code (which runs git commands) to run as a different user than the -# current user on the host machine (who will be the owner of the mounted git -# repository) -RUN git config --global --add safe.directory "${TASKS_MOUNT_DIR}" - -# TODO: Why are we copying these files to /tmp? -COPY --chown=$MAMBA_USER:$MAMBA_USER conda-lock.yml /tmp/conda-lock.yml -RUN micromamba install -y -n base -f /tmp/conda-lock.yml +ENV TASKS_DIR=/luigi/tasks/qgreenland +WORKDIR "${TASKS_DIR}" +COPY --chown=$MAMBA_USER:$MAMBA_USER . . -# Install mamba. It is missing after installing `conda-lock.yml` -RUN micromamba install -y -c conda-forge -n base conda mamba~=1.4.2 +# Our code needs to run git commands (for example, to determine a full version +# string), but if tasks repo is mounted from the host machine, the owner of +# the repo won't match the container user. A "safe directory" allows Git to +# tolerate this user mismatch. +RUN git config --global --add safe.directory "${TASKS_DIR}" -COPY --chown=$MAMBA_USER:$MAMBA_USER environment.cmd.yml /tmp/environment.cmd.yml -RUN micromamba create -y -f /tmp/environment.cmd.yml +# Set up the Luigi task environment and the command environment +RUN micromamba install -y -n base -f conda-lock.yml +RUN micromamba create -y -f environment.cmd.yml # Cleanup RUN micromamba clean --all --yes @@ -47,7 +46,7 @@ WORKDIR /luigi # gets populated. Additionally, /luigi/tasks is where we expect python code to # be mounted. # TODO: With modern micromamba, can we clean this up? -ENV PYTHONPATH "${TASKS_MOUNT_DIR}:/opt/conda/share/qgis/python/plugins:/opt/conda/share/qgis/python" +ENV PYTHONPATH "${TASKS_DIR}:/opt/conda/share/qgis/python/plugins:/opt/conda/share/qgis/python" ENV PATH "/opt/conda/bin:${PATH}" CMD ["/usr/local/bin/luigid"] diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..c4a5fd8c --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,10 @@ +version: "3.4" + +services: + + luigi: + image: "nsidc/luigi:dev" + build: "." + volumes: + # Code + - "./:/luigi/tasks/qgreenland:ro" diff --git a/docker-compose.yml b/docker-compose.yml index 40fb93ed..a0f51fe4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,8 +5,7 @@ services: # Luigi runs as a service and must have jobs submitted to it # (`scripts/run.sh`) luigi: - image: "nsidc/luigi:dev" - build: "." + image: "nsidc/luigi:local" container_name: "luigi" volumes: # Code @@ -23,10 +22,12 @@ services: # locations temporarily, and we haven't re-tested yet. - "${DATA_WORKING_STORAGE_TMP:-./data/working-storage}:/working-storage:rw" environment: - - "LUIGI_CONFIG_PARSER=toml" - "ENVIRONMENT" - "EARTHDATA_USERNAME" - "EARTHDATA_PASSWORD" + - "QGREENLAND_ENV_MANAGER=micromamba" + # Configure Luigi to find its config in luigi/conf/luigi.toml + - "LUIGI_CONFIG_PARSER=toml" # Set `export PYTHONBREAKPOINT=ipdb.set_trace` to use `ipdb` by default # instead of `pdb`. - "PYTHONBREAKPOINT" diff --git a/qgreenland/constants/project.py b/qgreenland/constants/project.py index 2ce0a372..00643f80 100644 --- a/qgreenland/constants/project.py +++ b/qgreenland/constants/project.py @@ -2,6 +2,7 @@ PROJECT = "QGreenland" ENVIRONMENT = os.environ.get("ENVIRONMENT", "dev") +ENV_MANAGER = os.environ.get("QGREENLAND_ENV_MANAGER", "conda") # In seconds. See # https://2.python-requests.org/en/master/user/quickstart/#timeouts diff --git a/qgreenland/util/command.py b/qgreenland/util/command.py index f4aca505..4ee365ec 100644 --- a/qgreenland/util/command.py +++ b/qgreenland/util/command.py @@ -3,6 +3,7 @@ from collections.abc import Sequence import qgreenland.exceptions as exc +from qgreenland.constants.project import ENV_MANAGER from qgreenland.util.runtime_vars import EvalStr logger = logging.getLogger("luigi-interface") @@ -16,19 +17,33 @@ def interpolate_args( return [arg.eval(**kwargs) for arg in args] -def run_qgr_command(args: list[str]): +def run_qgr_command(args: list[str]) -> None: """Run a command in the `qgreenland-cmd` environment.""" - cmd = [".", "activate", "qgreenland-cmd", "&&"] - cmd.extend(args) + conda_env_name = "qgreenland-cmd" + # With conda or mamba, `. activate myenv` works as expected, but with micromamba, we + # need something a little different. + if ENV_MANAGER == "micromamba": + cmd = [ + "eval", + '"$(micromamba shell hook -s posix)"', + "&&", + "micromamba", + "activate", + conda_env_name, + "&&", + *args, + ] + + else: + cmd = [".", "activate", conda_env_name, "&&", *args] run_cmd(cmd) + return -def run_cmd(args: list[str]): +def run_cmd(args: list[str]) -> subprocess.CompletedProcess: """Run a command and log it.""" - # Hack. The activation of a conda environment does not work as a list. - # `subprocess.run(..., shell=True, ...)` enables running commands from - # strings. + # Hack. The activation of a conda environment does not work without `shell=True`. cmd_str = " ".join(str(arg) for arg in args) logger.info("Running command:")