From c133d1d9cd7dbb7092c9ac79d6593be75d35c453 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Mon, 10 Jul 2023 19:28:04 -0600 Subject: [PATCH 1/6] Bake QGreenland tasks into Docker image * Remove mamba from docker image * Support micromamba-specific environment activation behavior * Stop mounting code into docker container * Add a dev compose file which mounts code --- Dockerfile | 33 ++++++++++++++++----------------- docker-compose.dev.yml | 10 ++++++++++ docker-compose.yml | 7 ++++--- qgreenland/constants/project.py | 1 + qgreenland/util/command.py | 29 ++++++++++++++++++++++------- 5 files changed, 53 insertions(+), 27 deletions(-) create mode 100644 docker-compose.dev.yml diff --git a/Dockerfile b/Dockerfile index 6ef0c7be..551d96a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,41 +1,40 @@ # This docker image simply runs luigi's centralized scheduler with the needed -# dependencies installed into a conda environment. It's expected that task code -# will always be mounted in using volumes! +# dependencies installed into a conda environment. All QGreenland tasks are +# available at `TASKS_DIR`. FROM axiom/docker-luigi:3.0.3-alpine AS luigi # This build stage only exists to grab the luigi run script. Luigi dependency # itself is specified in `environment.yml` +# TODO: Why is this necessary? Does `luigid` not come along with the conda package? FROM mambaorg/micromamba:1.4.2 AS micromamba COPY --from=luigi /bin/run /usr/local/bin/luigid USER root -ENV TASKS_MOUNT_DIR=/luigi/tasks/qgreenland - # `libgl1-mesa-glx` is required for pyqgis # `git` is required for analyzing the current version # `make` is required for building sphinx docs # `texlive-latex-extra` is required for pdf doc builds +# TODO: Remove `make` RUN apt-get update && apt-get install -y \ git \ make \ libgl1-mesa-glx \ texlive-latex-extra -# Enable our code (which runs git commands) to run as a different user than the -# current user on the host machine (who will be the owner of the mounted git -# repository) -RUN git config --global --add safe.directory "${TASKS_MOUNT_DIR}" - -# TODO: Why are we copying these files to /tmp? -COPY --chown=$MAMBA_USER:$MAMBA_USER conda-lock.yml /tmp/conda-lock.yml -RUN micromamba install -y -n base -f /tmp/conda-lock.yml +ENV TASKS_DIR=/luigi/tasks/qgreenland +WORKDIR "${TASKS_DIR}" +COPY --chown=$MAMBA_USER:$MAMBA_USER . . -# Install mamba. It is missing after installing `conda-lock.yml` -RUN micromamba install -y -c conda-forge -n base conda mamba~=1.4.2 +# Our code needs to run git commands (for example, to determine a full version +# string), but if tasks repo is mounted from the host machine, the owner of +# the repo won't match the container user. A "safe directory" allows Git to +# tolerate this user mismatch. +RUN git config --global --add safe.directory "${TASKS_DIR}" -COPY --chown=$MAMBA_USER:$MAMBA_USER environment.cmd.yml /tmp/environment.cmd.yml -RUN micromamba create -y -f /tmp/environment.cmd.yml +# Set up the Luigi task environment and the command environment +RUN micromamba install -y -n base -f conda-lock.yml +RUN micromamba create -y -f environment.cmd.yml # Cleanup RUN micromamba clean --all --yes @@ -47,7 +46,7 @@ WORKDIR /luigi # gets populated. Additionally, /luigi/tasks is where we expect python code to # be mounted. # TODO: With modern micromamba, can we clean this up? -ENV PYTHONPATH "${TASKS_MOUNT_DIR}:/opt/conda/share/qgis/python/plugins:/opt/conda/share/qgis/python" +ENV PYTHONPATH "${TASKS_DIR}:/opt/conda/share/qgis/python/plugins:/opt/conda/share/qgis/python" ENV PATH "/opt/conda/bin:${PATH}" CMD ["/usr/local/bin/luigid"] diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..c4a5fd8c --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,10 @@ +version: "3.4" + +services: + + luigi: + image: "nsidc/luigi:dev" + build: "." + volumes: + # Code + - "./:/luigi/tasks/qgreenland:ro" diff --git a/docker-compose.yml b/docker-compose.yml index 40fb93ed..a0f51fe4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,8 +5,7 @@ services: # Luigi runs as a service and must have jobs submitted to it # (`scripts/run.sh`) luigi: - image: "nsidc/luigi:dev" - build: "." + image: "nsidc/luigi:local" container_name: "luigi" volumes: # Code @@ -23,10 +22,12 @@ services: # locations temporarily, and we haven't re-tested yet. - "${DATA_WORKING_STORAGE_TMP:-./data/working-storage}:/working-storage:rw" environment: - - "LUIGI_CONFIG_PARSER=toml" - "ENVIRONMENT" - "EARTHDATA_USERNAME" - "EARTHDATA_PASSWORD" + - "QGREENLAND_ENV_MANAGER=micromamba" + # Configure Luigi to find its config in luigi/conf/luigi.toml + - "LUIGI_CONFIG_PARSER=toml" # Set `export PYTHONBREAKPOINT=ipdb.set_trace` to use `ipdb` by default # instead of `pdb`. - "PYTHONBREAKPOINT" diff --git a/qgreenland/constants/project.py b/qgreenland/constants/project.py index 2ce0a372..00643f80 100644 --- a/qgreenland/constants/project.py +++ b/qgreenland/constants/project.py @@ -2,6 +2,7 @@ PROJECT = "QGreenland" ENVIRONMENT = os.environ.get("ENVIRONMENT", "dev") +ENV_MANAGER = os.environ.get("QGREENLAND_ENV_MANAGER", "conda") # In seconds. See # https://2.python-requests.org/en/master/user/quickstart/#timeouts diff --git a/qgreenland/util/command.py b/qgreenland/util/command.py index f4aca505..4ee365ec 100644 --- a/qgreenland/util/command.py +++ b/qgreenland/util/command.py @@ -3,6 +3,7 @@ from collections.abc import Sequence import qgreenland.exceptions as exc +from qgreenland.constants.project import ENV_MANAGER from qgreenland.util.runtime_vars import EvalStr logger = logging.getLogger("luigi-interface") @@ -16,19 +17,33 @@ def interpolate_args( return [arg.eval(**kwargs) for arg in args] -def run_qgr_command(args: list[str]): +def run_qgr_command(args: list[str]) -> None: """Run a command in the `qgreenland-cmd` environment.""" - cmd = [".", "activate", "qgreenland-cmd", "&&"] - cmd.extend(args) + conda_env_name = "qgreenland-cmd" + # With conda or mamba, `. activate myenv` works as expected, but with micromamba, we + # need something a little different. + if ENV_MANAGER == "micromamba": + cmd = [ + "eval", + '"$(micromamba shell hook -s posix)"', + "&&", + "micromamba", + "activate", + conda_env_name, + "&&", + *args, + ] + + else: + cmd = [".", "activate", conda_env_name, "&&", *args] run_cmd(cmd) + return -def run_cmd(args: list[str]): +def run_cmd(args: list[str]) -> subprocess.CompletedProcess: """Run a command and log it.""" - # Hack. The activation of a conda environment does not work as a list. - # `subprocess.run(..., shell=True, ...)` enables running commands from - # strings. + # Hack. The activation of a conda environment does not work without `shell=True`. cmd_str = " ".join(str(arg) for arg in args) logger.info("Running command:") From 453b57e91136673bcea8f9675cd366c3dbdcdd26 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Mon, 10 Jul 2023 20:36:50 -0600 Subject: [PATCH 2/6] Add container image build and release job to GHA --- .github/workflows/test-and-build.yml | 44 ++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 04791f53..d8aca246 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -6,7 +6,7 @@ on: # instead? push: branches: - - "*" + - "main" tags: - "v[0-9]+.[0-9]+.[0-9]+*" pull_request: @@ -36,7 +36,6 @@ jobs: - name: "Install Conda environment" uses: "mamba-org/setup-micromamba@v1" with: - micromamba-version: "1.4.2-2" environment-file: "conda-lock.yml" # When using a lock-file, we have to set an environment name. environment-name: "qgreenland-ci" @@ -63,9 +62,48 @@ jobs: QT_QPA_PLATFORM: offscreen - build: + # TODO: Consider extracting everything below this line to a separate workflow + # which is triggered by pushes to the default branch and GitHub releases. + build-and-release-image: runs-on: "ubuntu-latest" needs: ["test"] + # if: "github.ref_type == 'tag' || github.ref == github.event.repository.default_branch" + if: "github.ref_type == 'tag' || github.ref_type == 'branch'" + env: + IMAGE_NAME: "nsidc/qgreenland" + IMAGE_TAG: "${{ github.ref_type == 'branch' && 'latest' || github.ref_name }}" + steps: + - name: "Check out repository" + uses: "actions/checkout@v3" + + - name: "Build Docker image" + run: | + docker build -t "${IMAGE_NAME}:${IMAGE_TAG}" . + + - name: "DockerHub login" + uses: "docker/login-action@v2" + with: + username: "${{secrets.DOCKER_USER}}" + password: "${{secrets.DOCKER_PASS}}" + + - name: "GHCR login" + uses: "docker/login-action@v2" + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: "Release to DockerHub and GHCR" + run: | + docker push "${IMAGE_NAME}:${IMAGE_TAG}" + + docker tag "${IMAGE_NAME}:${IMAGE_TAG}" "ghcr.io/${IMAGE_NAME}:${IMAGE_TAG}" + docker push "ghcr.io/${IMAGE_NAME}:${IMAGE_TAG}" + + + build-package: + runs-on: "ubuntu-latest" + needs: ["build-and-release-image"] if: "github.ref_type == 'tag'" steps: - name: "Trigger Jenkins to build QGreenland Core" From d1bb9d5ca7e8a7e8dfe8d7cfe84c23364a448d3d Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Tue, 11 Jul 2023 14:00:10 -0600 Subject: [PATCH 3/6] Prevent pushing images on pull requests Should only push images on tags (versions images) or main branch pushes ("latest"). --- .github/workflows/test-and-build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index d8aca246..3c1bb17c 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -67,7 +67,6 @@ jobs: build-and-release-image: runs-on: "ubuntu-latest" needs: ["test"] - # if: "github.ref_type == 'tag' || github.ref == github.event.repository.default_branch" if: "github.ref_type == 'tag' || github.ref_type == 'branch'" env: IMAGE_NAME: "nsidc/qgreenland" @@ -81,12 +80,14 @@ jobs: docker build -t "${IMAGE_NAME}:${IMAGE_TAG}" . - name: "DockerHub login" + if: "github.event_name != 'pull_request'" uses: "docker/login-action@v2" with: username: "${{secrets.DOCKER_USER}}" password: "${{secrets.DOCKER_PASS}}" - name: "GHCR login" + if: "github.event_name != 'pull_request'" uses: "docker/login-action@v2" with: registry: "ghcr.io" @@ -94,6 +95,7 @@ jobs: password: "${{ secrets.GITHUB_TOKEN }}" - name: "Release to DockerHub and GHCR" + if: "github.event_name != 'pull_request'" run: | docker push "${IMAGE_NAME}:${IMAGE_TAG}" From 62391c7b5634ab52865a5e0ebf8b2eb64b04e9b9 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Tue, 11 Jul 2023 17:24:25 -0600 Subject: [PATCH 4/6] Clarify docker image build conditionals --- .github/workflows/test-and-build.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 3c1bb17c..7d50e83c 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -67,7 +67,6 @@ jobs: build-and-release-image: runs-on: "ubuntu-latest" needs: ["test"] - if: "github.ref_type == 'tag' || github.ref_type == 'branch'" env: IMAGE_NAME: "nsidc/qgreenland" IMAGE_TAG: "${{ github.ref_type == 'branch' && 'latest' || github.ref_name }}" @@ -80,14 +79,14 @@ jobs: docker build -t "${IMAGE_NAME}:${IMAGE_TAG}" . - name: "DockerHub login" - if: "github.event_name != 'pull_request'" + if: "github.event_name == 'tag' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch)" uses: "docker/login-action@v2" with: username: "${{secrets.DOCKER_USER}}" password: "${{secrets.DOCKER_PASS}}" - name: "GHCR login" - if: "github.event_name != 'pull_request'" + if: "github.event_name == 'tag' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch)" uses: "docker/login-action@v2" with: registry: "ghcr.io" @@ -95,7 +94,7 @@ jobs: password: "${{ secrets.GITHUB_TOKEN }}" - name: "Release to DockerHub and GHCR" - if: "github.event_name != 'pull_request'" + if: "github.event_name == 'tag' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch)" run: | docker push "${IMAGE_NAME}:${IMAGE_TAG}" From eca53f34afe4c475114f0efc25f32ace661c2cf8 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Tue, 11 Jul 2023 17:43:43 -0600 Subject: [PATCH 5/6] Give more explicit names to workflow and jobs --- .github/workflows/test-and-build.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 7d50e83c..3461e8c1 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -1,4 +1,4 @@ -name: "Test and (if tag) build" +name: "Test, release, and build" on: # TODO: We currently get double workflows when pushing to pull request @@ -21,6 +21,7 @@ defaults: jobs: test: + name: "Run tests" runs-on: "ubuntu-latest" steps: - name: "Check out repository" @@ -65,6 +66,7 @@ jobs: # TODO: Consider extracting everything below this line to a separate workflow # which is triggered by pushes to the default branch and GitHub releases. build-and-release-image: + name: "Build and (if `main` or tag) release container image" runs-on: "ubuntu-latest" needs: ["test"] env: @@ -78,14 +80,14 @@ jobs: run: | docker build -t "${IMAGE_NAME}:${IMAGE_TAG}" . - - name: "DockerHub login" + - name: "DockerHub login (if `main` or tag)" if: "github.event_name == 'tag' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch)" uses: "docker/login-action@v2" with: username: "${{secrets.DOCKER_USER}}" password: "${{secrets.DOCKER_PASS}}" - - name: "GHCR login" + - name: "GHCR login (if `main` or tag)" if: "github.event_name == 'tag' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch)" uses: "docker/login-action@v2" with: @@ -93,7 +95,7 @@ jobs: username: "${{ github.repository_owner }}" password: "${{ secrets.GITHUB_TOKEN }}" - - name: "Release to DockerHub and GHCR" + - name: "Release to DockerHub and GHCR (if `main` or tag)" if: "github.event_name == 'tag' || (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch)" run: | docker push "${IMAGE_NAME}:${IMAGE_TAG}" @@ -103,6 +105,7 @@ jobs: build-package: + name: "Build QGreenland project (if tag)" runs-on: "ubuntu-latest" needs: ["build-and-release-image"] if: "github.ref_type == 'tag'" From e38c9d870a0891a5c74ee0777999cf5a7b14d4db Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Wed, 12 Jul 2023 11:05:39 -0600 Subject: [PATCH 6/6] Add comment explaining ternary expression --- .github/workflows/test-and-build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 3461e8c1..83ed2ef5 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -71,6 +71,11 @@ jobs: needs: ["test"] env: IMAGE_NAME: "nsidc/qgreenland" + # GitHub Actions expressions don't have great conditional support, so + # writing a ternary expression looks a lot like bash. In Python, this + # would read as: + # 'latest' if github.ref_type == 'branch' else github.ref_name + # https://docs.github.com/en/actions/learn-github-actions/expressions IMAGE_TAG: "${{ github.ref_type == 'branch' && 'latest' || github.ref_name }}" steps: - name: "Check out repository"