diff --git a/.github/workflows/container_build.yml b/.github/workflows/container_build.yml index 1e4b6f3b8..3b964d9e4 100644 --- a/.github/workflows/container_build.yml +++ b/.github/workflows/container_build.yml @@ -83,34 +83,34 @@ jobs: tags: ${{ steps.context_basename.outputs.basename }}_docker:latest outputs: type=docker,dest=/tmp/${{ steps.context_basename.outputs.basename }}_docker.tar - apptainer_container: - needs: setup - if: needs.setup.outputs.all_changed_files_count > 0 - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} - - steps: - - name: Checkout pull request branch - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Setup micromamba - uses: mamba-org/setup-micromamba@v1 - with: - environment-name: containers - create-args: apptainer - generate-run-shell: true - - - name: Get basename - id: context_basename - run: echo "basename=$(basename ${{ matrix.context }})" >> "$GITHUB_OUTPUT" - - - name: Build apptainer containers - shell: micromamba-shell {0} - run: | - pushd ${{ matrix.context}}; \ - apptainer build /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif ${{ github.workspace }}/.github/apptainer.def; \ - popd + # apptainer_container: + # needs: setup + # if: needs.setup.outputs.all_changed_files_count > 0 + # runs-on: ubuntu-latest + # strategy: + # fail-fast: false + # matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} + # + # steps: + # - name: Checkout pull request branch + # uses: actions/checkout@v3 + # with: + # fetch-depth: 0 + # + # - name: Setup micromamba + # uses: mamba-org/setup-micromamba@v1 + # with: + # environment-name: containers + # create-args: apptainer + # generate-run-shell: true + # + # - name: Get basename + # id: context_basename + # run: echo "basename=$(basename ${{ matrix.context }})" >> "$GITHUB_OUTPUT" + # + # - name: Build apptainer containers + # shell: micromamba-shell {0} + # run: | + # pushd ${{ matrix.context}}; \ + # apptainer build /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif ${{ github.workspace }}/.github/apptainer.def; \ + # popd diff --git a/.github/workflows/container_upload.yml b/.github/workflows/container_upload.yml index 332dafa03..2db1be9c4 100644 --- a/.github/workflows/container_upload.yml +++ b/.github/workflows/container_upload.yml @@ -88,79 +88,79 @@ jobs: quay.io/${{ vars.QUAYIO_USERNAME }}/${{ steps.context_basename.outputs.basename }}_docker:latest docker.io/${{ vars.DOCKERIO_USERNAME }}/${{ steps.context_basename.outputs.basename }}_docker:latest - push_apptainer_containers: - needs: setup - if: needs.setup.outputs.any_changed == 'true' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} - permissions: - contents: read - packages: write - - steps: - - name: Checkout pull request branch - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Setup micromamba - uses: mamba-org/setup-micromamba@v1 - with: - environment-name: containers - create-args: apptainer - generate-run-shell: true - - - name: Get basename - id: context_basename - run: echo "basename=$(basename ${{ matrix.context }})" >> "$GITHUB_OUTPUT" - - - name: Build apptainer containers - shell: micromamba-shell {0} - run: | - pushd ${{ matrix.context}}; \ - apptainer build /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif ${{ github.workspace }}/.github/apptainer.def; \ - popd - - - name: Log in to ghcr.io with apptainer - shell: micromamba-shell {0} - run: | - apptainer remote login --username ${{ github.repository_owner }} \ - --password ${{ secrets.GITHUB_TOKEN }} \ - docker://ghcr.io/${{ github.repository_owner }} - - - name: Push apptainer containers to ghcr.io - shell: micromamba-shell {0} - run: | - apptainer push \ - /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif \ - oras://ghcr.io/${{ github.repository_owner }}/${{ steps.context_basename.outputs.basename }}_apptainer:latest - - - name: Log in to quay.io with apptainer - shell: micromamba-shell {0} - run: | - apptainer remote login --username ${{ vars.QUAYIO_USERNAME }} \ - --password ${{ secrets.QUAYIO_TOKEN }} \ - docker://quay.io/${{ vars.QUAYIO_USERNAME }} - - - name: Push apptainer containers to quay.io - shell: micromamba-shell {0} - run: | - apptainer push \ - /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif \ - oras://quay.io/${{ vars.QUAYIO_USERNAME }}/${{ steps.context_basename.outputs.basename }}_apptainer:latest - - - name: Log in to docker.io with apptainer - shell: micromamba-shell {0} - run: | - apptainer remote login --username ${{ vars.DOCKERIO_USERNAME }} \ - --password ${{ secrets.DOCKERIO_TOKEN }} \ - docker://docker.io - - - name: Push apptainer containers to docker.io - shell: micromamba-shell {0} - run: | - apptainer push \ - /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif \ - oras://docker.io/${{ vars.DOCKERIO_USERNAME }}/${{ steps.context_basename.outputs.basename }}_apptainer:latest + # push_apptainer_containers: + # needs: setup + # if: needs.setup.outputs.any_changed == 'true' + # runs-on: ubuntu-latest + # strategy: + # fail-fast: false + # matrix: ${{ fromJSON(needs.setup.outputs.matrix) }} + # permissions: + # contents: read + # packages: write + # + # steps: + # - name: Checkout pull request branch + # uses: actions/checkout@v3 + # with: + # fetch-depth: 0 + # + # - name: Setup micromamba + # uses: mamba-org/setup-micromamba@v1 + # with: + # environment-name: containers + # create-args: apptainer + # generate-run-shell: true + # + # - name: Get basename + # id: context_basename + # run: echo "basename=$(basename ${{ matrix.context }})" >> "$GITHUB_OUTPUT" + # + # - name: Build apptainer containers + # shell: micromamba-shell {0} + # run: | + # pushd ${{ matrix.context}}; \ + # apptainer build /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif ${{ github.workspace }}/.github/apptainer.def; \ + # popd + # + # - name: Log in to ghcr.io with apptainer + # shell: micromamba-shell {0} + # run: | + # apptainer remote login --username ${{ github.repository_owner }} \ + # --password ${{ secrets.GITHUB_TOKEN }} \ + # docker://ghcr.io/${{ github.repository_owner }} + # + # - name: Push apptainer containers to ghcr.io + # shell: micromamba-shell {0} + # run: | + # apptainer push \ + # /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif \ + # oras://ghcr.io/${{ github.repository_owner }}/${{ steps.context_basename.outputs.basename }}_apptainer:latest + # + # - name: Log in to quay.io with apptainer + # shell: micromamba-shell {0} + # run: | + # apptainer remote login --username ${{ vars.QUAYIO_USERNAME }} \ + # --password ${{ secrets.QUAYIO_TOKEN }} \ + # docker://quay.io/${{ vars.QUAYIO_USERNAME }} + # + # - name: Push apptainer containers to quay.io + # shell: micromamba-shell {0} + # run: | + # apptainer push \ + # /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif \ + # oras://quay.io/${{ vars.QUAYIO_USERNAME }}/${{ steps.context_basename.outputs.basename }}_apptainer:latest + # + # - name: Log in to docker.io with apptainer + # shell: micromamba-shell {0} + # run: | + # apptainer remote login --username ${{ vars.DOCKERIO_USERNAME }} \ + # --password ${{ secrets.DOCKERIO_TOKEN }} \ + # docker://docker.io + # + # - name: Push apptainer containers to docker.io + # shell: micromamba-shell {0} + # run: | + # apptainer push \ + # /tmp/${{ steps.context_basename.outputs.basename }}_apptainer.sif \ + # oras://docker.io/${{ vars.DOCKERIO_USERNAME }}/${{ steps.context_basename.outputs.basename }}_apptainer:latest diff --git a/.github/workflows/dispatch_container_upload.yml b/.github/workflows/dispatch_container_upload.yml index ad23a9c18..4b55c913b 100644 --- a/.github/workflows/dispatch_container_upload.yml +++ b/.github/workflows/dispatch_container_upload.yml @@ -56,71 +56,71 @@ jobs: quay.io/${{ vars.QUAYIO_USERNAME }}/${{ github.event.inputs.container_name }}_docker:latest docker.io/${{ vars.DOCKERIO_USERNAME }}/${{ github.event.inputs.container_name }}_docker:latest - push_apptainer_containers: - runs-on: ubuntu-latest - strategy: - fail-fast: false - permissions: - contents: read - packages: write - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup micromamba - uses: mamba-org/setup-micromamba@v1 - with: - environment-name: containers - create-args: apptainer - generate-run-shell: true - - - name: Build apptainer containers - shell: micromamba-shell {0} - run: | - pushd container/${{ github.event.inputs.container_name }}; \ - apptainer build /tmp/${{ github.event.inputs.container_name }}_apptainer.sif ${{ github.workspace }}/.github/apptainer.def; \ - popd - - - name: Log in to ghcr.io with apptainer - shell: micromamba-shell {0} - run: | - apptainer remote login --username ${{ github.repository_owner }} \ - --password ${{ secrets.GITHUB_TOKEN }} \ - docker://ghcr.io/${{ github.repository_owner }} - - - name: Push apptainer containers to ghcr.io - shell: micromamba-shell {0} - run: | - apptainer push \ - /tmp/${{ github.event.inputs.container_name }}_apptainer.sif \ - oras://ghcr.io/${{ github.repository_owner }}/${{ github.event.inputs.container_name }}_apptainer:latest - - - name: Log in to quay.io with apptainer - shell: micromamba-shell {0} - run: | - apptainer remote login --username ${{ vars.QUAYIO_USERNAME }} \ - --password ${{ secrets.QUAYIO_TOKEN }} \ - docker://quay.io/${{ vars.QUAYIO_USERNAME }} - - - name: Push apptainer containers to quay.io - shell: micromamba-shell {0} - run: | - apptainer push \ - /tmp/${{ github.event.inputs.container_name }}_apptainer.sif \ - oras://quay.io/${{ vars.QUAYIO_USERNAME }}/${{ github.event.inputs.container_name }}_apptainer:latest - - - name: Log in to docker.io with apptainer - shell: micromamba-shell {0} - run: | - apptainer remote login --username ${{ vars.DOCKERIO_USERNAME }} \ - --password ${{ secrets.DOCKERIO_TOKEN }} \ - docker://docker.io - - - name: Push apptainer containers to docker.io - shell: micromamba-shell {0} - run: | - apptainer push \ - /tmp/${{ github.event.inputs.container_name }}_apptainer.sif \ - oras://docker.io/${{ vars.DOCKERIO_USERNAME }}/${{ github.event.inputs.container_name }}_apptainer:latest + # push_apptainer_containers: + # runs-on: ubuntu-latest + # strategy: + # fail-fast: false + # permissions: + # contents: read + # packages: write + # + # steps: + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # + # - name: Setup micromamba + # uses: mamba-org/setup-micromamba@v1 + # with: + # environment-name: containers + # create-args: apptainer + # generate-run-shell: true + # + # - name: Build apptainer containers + # shell: micromamba-shell {0} + # run: | + # pushd container/${{ github.event.inputs.container_name }}; \ + # apptainer build /tmp/${{ github.event.inputs.container_name }}_apptainer.sif ${{ github.workspace }}/.github/apptainer.def; \ + # popd + # + # - name: Log in to ghcr.io with apptainer + # shell: micromamba-shell {0} + # run: | + # apptainer remote login --username ${{ github.repository_owner }} \ + # --password ${{ secrets.GITHUB_TOKEN }} \ + # docker://ghcr.io/${{ github.repository_owner }} + # + # - name: Push apptainer containers to ghcr.io + # shell: micromamba-shell {0} + # run: | + # apptainer push \ + # /tmp/${{ github.event.inputs.container_name }}_apptainer.sif \ + # oras://ghcr.io/${{ github.repository_owner }}/${{ github.event.inputs.container_name }}_apptainer:latest + # + # - name: Log in to quay.io with apptainer + # shell: micromamba-shell {0} + # run: | + # apptainer remote login --username ${{ vars.QUAYIO_USERNAME }} \ + # --password ${{ secrets.QUAYIO_TOKEN }} \ + # docker://quay.io/${{ vars.QUAYIO_USERNAME }} + # + # - name: Push apptainer containers to quay.io + # shell: micromamba-shell {0} + # run: | + # apptainer push \ + # /tmp/${{ github.event.inputs.container_name }}_apptainer.sif \ + # oras://quay.io/${{ vars.QUAYIO_USERNAME }}/${{ github.event.inputs.container_name }}_apptainer:latest + # + # - name: Log in to docker.io with apptainer + # shell: micromamba-shell {0} + # run: | + # apptainer remote login --username ${{ vars.DOCKERIO_USERNAME }} \ + # --password ${{ secrets.DOCKERIO_TOKEN }} \ + # docker://docker.io + # + # - name: Push apptainer containers to docker.io + # shell: micromamba-shell {0} + # run: | + # apptainer push \ + # /tmp/${{ github.event.inputs.container_name }}_apptainer.sif \ + # oras://docker.io/${{ vars.DOCKERIO_USERNAME }}/${{ github.event.inputs.container_name }}_apptainer:latest diff --git a/container/README.md b/container/README.md index 95e71f2ff..452b022f1 100644 --- a/container/README.md +++ b/container/README.md @@ -1,7 +1,8 @@ # Container provisioning pipeline Containers in the xQTL pipeline are built from conda environments and pushed to container registry using GitHub Actions. We use the [micromamba container](https://hub.docker.com/r/mambaorg/micromamba) as our base image and install a Conda environment into the container. Conda packages are only used from [conda-forge](https://anaconda.org/conda-forge), [bioconda](https://anaconda.org/bioconda), and the [personal channel](https://anaconda.org/dnachun) of contributor @danielnachun. -Note: the minimum required version of Singularity for our containers is **3.6.0**. This release made backwards incompatibile changes to the container signature format that mean any containers built with Singularity 3.6.0 or newer will not work on 3.5.3 or older. We use the latest version of Apptainer (renamed from Singularity) to build and sign our containers. +~~Note: the minimum required version of Singularity for our containers is **3.6.0**. This release made backwards incompatibile changes to the container signature format that mean any containers built with Singularity 3.6.0 or newer will not work on 3.5.3 or older. We use the latest version of Apptainer (renamed from Singularity) to build and sign our containers.~~ +Support for apptainer is now dropped as we are migrating our software stack to `pixi` with Conda packages. Docker images should only be used for batch MemVerge jobs, and this will only continue until a containerless solution is found for this system. ## Pipeline description The GitHub Actions are located in the [.github/workflows](https://github.com/cumc/xqtl-protocol/tree/main/.github/workflows) folder, while the Conda environments and the CSV table used to generate those environments are found in this folder. The environments for a single container are each in their own folder e.g. `bioinfo/bioinfo.yml` – currently each container only has a single environment in it, but our pipeline allows for multiple environments in the same container should the need arise. We build containers for both Docker and Singularity and push the containers to the registries at [ghcr.io](https://ghcr.io), [quay.io](https://quay.io), and [docker.io](https://docker.io). diff --git a/container/readme/README.md b/container/readme/README.md index f31a13e67..3d4d9c518 100644 --- a/container/readme/README.md +++ b/container/readme/README.md @@ -1,7 +1,8 @@ # Container provisioning pipeline Containers in the xQTL pipeline are built from conda environments and pushed to container registry using GitHub Actions. We use the [micromamba container](https://hub.docker.com/r/mambaorg/micromamba) as our base image and install a Conda environment into the container. Conda packages are only used from [conda-forge](https://anaconda.org/conda-forge), [bioconda](https://anaconda.org/bioconda), and the [personal channel](https://anaconda.org/dnachun) of contributor @danielnachun. -Note: the minimum required version of Singularity for our containers is **3.6.0**. This release made backwards incompatibile changes to the container signature format that mean any containers built with Singularity 3.6.0 or newer will not work on 3.5.3 or older. We use the latest version of Apptainer (renamed from Singularity) to build and sign our containers. +~~Note: the minimum required version of Singularity for our containers is **3.6.0**. This release made backwards incompatibile changes to the container signature format that mean any containers built with Singularity 3.6.0 or newer will not work on 3.5.3 or older. We use the latest version of Apptainer (renamed from Singularity) to build and sign our containers.~~ +Support for apptainer is now dropped as we are migrating our software stack to `pixi` with Conda packages. Docker images should only be used for batch MemVerge jobs, and this will only continue until a containerless solution is found for this system. ## Pipeline description The GitHub Actions are located in the [.github/workflows](https://github.com/cumc/xqtl-protocol/tree/main/.github/workflows) folder, while the Conda environments and the CSV table used to generate those environments are found in this folder. The environments for a single container are each in their own folder e.g. `bioinfo/bioinfo.yml` – currently each container only has a single environment in it, but our pipeline allows for multiple environments in the same container should the need arise. We build containers for both Docker and Singularity and push the containers to the registries at [ghcr.io](https://ghcr.io), [quay.io](https://quay.io), and [docker.io](https://docker.io).