Merge pull request #3 from hpc4cmb/docker

Large changes to support docker deployment

.dockerignore

@@ -1,9 +1,2 @@
-cmbenv
 build
-configs
-pkgs
-templates
-tools
 pool
-README*
-LICENSE

.github/workflows/deploy.yml

@@ -0,0 +1,99 @@
+# Deploy an image with all tools and latest python.  Also deploy images
+# with just the toast dependencies for various versions of python.  These can
+# be used in toast CI testing.
+
+name:  CMBenv Deploy
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  full:
+    name: Docker Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Create Dockerfile
+        run: ./cmbenv -c docker-py3.8-debian -p /usr/local
+      - name: Build Docker Image
+        run: docker build -t hpc4cmb/cmbenv:${GITHUB_REF} -f Dockerfile_docker-py3.8-debian .
+      - name: Test Docker Image
+        run: docker run hpc4cmb/cmbenv:${GITHUB_REF} python -c 'import toast.tests; toast.tests.run()'
+      - name: Tag Latest
+        run: docker tag hpc4cmb/cmbenv:${GITHUB_REF} hpc4cmb/cmbenv:latest
+      - name: Push Docker Image
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/cmbenv:${GITHUB_REF} && docker push hpc4cmb/cmbenv:latest
+  toastdeps-py35:
+    name: TOAST Dependencies Python 3.5
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Create Dockerfile
+        run: ./cmbenv -c docker-toastdeps-py35 -p /usr/local
+      - name: Build Docker Image
+        run: docker build -t hpc4cmb/toast-deps-py35:${GITHUB_REF} -f Dockerfile_docker-toastdeps-py35 .
+      - name: Tag Latest
+        run: docker tag hpc4cmb/toast-deps-py35:${GITHUB_REF} hpc4cmb/toast-deps-py35:latest
+      - name: Push Docker Image
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/toast-deps-py35:${GITHUB_REF} && docker push hpc4cmb/toast-deps-py35:latest
+  toastdeps-py36:
+    name: TOAST Dependencies Python 3.6
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Create Dockerfile
+        run: ./cmbenv -c docker-toastdeps-py36 -p /usr/local
+      - name: Build Docker Image
+        run: docker build -t hpc4cmb/toast-deps-py36:${GITHUB_REF} -f Dockerfile_docker-toastdeps-py36 .
+      - name: Tag Latest
+        run: docker tag hpc4cmb/toast-deps-py36:${GITHUB_REF} hpc4cmb/toast-deps-py36:latest
+      - name: Push Docker Image
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/toast-deps-py36:${GITHUB_REF} && docker push hpc4cmb/toast-deps-py36:latest
+  toastdeps-py37:
+    name: TOAST Dependencies Python 3.7
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Create Dockerfile
+        run: ./cmbenv -c docker-toastdeps-py37 -p /usr/local
+      - name: Build Docker Image
+        run: docker build -t hpc4cmb/toast-deps-py37:${GITHUB_REF} -f Dockerfile_docker-toastdeps-py37 .
+      - name: Tag Latest
+        run: docker tag hpc4cmb/toast-deps-py37:${GITHUB_REF} hpc4cmb/toast-deps-py37:latest
+      - name: Push Docker Image
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/toast-deps-py37:${GITHUB_REF} && docker push hpc4cmb/toast-deps-py37:latest
+  toastdeps-py38:
+    name: TOAST Dependencies Python 3.8
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Create Dockerfile
+        run: ./cmbenv -c docker-toastdeps-py38 -p /usr/local
+      - name: Build Docker Image
+        run: docker build -t hpc4cmb/toast-deps-py38:${GITHUB_REF} -f Dockerfile_docker-toastdeps-py38 .
+      - name: Tag Latest
+        run: docker tag hpc4cmb/toast-deps-py38:${GITHUB_REF} hpc4cmb/toast-deps-py38:latest
+      - name: Push Docker Image
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/toast-deps-py38:${GITHUB_REF} && docker push hpc4cmb/toast-deps-py38:latest

.github/workflows/test.yml

@@ -0,0 +1,59 @@
+# Test action that attempts to generate new docker containers on push to master
+
+name:  CMBenv Docker Test
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  py35:
+    name: Docker Py3.5 Debian
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Create Dockerfile
+        run: ./cmbenv -c docker-py3.5-debian -p /usr/local
+      - name: Build Docker Image
+        run: docker build -t hpc4cmb/cmbenv:temp_py35 -f Dockerfile_docker-py3.5-debian .
+      - name: Test Docker Image
+        run: docker run hpc4cmb/cmbenv:temp_py35 python -c 'import toast.tests; toast.tests.run()' && docker run hpc4cmb/cmbenv:temp_py35 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()'
+  py36:
+    name: Docker Py3.6 Debian
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Create Dockerfile
+        run: ./cmbenv -c docker-py3.6-debian -p /usr/local
+      - name: Build Docker Image
+        run: docker build -t hpc4cmb/cmbenv:temp_py36 -f Dockerfile_docker-py3.6-debian .
+      - name: Test Docker Image
+        run: docker run hpc4cmb/cmbenv:temp_py36 python -c 'import toast.tests; toast.tests.run()' && docker run hpc4cmb/cmbenv:temp_py36 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()'
+  py37:
+    name: Docker Py3.7 Debian
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Create Dockerfile
+        run: ./cmbenv -c docker-py3.7-debian -p /usr/local
+      - name: Build Docker Image
+        run: docker build -t hpc4cmb/cmbenv:temp_py37 -f Dockerfile_docker-py3.7-debian .
+      - name: Test Docker Image
+        run: docker run hpc4cmb/cmbenv:temp_py37 python -c 'import toast.tests; toast.tests.run()' && docker run hpc4cmb/cmbenv:temp_py37 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()'
+  py38:
+    name: Docker Py3.8 Debian
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Create Dockerfile
+        run: ./cmbenv -c docker-py3.8-debian -p /usr/local
+      - name: Build Docker Image
+        run: docker build -t hpc4cmb/cmbenv:temp_py38 -f Dockerfile_docker-py3.8-debian .
+      - name: Test Docker Image
+        run: docker run hpc4cmb/cmbenv:temp_py38 python -c 'import toast.tests; toast.tests.run()' && docker run hpc4cmb/cmbenv:temp_py38 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()'

cmbenv

@@ -105,20 +105,26 @@ pushd $(dirname $0) > /dev/null
 topdir=$(pwd -P)
 popd > /dev/null
 
-conf_file="configs/${config}"
-conf_pkgs="configs/${config}.pkgs"
-conf_mod="configs/${config}.mod"
-conf_sh="configs/${config}.sh"
+conf_file="${topdir}/configs/${config}"
+conf_pkgs="${topdir}/configs/${config}.pkgs"
+conf_mod="${topdir}/configs/${config}.mod"
+conf_sh="${topdir}/configs/${config}.sh"
+
+# We need to get the name of the template from the input config.
+template=$(grep TEMPLATE "${conf_file}" | sed -e "s#[^=]*= *\(.*\)#\1#")
+if [ "x${template}" = "x" ]; then
+    template="linux"
+fi
 
 script=""
 if [ "${is_docker}" = "yes" ]; then
     script="Dockerfile_${config}"
-    eval "${topdir}/tools/gen_script.sh" Dockerfile.in "${config}" \
+    eval "${topdir}/tools/gen_script.sh" "${template}.in" "${config}" \
         "${conf_file}" "${conf_pkgs}" "${conf_mod}" "${conf_sh}" \
         "${script}" "${prefix}" "${version}" "${moduledir}" yes
 else
     script="install_${config}"
-    eval "${topdir}/tools/gen_script.sh" install.in "${config}" \
+    eval "${topdir}/tools/gen_script.sh" "install-${template}.in" "${config}" \
         "${conf_file}" "${conf_pkgs}" "${conf_mod}" "${conf_sh}" \
         "${script}" "${prefix}" "${version}" "${moduledir}" no
 fi

configs/cori-gcc

@@ -2,6 +2,18 @@
 # OS type (linux OR osx)
 OSTYPE = linux
 
+# Specify the template to use
+TEMPLATE = linux
+
+# Python version to use
+PYVERSION = 3.7
+
+# Packages to install with pip
+PIP_PKGS =
+
+# Packages to install with conda
+CONDA_PKGS = nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab
+
 # Serial compilers
 
 CC = gcc
@@ -41,9 +53,6 @@ CROSS =
 # Parallel builds
 MAKEJ = 4
 
-# Python version to use
-PYVERSION = 3.6
-
 # Define MKL location
 MKL =
 

configs/cori-gcc-gpu

@@ -2,6 +2,18 @@
 # OS type (linux OR osx)
 OSTYPE = linux
 
+# Specify the template to use
+TEMPLATE = linux
+
+# Python version to use
+PYVERSION = 3.7
+
+# Packages to install with pip
+PIP_PKGS =
+
+# Packages to install with conda
+CONDA_PKGS = nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab
+
 # Serial compilers
 
 CC = gcc
@@ -41,9 +53,6 @@ CROSS =
 # Parallel builds
 MAKEJ = 4
 
-# Python version to use
-PYVERSION = 3.6
-
 # Define MKL location
 MKL =
 

configs/cori-gcc-gpu.pkgs

@@ -4,14 +4,6 @@
 # These extra options are documented in the shell snippet for each package
 # in the top-level pkgs directory.
 #
-# Python 3.  Uncomment one of the following lines.  You should only use the
-# "default" option for Docker images.  All other configs should use either
-# a virtualenv or a conda environment.  This will also install a set of
-# common base packages.
-#python:virtualenv
-python:conda nomkl
-#python:default
-#
 # Autotools.  If you already have recent versions installed with your OS
 # package manager, then comment these out.
 m4
@@ -26,6 +18,17 @@ cmake
 # very ancient.
 zlib
 #
+# Will use libcublas
+#openblas
+#
+# Python 3.  Uncomment one of the following lines.  You should only use the
+# "default" option for Docker images.  All other configs should use either
+# a virtualenv or a conda environment.  This will also install a set of
+# common base packages.
+#python:virtualenv
+python:conda nomkl
+#python:default
+#
 # Install MPICH.  This step is optional and only needed in 2 cases:
 #   1.  You don't have MPI available through system packages that is
 #       compatible with the serial compilers you are using.
@@ -39,9 +42,6 @@ mpi4py
 # Install CFITSIO.
 cfitsio
 #
-# Will use libcublas
-#openblas
-#
 # Will use libcufftw
 #fftw
 #

configs/cori-gcc.pkgs

@@ -4,14 +4,6 @@
 # These extra options are documented in the shell snippet for each package
 # in the top-level pkgs directory.
 #
-# Python 3.  Uncomment one of the following lines.  You should only use the
-# "default" option for Docker images.  All other configs should use either
-# a virtualenv or a conda environment.  This will also install a set of
-# common base packages.
-#python:virtualenv
-python:conda nomkl
-#python:default
-#
 # Autotools.  If you already have recent versions installed with your OS
 # package manager, then comment these out.
 m4
@@ -26,6 +18,18 @@ cmake
 # very ancient.
 zlib
 #
+# Install OpenBLAS.  Comment out if you are using MKL with the Intel compilers
+# or some other vendor BLAS / LAPACK solution.
+openblas
+#
+# Python 3.  Uncomment one of the following lines.  You should only use the
+# "default" option for Docker images.  All other configs should use either
+# a virtualenv or a conda environment.  This will also install a set of
+# common base packages.
+#python:virtualenv
+python:conda nomkl
+#python:default
+#
 # Install MPICH.  This step is optional and only needed in 2 cases:
 #   1.  You don't have MPI available through system packages that is
 #       compatible with the serial compilers you are using.
@@ -39,10 +43,6 @@ mpi4py
 # Install CFITSIO.
 cfitsio
 #
-# Install OpenBLAS.  Comment out if you are using MKL with the Intel compilers
-# or some other vendor BLAS / LAPACK solution.
-openblas
-#
 # Install FFTW3.
 fftw
 #