diff --git a/.dockerignore b/.dockerignore index 54d6860..52710db 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,2 @@ -cmbenv build -configs -pkgs -templates -tools pool -README* -LICENSE diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..e17c34c --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,99 @@ +# Deploy an image with all tools and latest python. Also deploy images +# with just the toast dependencies for various versions of python. These can +# be used in toast CI testing. + +name: CMBenv Deploy + +on: + push: + tags: + - '*' + +jobs: + full: + name: Docker Release + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Create Dockerfile + run: ./cmbenv -c docker-py3.8-debian -p /usr/local + - name: Build Docker Image + run: docker build -t hpc4cmb/cmbenv:${GITHUB_REF} -f Dockerfile_docker-py3.8-debian . + - name: Test Docker Image + run: docker run hpc4cmb/cmbenv:${GITHUB_REF} python -c 'import toast.tests; toast.tests.run()' + - name: Tag Latest + run: docker tag hpc4cmb/cmbenv:${GITHUB_REF} hpc4cmb/cmbenv:latest + - name: Push Docker Image + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/cmbenv:${GITHUB_REF} && docker push hpc4cmb/cmbenv:latest + toastdeps-py35: + name: TOAST Dependencies Python 3.5 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Create Dockerfile + run: ./cmbenv -c docker-toastdeps-py35 -p /usr/local + - name: Build Docker Image + run: docker build -t hpc4cmb/toast-deps-py35:${GITHUB_REF} -f Dockerfile_docker-toastdeps-py35 . + - name: Tag Latest + run: docker tag hpc4cmb/toast-deps-py35:${GITHUB_REF} hpc4cmb/toast-deps-py35:latest + - name: Push Docker Image + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/toast-deps-py35:${GITHUB_REF} && docker push hpc4cmb/toast-deps-py35:latest + toastdeps-py36: + name: TOAST Dependencies Python 3.6 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Create Dockerfile + run: ./cmbenv -c docker-toastdeps-py36 -p /usr/local + - name: Build Docker Image + run: docker build -t hpc4cmb/toast-deps-py36:${GITHUB_REF} -f Dockerfile_docker-toastdeps-py36 . + - name: Tag Latest + run: docker tag hpc4cmb/toast-deps-py36:${GITHUB_REF} hpc4cmb/toast-deps-py36:latest + - name: Push Docker Image + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/toast-deps-py36:${GITHUB_REF} && docker push hpc4cmb/toast-deps-py36:latest + toastdeps-py37: + name: TOAST Dependencies Python 3.7 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Create Dockerfile + run: ./cmbenv -c docker-toastdeps-py37 -p /usr/local + - name: Build Docker Image + run: docker build -t hpc4cmb/toast-deps-py37:${GITHUB_REF} -f Dockerfile_docker-toastdeps-py37 . + - name: Tag Latest + run: docker tag hpc4cmb/toast-deps-py37:${GITHUB_REF} hpc4cmb/toast-deps-py37:latest + - name: Push Docker Image + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/toast-deps-py37:${GITHUB_REF} && docker push hpc4cmb/toast-deps-py37:latest + toastdeps-py38: + name: TOAST Dependencies Python 3.8 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Create Dockerfile + run: ./cmbenv -c docker-toastdeps-py38 -p /usr/local + - name: Build Docker Image + run: docker build -t hpc4cmb/toast-deps-py38:${GITHUB_REF} -f Dockerfile_docker-toastdeps-py38 . + - name: Tag Latest + run: docker tag hpc4cmb/toast-deps-py38:${GITHUB_REF} hpc4cmb/toast-deps-py38:latest + - name: Push Docker Image + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/toast-deps-py38:${GITHUB_REF} && docker push hpc4cmb/toast-deps-py38:latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..5ee66fe --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,59 @@ +# Test action that attempts to generate new docker containers on push to master + +name: CMBenv Docker Test + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + py35: + name: Docker Py3.5 Debian + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Create Dockerfile + run: ./cmbenv -c docker-py3.5-debian -p /usr/local + - name: Build Docker Image + run: docker build -t hpc4cmb/cmbenv:temp_py35 -f Dockerfile_docker-py3.5-debian . + - name: Test Docker Image + run: docker run hpc4cmb/cmbenv:temp_py35 python -c 'import toast.tests; toast.tests.run()' && docker run hpc4cmb/cmbenv:temp_py35 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()' + py36: + name: Docker Py3.6 Debian + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Create Dockerfile + run: ./cmbenv -c docker-py3.6-debian -p /usr/local + - name: Build Docker Image + run: docker build -t hpc4cmb/cmbenv:temp_py36 -f Dockerfile_docker-py3.6-debian . + - name: Test Docker Image + run: docker run hpc4cmb/cmbenv:temp_py36 python -c 'import toast.tests; toast.tests.run()' && docker run hpc4cmb/cmbenv:temp_py36 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()' + py37: + name: Docker Py3.7 Debian + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Create Dockerfile + run: ./cmbenv -c docker-py3.7-debian -p /usr/local + - name: Build Docker Image + run: docker build -t hpc4cmb/cmbenv:temp_py37 -f Dockerfile_docker-py3.7-debian . + - name: Test Docker Image + run: docker run hpc4cmb/cmbenv:temp_py37 python -c 'import toast.tests; toast.tests.run()' && docker run hpc4cmb/cmbenv:temp_py37 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()' + py38: + name: Docker Py3.8 Debian + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Create Dockerfile + run: ./cmbenv -c docker-py3.8-debian -p /usr/local + - name: Build Docker Image + run: docker build -t hpc4cmb/cmbenv:temp_py38 -f Dockerfile_docker-py3.8-debian . + - name: Test Docker Image + run: docker run hpc4cmb/cmbenv:temp_py38 python -c 'import toast.tests; toast.tests.run()' && docker run hpc4cmb/cmbenv:temp_py38 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()' diff --git a/cmbenv b/cmbenv index a473547..3377b17 100755 --- a/cmbenv +++ b/cmbenv @@ -105,20 +105,26 @@ pushd $(dirname $0) > /dev/null topdir=$(pwd -P) popd > /dev/null -conf_file="configs/${config}" -conf_pkgs="configs/${config}.pkgs" -conf_mod="configs/${config}.mod" -conf_sh="configs/${config}.sh" +conf_file="${topdir}/configs/${config}" +conf_pkgs="${topdir}/configs/${config}.pkgs" +conf_mod="${topdir}/configs/${config}.mod" +conf_sh="${topdir}/configs/${config}.sh" + +# We need to get the name of the template from the input config. +template=$(grep TEMPLATE "${conf_file}" | sed -e "s#[^=]*= *\(.*\)#\1#") +if [ "x${template}" = "x" ]; then + template="linux" +fi script="" if [ "${is_docker}" = "yes" ]; then script="Dockerfile_${config}" - eval "${topdir}/tools/gen_script.sh" Dockerfile.in "${config}" \ + eval "${topdir}/tools/gen_script.sh" "${template}.in" "${config}" \ "${conf_file}" "${conf_pkgs}" "${conf_mod}" "${conf_sh}" \ "${script}" "${prefix}" "${version}" "${moduledir}" yes else script="install_${config}" - eval "${topdir}/tools/gen_script.sh" install.in "${config}" \ + eval "${topdir}/tools/gen_script.sh" "install-${template}.in" "${config}" \ "${conf_file}" "${conf_pkgs}" "${conf_mod}" "${conf_sh}" \ "${script}" "${prefix}" "${version}" "${moduledir}" no fi diff --git a/configs/cori-gcc b/configs/cori-gcc index 7337a0a..a8e4b0c 100644 --- a/configs/cori-gcc +++ b/configs/cori-gcc @@ -2,6 +2,18 @@ # OS type (linux OR osx) OSTYPE = linux +# Specify the template to use +TEMPLATE = linux + +# Python version to use +PYVERSION = 3.7 + +# Packages to install with pip +PIP_PKGS = + +# Packages to install with conda +CONDA_PKGS = nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab + # Serial compilers CC = gcc @@ -41,9 +53,6 @@ CROSS = # Parallel builds MAKEJ = 4 -# Python version to use -PYVERSION = 3.6 - # Define MKL location MKL = diff --git a/configs/cori-gcc-gpu b/configs/cori-gcc-gpu index 6723f69..0f29cf3 100644 --- a/configs/cori-gcc-gpu +++ b/configs/cori-gcc-gpu @@ -2,6 +2,18 @@ # OS type (linux OR osx) OSTYPE = linux +# Specify the template to use +TEMPLATE = linux + +# Python version to use +PYVERSION = 3.7 + +# Packages to install with pip +PIP_PKGS = + +# Packages to install with conda +CONDA_PKGS = nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab + # Serial compilers CC = gcc @@ -41,9 +53,6 @@ CROSS = # Parallel builds MAKEJ = 4 -# Python version to use -PYVERSION = 3.6 - # Define MKL location MKL = diff --git a/configs/cori-gcc-gpu.pkgs b/configs/cori-gcc-gpu.pkgs index 02555a8..bac468e 100644 --- a/configs/cori-gcc-gpu.pkgs +++ b/configs/cori-gcc-gpu.pkgs @@ -4,14 +4,6 @@ # These extra options are documented in the shell snippet for each package # in the top-level pkgs directory. # -# Python 3. Uncomment one of the following lines. You should only use the -# "default" option for Docker images. All other configs should use either -# a virtualenv or a conda environment. This will also install a set of -# common base packages. -#python:virtualenv -python:conda nomkl -#python:default -# # Autotools. If you already have recent versions installed with your OS # package manager, then comment these out. m4 @@ -26,6 +18,17 @@ cmake # very ancient. zlib # +# Will use libcublas +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +python:conda nomkl +#python:default +# # Install MPICH. This step is optional and only needed in 2 cases: # 1. You don't have MPI available through system packages that is # compatible with the serial compilers you are using. @@ -39,9 +42,6 @@ mpi4py # Install CFITSIO. cfitsio # -# Will use libcublas -#openblas -# # Will use libcufftw #fftw # diff --git a/configs/cori-gcc-knl.mod b/configs/cori-gcc-knl.mod deleted file mode 100644 index c500b77..0000000 --- a/configs/cori-gcc-knl.mod +++ /dev/null @@ -1,23 +0,0 @@ - -if [ module-info mode load ] { - if [ is-loaded PrgEnv-gnu ] { - } else { - if [ is-loaded PrgEnv-cray ] { - module swap PrgEnv-cray PrgEnv-gnu - } - if [ is-loaded PrgEnv-intel ] { - module swap PrgEnv-intel PrgEnv-gnu - } - } - # altd may cause random job hangs - if [ is-loaded altd ] { - module unload altd - } - # darshan may cause overhead - if [ is-loaded darshan ] { - module unload darshan - } - module unload cray-libsci - module unload craype-hugepages2M - setenv CRAYPE_LINK_TYPE dynamic -} diff --git a/configs/cori-gcc-knl.sh b/configs/cori-gcc-knl.sh deleted file mode 100644 index 31ad8ac..0000000 --- a/configs/cori-gcc-knl.sh +++ /dev/null @@ -1,24 +0,0 @@ -loadedgnu=`${MODULESHOME}/bin/modulecmd sh -t list 2>&1 | grep PrgEnv-gnu` -loadedintel=`${MODULESHOME}/bin/modulecmd sh -t list 2>&1 | grep PrgEnv-intel` -loadedcray=`${MODULESHOME}/bin/modulecmd sh -t list 2>&1 | grep PrgEnv-cray` -loadeddarshan=`${MODULESHOME}/bin/modulecmd sh -t list 2>&1 | grep darshan` -loadedaltd=`${MODULESHOME}/bin/modulecmd sh -t list 2>&1 | grep altd` -if [ "x${loadedgnu}" = x ]; then - if [ "x${loadedcray}" != x ]; then - module swap PrgEnv-cray PrgEnv-gnu - fi - if [ "x${loadedintel}" != x ]; then - module swap PrgEnv-intel PrgEnv-gnu - fi -fi -# altd may cause random job hangs -if [ "x${loadedaltd}" != x ]; then - module unload altd -fi -# darshan may cause overhead -if [ "x${loadeddarshan}" != x ]; then - module unload darshan -fi -module unload cray-libsci -module unload craype-hugepages2M -export CRAYPE_LINK_TYPE=dynamic diff --git a/configs/cori-gcc.pkgs b/configs/cori-gcc.pkgs index 85d8842..a39f760 100644 --- a/configs/cori-gcc.pkgs +++ b/configs/cori-gcc.pkgs @@ -4,14 +4,6 @@ # These extra options are documented in the shell snippet for each package # in the top-level pkgs directory. # -# Python 3. Uncomment one of the following lines. You should only use the -# "default" option for Docker images. All other configs should use either -# a virtualenv or a conda environment. This will also install a set of -# common base packages. -#python:virtualenv -python:conda nomkl -#python:default -# # Autotools. If you already have recent versions installed with your OS # package manager, then comment these out. m4 @@ -26,6 +18,18 @@ cmake # very ancient. zlib # +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +python:conda nomkl +#python:default +# # Install MPICH. This step is optional and only needed in 2 cases: # 1. You don't have MPI available through system packages that is # compatible with the serial compilers you are using. @@ -39,10 +43,6 @@ mpi4py # Install CFITSIO. cfitsio # -# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers -# or some other vendor BLAS / LAPACK solution. -openblas -# # Install FFTW3. fftw # diff --git a/configs/cori-intel b/configs/cori-intel index 6aeed97..d1c637e 100644 --- a/configs/cori-intel +++ b/configs/cori-intel @@ -2,6 +2,18 @@ # OS type (linux OR osx) OSTYPE = linux +# Specify the template to use +TEMPLATE = linux + +# Python version to use +PYVERSION = 3.7 + +# Packages to install with pip +PIP_PKGS = + +# Packages to install with conda +CONDA_PKGS = nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab + # Serial compilers CC = ${INTEL_PATH}/linux/bin/intel64/icc @@ -42,9 +54,6 @@ CROSS = # Parallel builds MAKEJ = 4 -# Python version to use -PYVERSION = 3.6 - # Define MKL location MKL = ${MKLROOT} diff --git a/configs/cori-intel.pkgs b/configs/cori-intel.pkgs index 436f644..004b44e 100644 --- a/configs/cori-intel.pkgs +++ b/configs/cori-intel.pkgs @@ -4,14 +4,6 @@ # These extra options are documented in the shell snippet for each package # in the top-level pkgs directory. # -# Python 3. Uncomment one of the following lines. You should only use the -# "default" option for Docker images. All other configs should use either -# a virtualenv or a conda environment. This will also install a set of -# common base packages. -#python:virtualenv -python:conda nomkl -#python:default -# # Autotools. If you already have recent versions installed with your OS # package manager, then comment these out. m4 @@ -26,6 +18,18 @@ cmake # very ancient. zlib # +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +python:conda nomkl +#python:default +# # Install MPICH. This step is optional and only needed in 2 cases: # 1. You don't have MPI available through system packages that is # compatible with the serial compilers you are using. @@ -39,10 +43,6 @@ mpi4py # Install CFITSIO. cfitsio # -# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers -# or some other vendor BLAS / LAPACK solution. -#openblas -# # Install FFTW3. fftw # diff --git a/configs/docker-py3.5-alpine3.11 b/configs/docker-py3.5-alpine3.11 new file mode 100644 index 0000000..7a1d14b --- /dev/null +++ b/configs/docker-py3.5-alpine3.11 @@ -0,0 +1,77 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the template to use +TEMPLATE = docker-alpine + +# Python version +PYVERSION = 3.5 + +# Specify the docker base image +DOCKER_BASE = python:3.5-alpine3.11 + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS = cmake m4 libtool autoconf automake zlib llvm8-dev suitesparse-dev fftw-dev flac-dev + +# Packages to install with pip +PIP_PKGS = setuptools six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.0.1 numpy==1.11.3 scipy==0.18.1 matplotlib==1.5.3 pandas==0.19.2 memory_profiler==0.42 astropy==1.2.2 ephem==3.7.6.0 ipython==5.2.2 numba==0.30.1 + +# Packages to install with conda +CONDA_PKGS = + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = @AUX_PREFIX@/include +MPI_LDFLAGS = @AUX_PREFIX@/lib +MPI_CXXLIB = mpichcxx +MPI_LIB = mpich +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O2 -g -fPIC -pthread +CXXFLAGS = -O2 -g -fPIC -pthread -std=c++11 +FCFLAGS = -O2 -g -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? +CROSS = + +# Parallel builds +MAKEJ = 1 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/docker-py3.5-alpine3.11.pkgs b/configs/docker-py3.5-alpine3.11.pkgs new file mode 100644 index 0000000..e364c98 --- /dev/null +++ b/configs/docker-py3.5-alpine3.11.pkgs @@ -0,0 +1,103 @@ +# This is the full set of packages that can be installed. This can be +# used as a starting point for the package list for a particular config. +# Some packages accept additional options which can be added after a colon ":". +# These extra options are documented in the shell snippet for each package +# in the top-level pkgs directory. +# +# Autotools. If you already have recent versions installed with your OS +# package manager, then comment these out. +#m4 +#libtool +#autoconf +#automake +# +# CMake. +#cmake +# +# Install zlib. This is needed on some systems where the OS-provided zlib is +# very ancient. +#zlib +# +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +#python:conda +python:default +# +# Install MPICH. This step is optional and only needed in 2 cases: +# 1. You don't have MPI available through system packages that is +# compatible with the serial compilers you are using. +# 2. You are building a docker container for use at NERSC with shifter +# and need an MPICH compatible with the Cray one. +mpich +# +# Install mpi4py. This will use the MPI compilers specified in your config. +mpi4py +# +# Install CFITSIO. +cfitsio +# +# Install FFTW3. +#fftw +# +# Install libbz2. This is required to enable some features in the boost +# build which are in turn needed by spt3g. +bzip2 +# +# Install Boost. +#boost +# +# Install HDF5 (serial version). +hdf5 +# +# Install h5py (using our HDF5). +h5py +# +# Install healpy +healpy +# +# Install quaternionarray +qarray +# +# Install fitsio +fitsio +# +# Install aatm. Needed for some features of atmosphere simulations. +aatm +# +# Install suitesparse +#suitesparse +# +# Install libconviqt. 4-PI a_lm space beam convolution. +conviqt +# +# Install libsharp. Distributed harmonic transforms. +libsharp +# +# Install libmadam. Destriping map-maker. +madam +# +# Install FLAC, needed by spt3g +#libflac +# +# Install SPT3G. Frame based data format. +#spt3g +# +# Install TIDAS. HDF5-based timestream data format. +tidas +# +# Install PySM. Python Sky Model. +pysm +# +# Install PyMPIT for environment testing. Useful for testing working +# mpi4py stack. +pympit +# +# Install TOAST. +toast diff --git a/configs/docker-py3.5-debian b/configs/docker-py3.5-debian new file mode 100644 index 0000000..ce0bd69 --- /dev/null +++ b/configs/docker-py3.5-debian @@ -0,0 +1,80 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the template to use +TEMPLATE = docker-debian + +# Python version +PYVERSION = 3.5 + +# Specify the docker base image +DOCKER_BASE = python:3.5-slim-buster + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS = m4 libtool autoconf automake llvm libsuitesparse-dev libfftw3-dev libcfitsio-dev libflac-dev zlib1g-dev libbz2-dev libopenblas-dev liblapack-dev libhdf5-dev + +# Packages to install with pip. Note that we must force older versions of packages +# to ones that have wheels available for python3.5. The numpy 1.13 requirement comes +# from healpy and the others are chosen to be from around the same time period. +# +PIP_PKGS = setuptools six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.3.1 numpy==1.13.3 scipy==0.19.1 matplotlib==2.1.0 pandas==0.20.3 memory_profiler==0.47 astropy==1.3.3 ephem==3.7.7.0 ipython==6.2.1 numba==0.36.2 cmake + +# Packages to install with conda +CONDA_PKGS = + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = @AUX_PREFIX@/include +MPI_LDFLAGS = @AUX_PREFIX@/lib +MPI_CXXLIB = mpichcxx +MPI_LIB = mpich +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O2 -g -fPIC -pthread +CXXFLAGS = -O2 -g -fPIC -pthread -std=c++11 +FCFLAGS = -O2 -g -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? +CROSS = + +# Parallel builds +MAKEJ = 1 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/docker-py3.5-debian.pkgs b/configs/docker-py3.5-debian.pkgs new file mode 100644 index 0000000..1ebe1d0 --- /dev/null +++ b/configs/docker-py3.5-debian.pkgs @@ -0,0 +1,103 @@ +# This is the full set of packages that can be installed. This can be +# used as a starting point for the package list for a particular config. +# Some packages accept additional options which can be added after a colon ":". +# These extra options are documented in the shell snippet for each package +# in the top-level pkgs directory. +# +# Autotools. If you already have recent versions installed with your OS +# package manager, then comment these out. +#m4 +#libtool +#autoconf +#automake +# +# CMake. +#cmake +# +# Install zlib. This is needed on some systems where the OS-provided zlib is +# very ancient. +#zlib +# +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +#python:conda +python:default +# +# Install MPICH. This step is optional and only needed in 2 cases: +# 1. You don't have MPI available through system packages that is +# compatible with the serial compilers you are using. +# 2. You are building a docker container for use at NERSC with shifter +# and need an MPICH compatible with the Cray one. +mpich +# +# Install mpi4py. This will use the MPI compilers specified in your config. +mpi4py +# +# Install CFITSIO. +#cfitsio +# +# Install FFTW3. +#fftw +# +# Install libbz2. This is required to enable some features in the boost +# build which are in turn needed by spt3g. +#bzip2 +# +# Install Boost. +#boost +# +# Install HDF5 (serial version). +#hdf5 +# +# Install h5py (using our HDF5). +h5py:pkg-config +# +# Install healpy +healpy +# +# Install quaternionarray +qarray +# +# Install fitsio +fitsio +# +# Install aatm. Needed for some features of atmosphere simulations. +aatm +# +# Install suitesparse +#suitesparse +# +# Install libconviqt. 4-PI a_lm space beam convolution. +conviqt +# +# Install libsharp. Distributed harmonic transforms. +libsharp +# +# Install libmadam. Destriping map-maker. +madam:--with-cfitsio=/usr --with-fftw=/usr +# +# Install FLAC, needed by spt3g +#libflac +# +# Install SPT3G. Frame based data format. +#spt3g:boost=/usr/local flaclib=/usr/lib/x86_64-linux-gnu/libFLAC.so +# +# Install TIDAS. HDF5-based timestream data format. +tidas +# +# Install PySM. Python Sky Model. (not supported for python < 3.6) +#pysm +# +# Install PyMPIT for environment testing. Useful for testing working +# mpi4py stack. +pympit +# +# Install TOAST. +toast diff --git a/configs/docker-py3.6-alpine3.11 b/configs/docker-py3.6-alpine3.11 new file mode 100644 index 0000000..1c602e4 --- /dev/null +++ b/configs/docker-py3.6-alpine3.11 @@ -0,0 +1,77 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the template to use +TEMPLATE = docker-alpine + +# Python version +PYVERSION = 3.6 + +# Specify the docker base image +DOCKER_BASE = python:3.6-alpine3.11 + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS = python3 python3-dev cython py3-nose py3-yaml py3-numpy py3-numpy-dev py3-scipy cmake m4 libtool autoconf automake zlib llvm8-dev + +# Packages to install with pip +PIP_PKGS = setuptools six cycler kiwisolver pyparsing python-dateutil toml pkgconfig matplotlib astropy psutil ephem pandas memory_profiler ipython numba + +# Packages to install with conda +CONDA_PKGS = + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = @AUX_PREFIX@/include +MPI_LDFLAGS = @AUX_PREFIX@/lib +MPI_CXXLIB = mpichcxx +MPI_LIB = mpich +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O2 -g -fPIC -pthread +CXXFLAGS = -O2 -g -fPIC -pthread -std=c++11 +FCFLAGS = -O2 -g -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? +CROSS = + +# Parallel builds +MAKEJ = 1 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/docker-py3.6-alpine3.11.pkgs b/configs/docker-py3.6-alpine3.11.pkgs new file mode 120000 index 0000000..df13829 --- /dev/null +++ b/configs/docker-py3.6-alpine3.11.pkgs @@ -0,0 +1 @@ +docker-py3.5-alpine3.11.pkgs \ No newline at end of file diff --git a/configs/docker-py3.6-debian b/configs/docker-py3.6-debian new file mode 100644 index 0000000..2c663d0 --- /dev/null +++ b/configs/docker-py3.6-debian @@ -0,0 +1,77 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the template to use +TEMPLATE = docker-debian + +# Python version +PYVERSION = 3.6 + +# Specify the docker base image +DOCKER_BASE = python:3.6-slim-buster + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS = m4 libtool autoconf automake llvm libsuitesparse-dev libfftw3-dev libcfitsio-dev libflac-dev zlib1g-dev libbz2-dev libopenblas-dev liblapack-dev libhdf5-dev + +# Packages to install with pip +PIP_PKGS = setuptools six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 cmake + +# Packages to install with conda +CONDA_PKGS = + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = @AUX_PREFIX@/include +MPI_LDFLAGS = @AUX_PREFIX@/lib +MPI_CXXLIB = mpichcxx +MPI_LIB = mpich +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O2 -g -fPIC -pthread +CXXFLAGS = -O2 -g -fPIC -pthread -std=c++11 +FCFLAGS = -O2 -g -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? +CROSS = + +# Parallel builds +MAKEJ = 1 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/docker-py3.6-debian.pkgs b/configs/docker-py3.6-debian.pkgs new file mode 100644 index 0000000..c719166 --- /dev/null +++ b/configs/docker-py3.6-debian.pkgs @@ -0,0 +1,103 @@ +# This is the full set of packages that can be installed. This can be +# used as a starting point for the package list for a particular config. +# Some packages accept additional options which can be added after a colon ":". +# These extra options are documented in the shell snippet for each package +# in the top-level pkgs directory. +# +# Autotools. If you already have recent versions installed with your OS +# package manager, then comment these out. +#m4 +#libtool +#autoconf +#automake +# +# CMake. +#cmake +# +# Install zlib. This is needed on some systems where the OS-provided zlib is +# very ancient. +#zlib +# +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +#python:conda +python:default +# +# Install MPICH. This step is optional and only needed in 2 cases: +# 1. You don't have MPI available through system packages that is +# compatible with the serial compilers you are using. +# 2. You are building a docker container for use at NERSC with shifter +# and need an MPICH compatible with the Cray one. +mpich +# +# Install mpi4py. This will use the MPI compilers specified in your config. +mpi4py +# +# Install CFITSIO. +#cfitsio +# +# Install FFTW3. +#fftw +# +# Install libbz2. This is required to enable some features in the boost +# build which are in turn needed by spt3g. +#bzip2 +# +# Install Boost. +#boost +# +# Install HDF5 (serial version). +#hdf5 +# +# Install h5py (using our HDF5). +h5py:pkg-config +# +# Install healpy +healpy +# +# Install quaternionarray +qarray +# +# Install fitsio +fitsio +# +# Install aatm. Needed for some features of atmosphere simulations. +aatm +# +# Install suitesparse +#suitesparse +# +# Install libconviqt. 4-PI a_lm space beam convolution. +conviqt +# +# Install libsharp. Distributed harmonic transforms. +libsharp +# +# Install libmadam. Destriping map-maker. +madam:--with-cfitsio=/usr --with-fftw=/usr +# +# Install FLAC, needed by spt3g +#libflac +# +# Install SPT3G. Frame based data format. +#spt3g:boost=/usr/local flaclib=/usr/lib/x86_64-linux-gnu/libFLAC.so +# +# Install TIDAS. HDF5-based timestream data format. +tidas +# +# Install PySM. Python Sky Model. +pysm +# +# Install PyMPIT for environment testing. Useful for testing working +# mpi4py stack. +pympit +# +# Install TOAST. +toast diff --git a/configs/docker-py3.7-alpine3.11 b/configs/docker-py3.7-alpine3.11 new file mode 100644 index 0000000..9a2853d --- /dev/null +++ b/configs/docker-py3.7-alpine3.11 @@ -0,0 +1,77 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the template to use +TEMPLATE = docker-alpine + +# Python version +PYVERSION = 3.7 + +# Specify the docker base image +DOCKER_BASE = python:3.7-alpine3.11 + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS = python3 python3-dev cython py3-nose py3-yaml py3-numpy py3-numpy-dev py3-scipy cmake m4 libtool autoconf automake zlib llvm8-dev + +# Packages to install with pip +PIP_PKGS = setuptools six cycler kiwisolver pyparsing python-dateutil toml pkgconfig matplotlib astropy psutil ephem pandas memory_profiler ipython numba + +# Packages to install with conda +CONDA_PKGS = + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = @AUX_PREFIX@/include +MPI_LDFLAGS = @AUX_PREFIX@/lib +MPI_CXXLIB = mpichcxx +MPI_LIB = mpich +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O2 -g -fPIC -pthread +CXXFLAGS = -O2 -g -fPIC -pthread -std=c++11 +FCFLAGS = -O2 -g -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? +CROSS = + +# Parallel builds +MAKEJ = 1 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/docker-py3.7-alpine3.11.pkgs b/configs/docker-py3.7-alpine3.11.pkgs new file mode 120000 index 0000000..df13829 --- /dev/null +++ b/configs/docker-py3.7-alpine3.11.pkgs @@ -0,0 +1 @@ +docker-py3.5-alpine3.11.pkgs \ No newline at end of file diff --git a/configs/docker-py3.7-debian b/configs/docker-py3.7-debian new file mode 100644 index 0000000..9603db2 --- /dev/null +++ b/configs/docker-py3.7-debian @@ -0,0 +1,77 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the template to use +TEMPLATE = docker-debian + +# Python version +PYVERSION = 3.7 + +# Specify the docker base image +DOCKER_BASE = python:3.7-slim-buster + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS = m4 libtool autoconf automake llvm libsuitesparse-dev libfftw3-dev libcfitsio-dev libflac-dev zlib1g-dev libbz2-dev libopenblas-dev liblapack-dev libhdf5-dev + +# Packages to install with pip +PIP_PKGS = setuptools six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 cmake + +# Packages to install with conda +CONDA_PKGS = + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = @AUX_PREFIX@/include +MPI_LDFLAGS = @AUX_PREFIX@/lib +MPI_CXXLIB = mpichcxx +MPI_LIB = mpich +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O2 -g -fPIC -pthread +CXXFLAGS = -O2 -g -fPIC -pthread -std=c++11 +FCFLAGS = -O2 -g -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? +CROSS = + +# Parallel builds +MAKEJ = 1 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/docker-py3.7-debian.pkgs b/configs/docker-py3.7-debian.pkgs new file mode 120000 index 0000000..1354c8d --- /dev/null +++ b/configs/docker-py3.7-debian.pkgs @@ -0,0 +1 @@ +docker-py3.6-debian.pkgs \ No newline at end of file diff --git a/configs/docker-py3.8-alpine3.11 b/configs/docker-py3.8-alpine3.11 new file mode 100644 index 0000000..e591fdb --- /dev/null +++ b/configs/docker-py3.8-alpine3.11 @@ -0,0 +1,77 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the template to use +TEMPLATE = docker-alpine + +# Python version +PYVERSION = 3.8 + +# Specify the docker base image +DOCKER_BASE = python:3.8-alpine3.11 + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS = python3 python3-dev cython py3-nose py3-yaml py3-numpy py3-numpy-dev py3-scipy cmake m4 libtool autoconf automake zlib llvm8-dev + +# Packages to install with pip +PIP_PKGS = setuptools six cycler kiwisolver pyparsing python-dateutil toml pkgconfig matplotlib astropy psutil ephem pandas memory_profiler ipython numba + +# Packages to install with conda +CONDA_PKGS = + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = @AUX_PREFIX@/include +MPI_LDFLAGS = @AUX_PREFIX@/lib +MPI_CXXLIB = mpichcxx +MPI_LIB = mpich +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O2 -g -fPIC -pthread +CXXFLAGS = -O2 -g -fPIC -pthread -std=c++11 +FCFLAGS = -O2 -g -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? +CROSS = + +# Parallel builds +MAKEJ = 1 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/docker-py3.8-alpine3.11.pkgs b/configs/docker-py3.8-alpine3.11.pkgs new file mode 120000 index 0000000..df13829 --- /dev/null +++ b/configs/docker-py3.8-alpine3.11.pkgs @@ -0,0 +1 @@ +docker-py3.5-alpine3.11.pkgs \ No newline at end of file diff --git a/configs/docker-py3.8-debian b/configs/docker-py3.8-debian new file mode 100644 index 0000000..2d49d9f --- /dev/null +++ b/configs/docker-py3.8-debian @@ -0,0 +1,77 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the template to use +TEMPLATE = docker-debian + +# Python version +PYVERSION = 3.8 + +# Specify the docker base image +DOCKER_BASE = python:3.8-slim-buster + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS = m4 libtool autoconf automake llvm libsuitesparse-dev libfftw3-dev libcfitsio-dev libflac-dev zlib1g-dev libbz2-dev libopenblas-dev liblapack-dev libhdf5-dev + +# Packages to install with pip +PIP_PKGS = setuptools six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 cmake + +# Packages to install with conda +CONDA_PKGS = + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = @AUX_PREFIX@/include +MPI_LDFLAGS = @AUX_PREFIX@/lib +MPI_CXXLIB = mpichcxx +MPI_LIB = mpich +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O2 -g -fPIC -pthread +CXXFLAGS = -O2 -g -fPIC -pthread -std=c++11 +FCFLAGS = -O2 -g -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? +CROSS = + +# Parallel builds +MAKEJ = 1 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/docker-py3.8-debian.pkgs b/configs/docker-py3.8-debian.pkgs new file mode 120000 index 0000000..1354c8d --- /dev/null +++ b/configs/docker-py3.8-debian.pkgs @@ -0,0 +1 @@ +docker-py3.6-debian.pkgs \ No newline at end of file diff --git a/configs/docker-toastdeps-py35 b/configs/docker-toastdeps-py35 new file mode 120000 index 0000000..1549483 --- /dev/null +++ b/configs/docker-toastdeps-py35 @@ -0,0 +1 @@ +docker-py3.5-debian \ No newline at end of file diff --git a/configs/docker-toastdeps-py35.pkgs b/configs/docker-toastdeps-py35.pkgs new file mode 100644 index 0000000..0b10d96 --- /dev/null +++ b/configs/docker-toastdeps-py35.pkgs @@ -0,0 +1,101 @@ +# This is the full set of packages that can be installed. This can be +# used as a starting point for the package list for a particular config. +# Some packages accept additional options which can be added after a colon ":". +# These extra options are documented in the shell snippet for each package +# in the top-level pkgs directory. +# +# Autotools. If you already have recent versions installed with your OS +# package manager, then comment these out. +#m4 +#libtool +#autoconf +#automake +# +# CMake. +#cmake +# +# Install zlib. This is needed on some systems where the OS-provided zlib is +# very ancient. +#zlib +# +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +#python:conda +python:default +# +# Install MPICH. This step is optional and only needed in 2 cases: +# 1. You don't have MPI available through system packages that is +# compatible with the serial compilers you are using. +# 2. You are building a docker container for use at NERSC with shifter +# and need an MPICH compatible with the Cray one. +mpich +# +# Install mpi4py. This will use the MPI compilers specified in your config. +mpi4py +# +# Install CFITSIO. +#cfitsio +# +# Install FFTW3. +#fftw +# +# Install libbz2. This is required to enable some features in the boost +# build which are in turn needed by spt3g. +#bzip2 +# +# Install Boost. +#boost +# +# Install HDF5 (serial version). +#hdf5 +# +# Install h5py (using our HDF5). +h5py:pkg-config +# +# Install healpy +healpy +# +# Install quaternionarray +qarray +# +# Install fitsio +fitsio +# +# Install aatm. Needed for some features of atmosphere simulations. +aatm +# +# Install suitesparse +#suitesparse +# +# Install libconviqt. 4-PI a_lm space beam convolution. +conviqt +# +# Install libsharp. Distributed harmonic transforms. +libsharp +# +# Install libmadam. Destriping map-maker. +madam:--with-cfitsio=/usr --with-fftw=/usr +# +# Install FLAC, needed by spt3g +#libflac +# +# Install SPT3G. Frame based data format. +#spt3g:boost=/usr/local flaclib=/usr/lib/x86_64-linux-gnu/libFLAC.so +# +# Install TIDAS. HDF5-based timestream data format. +tidas +# +# Install PySM. Python Sky Model. (not supported with python < 3.6) +#pysm +# +# Install PyMPIT for environment testing. Useful for testing working +# mpi4py stack. +pympit +# diff --git a/configs/docker-toastdeps-py36 b/configs/docker-toastdeps-py36 new file mode 120000 index 0000000..8b77647 --- /dev/null +++ b/configs/docker-toastdeps-py36 @@ -0,0 +1 @@ +docker-py3.6-debian \ No newline at end of file diff --git a/configs/docker-toastdeps-py36.pkgs b/configs/docker-toastdeps-py36.pkgs new file mode 100644 index 0000000..560687a --- /dev/null +++ b/configs/docker-toastdeps-py36.pkgs @@ -0,0 +1,101 @@ +# This is the full set of packages that can be installed. This can be +# used as a starting point for the package list for a particular config. +# Some packages accept additional options which can be added after a colon ":". +# These extra options are documented in the shell snippet for each package +# in the top-level pkgs directory. +# +# Autotools. If you already have recent versions installed with your OS +# package manager, then comment these out. +#m4 +#libtool +#autoconf +#automake +# +# CMake. +#cmake +# +# Install zlib. This is needed on some systems where the OS-provided zlib is +# very ancient. +#zlib +# +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +#python:conda +python:default +# +# Install MPICH. This step is optional and only needed in 2 cases: +# 1. You don't have MPI available through system packages that is +# compatible with the serial compilers you are using. +# 2. You are building a docker container for use at NERSC with shifter +# and need an MPICH compatible with the Cray one. +mpich +# +# Install mpi4py. This will use the MPI compilers specified in your config. +mpi4py +# +# Install CFITSIO. +#cfitsio +# +# Install FFTW3. +#fftw +# +# Install libbz2. This is required to enable some features in the boost +# build which are in turn needed by spt3g. +#bzip2 +# +# Install Boost. +#boost +# +# Install HDF5 (serial version). +#hdf5 +# +# Install h5py (using our HDF5). +h5py:pkg-config +# +# Install healpy +healpy +# +# Install quaternionarray +qarray +# +# Install fitsio +fitsio +# +# Install aatm. Needed for some features of atmosphere simulations. +aatm +# +# Install suitesparse +#suitesparse +# +# Install libconviqt. 4-PI a_lm space beam convolution. +conviqt +# +# Install libsharp. Distributed harmonic transforms. +libsharp +# +# Install libmadam. Destriping map-maker. +madam:--with-cfitsio=/usr --with-fftw=/usr +# +# Install FLAC, needed by spt3g +#libflac +# +# Install SPT3G. Frame based data format. +#spt3g:boost=/usr/local flaclib=/usr/lib/x86_64-linux-gnu/libFLAC.so +# +# Install TIDAS. HDF5-based timestream data format. +tidas +# +# Install PySM. Python Sky Model. +pysm +# +# Install PyMPIT for environment testing. Useful for testing working +# mpi4py stack. +pympit +# diff --git a/configs/docker-toastdeps-py37 b/configs/docker-toastdeps-py37 new file mode 120000 index 0000000..5791b1b --- /dev/null +++ b/configs/docker-toastdeps-py37 @@ -0,0 +1 @@ +docker-py3.7-debian \ No newline at end of file diff --git a/configs/docker-toastdeps-py37.pkgs b/configs/docker-toastdeps-py37.pkgs new file mode 120000 index 0000000..0a373e7 --- /dev/null +++ b/configs/docker-toastdeps-py37.pkgs @@ -0,0 +1 @@ +docker-toastdeps-py36.pkgs \ No newline at end of file diff --git a/configs/docker-toastdeps-py38 b/configs/docker-toastdeps-py38 new file mode 120000 index 0000000..13f08c2 --- /dev/null +++ b/configs/docker-toastdeps-py38 @@ -0,0 +1 @@ +docker-py3.8-debian \ No newline at end of file diff --git a/configs/docker-toastdeps-py38.pkgs b/configs/docker-toastdeps-py38.pkgs new file mode 120000 index 0000000..0a373e7 --- /dev/null +++ b/configs/docker-toastdeps-py38.pkgs @@ -0,0 +1 @@ +docker-toastdeps-py36.pkgs \ No newline at end of file diff --git a/configs/docker-ubuntu_20.04_gcc9_py38 b/configs/docker-ubuntu_20.04_gcc9_py38 new file mode 100644 index 0000000..68cb2e4 --- /dev/null +++ b/configs/docker-ubuntu_20.04_gcc9_py38 @@ -0,0 +1,77 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the template to use +TEMPLATE = docker-ubuntu + +# Specify the docker base image +DOCKER_BASE = ubuntu:20.04 + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS = m4 libtool autoconf automake cmake libcfitsio-dev libfftw3-dev libopenblas-dev liblapack-dev libhdf5-dev libboost-all-dev libopenmpi-dev python3 python3-dev python3-setuptools python3-pkg-resources python3-pip python3-tk python3-numpy python3-scipy python3-matplotlib python3-yaml python3-astropy python3-six python3-psutil python3-ephem python3-pandas python3-memory-profiler python3-ipython cython3 python3-cycler python3-kiwisolver python3-dateutil python3-toml python3-numba black python3-jupyter-client python3-h5py python3-mpi4py libsuitesparse-dev + +# Packages to install with pip +PIP_PKGS = + +# Packages to install with conda +CONDA_PKGS = + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = /usr/include +MPI_LDFLAGS = /usr/lib/x86_64-linux-gnu +MPI_CXXLIB = mpi_cxx +MPI_LIB = mpi +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O2 -g -fPIC -pthread +CXXFLAGS = -O2 -g -fPIC -pthread -std=c++11 +FCFLAGS = -O2 -g -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? +CROSS = + +# Python version +PYVERSION = 3.8 + +# Parallel builds +MAKEJ = 2 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/docker-ubuntu_20.04_gcc9_py38.pkgs b/configs/docker-ubuntu_20.04_gcc9_py38.pkgs new file mode 100644 index 0000000..21c89bc --- /dev/null +++ b/configs/docker-ubuntu_20.04_gcc9_py38.pkgs @@ -0,0 +1,103 @@ +# This is the full set of packages that can be installed. This can be +# used as a starting point for the package list for a particular config. +# Some packages accept additional options which can be added after a colon ":". +# These extra options are documented in the shell snippet for each package +# in the top-level pkgs directory. +# +# Autotools. If you already have recent versions installed with your OS +# package manager, then comment these out. +#m4 +#libtool +#autoconf +#automake +# +# CMake. +#cmake +# +# Install zlib. This is needed on some systems where the OS-provided zlib is +# very ancient. +#zlib +# +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +#python:conda +python:default +# +# Install MPICH. This step is optional and only needed in 2 cases: +# 1. You don't have MPI available through system packages that is +# compatible with the serial compilers you are using. +# 2. You are building a docker container for use at NERSC with shifter +# and need an MPICH compatible with the Cray one. +#mpich +# +# Install mpi4py. This will use the MPI compilers specified in your config. +#mpi4py +# +# Install CFITSIO. +#cfitsio +# +# Install FFTW3. +#fftw +# +# Install libbz2. This is required to enable some features in the boost +# build which are in turn needed by spt3g. +#bzip2 +# +# Install Boost. +#boost +# +# Install HDF5 (serial version). +#hdf5 +# +# Install h5py (using our HDF5). +#h5py +# +# Install healpy +healpy +# +# Install quaternionarray +qarray +# +# Install fitsio +fitsio +# +# Install aatm. Needed for some features of atmosphere simulations. +aatm +# +# Install suitesparse +#suitesparse +# +# Install libconviqt. 4-PI a_lm space beam convolution. +conviqt +# +# Install libsharp. Distributed harmonic transforms. +libsharp +# +# Install libmadam. Destriping map-maker. +madam +# +# Install FLAC, needed by spt3g +libflac +# +# Install SPT3G. Frame based data format. +spt3g +# +# Install TIDAS. HDF5-based timestream data format. +tidas +# +# Install PySM. Python Sky Model. +pysm +# +# Install PyMPIT for environment testing. Useful for testing working +# mpi4py stack. +pympit +# +# Install TOAST. +toast diff --git a/configs/linux-conda b/configs/linux-conda index fb5589d..a53a08e 100644 --- a/configs/linux-conda +++ b/configs/linux-conda @@ -2,6 +2,18 @@ # OS type (linux OR osx) OSTYPE = linux +# Specify the template to use +TEMPLATE = linux + +# Python version to use +PYVERSION = 3.7 + +# Packages to install with pip +PIP_PKGS = + +# Packages to install with conda +CONDA_PKGS = nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab + # Serial compilers CC = gcc @@ -42,9 +54,6 @@ CROSS = # Parallel builds MAKEJ = 4 -# Python version to use -PYVERSION = 3.6 - # Define MKL location MKL = diff --git a/configs/linux-conda-nomkl b/configs/linux-conda-nomkl index d82a78d..a53a08e 100644 --- a/configs/linux-conda-nomkl +++ b/configs/linux-conda-nomkl @@ -2,6 +2,18 @@ # OS type (linux OR osx) OSTYPE = linux +# Specify the template to use +TEMPLATE = linux + +# Python version to use +PYVERSION = 3.7 + +# Packages to install with pip +PIP_PKGS = + +# Packages to install with conda +CONDA_PKGS = nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab + # Serial compilers CC = gcc @@ -42,9 +54,6 @@ CROSS = # Parallel builds MAKEJ = 4 -# Python version to use -PYVERSION = 3.7 - # Define MKL location MKL = diff --git a/configs/linux-conda-nomkl.pkgs b/configs/linux-conda-nomkl.pkgs index 0548ee8..5465fb0 100644 --- a/configs/linux-conda-nomkl.pkgs +++ b/configs/linux-conda-nomkl.pkgs @@ -4,14 +4,6 @@ # These extra options are documented in the shell snippet for each package # in the top-level pkgs directory. # -# Python 3. Uncomment one of the following lines. You should only use the -# "default" option for Docker images. All other configs should use either -# a virtualenv or a conda environment. This will also install a set of -# common base packages. -#python:virtualenv -python:conda nomkl -#python:default -# # Autotools. If you already have recent versions installed with your OS # package manager, then comment these out. m4 @@ -26,6 +18,18 @@ cmake # very ancient. zlib # +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +python:conda nomkl +#python:default +# # Install MPICH. This step is optional and only needed in 2 cases: # 1. You don't have MPI available through system packages that is # compatible with the serial compilers you are using. @@ -39,10 +43,6 @@ mpi4py # Install CFITSIO. cfitsio # -# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers -# or some other vendor BLAS / LAPACK solution. -openblas -# # Install FFTW3. fftw # diff --git a/configs/linux-conda.pkgs b/configs/linux-conda.pkgs index c6973c9..6cb7917 100644 --- a/configs/linux-conda.pkgs +++ b/configs/linux-conda.pkgs @@ -4,14 +4,6 @@ # These extra options are documented in the shell snippet for each package # in the top-level pkgs directory. # -# Python 3. Uncomment one of the following lines. You should only use the -# "default" option for Docker images. All other configs should use either -# a virtualenv or a conda environment. This will also install a set of -# common base packages. -#python:virtualenv -python:conda -#python:default -# # Autotools. If you already have recent versions installed with your OS # package manager, then comment these out. m4 @@ -26,6 +18,18 @@ cmake # very ancient. zlib # +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +python:conda +#python:default +# # Install MPICH. This step is optional and only needed in 2 cases: # 1. You don't have MPI available through system packages that is # compatible with the serial compilers you are using. @@ -39,10 +43,6 @@ mpi4py # Install CFITSIO. cfitsio # -# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers -# or some other vendor BLAS / LAPACK solution. -openblas -# # Install FFTW3. fftw # diff --git a/configs/linux-venv b/configs/linux-venv index 552b6db..f745497 100644 --- a/configs/linux-venv +++ b/configs/linux-venv @@ -2,6 +2,18 @@ # OS type (linux OR osx) OSTYPE = linux +# Specify the template to use +TEMPLATE = linux + +# Python version to use +PYVERSION = auto + +# Packages to install with pip +PIP_PKGS = nose Cython numpy scipy matplotlib pyyaml astropy six psutil ephem pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab + +# Packages to install with conda +CONDA_PKGS = + # Serial compilers CC = gcc @@ -36,15 +48,11 @@ OPENMP_CXXFLAGS = -fopenmp LDFLAGS = -lpthread -fopenmp # Are we doing a cross-compile? - CROSS = # Parallel builds MAKEJ = 4 -# Python version to use -PYVERSION = auto - # Define MKL location MKL = diff --git a/configs/linux-venv.pkgs b/configs/linux-venv.pkgs index 1f48e51..3a32805 100644 --- a/configs/linux-venv.pkgs +++ b/configs/linux-venv.pkgs @@ -4,14 +4,6 @@ # These extra options are documented in the shell snippet for each package # in the top-level pkgs directory. # -# Python 3. Uncomment one of the following lines. You should only use the -# "default" option for Docker images. All other configs should use either -# a virtualenv or a conda environment. This will also install a set of -# common base packages. -python:virtualenv -#python:conda -#python:default -# # Autotools. If you already have recent versions installed with your OS # package manager, then comment these out. m4 @@ -26,6 +18,18 @@ cmake # very ancient. zlib # +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +python:virtualenv +#python:conda +#python:default +# # Install MPICH. This step is optional and only needed in 2 cases: # 1. You don't have MPI available through system packages that is # compatible with the serial compilers you are using. @@ -39,10 +43,6 @@ mpi4py # Install CFITSIO. cfitsio # -# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers -# or some other vendor BLAS / LAPACK solution. -openblas -# # Install FFTW3. fftw # diff --git a/configs/docker-gcc b/configs/old/docker-cori-gcc similarity index 72% rename from configs/docker-gcc rename to configs/old/docker-cori-gcc index 3347e07..17a0c99 100644 --- a/configs/docker-gcc +++ b/configs/old/docker-cori-gcc @@ -2,6 +2,26 @@ # OS type (linux OR osx) OSTYPE = linux +# Specify the docker base image +DOCKER_BASE=ubuntu:20.04 + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS=" \ +gfortran \ +autoconf \ +automake \ +libtool \ +m4 \ +cmake \ +python3 \ +libpython3-dev \ +python3-pip \ + +" + +# On linux, optionally set the alternate compiler toolchain. +GCC_VER = + # Serial compilers CC = gcc @@ -43,7 +63,7 @@ CROSS = MAKEJ = 4 # Python version to use -PYVERSION = 3.7 +PYVERSION = 3.8 # Define MKL location MKL = diff --git a/configs/old/docker-cori-gcc.pkgs b/configs/old/docker-cori-gcc.pkgs new file mode 100644 index 0000000..4aea2aa --- /dev/null +++ b/configs/old/docker-cori-gcc.pkgs @@ -0,0 +1,103 @@ +# This is the full set of packages that can be installed. This can be +# used as a starting point for the package list for a particular config. +# Some packages accept additional options which can be added after a colon ":". +# These extra options are documented in the shell snippet for each package +# in the top-level pkgs directory. +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +#python:conda +python:default +# +# Autotools. If you already have recent versions installed with your OS +# package manager, then comment these out. +#m4 +#libtool +#autoconf +#automake +# +# CMake. +#cmake +# +# Install zlib. This is needed on some systems where the OS-provided zlib is +# very ancient. +#zlib +# +# Install MPICH. This step is optional and only needed in 2 cases: +# 1. You don't have MPI available through system packages that is +# compatible with the serial compilers you are using. +# 2. You are building a docker container for use at NERSC with shifter +# and need an MPICH compatible with the Cray one. +mpich +# +# Install mpi4py. This will use the MPI compilers specified in your config. +mpi4py +# +# Install CFITSIO. +cfitsio +# +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +openblas +# +# Install FFTW3. +fftw +# +# Install libbz2. This is required to enable some features in the boost +# build which are in turn needed by spt3g. +bzip2 +# +# Install Boost. +boost +# +# Install HDF5 (serial version). +hdf5 +# +# Install h5py (using our HDF5). +h5py +# +# Install healpy +healpy +# +# Install quaternionarray +qarray +# +# Install fitsio +fitsio +# +# Install aatm. Needed for some features of atmosphere simulations. +aatm +# +# Install suitesparse +suitesparse +# +# Install libconviqt. 4-PI a_lm space beam convolution. +conviqt +# +# Install libsharp. Distributed harmonic transforms. +libsharp +# +# Install libmadam. Destriping map-maker. +madam +# +# Install FLAC, needed by spt3g +libflac +# +# Install SPT3G. Frame based data format. +spt3g +# +# Install TIDAS. HDF5-based timestream data format. +tidas +# +# Install PySM. Python Sky Model. +pysm +# +# Install PyMPIT for environment testing. Useful for testing working +# mpi4py stack. +pympit +# +# Install TOAST. +toast diff --git a/configs/old/docker-gcc b/configs/old/docker-gcc new file mode 100644 index 0000000..8733483 --- /dev/null +++ b/configs/old/docker-gcc @@ -0,0 +1,72 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the docker base image +DOCKER_BASE = ubuntu:20.04 + +# Python version to use +PYVERSION = 3.8 + +# On linux, optionally set the alternate compiler toolchain. +GCC_VER = + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS = gfortran autoconf automake libtool m4 cmake python3 python3-dev python3-tk python3-pip zlib1g-dev libbz2-dev libopenblas-dev liblapack-dev libboost-all-dev libcfitsio-dev libfftw3-dev libhdf5-dev libflac-dev libsuitesparse-dev + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = /usr/include +MPI_LDFLAGS = /usr/lib/x86_64-linux-gnu +MPI_CXXLIB = mpi_cxx +MPI_LIB = mpi +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O3 -fPIC -pthread +CXXFLAGS = -O3 -fPIC -pthread -std=c++11 +FCFLAGS = -O3 -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? + +CROSS = + +# Parallel builds +MAKEJ = 2 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/docker-gcc.pkgs b/configs/old/docker-gcc.pkgs similarity index 98% rename from configs/docker-gcc.pkgs rename to configs/old/docker-gcc.pkgs index 469d7aa..2db0df3 100644 --- a/configs/docker-gcc.pkgs +++ b/configs/old/docker-gcc.pkgs @@ -20,18 +20,18 @@ python:default #automake # # CMake. -cmake +#cmake # # Install zlib. This is needed on some systems where the OS-provided zlib is # very ancient. -zlib +#zlib # # Install MPICH. This step is optional and only needed in 2 cases: # 1. You don't have MPI available through system packages that is # compatible with the serial compilers you are using. # 2. You are building a docker container for use at NERSC with shifter # and need an MPICH compatible with the Cray one. -mpich +#mpich # # Install mpi4py. This will use the MPI compilers specified in your config. mpi4py @@ -48,13 +48,13 @@ fftw # # Install libbz2. This is required to enable some features in the boost # build which are in turn needed by spt3g. -bzip2 +#bzip2 # # Install Boost. -boost +#boost # # Install HDF5 (serial version). -hdf5 +#hdf5 # # Install h5py (using our HDF5). h5py diff --git a/configs/old/docker-toastdeps_14.04_gcc49_py34 b/configs/old/docker-toastdeps_14.04_gcc49_py34 new file mode 100644 index 0000000..9f07a0c --- /dev/null +++ b/configs/old/docker-toastdeps_14.04_gcc49_py34 @@ -0,0 +1,72 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the docker base image +DOCKER_BASE = ubuntu:14.04 + +# Python version to use +PYVERSION = 3.4 + +# On linux, optionally set the alternate compiler toolchain. +GCC_VER = + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS = python3.4 python3.4-dev python3.4-tk + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = /usr/include +MPI_LDFLAGS = /usr/lib +MPI_CXXLIB = mpichcxx +MPI_LIB = mpich +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O2 -g -fPIC -pthread +CXXFLAGS = -O2 -g -fPIC -pthread -std=c++11 +FCFLAGS = -O2 -g -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? + +CROSS = + +# Parallel builds +MAKEJ = 2 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/cori-gcc-knl.pkgs b/configs/old/docker-toastdeps_14.04_gcc49_py34.pkgs similarity index 98% rename from configs/cori-gcc-knl.pkgs rename to configs/old/docker-toastdeps_14.04_gcc49_py34.pkgs index 85d8842..32ad17a 100644 --- a/configs/cori-gcc-knl.pkgs +++ b/configs/old/docker-toastdeps_14.04_gcc49_py34.pkgs @@ -9,8 +9,8 @@ # a virtualenv or a conda environment. This will also install a set of # common base packages. #python:virtualenv -python:conda nomkl -#python:default +#python:conda +python:default # # Autotools. If you already have recent versions installed with your OS # package manager, then comment these out. @@ -31,7 +31,7 @@ zlib # compatible with the serial compilers you are using. # 2. You are building a docker container for use at NERSC with shifter # and need an MPICH compatible with the Cray one. -#mpich +mpich # # Install mpi4py. This will use the MPI compilers specified in your config. mpi4py @@ -100,4 +100,4 @@ pysm pympit # # Install TOAST. -toast +#toast diff --git a/configs/old/docker-toastdeps_16.04_gcc5_py35 b/configs/old/docker-toastdeps_16.04_gcc5_py35 new file mode 100644 index 0000000..1620806 --- /dev/null +++ b/configs/old/docker-toastdeps_16.04_gcc5_py35 @@ -0,0 +1,72 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the docker base image +DOCKER_BASE = ubuntu:16.04 + +# Python version to use +PYVERSION = 3.5 + +# On linux, optionally set the alternate compiler toolchain. +GCC_VER = + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS = python3.5 python3.5-dev python3.5-tk + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = /usr/include +MPI_LDFLAGS = /usr/lib +MPI_CXXLIB = mpichcxx +MPI_LIB = mpich +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O2 -g -fPIC -pthread +CXXFLAGS = -O2 -g -fPIC -pthread -std=c++11 +FCFLAGS = -O2 -g -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? + +CROSS = + +# Parallel builds +MAKEJ = 2 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/old/docker-toastdeps_16.04_gcc5_py35.pkgs b/configs/old/docker-toastdeps_16.04_gcc5_py35.pkgs new file mode 100644 index 0000000..32ad17a --- /dev/null +++ b/configs/old/docker-toastdeps_16.04_gcc5_py35.pkgs @@ -0,0 +1,103 @@ +# This is the full set of packages that can be installed. This can be +# used as a starting point for the package list for a particular config. +# Some packages accept additional options which can be added after a colon ":". +# These extra options are documented in the shell snippet for each package +# in the top-level pkgs directory. +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +#python:conda +python:default +# +# Autotools. If you already have recent versions installed with your OS +# package manager, then comment these out. +m4 +libtool +autoconf +automake +# +# CMake. +cmake +# +# Install zlib. This is needed on some systems where the OS-provided zlib is +# very ancient. +zlib +# +# Install MPICH. This step is optional and only needed in 2 cases: +# 1. You don't have MPI available through system packages that is +# compatible with the serial compilers you are using. +# 2. You are building a docker container for use at NERSC with shifter +# and need an MPICH compatible with the Cray one. +mpich +# +# Install mpi4py. This will use the MPI compilers specified in your config. +mpi4py +# +# Install CFITSIO. +cfitsio +# +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +openblas +# +# Install FFTW3. +fftw +# +# Install libbz2. This is required to enable some features in the boost +# build which are in turn needed by spt3g. +bzip2 +# +# Install Boost. +boost +# +# Install HDF5 (serial version). +hdf5 +# +# Install h5py (using our HDF5). +h5py +# +# Install healpy +healpy +# +# Install quaternionarray +qarray +# +# Install fitsio +fitsio +# +# Install aatm. Needed for some features of atmosphere simulations. +aatm +# +# Install suitesparse +suitesparse +# +# Install libconviqt. 4-PI a_lm space beam convolution. +conviqt +# +# Install libsharp. Distributed harmonic transforms. +libsharp +# +# Install libmadam. Destriping map-maker. +madam +# +# Install FLAC, needed by spt3g +libflac +# +# Install SPT3G. Frame based data format. +spt3g +# +# Install TIDAS. HDF5-based timestream data format. +tidas +# +# Install PySM. Python Sky Model. +pysm +# +# Install PyMPIT for environment testing. Useful for testing working +# mpi4py stack. +pympit +# +# Install TOAST. +#toast diff --git a/configs/cori-gcc-knl b/configs/old/docker-toastdeps_18.04_gcc7_py36 similarity index 50% rename from configs/cori-gcc-knl rename to configs/old/docker-toastdeps_18.04_gcc7_py36 index af645a1..d75af5d 100644 --- a/configs/cori-gcc-knl +++ b/configs/old/docker-toastdeps_18.04_gcc7_py36 @@ -2,6 +2,18 @@ # OS type (linux OR osx) OSTYPE = linux +# Specify the docker base image +DOCKER_BASE = ubuntu:18.04 + +# Python version to use +PYVERSION = 3.6 + +# On linux, optionally set the alternate compiler toolchain. +GCC_VER = + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS = python3 python3-dev python3-tk + # Serial compilers CC = gcc @@ -15,34 +27,32 @@ BUILD_CXX = g++ # MPI compilers -MPICC = ${CRAYPE_DIR}/bin/cc -MPICXX = ${CRAYPE_DIR}/bin/CC -MPIFC = ${CRAYPE_DIR}/bin/ftn -MPI_CPPFLAGS = ${CRAY_MPICH2_DIR}/include -MPI_LDFLAGS = ${CRAY_MPICH2_DIR}/lib +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = /usr/include +MPI_LDFLAGS = /usr/lib MPI_CXXLIB = mpichcxx MPI_LIB = mpich MPI_EXTRA_COMP = -MPI_EXTRA_LINK = -shared +MPI_EXTRA_LINK = # Compile flags -CFLAGS = -O3 -g -fPIC -pthread -march=knl -CXXFLAGS = -O3 -g -fPIC -pthread -march=knl -std=c++11 -FCFLAGS = -O3 -g -fPIC -pthread -march=knl +CFLAGS = -O2 -g -fPIC -pthread +CXXFLAGS = -O2 -g -fPIC -pthread -std=c++11 +FCFLAGS = -O2 -g -fPIC -pthread OPENMP_CFLAGS = -fopenmp OPENMP_CXXFLAGS = -fopenmp -LDFLAGS = -fopenmp -lpthread +LDFLAGS = -lpthread -fopenmp -# We are cross-compiling for KNL on Haswell login nodes -CROSS = --build x86_64-pc-linux-gnu --host x86_64-unknown-linux-gnu +# Are we doing a cross-compile? -# Parallel builds -MAKEJ = 4 +CROSS = -# Python version to use -PYVERSION = 3.6 +# Parallel builds +MAKEJ = 2 # Define MKL location MKL = @@ -58,5 +68,5 @@ BOOSTCHAIN = gcc # Group and permissions to set -CHGRP = hpcosmo -CHMOD = a+rX,g-w,o-w +CHGRP = +CHMOD = diff --git a/configs/old/docker-toastdeps_18.04_gcc7_py36.pkgs b/configs/old/docker-toastdeps_18.04_gcc7_py36.pkgs new file mode 100644 index 0000000..32ad17a --- /dev/null +++ b/configs/old/docker-toastdeps_18.04_gcc7_py36.pkgs @@ -0,0 +1,103 @@ +# This is the full set of packages that can be installed. This can be +# used as a starting point for the package list for a particular config. +# Some packages accept additional options which can be added after a colon ":". +# These extra options are documented in the shell snippet for each package +# in the top-level pkgs directory. +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +#python:conda +python:default +# +# Autotools. If you already have recent versions installed with your OS +# package manager, then comment these out. +m4 +libtool +autoconf +automake +# +# CMake. +cmake +# +# Install zlib. This is needed on some systems where the OS-provided zlib is +# very ancient. +zlib +# +# Install MPICH. This step is optional and only needed in 2 cases: +# 1. You don't have MPI available through system packages that is +# compatible with the serial compilers you are using. +# 2. You are building a docker container for use at NERSC with shifter +# and need an MPICH compatible with the Cray one. +mpich +# +# Install mpi4py. This will use the MPI compilers specified in your config. +mpi4py +# +# Install CFITSIO. +cfitsio +# +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +openblas +# +# Install FFTW3. +fftw +# +# Install libbz2. This is required to enable some features in the boost +# build which are in turn needed by spt3g. +bzip2 +# +# Install Boost. +boost +# +# Install HDF5 (serial version). +hdf5 +# +# Install h5py (using our HDF5). +h5py +# +# Install healpy +healpy +# +# Install quaternionarray +qarray +# +# Install fitsio +fitsio +# +# Install aatm. Needed for some features of atmosphere simulations. +aatm +# +# Install suitesparse +suitesparse +# +# Install libconviqt. 4-PI a_lm space beam convolution. +conviqt +# +# Install libsharp. Distributed harmonic transforms. +libsharp +# +# Install libmadam. Destriping map-maker. +madam +# +# Install FLAC, needed by spt3g +libflac +# +# Install SPT3G. Frame based data format. +spt3g +# +# Install TIDAS. HDF5-based timestream data format. +tidas +# +# Install PySM. Python Sky Model. +pysm +# +# Install PyMPIT for environment testing. Useful for testing working +# mpi4py stack. +pympit +# +# Install TOAST. +#toast diff --git a/configs/travis_14.04 b/configs/old/travis_14.04 similarity index 100% rename from configs/travis_14.04 rename to configs/old/travis_14.04 diff --git a/configs/travis_14.04.pkgs b/configs/old/travis_14.04.pkgs similarity index 100% rename from configs/travis_14.04.pkgs rename to configs/old/travis_14.04.pkgs diff --git a/configs/osx-homebrew-clang.pkgs b/configs/osx-homebrew-clang.pkgs index 64a9f73..ca7ab90 100644 --- a/configs/osx-homebrew-clang.pkgs +++ b/configs/osx-homebrew-clang.pkgs @@ -4,14 +4,6 @@ # These extra options are documented in the shell snippet for each package # in the top-level pkgs directory. # -# Python 3. Uncomment one of the following lines. You should only use the -# "default" option for Docker images. All other configs should use either -# a virtualenv or a conda environment. This will also install a set of -# common base packages. -python:virtualenv -#python:conda -#python:default -# # Autotools. If you already have recent versions installed with your OS # package manager, then comment these out. #m4 @@ -26,6 +18,18 @@ python:virtualenv # very ancient. #zlib # +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +python:virtualenv +#python:conda +#python:default +# # Install MPICH. This step is optional and only needed in 2 cases: # 1. You don't have MPI available through system packages that is # compatible with the serial compilers you are using. @@ -39,10 +43,6 @@ mpi4py # Install CFITSIO. #cfitsio # -# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers -# or some other vendor BLAS / LAPACK solution. -#openblas -# # Install FFTW3. #fftw # diff --git a/configs/osx-macports-gcc.pkgs b/configs/osx-macports-gcc.pkgs index 86c3a38..c1803c3 100644 --- a/configs/osx-macports-gcc.pkgs +++ b/configs/osx-macports-gcc.pkgs @@ -4,14 +4,6 @@ # These extra options are documented in the shell snippet for each package # in the top-level pkgs directory. # -# Python 3. Uncomment one of the following lines. You should only use the -# "default" option for Docker images. All other configs should use either -# a virtualenv or a conda environment. This will also install a set of -# common base packages. -python:virtualenv -#python:conda -#python:default -# # Autotools. If you already have recent versions installed with your OS # package manager, then comment these out. #m4 @@ -26,6 +18,18 @@ python:virtualenv # very ancient. #zlib # +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +python:virtualenv +#python:conda +#python:default +# # Install MPICH. This step is optional and only needed in 2 cases: # 1. You don't have MPI available through system packages that is # compatible with the serial compilers you are using. @@ -39,10 +43,6 @@ mpi4py # Install CFITSIO. #cfitsio # -# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers -# or some other vendor BLAS / LAPACK solution. -#openblas -# # Install FFTW3. #fftw # diff --git a/configs/simons-intel b/configs/simons-intel index df73144..485b970 100644 --- a/configs/simons-intel +++ b/configs/simons-intel @@ -2,6 +2,18 @@ # OS type (linux OR osx) OSTYPE = linux +# Specify the template to use +TEMPLATE = linux + +# Python version to use +PYVERSION = 3.7 + +# Packages to install with pip +PIP_PKGS = + +# Packages to install with conda +CONDA_PKGS = nose Cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab + # Serial compilers CC = icc @@ -42,9 +54,6 @@ CROSS = # Parallel builds MAKEJ = 4 -# Python version to use -PYVERSION = 3.6 - # Define MKL location MKL = ${MKLROOT} diff --git a/configs/simons-intel.pkgs b/configs/simons-intel.pkgs index e85e964..249606c 100644 --- a/configs/simons-intel.pkgs +++ b/configs/simons-intel.pkgs @@ -4,14 +4,6 @@ # These extra options are documented in the shell snippet for each package # in the top-level pkgs directory. # -# Python 3. Uncomment one of the following lines. You should only use the -# "default" option for Docker images. All other configs should use either -# a virtualenv or a conda environment. This will also install a set of -# common base packages. -python:virtualenv -#python:conda nomkl -#python:default -# # Autotools. If you already have recent versions installed with your OS # package manager, then comment these out. m4 @@ -26,6 +18,18 @@ cmake # very ancient. zlib # +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +python:virtualenv +#python:conda nomkl +#python:default +# # Install MPICH. This step is optional and only needed in 2 cases: # 1. You don't have MPI available through system packages that is # compatible with the serial compilers you are using. @@ -39,10 +43,6 @@ mpi4py # Install CFITSIO. cfitsio # -# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers -# or some other vendor BLAS / LAPACK solution. -#openblas -# # Install FFTW3. fftw # diff --git a/configs/ubuntu-18.04-venv b/configs/ubuntu-18.04-venv new file mode 100644 index 0000000..788678d --- /dev/null +++ b/configs/ubuntu-18.04-venv @@ -0,0 +1,72 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the template to use +TEMPLATE = linux + +# Python version to use +PYVERSION = auto + +# Packages to install with pip +PIP_PKGS = setuptools six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 cmake + +# Packages to install with conda +CONDA_PKGS = + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = /usr/include +MPI_LDFLAGS = /usr/lib/x86_64-linux-gnu +MPI_CXXLIB = mpi_cxx +MPI_LIB = mpi +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O3 -fPIC -pthread +CXXFLAGS = -O3 -fPIC -pthread -std=c++11 +FCFLAGS = -O3 -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? + +CROSS = + +# Parallel builds +MAKEJ = 2 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = -llapack + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/ubuntu-18.04-venv.notes b/configs/ubuntu-18.04-venv.notes new file mode 100644 index 0000000..f6310f5 --- /dev/null +++ b/configs/ubuntu-18.04-venv.notes @@ -0,0 +1,28 @@ + +# Before running the install, run these commands once +# to get dependencies from OS packages. + +apt update + +apt install \ +build-essential \ +gfortran \ +python3-dev \ +virtualenv \ +pkg-config \ +curl \ +m4 \ +git \ +libtool \ +autoconf \ +automake \ +zlib1g-dev \ +libopenblas-dev \ +liblapack-dev \ +libboost-all-dev \ +libopenmpi-dev \ +libcfitsio-dev \ +libfftw3-dev \ +libhdf5-dev \ +libflac-dev \ +libsuitesparse-dev diff --git a/configs/ubuntu-18.04-venv.pkgs b/configs/ubuntu-18.04-venv.pkgs new file mode 100644 index 0000000..61b088a --- /dev/null +++ b/configs/ubuntu-18.04-venv.pkgs @@ -0,0 +1,103 @@ +# This is the full set of packages that can be installed. This can be +# used as a starting point for the package list for a particular config. +# Some packages accept additional options which can be added after a colon ":". +# These extra options are documented in the shell snippet for each package +# in the top-level pkgs directory. +# +# Autotools. If you already have recent versions installed with your OS +# package manager, then comment these out. +#m4 +#libtool +#autoconf +#automake +# +# CMake. +#cmake +# +# Install zlib. This is needed on some systems where the OS-provided zlib is +# very ancient. +#zlib +# +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +python:virtualenv +#python:conda +#python:default +# +# Install MPICH. This step is optional and only needed in 2 cases: +# 1. You don't have MPI available through system packages that is +# compatible with the serial compilers you are using. +# 2. You are building a docker container for use at NERSC with shifter +# and need an MPICH compatible with the Cray one. +#mpich +# +# Install mpi4py. This will use the MPI compilers specified in your config. +mpi4py +# +# Install CFITSIO. +#cfitsio +# +# Install FFTW3. +#fftw +# +# Install libbz2. This is required to enable some features in the boost +# build which are in turn needed by spt3g. +#bzip2 +# +# Install Boost. +#boost +# +# Install HDF5 (serial version). +#hdf5 +# +# Install h5py (using our HDF5). +h5py:pkg-config +# +# Install healpy +healpy +# +# Install quaternionarray +qarray +# +# Install fitsio +fitsio +# +# Install aatm. Needed for some features of atmosphere simulations. +aatm +# +# Install suitesparse +#suitesparse +# +# Install libconviqt. 4-PI a_lm space beam convolution. +conviqt +# +# Install libsharp. Distributed harmonic transforms. +libsharp +# +# Install libmadam. Destriping map-maker. +madam:--with-cfitsio=/usr --with-fftw=/usr +# +# Install FLAC, needed by spt3g +#libflac +# +# Install SPT3G. Frame based data format. +spt3g:boost=/usr flaclib=/usr/lib/x86_64-linux-gnu/libFLAC.so +# +# Install TIDAS. HDF5-based timestream data format. +tidas +# +# Install PySM. Python Sky Model. +pysm +# +# Install PyMPIT for environment testing. Useful for testing working +# mpi4py stack. +pympit +# +# Install TOAST. +toast diff --git a/configs/ubuntu-19.04-venv b/configs/ubuntu-19.04-venv index 43bca33..0960f46 100644 --- a/configs/ubuntu-19.04-venv +++ b/configs/ubuntu-19.04-venv @@ -2,6 +2,18 @@ # OS type (linux OR osx) OSTYPE = linux +# Specify the template to use +TEMPLATE = linux + +# Python version to use +PYVERSION = auto + +# Packages to install with pip +PIP_PKGS = nose Cython numpy scipy matplotlib pyyaml astropy six psutil ephem pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab + +# Packages to install with conda +CONDA_PKGS = + # Serial compilers CC = gcc @@ -42,9 +54,6 @@ CROSS = # Parallel builds MAKEJ = 2 -# Python version to use -PYVERSION = auto - # Define MKL location MKL = diff --git a/configs/ubuntu-19.04-venv.pkgs b/configs/ubuntu-19.04-venv.pkgs index 5c21272..61b088a 100644 --- a/configs/ubuntu-19.04-venv.pkgs +++ b/configs/ubuntu-19.04-venv.pkgs @@ -4,14 +4,6 @@ # These extra options are documented in the shell snippet for each package # in the top-level pkgs directory. # -# Python 3. Uncomment one of the following lines. You should only use the -# "default" option for Docker images. All other configs should use either -# a virtualenv or a conda environment. This will also install a set of -# common base packages. -python:virtualenv -#python:conda -#python:default -# # Autotools. If you already have recent versions installed with your OS # package manager, then comment these out. #m4 @@ -26,6 +18,18 @@ python:virtualenv # very ancient. #zlib # +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +python:virtualenv +#python:conda +#python:default +# # Install MPICH. This step is optional and only needed in 2 cases: # 1. You don't have MPI available through system packages that is # compatible with the serial compilers you are using. @@ -39,10 +43,6 @@ mpi4py # Install CFITSIO. #cfitsio # -# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers -# or some other vendor BLAS / LAPACK solution. -#openblas -# # Install FFTW3. #fftw # @@ -100,4 +100,4 @@ pysm pympit # # Install TOAST. -toast:lapack=/usr/lib/x86_64-linux-gnu/liblapack.so blas=/usr/lib/x86_64-linux-gnu/libopenblas.so +toast diff --git a/configs/ubuntu-19.10-venv b/configs/ubuntu-19.10-venv index d789175..57eb603 100644 --- a/configs/ubuntu-19.10-venv +++ b/configs/ubuntu-19.10-venv @@ -2,6 +2,18 @@ # OS type (linux OR osx) OSTYPE = linux +# Specify the template to use +TEMPLATE = linux + +# Python version to use +PYVERSION = auto + +# Packages to install with pip +PIP_PKGS = nose Cython numpy scipy matplotlib pyyaml astropy six psutil ephem pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab + +# Packages to install with conda +CONDA_PKGS = + # Serial compilers CC = gcc @@ -42,9 +54,6 @@ CROSS = # Parallel builds MAKEJ = 1 -# Python version to use -PYVERSION = auto - # Define MKL location MKL = diff --git a/configs/ubuntu-19.10-venv.pkgs b/configs/ubuntu-19.10-venv.pkgs index 5c21272..61b088a 100644 --- a/configs/ubuntu-19.10-venv.pkgs +++ b/configs/ubuntu-19.10-venv.pkgs @@ -4,14 +4,6 @@ # These extra options are documented in the shell snippet for each package # in the top-level pkgs directory. # -# Python 3. Uncomment one of the following lines. You should only use the -# "default" option for Docker images. All other configs should use either -# a virtualenv or a conda environment. This will also install a set of -# common base packages. -python:virtualenv -#python:conda -#python:default -# # Autotools. If you already have recent versions installed with your OS # package manager, then comment these out. #m4 @@ -26,6 +18,18 @@ python:virtualenv # very ancient. #zlib # +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +python:virtualenv +#python:conda +#python:default +# # Install MPICH. This step is optional and only needed in 2 cases: # 1. You don't have MPI available through system packages that is # compatible with the serial compilers you are using. @@ -39,10 +43,6 @@ mpi4py # Install CFITSIO. #cfitsio # -# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers -# or some other vendor BLAS / LAPACK solution. -#openblas -# # Install FFTW3. #fftw # @@ -100,4 +100,4 @@ pysm pympit # # Install TOAST. -toast:lapack=/usr/lib/x86_64-linux-gnu/liblapack.so blas=/usr/lib/x86_64-linux-gnu/libopenblas.so +toast diff --git a/configs/ubuntu-20.04-venv b/configs/ubuntu-20.04-venv new file mode 100644 index 0000000..57eb603 --- /dev/null +++ b/configs/ubuntu-20.04-venv @@ -0,0 +1,72 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the template to use +TEMPLATE = linux + +# Python version to use +PYVERSION = auto + +# Packages to install with pip +PIP_PKGS = nose Cython numpy scipy matplotlib pyyaml astropy six psutil ephem pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab + +# Packages to install with conda +CONDA_PKGS = + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = /usr/include +MPI_LDFLAGS = /usr/lib/x86_64-linux-gnu +MPI_CXXLIB = mpi_cxx +MPI_LIB = mpi +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O3 -fPIC -pthread +CXXFLAGS = -O3 -fPIC -pthread -std=c++11 +FCFLAGS = -O3 -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? + +CROSS = + +# Parallel builds +MAKEJ = 1 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = -llapack + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/ubuntu-20.04-venv.notes b/configs/ubuntu-20.04-venv.notes new file mode 100644 index 0000000..f7f2e3a --- /dev/null +++ b/configs/ubuntu-20.04-venv.notes @@ -0,0 +1,29 @@ + +# Before running the install, run these commands once +# to get dependencies from OS packages. + +apt update + +apt install \ +build-essential \ +gfortran \ +python3-dev \ +python3-pip \ +python3-virtualenv \ +virtualenv \ +cmake \ +pkg-config \ +curl \ +m4 \ +libtool \ +autoconf \ +automake \ +zlib1g-dev \ +libopenblas-dev \ +liblapack-dev \ +libboost-all-dev \ +libcfitsio-dev \ +libfftw3-dev \ +libhdf5-dev \ +libflac-dev \ +libsuitesparse-dev diff --git a/configs/ubuntu-20.04-venv.pkgs b/configs/ubuntu-20.04-venv.pkgs new file mode 100644 index 0000000..e949256 --- /dev/null +++ b/configs/ubuntu-20.04-venv.pkgs @@ -0,0 +1,103 @@ +# This is the full set of packages that can be installed. This can be +# used as a starting point for the package list for a particular config. +# Some packages accept additional options which can be added after a colon ":". +# These extra options are documented in the shell snippet for each package +# in the top-level pkgs directory. +# +# Autotools. If you already have recent versions installed with your OS +# package manager, then comment these out. +#m4 +#libtool +#autoconf +#automake +# +# CMake. +#cmake +# +# Install zlib. This is needed on some systems where the OS-provided zlib is +# very ancient. +#zlib +# +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +python:virtualenv +#python:conda +#python:default +# +# Install MPICH. This step is optional and only needed in 2 cases: +# 1. You don't have MPI available through system packages that is +# compatible with the serial compilers you are using. +# 2. You are building a docker container for use at NERSC with shifter +# and need an MPICH compatible with the Cray one. +#mpich +# +# Install mpi4py. This will use the MPI compilers specified in your config. +#mpi4py +# +# Install CFITSIO. +#cfitsio +# +# Install FFTW3. +#fftw +# +# Install libbz2. This is required to enable some features in the boost +# build which are in turn needed by spt3g. +#bzip2 +# +# Install Boost. +#boost +# +# Install HDF5 (serial version). +#hdf5 +# +# Install h5py (using our HDF5). +#h5py:pkg-config +# +# Install healpy +healpy +# +# Install quaternionarray +qarray +# +# Install fitsio +fitsio +# +# Install aatm. Needed for some features of atmosphere simulations. +aatm +# +# Install suitesparse +#suitesparse +# +# Install libconviqt. 4-PI a_lm space beam convolution. +conviqt +# +# Install libsharp. Distributed harmonic transforms. +libsharp +# +# Install libmadam. Destriping map-maker. +madam:--with-cfitsio=/usr --with-fftw=/usr +# +# Install FLAC, needed by spt3g +#libflac +# +# Install SPT3G. Frame based data format. +spt3g:boost=/usr flaclib=/usr/lib/x86_64-linux-gnu/libFLAC.so +# +# Install TIDAS. HDF5-based timestream data format. +tidas +# +# Install PySM. Python Sky Model. +pysm +# +# Install PyMPIT for environment testing. Useful for testing working +# mpi4py stack. +pympit +# +# Install TOAST. +toast:lapack=/usr/lib/x86_64-linux-gnu/liblapack.so blas=/usr/lib/x86_64-linux-gnu/libopenblas.so diff --git a/pkgs/boost.sh b/pkgs/boost.sh index 960e6bf..1ea7275 100644 --- a/pkgs/boost.sh +++ b/pkgs/boost.sh @@ -24,7 +24,6 @@ rm -rf boost_${version} tar xjf ${src} \ && cd boost_${version} \ && cleanup="${cleanup} $(pwd)" \ - && echo "" > tools/build/user-config.jam \ && echo "using mpi : @MPICXX@ : \"@MPI_CPPFLAGS@\" \"@MPI_LDFLAGS@\" \"@MPI_CXXLIB@\" \"@MPI_LIB@\" ;" >> tools/build/user-config.jam \ && echo "option jobs : @MAKEJ@ ;" >> tools/build/user-config.jam \ && BOOST_BUILD_USER_CONFIG=tools/build/user-config.jam \ @@ -32,7 +31,7 @@ tar xjf ${src} \ BZIP2_LIBPATH="@AUX_PREFIX@/lib" \ ./bootstrap.sh \ --with-toolset=@BOOSTCHAIN@ \ - --with-python=python@PYVERSION@ \ + --with-python=python3 \ --prefix=@AUX_PREFIX@ > ${log} 2>&1 \ && ./b2 --layout=tagged --user-config=./tools/build/user-config.jam \ $(python3-config --includes | sed -e 's/-I//g' -e 's/\([^[:space:]]\+\)/ include=\1/g') \ diff --git a/pkgs/conviqt.sh b/pkgs/conviqt.sh index ae4cf6d..3027d95 100644 --- a/pkgs/conviqt.sh +++ b/pkgs/conviqt.sh @@ -22,13 +22,13 @@ rm -rf libconviqt-${version} tar xzf ${src} \ && cd libconviqt-${version} \ && cleanup="${cleanup} $(pwd)" \ - && ./autogen.sh \ + && ./autogen.sh > ${log} 2>&1 \ && CC="@MPICC@" CXX="@MPICXX@" MPICC="@MPICC@" MPICXX="@MPICXX@" \ CFLAGS="@CFLAGS@ -std=gnu99" CXXFLAGS="@CXXFLAGS@" \ OPENMP_CFLAGS="@OPENMP_CFLAGS@" OPENMP_CXXFLAGS="@OPENMP_CXXFLAGS@" \ LDFLAGS="@LDFLAGS@" \ ./configure @CROSS@ \ - --with-cfitsio="@AUX_PREFIX@" --prefix="@AUX_PREFIX@" > ${log} 2>&1 \ + --with-cfitsio="@AUX_PREFIX@" --prefix="@AUX_PREFIX@" >> ${log} 2>&1 \ && make -j @MAKEJ@ >> ${log} 2>&1 \ && make install >> ${log} 2>&1 \ && cd python \ diff --git a/pkgs/h5py.sh b/pkgs/h5py.sh index 3b05c40..113663c 100644 --- a/pkgs/h5py.sh +++ b/pkgs/h5py.sh @@ -15,10 +15,10 @@ fi # Note- the download URL includes a checksum and will need to # be updated when you change this version string. -version=2.9.0 +version=2.10.0 pfile=h5py-${version}.tar.gz -src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://files.pythonhosted.org/packages/43/27/a6e7dcb8ae20a4dbf3725321058923fec262b6f7835179d78ccc8d98deec/${pfile} ${pfile}) +src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://files.pythonhosted.org/packages/5f/97/a58afbcf40e8abecededd9512978b4e4915374e5b80049af082f49cebe9a/${pfile} ${pfile}) if [ "x${src}" = "x" ]; then echo "Failed to fetch ${pkg}" >&2 diff --git a/pkgs/hdf5.sh b/pkgs/hdf5.sh index 48da8e9..aa23286 100644 --- a/pkgs/hdf5.sh +++ b/pkgs/hdf5.sh @@ -4,10 +4,10 @@ pkg="hdf5" pkgopts=$@ cleanup="" -version=1.10.5 +version=1.12.0 pfile=hdf5-${version}.tar.bz2 -src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-${version}/src/${pfile} ${pfile}) +src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-${version}/src/${pfile} ${pfile}) if [ "x${src}" = "x" ]; then echo "Failed to fetch ${pkg}" >&2 @@ -30,11 +30,12 @@ tar xjf ${src} \ FC="@FC@" FCFLAGS=$(if [ "x@CROSS@" = x ]; then echo "@FCFLAGS@"; \ else echo "-O3"; fi) \ ./configure \ + --enable-build-mode=production \ + --with-default-api-version=v110 \ --disable-silent-rules \ --disable-parallel \ --enable-cxx \ - --enable-fortran \ - --enable-fortran2003 \ + --disable-fortran \ --prefix="@AUX_PREFIX@" > ${log} 2>&1 \ && make -j @MAKEJ@ >> ${log} 2>&1 \ && make install >> ${log} 2>&1 diff --git a/pkgs/healpy.sh b/pkgs/healpy.sh index 65c856a..31303ce 100644 --- a/pkgs/healpy.sh +++ b/pkgs/healpy.sh @@ -4,9 +4,9 @@ pkg="healpy" pkgopts=$@ cleanup="" -version=1.12.10 +version=1.13.0 pfile=healpy-${version}.tar.gz -src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://files.pythonhosted.org/packages/e6/34/da61364cd7826ae6da0dd5cdc0ac1abaa933bf6bee4381986f326bc87a23/healpy-1.12.10.tar.gz ${pfile}) +src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://files.pythonhosted.org/packages/26/74/0c8592686027a8196e275cb81999e8aae88d0416c223fa55a7f0cb5bdd26/${pfile} ${pfile}) if [ "x${src}" = "x" ]; then echo "Failed to fetch ${pkg}" >&2 diff --git a/pkgs/mpi4py.sh b/pkgs/mpi4py.sh index 3106cbf..c00a5b6 100644 --- a/pkgs/mpi4py.sh +++ b/pkgs/mpi4py.sh @@ -5,10 +5,10 @@ pkgopts=$@ cleanup="" # Note: update the URL when you change the version. -version=3.0.1 +version=3.0.3 pfile=mpi4py-${version}.tar.gz -src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://files.pythonhosted.org/packages/55/a2/c827b196070e161357b49287fa46d69f25641930fd5f854722319d431843/${pfile} ${pfile}) +src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://files.pythonhosted.org/packages/ec/8f/bbd8de5ba566dd77e408d8136e2bab7fdf2b97ce06cab830ba8b50a2f588/${pfile} ${pfile}) if [ "x${src}" = "x" ]; then echo "Failed to fetch ${pkg}" >&2 diff --git a/pkgs/python.sh b/pkgs/python.sh index 76e8ed2..07cf82e 100644 --- a/pkgs/python.sh +++ b/pkgs/python.sh @@ -7,7 +7,7 @@ cleanup="" pkg="python" export pytype=$(echo "$pkgopts" | awk '{print $1}') -export cdatype=$(echo "$pkgopts" | awk '{print $2}') +export pextra=$(echo "$pkgopts" | awk '{print $2}') mkdir -p "@PYTHON_PREFIX@/bin" @@ -31,12 +31,12 @@ if [ "${pytype}" = "conda" ]; then bash "${inst}" -b -f -p "@PYTHON_PREFIX@" \ && echo "# condarc to force channel order" > "@PYTHON_PREFIX@/.condarc" \ && echo "channels:" >> "@PYTHON_PREFIX@/.condarc" \ - && if [ "x${cdatype}" = "xnomkl" ]; then + && if [ "x${pextra}" = "xnomkl" ]; then echo " - conda-forge" >> "@PYTHON_PREFIX@/.condarc" fi \ && echo " - defaults" >> "@PYTHON_PREFIX@/.condarc" \ && echo "changeps1: false" >> "@PYTHON_PREFIX@/.condarc" \ - && eval "@SRCDIR@/tools/gen_activate.sh" "@VERSION@" "@PREFIX@" "@PYTHON_PREFIX@" "@AUX_PREFIX@" "@PYVERSION@" "${pytype}" "${cdatype}" \ + && eval "@TOP_DIR@/tools/gen_activate.sh" "@VERSION@" "@PREFIX@" "@PYTHON_PREFIX@" "@AUX_PREFIX@" "@PYVERSION@" "${pytype}" "${pextra}" \ && source "@PYTHON_PREFIX@/bin/cmbenv" \ && conda install --copy --yes python=@PYVERSION@ \ && ln -s "@PYTHON_PREFIX@"/include/python* "@AUX_PREFIX@/include/" \ @@ -46,7 +46,7 @@ if [ "${pytype}" = "conda" ]; then echo "conda python install failed" >&2 exit 1 fi - if [ "x${cdatype}" = "xnomkl" ]; then + if [ "x${pextra}" = "xnomkl" ]; then conda install --copy --yes "blas=*=openblas" else conda install --copy --yes blas @@ -55,80 +55,33 @@ if [ "${pytype}" = "conda" ]; then echo "conda blas install failed" >&2 exit 1 fi - conda install --copy --yes \ - nose \ - cython \ - numpy \ - scipy \ - matplotlib \ - pyyaml \ - astropy \ - six \ - psutil \ - ephem \ - virtualenv \ - pandas \ - memory_profiler \ - ipython \ - cython \ - cycler \ - kiwisolver \ - python-dateutil \ - toml \ - numba \ - tbb \ - nbstripout \ - black \ - wurlitzer \ - ipympl \ - ipykernel \ - jupyterlab \ - && rm -rf "@PYTHON_PREFIX@/pkgs/*" + if [ "x@CONDA_PKGS@" != "x" ]; then + conda install --copy --yes @CONDA_PKGS@ + fi if [ $? -ne 0 ]; then echo "conda install packages failed" >&2 exit 1 fi + rm -rf "@PYTHON_PREFIX@/pkgs/*" else if [ "${pytype}" = "virtualenv" ]; then echo "Python using virtualenv" >&2 virtualenv -p python@PYVERSION@ "@PYTHON_PREFIX@" \ - && eval "@SRCDIR@/tools/gen_activate.sh" "@VERSION@" "@PREFIX@" "@PYTHON_PREFIX@" "@AUX_PREFIX@" "@PYVERSION@" "${pytype}" "${cdatype}" \ + && eval "@TOP_DIR@/tools/gen_activate.sh" "@VERSION@" "@PREFIX@" "@PYTHON_PREFIX@" "@AUX_PREFIX@" "@PYVERSION@" "${pytype}" "${pextra}" \ && source "@PYTHON_PREFIX@/bin/cmbenv" else echo "Python using default" >&2 - eval "@SRCDIR@/tools/gen_activate.sh" "@VERSION@" "@PREFIX@" "@PYTHON_PREFIX@" "@AUX_PREFIX@" "@PYVERSION@" "${pytype}" "${cdatype}" \ + eval "@TOP_DIR@/tools/gen_activate.sh" "@VERSION@" "@PREFIX@" "@PYTHON_PREFIX@" "@AUX_PREFIX@" "@PYVERSION@" "${pytype}" "${pextra}" \ && source "@PYTHON_PREFIX@/bin/cmbenv" fi - pip3 install \ - nose \ - cython \ - numpy \ - scipy \ - matplotlib \ - pyyaml \ - astropy \ - six \ - psutil \ - ephem \ - pandas \ - memory_profiler \ - ipython \ - cython \ - cycler \ - kiwisolver \ - python-dateutil \ - toml \ - numba \ - tbb \ - nbstripout \ - black \ - wurlitzer \ - ipympl \ - ipykernel \ - jupyterlab - if [ $? -ne 0 ]; then - echo "pip install failed" >&2 - exit 1 + if [ "x@PIP_PKGS@" != "x" ]; then + for pip_pkg in @PIP_PKGS@; do + pip3 install ${pip_pkg} >&2 + if [ $? -ne 0 ]; then + echo "pip install of ${pip_pkg} failed" >&2 + exit 1 + fi + done fi fi python3 -c "import matplotlib.font_manager" diff --git a/pkgs/spt3g.sh b/pkgs/spt3g.sh index 497d057..d4b1e75 100644 --- a/pkgs/spt3g.sh +++ b/pkgs/spt3g.sh @@ -61,7 +61,7 @@ tar xzf ${src} \ -DBoost_ARCHITECTURE=-x64 \ -DFLAC_LIBRARIES="${flaclib}" \ -DFLAC_INCLUDE_DIR="${flacinc}" \ - -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \ + -DPython_EXECUTABLE:FILEPATH=$(which python3) \ .. >> ${log} 2>&1 \ && make -j @MAKEJ@ >> ${log} 2>&1 \ && ln -s @AUX_PREFIX@/spt3g/build/bin/* @AUX_PREFIX@/bin/ \ diff --git a/pkgs/suitesparse.sh b/pkgs/suitesparse.sh index 3067385..dae12ad 100644 --- a/pkgs/suitesparse.sh +++ b/pkgs/suitesparse.sh @@ -4,7 +4,7 @@ pkg="suitesparse" pkgopts=$@ cleanup="" -version=5.4.0 +version=5.6.0 pfile=SuiteSparse-${version}.tar.gz src=$(eval "@TOP_DIR@/tools/fetch_check.sh" http://faculty.cse.tamu.edu/davis/SuiteSparse/${pfile} ${pfile}) diff --git a/pkgs/toast.sh b/pkgs/toast.sh index 17bdc0f..0acdb9f 100644 --- a/pkgs/toast.sh +++ b/pkgs/toast.sh @@ -19,14 +19,16 @@ log="../../log_${pkg}" echo "Building ${pkg}..." >&2 # Parse options for dependency install prefix -fftw="@AUX_PREFIX@" -ssparse="@AUX_PREFIX@" -blas="@AUX_PREFIX@/lib/libopenblas.a" -lapack="@AUX_PREFIX@/lib/libopenblas.a" +fftw="" +ssparse="" +blas="" +lapack="" + if [ "x@MKL@" != "x" ]; then blas="@MKL@/lib/intel64/libmkl_rt.so" lapack="@MKL@/lib/intel64/libmkl_rt.so" fi + for opt in $pkgopts; do chkfftw=$(echo $opt | sed -e "s/fftw=\(.*\)/\1/") if [ "x$chkfftw" != "x$opt" ]; then @@ -50,13 +52,21 @@ fftw_root="" if [ "x$fftw" != "x" ]; then fftw_root="-DFFTW_ROOT=$fftw" fi + suitesparse="" if [ "x$ssparse" != "x" ]; then suitesparse="-DSUITESPARSE_INCLUDE_DIR_HINTS=$ssparse/include -DSUITESPARSE_LIBRARY_DIR_HINTS=$ssparse/lib" fi -blaslib="-DBLAS_LIBRARIES=${blas}" -lapacklib="-DLAPACK_LIBRARIES=${lapack}" +blaslib="" +if [ "x$blas" != "x" ]; then + blaslib="-DBLAS_LIBRARIES=${blas}" +fi + +lapacklib="" +if [ "x$lapack" != "x" ]; then + lapacklib="-DLAPACK_LIBRARIES=${lapack}" +fi rm -rf toast-${version} tar xzf ${src} \ diff --git a/templates/Dockerfile.in b/templates/Dockerfile.in deleted file mode 100644 index 98e2ed6..0000000 --- a/templates/Dockerfile.in +++ /dev/null @@ -1,83 +0,0 @@ -FROM ubuntu:19.04 - -MAINTAINER Theodore Kisner - -# Use bash - -SHELL ["/bin/bash", "-c"] - -# Install system dependencies. - -RUN apt-get update \ - && apt-get install -y \ - curl \ - procps \ - build-essential \ - gfortran \ - git \ - autoconf \ - automake \ - libtool \ - m4 \ - locales \ - libgl1-mesa-glx \ - xvfb \ - python3 \ - libpython3-dev \ - python3-pip \ - && rm -fr /var/lib/apt/lists/* - -# Set up locales, to workaround a pip bug - -RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - -# We install everything directly into /usr so that we do -# not need to modify the default library and executable -# search paths. Shifter will manipulate LD_LIBRARY_PATH, -# so it is important not to use that variable. - -# Create a home directory for builds - -RUN mkdir /home/cmbenv -RUN mkdir /home/cmbenv/.astropy -WORKDIR /home/cmbenv -ENV HOME /home/cmbenv - -# Copy files into place - -RUN mkdir -p /home/cmbenv/Dockerfile_@CONFIG@_pkgs -COPY Dockerfile_@CONFIG@_pkgs /home/cmbenv/Dockerfile_@CONFIG@_pkgs/ - -RUN ls -al - -# Make site-packages directory - -RUN mkdir -p /usr/lib/python@PYVERSION@/site-packages - -# Remove conflicting python stuff - -ENV PYTHONPATH "" -ENV PYTHONSTARTUP "" -ENV PYTHONNOUSERSITE "1" -ENV PYTHONUSERBASE "/tmp" - -# Install packages - -@PACKAGES@ - -# Precompile all python modules - -RUN python3 -m compileall -q "/usr/lib/python@PYVERSION@/site-packages" - -# Imports to create config files - -RUN python3 -c "import astropy" -RUN python3 -c "import matplotlib.font_manager as fm; f = fm.FontManager" - -# Set the entrypoint and default command - -ENTRYPOINT ["/bin/bash", "-c"] -CMD ["/bin/bash"] diff --git a/templates/docker-alpine.in b/templates/docker-alpine.in new file mode 100644 index 0000000..f84dd49 --- /dev/null +++ b/templates/docker-alpine.in @@ -0,0 +1,105 @@ +FROM @DOCKER_BASE@ as builder + +MAINTAINER Theodore Kisner + +# Use bash + +SHELL ["/bin/sh", "-c"] + +# Install system dependencies. + +RUN apk add --no-cache --update-cache @DOCKER_SYS_PKGS@ \ + bash \ + build-base \ + libgomp \ + libgcc \ + libquadmath \ + linux-headers \ + openssl-dev \ + gfortran \ + libgfortran \ + curl \ + perl \ + git \ + less \ + make \ + pkgconf \ + libpng-dev \ + freetype-dev + +RUN if [ ! -e /usr/include/xlocale.h ]; then ln -s /usr/include/locale.h /usr/include/xlocale.h; fi + +RUN if [ ! -e /usr/bin/llvm-config ]; then ln -s /usr/bin/llvm*-config /usr/bin/llvm-config; fi + +# We install everything directly into /usr so that we do +# not need to modify the default library and executable +# search paths. Shifter will manipulate LD_LIBRARY_PATH, +# so it is important not to use that variable. + +# Create a home directory for builds + +RUN mkdir -p /home/cmbenv +WORKDIR /home/cmbenv +ENV HOME /home/cmbenv + +# Copy files into place + +RUN mkdir -p /home/cmbenv/Dockerfile_@CONFIG@_pkgs +COPY Dockerfile_@CONFIG@_pkgs /home/cmbenv/Dockerfile_@CONFIG@_pkgs/ + +RUN mkdir -p /home/cmbenv/tools +COPY tools /home/cmbenv/tools/ + +RUN mkdir -p /home/cmbenv/pkgs +COPY pkgs /home/cmbenv/pkgs/ + +RUN mkdir -p /home/cmbenv/templates +COPY templates /home/cmbenv/templates/ + +RUN mkdir -p /home/cmbenv/configs +COPY configs /home/cmbenv/configs/ + +COPY cmbenv /home/cmbenv/ + +# Install packages + +@PACKAGES@ + +# Precompile all python modules + +RUN python3 -m compileall -q "@AUX_PREFIX@/lib/python@PYVERSION@/site-packages" + +# ====================================== + +FROM @DOCKER_BASE@ + +# Copy our installed software binaries and libraries + +COPY --from=builder /usr /usr/ + +# Home directory + +RUN mkdir -p /home/cmbenv +RUN mkdir -p /home/cmbenv/.astropy +WORKDIR /home/cmbenv +ENV HOME /home/cmbenv + +# Imports to create config files + +RUN python3 -c "import astropy" +RUN python3 -c "import matplotlib.font_manager as fm; f = fm.FontManager" + +# Remove conflicting python stuff + +ENV PYTHONPATH "" +ENV PYTHONSTARTUP "" +ENV PYTHONNOUSERSITE "1" +ENV PYTHONUSERBASE "/tmp" + +# Set the entrypoint and default command + +COPY tools/docker-entrypoint.sh /usr/bin/ + +ENTRYPOINT ["docker-entrypoint.sh"] + +CMD ["/bin/bash"] diff --git a/templates/docker-debian.in b/templates/docker-debian.in new file mode 100644 index 0000000..f17b199 --- /dev/null +++ b/templates/docker-debian.in @@ -0,0 +1,103 @@ +FROM @DOCKER_BASE@ as builder + +MAINTAINER Theodore Kisner + +# Use bash + +SHELL ["/bin/bash", "-c"] + +# Install system dependencies. + +ENV TERM linux + +RUN apt-get -y update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y @DOCKER_SYS_PKGS@ \ + curl \ + procps \ + build-essential \ + gfortran \ + git \ + pkg-config \ + lsb-release \ + libssl-dev \ + libgl1-mesa-glx \ + xvfb \ + && rm -fr /var/lib/apt/lists/* + +# Set up locales + +ENV LANG C.UTF-8 + +RUN if [ ! -e /usr/include/xlocale.h ]; then ln -s /usr/include/locale.h /usr/include/xlocale.h; fi + +# Create a home directory for builds + +RUN mkdir -p /home/cmbenv +WORKDIR /home/cmbenv +ENV HOME /home/cmbenv + +# Copy files into place + +RUN mkdir -p /home/cmbenv/Dockerfile_@CONFIG@_pkgs +COPY Dockerfile_@CONFIG@_pkgs /home/cmbenv/Dockerfile_@CONFIG@_pkgs/ + +RUN mkdir -p /home/cmbenv/tools +COPY tools /home/cmbenv/tools/ + +RUN mkdir -p /home/cmbenv/pkgs +COPY pkgs /home/cmbenv/pkgs/ + +RUN mkdir -p /home/cmbenv/templates +COPY templates /home/cmbenv/templates/ + +RUN mkdir -p /home/cmbenv/configs +COPY configs /home/cmbenv/configs/ + +COPY cmbenv /home/cmbenv/ + +# Install packages + +@PACKAGES@ + +# Precompile all python modules + +RUN python3 -m compileall -q "@AUX_PREFIX@/lib/python@PYVERSION@/site-packages" + +# ====================================== + +FROM @DOCKER_BASE@ + +# Copy our installed software binaries and libraries + +COPY --from=builder /usr /usr/ +COPY --from=builder /lib /lib/ +COPY --from=builder /bin /bin/ +COPY --from=builder /sbin /sbin/ +COPY --from=builder /etc /etc/ + +# Home directory + +RUN mkdir -p /home/cmbenv +RUN mkdir -p /home/cmbenv/.astropy +WORKDIR /home/cmbenv +ENV HOME /home/cmbenv + +# Imports to create config files + +RUN python3 -c "import astropy" +RUN python3 -c "import matplotlib.font_manager as fm; f = fm.FontManager" + +# Remove conflicting python stuff + +ENV PYTHONPATH "" +ENV PYTHONSTARTUP "" +ENV PYTHONNOUSERSITE "1" +ENV PYTHONUSERBASE "/tmp" + +# Set the entrypoint and default command + +COPY tools/docker-entrypoint.sh /usr/bin/ + +ENTRYPOINT ["docker-entrypoint.sh"] + +CMD ["/bin/bash"] diff --git a/templates/Dockerfile-intel.in b/templates/docker-intel.in similarity index 100% rename from templates/Dockerfile-intel.in rename to templates/docker-intel.in diff --git a/templates/docker-ubuntu.in b/templates/docker-ubuntu.in new file mode 100644 index 0000000..6d354c9 --- /dev/null +++ b/templates/docker-ubuntu.in @@ -0,0 +1,114 @@ +FROM @DOCKER_BASE@ as builder + +MAINTAINER Theodore Kisner + +# Use bash + +SHELL ["/bin/bash", "-c"] + +# Install system dependencies. Optionally install an alternate +# compiler toolchain. + +ENV TERM linux + +RUN ln -fs /usr/share/zoneinfo/UTC /etc/localtime + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common \ + && add-apt-repository -y "ppa:ubuntu-toolchain-r/test" \ + && apt-get -y update \ + && apt-get install -y @DOCKER_SYS_PKGS@ \ + curl \ + procps \ + build-essential \ + gfortran \ + git \ + pkg-config \ + lsb-release \ + libssl-dev \ + libgl1-mesa-glx \ + xvfb \ + && rm -fr /var/lib/apt/lists/* + +# Set up locales + +ENV LANG C.UTF-8 + +RUN if [ ! -e /usr/include/xlocale.h ]; then ln -s /usr/include/locale.h /usr/include/xlocale.h; fi + +# Create a home directory for builds + +RUN mkdir -p /home/cmbenv +RUN mkdir -p /home/cmbenv/.astropy +WORKDIR /home/cmbenv +ENV HOME /home/cmbenv + +# Copy files into place + +RUN mkdir -p /home/cmbenv/Dockerfile_@CONFIG@_pkgs +COPY Dockerfile_@CONFIG@_pkgs /home/cmbenv/Dockerfile_@CONFIG@_pkgs/ + +RUN mkdir -p /home/cmbenv/tools +COPY tools /home/cmbenv/tools/ + +RUN mkdir -p /home/cmbenv/pkgs +COPY pkgs /home/cmbenv/pkgs/ + +RUN mkdir -p /home/cmbenv/templates +COPY templates /home/cmbenv/templates/ + +RUN mkdir -p /home/cmbenv/configs +COPY configs /home/cmbenv/configs/ + +COPY cmbenv /home/cmbenv/ + +# Make site-packages directory + +RUN mkdir -p "@AUX_PREFIX@/lib/python@PYVERSION@/site-packages" + +# Install packages + +@PACKAGES@ + +# Precompile all python modules + +RUN python3 -m compileall -q "@AUX_PREFIX@/lib/python@PYVERSION@/site-packages" + +# ====================================== + +FROM @DOCKER_BASE@ + +# Copy our installed software binaries and libraries + +COPY --from=builder /usr /usr/ +COPY --from=builder /lib /lib/ +COPY --from=builder /bin /bin/ +COPY --from=builder /sbin /sbin/ +COPY --from=builder /etc /etc/ + +# Home directory + +RUN mkdir -p /home/cmbenv +RUN mkdir -p /home/cmbenv/.astropy +WORKDIR /home/cmbenv +ENV HOME /home/cmbenv + +# Imports to create config files + +RUN python3 -c "import astropy" +RUN python3 -c "import matplotlib.font_manager as fm; f = fm.FontManager" + +# Remove conflicting python stuff + +ENV PYTHONPATH "" +ENV PYTHONSTARTUP "" +ENV PYTHONNOUSERSITE "1" +ENV PYTHONUSERBASE "/tmp" + +# Set the entrypoint and default command + +COPY tools/docker-entrypoint.sh /usr/bin/ + +ENTRYPOINT ["docker-entrypoint.sh"] + +CMD ["/bin/bash"] diff --git a/templates/install.in b/templates/install-linux.in similarity index 100% rename from templates/install.in rename to templates/install-linux.in diff --git a/templates/packages.txt b/templates/packages.txt index 78297aa..ba5d84a 100644 --- a/templates/packages.txt +++ b/templates/packages.txt @@ -4,14 +4,6 @@ # These extra options are documented in the shell snippet for each package # in the top-level pkgs directory. # -# Python 3. Uncomment one of the following lines. You should only use the -# "default" option for Docker images. All other configs should use either -# a virtualenv or a conda environment. This will also install a set of -# common base packages. -python:virtualenv -#python:conda -#python:default -# # Autotools. If you already have recent versions installed with your OS # package manager, then comment these out. m4 @@ -19,10 +11,25 @@ libtool autoconf automake # +# CMake. +cmake +# # Install zlib. This is needed on some systems where the OS-provided zlib is # very ancient. zlib # +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +python:virtualenv +#python:conda +#python:default +# # Install MPICH. This step is optional and only needed in 2 cases: # 1. You don't have MPI available through system packages that is # compatible with the serial compilers you are using. @@ -36,10 +43,6 @@ mpi4py # Install CFITSIO. cfitsio # -# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers -# or some other vendor BLAS / LAPACK solution. -openblas -# # Install FFTW3. fftw # diff --git a/tools/docker-entrypoint.sh b/tools/docker-entrypoint.sh new file mode 100755 index 0000000..8578277 --- /dev/null +++ b/tools/docker-entrypoint.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -Eeo pipefail + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +# Loads various settings that are used elsewhere in the script +# This should be called before any other functions +docker_setup_env() { + source cmbenv +} + +_main() { + docker_setup_env + exec "$@" +} + +if ! _is_sourced; then + _main "$@" +fi diff --git a/tools/gen_script.sh b/tools/gen_script.sh index fcab684..2a70517 100755 --- a/tools/gen_script.sh +++ b/tools/gen_script.sh @@ -24,7 +24,6 @@ moddir=${10} # polluting the image. docker=${11} - # Top level cmbenv git checkout pushd $(dirname $0) > /dev/null topdir=$(dirname $(pwd)) @@ -87,8 +86,13 @@ python_prefix="${prefix}/cmbenv_python" module_dir="${moddir}/cmbenv" if [ "x${docker}" = "xyes" ]; then - compiled_prefix="/usr" - python_prefix="/usr" + if [ "x${prefix}" = "x" ]; then + compiled_prefix="/usr" + python_prefix="/usr" + else + compiled_prefix="${prefix}" + python_prefix="${prefix}" + fi fi confsub="${confsub} -e 's#@SRCDIR@#${topdir}#g'" @@ -97,8 +101,14 @@ confsub="${confsub} -e 's#@AUX_PREFIX@#${compiled_prefix}#g'" confsub="${confsub} -e 's#@PYTHON_PREFIX@#${python_prefix}#g'" confsub="${confsub} -e 's#@VERSION@#${version}#g'" confsub="${confsub} -e 's#@MODULE_DIR@#${module_dir}#g'" -confsub="${confsub} -e 's#@TOP_DIR@#${topdir}#g'" +# If we are using docker, then the package scripts need to be able to find +# the tools that we have copied into the container. +if [ "x${docker}" = "xyes" ]; then + confsub="${confsub} -e 's#@TOP_DIR@#/home/cmbenv#g'" +else + confsub="${confsub} -e 's#@TOP_DIR@#${topdir}#g'" +fi # Process each selected package for this config. Copy each package file into # the output location while substituting config variables. Also build up the