From f4ea699da4eea69776abca4d5a3f9379af46bd6d Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Thu, 22 Apr 2021 12:28:02 -0700 Subject: [PATCH 1/8] Update package versions. Add Pixell version. --- pkgs/cmake.sh | 2 +- pkgs/fftw.sh | 2 +- pkgs/h5py.sh | 4 ++-- pkgs/healpy.sh | 4 ++-- pkgs/openblas.sh | 2 +- pkgs/pixell.sh | 31 +++++++++++++++++++++++++++++++ pkgs/pysm.sh | 4 ++-- pkgs/spt3g.sh | 2 +- pkgs/suitesparse.sh | 2 +- pkgs/toast.sh | 2 +- 10 files changed, 43 insertions(+), 12 deletions(-) create mode 100644 pkgs/pixell.sh diff --git a/pkgs/cmake.sh b/pkgs/cmake.sh index 9cd6729..b1e988e 100644 --- a/pkgs/cmake.sh +++ b/pkgs/cmake.sh @@ -4,7 +4,7 @@ pkg="cmake" pkgopts=$@ cleanup="" -version=3.16.4 +version=3.20.1 pfile=cmake-${version}.tar.gz src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://github.com/Kitware/CMake/releases/download/v${version}/${pfile} ${pfile}) diff --git a/pkgs/fftw.sh b/pkgs/fftw.sh index 1942eb5..346cd8a 100644 --- a/pkgs/fftw.sh +++ b/pkgs/fftw.sh @@ -4,7 +4,7 @@ pkg="fftw" pkgopts=$@ cleanup="" -version=3.3.8 +version=3.3.9 pfile=fftw-${version}.tar.gz src=$(eval "@TOP_DIR@/tools/fetch_check.sh" http://www.fftw.org/${pfile} ${pfile}) diff --git a/pkgs/h5py.sh b/pkgs/h5py.sh index 113663c..a7981af 100644 --- a/pkgs/h5py.sh +++ b/pkgs/h5py.sh @@ -15,10 +15,10 @@ fi # Note- the download URL includes a checksum and will need to # be updated when you change this version string. -version=2.10.0 +version=3.2.1 pfile=h5py-${version}.tar.gz -src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://files.pythonhosted.org/packages/5f/97/a58afbcf40e8abecededd9512978b4e4915374e5b80049af082f49cebe9a/${pfile} ${pfile}) +src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://files.pythonhosted.org/packages/ea/00/d0606cc0d6107a98f75b98367dc42917a67e3a7ec881636835f8e6987e6b/h5py-3.2.1.tar.gz ${pfile}) if [ "x${src}" = "x" ]; then echo "Failed to fetch ${pkg}" >&2 diff --git a/pkgs/healpy.sh b/pkgs/healpy.sh index 31303ce..a0317c8 100644 --- a/pkgs/healpy.sh +++ b/pkgs/healpy.sh @@ -4,9 +4,9 @@ pkg="healpy" pkgopts=$@ cleanup="" -version=1.13.0 +version=1.14.0 pfile=healpy-${version}.tar.gz -src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://files.pythonhosted.org/packages/26/74/0c8592686027a8196e275cb81999e8aae88d0416c223fa55a7f0cb5bdd26/${pfile} ${pfile}) +src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://files.pythonhosted.org/packages/52/bb/21e57f6b3a4c2a3bb59fb2a284fccf6ea15241a180e86ace1f9b891e251b/${pfile} ${pfile}) if [ "x${src}" = "x" ]; then echo "Failed to fetch ${pkg}" >&2 diff --git a/pkgs/openblas.sh b/pkgs/openblas.sh index 5caa882..8d1ff85 100644 --- a/pkgs/openblas.sh +++ b/pkgs/openblas.sh @@ -4,7 +4,7 @@ pkg="openblas" pkgopts=$@ cleanup="" -version=0.3.9 +version=0.3.13 pfile=OpenBLAS-${version}.tar.gz src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://github.com/xianyi/OpenBLAS/archive/v${version}.tar.gz ${pfile}) diff --git a/pkgs/pixell.sh b/pkgs/pixell.sh new file mode 100644 index 0000000..0adc2e5 --- /dev/null +++ b/pkgs/pixell.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +pkg="pixell" +version=0.12.0 +psrc=${pkg}-${version} +pfile=${psrc}.tar.gz + +fetched=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://github.com/simonsobs/pixell/archive/v${version}.tar.gz ${pfile}) + +if [ "x${fetched}" = "x" ]; then + echo "Failed to fetch ${pkg}" >&2 + exit 1 +fi + +log="../log_${pkg}" + +echo "Building ${pkg}..." >&2 + +rm -rf ${psrc} +tar xzf ${fetched} \ + && cd ${psrc} \ + && python3 setup.py build > ${log} 2>&1 \ + && python3 setup.py install --prefix "@PREFIX@" >> ${log} 2>&1 + +if [ $? -ne 0 ]; then + echo "Failed to build ${pkg}" >&2 + exit 1 +fi + +echo "Finished building ${pkg}" >&2 +exit 0 diff --git a/pkgs/pysm.sh b/pkgs/pysm.sh index 46332e7..7c134a1 100644 --- a/pkgs/pysm.sh +++ b/pkgs/pysm.sh @@ -4,9 +4,9 @@ pkg="pysm" pkgopts=$@ cleanup="" -version=3.2.0 +version=3.3.0 pfile=pysm3-${version}.tar.gz -src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://files.pythonhosted.org/packages/60/b3/daf8e62af528d8d749673069d9eee6006d6c9f9206531b80f3e6145b801d/pysm3-3.2.0.tar.gz ${pfile}) +src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://files.pythonhosted.org/packages/0a/bd/8a3d2306f3ab1ce06f3bf22139875ad01e7c9671286baebd016cd9662fa7/pysm3-3.3.0.tar.gz ${pfile}) if [ "x${src}" = "x" ]; then echo "Failed to fetch ${pkg}" >&2 diff --git a/pkgs/spt3g.sh b/pkgs/spt3g.sh index 9dec2a6..0e2a442 100644 --- a/pkgs/spt3g.sh +++ b/pkgs/spt3g.sh @@ -4,7 +4,7 @@ pkg="spt3g" pkgopts=$@ cleanup="" -version=ce2dd75675229475286d9867b70eac57bbcdd7f5 +version=c2c60207458869faacecd4157baba5d65d715760 pfile=spt3g_software-${version}.tar.gz src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://github.com/CMB-S4/spt3g_software/archive/${version}.tar.gz ${pfile}) diff --git a/pkgs/suitesparse.sh b/pkgs/suitesparse.sh index dc4e855..cdb0c73 100644 --- a/pkgs/suitesparse.sh +++ b/pkgs/suitesparse.sh @@ -4,7 +4,7 @@ pkg="suitesparse" pkgopts=$@ cleanup="" -version=5.7.2 +version=5.9.0 pfile=SuiteSparse-${version}.tar.gz src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://github.com/DrTimothyAldenDavis/SuiteSparse/archive/v${version}.tar.gz ${pfile}) diff --git a/pkgs/toast.sh b/pkgs/toast.sh index c3b9528..8d78e51 100644 --- a/pkgs/toast.sh +++ b/pkgs/toast.sh @@ -4,7 +4,7 @@ pkg="toast" pkgopts=$@ cleanup="" -version=2.3.8 +version=2.3.12 pfile=toast-${version}.tar.gz src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://github.com/hpc4cmb/toast/archive/${version}.tar.gz ${pfile}) From bd2f452b3ade72f2b473fef4c45e8dcacb016a15 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Thu, 13 May 2021 10:15:07 -0700 Subject: [PATCH 2/8] Clean up old configs. Add new conda / pip dependencies --- configs/cori-gcc | 4 +- configs/cori-gcc-gpu | 2 +- configs/cori-gcc.pkgs | 3 + configs/cori-intel | 4 +- configs/cori-intel.pkgs | 3 + configs/docker-py3.6-debian | 77 ------------- configs/docker-py3.6-debian.pkgs | 103 ----------------- configs/docker-py3.7-debian | 2 +- configs/docker-py3.7-debian.pkgs | 104 +++++++++++++++++- configs/docker-py3.8-debian | 2 +- configs/docker-py3.8-debian.pkgs | 2 +- configs/docker-toastdeps-py36 | 1 - configs/docker-toastdeps-py36.pkgs | 101 ----------------- configs/docker-toastdeps-py37.pkgs | 102 ++++++++++++++++- configs/docker-toastdeps-py38.pkgs | 2 +- configs/docker-ubuntu_20.04_gcc9_py38 | 2 +- configs/linux-conda | 2 +- configs/linux-conda-nomkl | 2 +- configs/linux-venv | 2 +- configs/{ => old}/docker-py3.5-alpine3.11 | 0 .../{ => old}/docker-py3.5-alpine3.11.pkgs | 0 configs/{ => old}/docker-py3.5-debian | 0 configs/{ => old}/docker-py3.5-debian.pkgs | 0 configs/{ => old}/docker-py3.6-alpine3.11 | 0 .../{ => old}/docker-py3.6-alpine3.11.pkgs | 0 configs/{ => old}/docker-py3.7-alpine3.11 | 0 .../{ => old}/docker-py3.7-alpine3.11.pkgs | 0 configs/{ => old}/docker-py3.8-alpine3.11 | 0 .../{ => old}/docker-py3.8-alpine3.11.pkgs | 0 configs/{ => old}/docker-toastdeps-py35 | 0 configs/{ => old}/docker-toastdeps-py35.pkgs | 0 configs/{ => old}/ubuntu-19.04-venv | 0 configs/{ => old}/ubuntu-19.04-venv.notes | 0 configs/{ => old}/ubuntu-19.04-venv.pkgs | 0 configs/{ => old}/ubuntu-19.10-venv | 0 configs/{ => old}/ubuntu-19.10-venv.notes | 0 configs/{ => old}/ubuntu-19.10-venv.pkgs | 0 configs/ubuntu-18.04-venv | 2 +- configs/ubuntu-20.04-venv | 2 +- pkgs/boost.sh | 9 +- pkgs/h5py.sh | 3 +- pkgs/libflac.sh | 2 +- pkgs/spt3g.sh | 14 ++- templates/packages.txt | 3 + 44 files changed, 245 insertions(+), 310 deletions(-) delete mode 100644 configs/docker-py3.6-debian delete mode 100644 configs/docker-py3.6-debian.pkgs mode change 120000 => 100644 configs/docker-py3.7-debian.pkgs delete mode 120000 configs/docker-toastdeps-py36 delete mode 100644 configs/docker-toastdeps-py36.pkgs mode change 120000 => 100644 configs/docker-toastdeps-py37.pkgs rename configs/{ => old}/docker-py3.5-alpine3.11 (100%) rename configs/{ => old}/docker-py3.5-alpine3.11.pkgs (100%) rename configs/{ => old}/docker-py3.5-debian (100%) rename configs/{ => old}/docker-py3.5-debian.pkgs (100%) rename configs/{ => old}/docker-py3.6-alpine3.11 (100%) rename configs/{ => old}/docker-py3.6-alpine3.11.pkgs (100%) rename configs/{ => old}/docker-py3.7-alpine3.11 (100%) rename configs/{ => old}/docker-py3.7-alpine3.11.pkgs (100%) rename configs/{ => old}/docker-py3.8-alpine3.11 (100%) rename configs/{ => old}/docker-py3.8-alpine3.11.pkgs (100%) rename configs/{ => old}/docker-toastdeps-py35 (100%) rename configs/{ => old}/docker-toastdeps-py35.pkgs (100%) rename configs/{ => old}/ubuntu-19.04-venv (100%) rename configs/{ => old}/ubuntu-19.04-venv.notes (100%) rename configs/{ => old}/ubuntu-19.04-venv.pkgs (100%) rename configs/{ => old}/ubuntu-19.10-venv (100%) rename configs/{ => old}/ubuntu-19.10-venv.notes (100%) rename configs/{ => old}/ubuntu-19.10-venv.pkgs (100%) diff --git a/configs/cori-gcc b/configs/cori-gcc index ac36e60..d9ed6a5 100644 --- a/configs/cori-gcc +++ b/configs/cori-gcc @@ -12,7 +12,7 @@ PYVERSION = 3.7 PIP_PKGS = # Packages to install with conda -CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow +CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets>=5.0 # Serial compilers @@ -67,5 +67,5 @@ BOOSTCHAIN = gcc # Group and permissions to set -CHGRP = hpcosmo +CHGRP = cmb CHMOD = a+rX,g-w,o-w diff --git a/configs/cori-gcc-gpu b/configs/cori-gcc-gpu index 88f0f8e..f936837 100644 --- a/configs/cori-gcc-gpu +++ b/configs/cori-gcc-gpu @@ -12,7 +12,7 @@ PYVERSION = 3.7 PIP_PKGS = # Packages to install with conda -CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow +CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets>=5.0 # Serial compilers diff --git a/configs/cori-gcc.pkgs b/configs/cori-gcc.pkgs index a39f760..cd0597a 100644 --- a/configs/cori-gcc.pkgs +++ b/configs/cori-gcc.pkgs @@ -62,6 +62,9 @@ h5py # Install healpy healpy # +# Install Pixell +pixell +# # Install quaternionarray qarray # diff --git a/configs/cori-intel b/configs/cori-intel index 5b4ec92..f1f77dc 100644 --- a/configs/cori-intel +++ b/configs/cori-intel @@ -12,7 +12,7 @@ PYVERSION = 3.7 PIP_PKGS = # Packages to install with conda -CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow +CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets>=5.0 # Serial compilers @@ -68,5 +68,5 @@ BOOSTCHAIN = intel-linux # Group and permissions to set -CHGRP = hpcosmo +CHGRP = cmb CHMOD = a+rX,g-w,o-w diff --git a/configs/cori-intel.pkgs b/configs/cori-intel.pkgs index 004b44e..ad74cfc 100644 --- a/configs/cori-intel.pkgs +++ b/configs/cori-intel.pkgs @@ -62,6 +62,9 @@ h5py # Install healpy healpy # +# Install Pixell +pixell +# # Install quaternionarray qarray # diff --git a/configs/docker-py3.6-debian b/configs/docker-py3.6-debian deleted file mode 100644 index 3e6fbbd..0000000 --- a/configs/docker-py3.6-debian +++ /dev/null @@ -1,77 +0,0 @@ - -# OS type (linux OR osx) -OSTYPE = linux - -# Specify the template to use -TEMPLATE = docker-debian - -# Python version -PYVERSION = 3.6 - -# Specify the docker base image -DOCKER_BASE = python:3.6-slim-buster - -# Specify additional packages to install for docker builds -DOCKER_SYS_PKGS = m4 libtool autoconf automake llvm libsuitesparse-dev libfftw3-dev libcfitsio-dev libflac-dev zlib1g-dev libbz2-dev libopenblas-dev liblapack-dev libhdf5-dev - -# Packages to install with pip -PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 tbb cmake sphinx sphinx-rtd-theme pyfftw - -# Packages to install with conda -CONDA_PKGS = - -# Serial compilers - -CC = gcc -CXX = g++ -FC = gfortran - -# Compiler to use for packages needed on the build machine - -BUILD_CC = gcc -BUILD_CXX = g++ - -# MPI compilers - -MPICC = mpicc -MPICXX = mpicxx -MPIFC = mpifort -MPI_CPPFLAGS = @AUX_PREFIX@/include -MPI_LDFLAGS = @AUX_PREFIX@/lib -MPI_CXXLIB = mpichcxx -MPI_LIB = mpich -MPI_EXTRA_COMP = -MPI_EXTRA_LINK = - -# Compile flags - -CFLAGS = -O2 -g -fPIC -pthread -CXXFLAGS = -O2 -g -fPIC -pthread -std=c++11 -FCFLAGS = -O2 -g -fPIC -pthread - -OPENMP_CFLAGS = -fopenmp -OPENMP_CXXFLAGS = -fopenmp -LDFLAGS = -lpthread -fopenmp - -# Are we doing a cross-compile? -CROSS = - -# Parallel builds -MAKEJ = 1 - -# Define MKL location -MKL = - -# For BLAS/LAPACK, we use openblas - -BLAS = -lopenblas -fopenmp -lm -LAPACK = - -# Boost toolchain name - -BOOSTCHAIN = gcc - -# Group and permissions to set - -CHGRP = -CHMOD = diff --git a/configs/docker-py3.6-debian.pkgs b/configs/docker-py3.6-debian.pkgs deleted file mode 100644 index c719166..0000000 --- a/configs/docker-py3.6-debian.pkgs +++ /dev/null @@ -1,103 +0,0 @@ -# This is the full set of packages that can be installed. This can be -# used as a starting point for the package list for a particular config. -# Some packages accept additional options which can be added after a colon ":". -# These extra options are documented in the shell snippet for each package -# in the top-level pkgs directory. -# -# Autotools. If you already have recent versions installed with your OS -# package manager, then comment these out. -#m4 -#libtool -#autoconf -#automake -# -# CMake. -#cmake -# -# Install zlib. This is needed on some systems where the OS-provided zlib is -# very ancient. -#zlib -# -# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers -# or some other vendor BLAS / LAPACK solution. -#openblas -# -# Python 3. Uncomment one of the following lines. You should only use the -# "default" option for Docker images. All other configs should use either -# a virtualenv or a conda environment. This will also install a set of -# common base packages. -#python:virtualenv -#python:conda -python:default -# -# Install MPICH. This step is optional and only needed in 2 cases: -# 1. You don't have MPI available through system packages that is -# compatible with the serial compilers you are using. -# 2. You are building a docker container for use at NERSC with shifter -# and need an MPICH compatible with the Cray one. -mpich -# -# Install mpi4py. This will use the MPI compilers specified in your config. -mpi4py -# -# Install CFITSIO. -#cfitsio -# -# Install FFTW3. -#fftw -# -# Install libbz2. This is required to enable some features in the boost -# build which are in turn needed by spt3g. -#bzip2 -# -# Install Boost. -#boost -# -# Install HDF5 (serial version). -#hdf5 -# -# Install h5py (using our HDF5). -h5py:pkg-config -# -# Install healpy -healpy -# -# Install quaternionarray -qarray -# -# Install fitsio -fitsio -# -# Install aatm. Needed for some features of atmosphere simulations. -aatm -# -# Install suitesparse -#suitesparse -# -# Install libconviqt. 4-PI a_lm space beam convolution. -conviqt -# -# Install libsharp. Distributed harmonic transforms. -libsharp -# -# Install libmadam. Destriping map-maker. -madam:--with-cfitsio=/usr --with-fftw=/usr -# -# Install FLAC, needed by spt3g -#libflac -# -# Install SPT3G. Frame based data format. -#spt3g:boost=/usr/local flaclib=/usr/lib/x86_64-linux-gnu/libFLAC.so -# -# Install TIDAS. HDF5-based timestream data format. -tidas -# -# Install PySM. Python Sky Model. -pysm -# -# Install PyMPIT for environment testing. Useful for testing working -# mpi4py stack. -pympit -# -# Install TOAST. -toast diff --git a/configs/docker-py3.7-debian b/configs/docker-py3.7-debian index a38da42..706fb44 100644 --- a/configs/docker-py3.7-debian +++ b/configs/docker-py3.7-debian @@ -15,7 +15,7 @@ DOCKER_BASE = python:3.7-slim-buster DOCKER_SYS_PKGS = m4 libtool autoconf automake llvm libsuitesparse-dev libfftw3-dev libcfitsio-dev libflac-dev zlib1g-dev libbz2-dev libopenblas-dev liblapack-dev libhdf5-dev # Packages to install with pip -PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 tbb cmake sphinx sphinx-rtd-theme pyfftw +PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 tbb cmake sphinx sphinx-rtd-theme pyfftw pshmem tomlkit traitlets>=5.0 # Packages to install with conda CONDA_PKGS = diff --git a/configs/docker-py3.7-debian.pkgs b/configs/docker-py3.7-debian.pkgs deleted file mode 120000 index 1354c8d..0000000 --- a/configs/docker-py3.7-debian.pkgs +++ /dev/null @@ -1 +0,0 @@ -docker-py3.6-debian.pkgs \ No newline at end of file diff --git a/configs/docker-py3.7-debian.pkgs b/configs/docker-py3.7-debian.pkgs new file mode 100644 index 0000000..c719166 --- /dev/null +++ b/configs/docker-py3.7-debian.pkgs @@ -0,0 +1,103 @@ +# This is the full set of packages that can be installed. This can be +# used as a starting point for the package list for a particular config. +# Some packages accept additional options which can be added after a colon ":". +# These extra options are documented in the shell snippet for each package +# in the top-level pkgs directory. +# +# Autotools. If you already have recent versions installed with your OS +# package manager, then comment these out. +#m4 +#libtool +#autoconf +#automake +# +# CMake. +#cmake +# +# Install zlib. This is needed on some systems where the OS-provided zlib is +# very ancient. +#zlib +# +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +#python:conda +python:default +# +# Install MPICH. This step is optional and only needed in 2 cases: +# 1. You don't have MPI available through system packages that is +# compatible with the serial compilers you are using. +# 2. You are building a docker container for use at NERSC with shifter +# and need an MPICH compatible with the Cray one. +mpich +# +# Install mpi4py. This will use the MPI compilers specified in your config. +mpi4py +# +# Install CFITSIO. +#cfitsio +# +# Install FFTW3. +#fftw +# +# Install libbz2. This is required to enable some features in the boost +# build which are in turn needed by spt3g. +#bzip2 +# +# Install Boost. +#boost +# +# Install HDF5 (serial version). +#hdf5 +# +# Install h5py (using our HDF5). +h5py:pkg-config +# +# Install healpy +healpy +# +# Install quaternionarray +qarray +# +# Install fitsio +fitsio +# +# Install aatm. Needed for some features of atmosphere simulations. +aatm +# +# Install suitesparse +#suitesparse +# +# Install libconviqt. 4-PI a_lm space beam convolution. +conviqt +# +# Install libsharp. Distributed harmonic transforms. +libsharp +# +# Install libmadam. Destriping map-maker. +madam:--with-cfitsio=/usr --with-fftw=/usr +# +# Install FLAC, needed by spt3g +#libflac +# +# Install SPT3G. Frame based data format. +#spt3g:boost=/usr/local flaclib=/usr/lib/x86_64-linux-gnu/libFLAC.so +# +# Install TIDAS. HDF5-based timestream data format. +tidas +# +# Install PySM. Python Sky Model. +pysm +# +# Install PyMPIT for environment testing. Useful for testing working +# mpi4py stack. +pympit +# +# Install TOAST. +toast diff --git a/configs/docker-py3.8-debian b/configs/docker-py3.8-debian index 7f37def..68c4c7f 100644 --- a/configs/docker-py3.8-debian +++ b/configs/docker-py3.8-debian @@ -15,7 +15,7 @@ DOCKER_BASE = python:3.8-slim-buster DOCKER_SYS_PKGS = m4 libtool autoconf automake llvm libsuitesparse-dev libfftw3-dev libcfitsio-dev libflac-dev zlib1g-dev libbz2-dev libopenblas-dev liblapack-dev libhdf5-dev # Packages to install with pip -PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 tbb cmake sphinx sphinx-rtd-theme pyfftw +PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 tbb cmake sphinx sphinx-rtd-theme pyfftw pshmem tomlkit traitlets>=5.0 # Packages to install with conda CONDA_PKGS = diff --git a/configs/docker-py3.8-debian.pkgs b/configs/docker-py3.8-debian.pkgs index 1354c8d..70e64f4 120000 --- a/configs/docker-py3.8-debian.pkgs +++ b/configs/docker-py3.8-debian.pkgs @@ -1 +1 @@ -docker-py3.6-debian.pkgs \ No newline at end of file +docker-py3.7-debian.pkgs \ No newline at end of file diff --git a/configs/docker-toastdeps-py36 b/configs/docker-toastdeps-py36 deleted file mode 120000 index 8b77647..0000000 --- a/configs/docker-toastdeps-py36 +++ /dev/null @@ -1 +0,0 @@ -docker-py3.6-debian \ No newline at end of file diff --git a/configs/docker-toastdeps-py36.pkgs b/configs/docker-toastdeps-py36.pkgs deleted file mode 100644 index 560687a..0000000 --- a/configs/docker-toastdeps-py36.pkgs +++ /dev/null @@ -1,101 +0,0 @@ -# This is the full set of packages that can be installed. This can be -# used as a starting point for the package list for a particular config. -# Some packages accept additional options which can be added after a colon ":". -# These extra options are documented in the shell snippet for each package -# in the top-level pkgs directory. -# -# Autotools. If you already have recent versions installed with your OS -# package manager, then comment these out. -#m4 -#libtool -#autoconf -#automake -# -# CMake. -#cmake -# -# Install zlib. This is needed on some systems where the OS-provided zlib is -# very ancient. -#zlib -# -# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers -# or some other vendor BLAS / LAPACK solution. -#openblas -# -# Python 3. Uncomment one of the following lines. You should only use the -# "default" option for Docker images. All other configs should use either -# a virtualenv or a conda environment. This will also install a set of -# common base packages. -#python:virtualenv -#python:conda -python:default -# -# Install MPICH. This step is optional and only needed in 2 cases: -# 1. You don't have MPI available through system packages that is -# compatible with the serial compilers you are using. -# 2. You are building a docker container for use at NERSC with shifter -# and need an MPICH compatible with the Cray one. -mpich -# -# Install mpi4py. This will use the MPI compilers specified in your config. -mpi4py -# -# Install CFITSIO. -#cfitsio -# -# Install FFTW3. -#fftw -# -# Install libbz2. This is required to enable some features in the boost -# build which are in turn needed by spt3g. -#bzip2 -# -# Install Boost. -#boost -# -# Install HDF5 (serial version). -#hdf5 -# -# Install h5py (using our HDF5). -h5py:pkg-config -# -# Install healpy -healpy -# -# Install quaternionarray -qarray -# -# Install fitsio -fitsio -# -# Install aatm. Needed for some features of atmosphere simulations. -aatm -# -# Install suitesparse -#suitesparse -# -# Install libconviqt. 4-PI a_lm space beam convolution. -conviqt -# -# Install libsharp. Distributed harmonic transforms. -libsharp -# -# Install libmadam. Destriping map-maker. -madam:--with-cfitsio=/usr --with-fftw=/usr -# -# Install FLAC, needed by spt3g -#libflac -# -# Install SPT3G. Frame based data format. -#spt3g:boost=/usr/local flaclib=/usr/lib/x86_64-linux-gnu/libFLAC.so -# -# Install TIDAS. HDF5-based timestream data format. -tidas -# -# Install PySM. Python Sky Model. -pysm -# -# Install PyMPIT for environment testing. Useful for testing working -# mpi4py stack. -pympit -# diff --git a/configs/docker-toastdeps-py37.pkgs b/configs/docker-toastdeps-py37.pkgs deleted file mode 120000 index 0a373e7..0000000 --- a/configs/docker-toastdeps-py37.pkgs +++ /dev/null @@ -1 +0,0 @@ -docker-toastdeps-py36.pkgs \ No newline at end of file diff --git a/configs/docker-toastdeps-py37.pkgs b/configs/docker-toastdeps-py37.pkgs new file mode 100644 index 0000000..560687a --- /dev/null +++ b/configs/docker-toastdeps-py37.pkgs @@ -0,0 +1,101 @@ +# This is the full set of packages that can be installed. This can be +# used as a starting point for the package list for a particular config. +# Some packages accept additional options which can be added after a colon ":". +# These extra options are documented in the shell snippet for each package +# in the top-level pkgs directory. +# +# Autotools. If you already have recent versions installed with your OS +# package manager, then comment these out. +#m4 +#libtool +#autoconf +#automake +# +# CMake. +#cmake +# +# Install zlib. This is needed on some systems where the OS-provided zlib is +# very ancient. +#zlib +# +# Install OpenBLAS. Comment out if you are using MKL with the Intel compilers +# or some other vendor BLAS / LAPACK solution. +#openblas +# +# Python 3. Uncomment one of the following lines. You should only use the +# "default" option for Docker images. All other configs should use either +# a virtualenv or a conda environment. This will also install a set of +# common base packages. +#python:virtualenv +#python:conda +python:default +# +# Install MPICH. This step is optional and only needed in 2 cases: +# 1. You don't have MPI available through system packages that is +# compatible with the serial compilers you are using. +# 2. You are building a docker container for use at NERSC with shifter +# and need an MPICH compatible with the Cray one. +mpich +# +# Install mpi4py. This will use the MPI compilers specified in your config. +mpi4py +# +# Install CFITSIO. +#cfitsio +# +# Install FFTW3. +#fftw +# +# Install libbz2. This is required to enable some features in the boost +# build which are in turn needed by spt3g. +#bzip2 +# +# Install Boost. +#boost +# +# Install HDF5 (serial version). +#hdf5 +# +# Install h5py (using our HDF5). +h5py:pkg-config +# +# Install healpy +healpy +# +# Install quaternionarray +qarray +# +# Install fitsio +fitsio +# +# Install aatm. Needed for some features of atmosphere simulations. +aatm +# +# Install suitesparse +#suitesparse +# +# Install libconviqt. 4-PI a_lm space beam convolution. +conviqt +# +# Install libsharp. Distributed harmonic transforms. +libsharp +# +# Install libmadam. Destriping map-maker. +madam:--with-cfitsio=/usr --with-fftw=/usr +# +# Install FLAC, needed by spt3g +#libflac +# +# Install SPT3G. Frame based data format. +#spt3g:boost=/usr/local flaclib=/usr/lib/x86_64-linux-gnu/libFLAC.so +# +# Install TIDAS. HDF5-based timestream data format. +tidas +# +# Install PySM. Python Sky Model. +pysm +# +# Install PyMPIT for environment testing. Useful for testing working +# mpi4py stack. +pympit +# diff --git a/configs/docker-toastdeps-py38.pkgs b/configs/docker-toastdeps-py38.pkgs index 0a373e7..c01b8fa 120000 --- a/configs/docker-toastdeps-py38.pkgs +++ b/configs/docker-toastdeps-py38.pkgs @@ -1 +1 @@ -docker-toastdeps-py36.pkgs \ No newline at end of file +docker-toastdeps-py37.pkgs \ No newline at end of file diff --git a/configs/docker-ubuntu_20.04_gcc9_py38 b/configs/docker-ubuntu_20.04_gcc9_py38 index 68cb2e4..bf7addc 100644 --- a/configs/docker-ubuntu_20.04_gcc9_py38 +++ b/configs/docker-ubuntu_20.04_gcc9_py38 @@ -12,7 +12,7 @@ DOCKER_BASE = ubuntu:20.04 DOCKER_SYS_PKGS = m4 libtool autoconf automake cmake libcfitsio-dev libfftw3-dev libopenblas-dev liblapack-dev libhdf5-dev libboost-all-dev libopenmpi-dev python3 python3-dev python3-setuptools python3-pkg-resources python3-pip python3-tk python3-numpy python3-scipy python3-matplotlib python3-yaml python3-astropy python3-six python3-psutil python3-ephem python3-pandas python3-memory-profiler python3-ipython cython3 python3-cycler python3-kiwisolver python3-dateutil python3-toml python3-numba black python3-jupyter-client python3-h5py python3-mpi4py libsuitesparse-dev # Packages to install with pip -PIP_PKGS = +PIP_PKGS = pshmem tomlkit traitlets>=5.0 # Packages to install with conda CONDA_PKGS = diff --git a/configs/linux-conda b/configs/linux-conda index b3262f0..6754b93 100644 --- a/configs/linux-conda +++ b/configs/linux-conda @@ -12,7 +12,7 @@ PYVERSION = 3.7 PIP_PKGS = # Packages to install with conda -CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow +CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets>=5.0 # Serial compilers diff --git a/configs/linux-conda-nomkl b/configs/linux-conda-nomkl index b3262f0..6754b93 100644 --- a/configs/linux-conda-nomkl +++ b/configs/linux-conda-nomkl @@ -12,7 +12,7 @@ PYVERSION = 3.7 PIP_PKGS = # Packages to install with conda -CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow +CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets>=5.0 # Serial compilers diff --git a/configs/linux-venv b/configs/linux-venv index 8f63a2e..24e82f8 100644 --- a/configs/linux-venv +++ b/configs/linux-venv @@ -9,7 +9,7 @@ TEMPLATE = linux PYVERSION = auto # Packages to install with pip -PIP_PKGS = future nose Cython numpy scipy matplotlib pyyaml astropy six psutil ephem pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow +PIP_PKGS = future nose Cython numpy scipy matplotlib pyyaml astropy six psutil ephem pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets>=5.0 # Packages to install with conda CONDA_PKGS = diff --git a/configs/docker-py3.5-alpine3.11 b/configs/old/docker-py3.5-alpine3.11 similarity index 100% rename from configs/docker-py3.5-alpine3.11 rename to configs/old/docker-py3.5-alpine3.11 diff --git a/configs/docker-py3.5-alpine3.11.pkgs b/configs/old/docker-py3.5-alpine3.11.pkgs similarity index 100% rename from configs/docker-py3.5-alpine3.11.pkgs rename to configs/old/docker-py3.5-alpine3.11.pkgs diff --git a/configs/docker-py3.5-debian b/configs/old/docker-py3.5-debian similarity index 100% rename from configs/docker-py3.5-debian rename to configs/old/docker-py3.5-debian diff --git a/configs/docker-py3.5-debian.pkgs b/configs/old/docker-py3.5-debian.pkgs similarity index 100% rename from configs/docker-py3.5-debian.pkgs rename to configs/old/docker-py3.5-debian.pkgs diff --git a/configs/docker-py3.6-alpine3.11 b/configs/old/docker-py3.6-alpine3.11 similarity index 100% rename from configs/docker-py3.6-alpine3.11 rename to configs/old/docker-py3.6-alpine3.11 diff --git a/configs/docker-py3.6-alpine3.11.pkgs b/configs/old/docker-py3.6-alpine3.11.pkgs similarity index 100% rename from configs/docker-py3.6-alpine3.11.pkgs rename to configs/old/docker-py3.6-alpine3.11.pkgs diff --git a/configs/docker-py3.7-alpine3.11 b/configs/old/docker-py3.7-alpine3.11 similarity index 100% rename from configs/docker-py3.7-alpine3.11 rename to configs/old/docker-py3.7-alpine3.11 diff --git a/configs/docker-py3.7-alpine3.11.pkgs b/configs/old/docker-py3.7-alpine3.11.pkgs similarity index 100% rename from configs/docker-py3.7-alpine3.11.pkgs rename to configs/old/docker-py3.7-alpine3.11.pkgs diff --git a/configs/docker-py3.8-alpine3.11 b/configs/old/docker-py3.8-alpine3.11 similarity index 100% rename from configs/docker-py3.8-alpine3.11 rename to configs/old/docker-py3.8-alpine3.11 diff --git a/configs/docker-py3.8-alpine3.11.pkgs b/configs/old/docker-py3.8-alpine3.11.pkgs similarity index 100% rename from configs/docker-py3.8-alpine3.11.pkgs rename to configs/old/docker-py3.8-alpine3.11.pkgs diff --git a/configs/docker-toastdeps-py35 b/configs/old/docker-toastdeps-py35 similarity index 100% rename from configs/docker-toastdeps-py35 rename to configs/old/docker-toastdeps-py35 diff --git a/configs/docker-toastdeps-py35.pkgs b/configs/old/docker-toastdeps-py35.pkgs similarity index 100% rename from configs/docker-toastdeps-py35.pkgs rename to configs/old/docker-toastdeps-py35.pkgs diff --git a/configs/ubuntu-19.04-venv b/configs/old/ubuntu-19.04-venv similarity index 100% rename from configs/ubuntu-19.04-venv rename to configs/old/ubuntu-19.04-venv diff --git a/configs/ubuntu-19.04-venv.notes b/configs/old/ubuntu-19.04-venv.notes similarity index 100% rename from configs/ubuntu-19.04-venv.notes rename to configs/old/ubuntu-19.04-venv.notes diff --git a/configs/ubuntu-19.04-venv.pkgs b/configs/old/ubuntu-19.04-venv.pkgs similarity index 100% rename from configs/ubuntu-19.04-venv.pkgs rename to configs/old/ubuntu-19.04-venv.pkgs diff --git a/configs/ubuntu-19.10-venv b/configs/old/ubuntu-19.10-venv similarity index 100% rename from configs/ubuntu-19.10-venv rename to configs/old/ubuntu-19.10-venv diff --git a/configs/ubuntu-19.10-venv.notes b/configs/old/ubuntu-19.10-venv.notes similarity index 100% rename from configs/ubuntu-19.10-venv.notes rename to configs/old/ubuntu-19.10-venv.notes diff --git a/configs/ubuntu-19.10-venv.pkgs b/configs/old/ubuntu-19.10-venv.pkgs similarity index 100% rename from configs/ubuntu-19.10-venv.pkgs rename to configs/old/ubuntu-19.10-venv.pkgs diff --git a/configs/ubuntu-18.04-venv b/configs/ubuntu-18.04-venv index 446ce22..98b9d31 100644 --- a/configs/ubuntu-18.04-venv +++ b/configs/ubuntu-18.04-venv @@ -9,7 +9,7 @@ TEMPLATE = linux PYVERSION = auto # Packages to install with pip -PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pyyaml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 cmake pyfftw pillow +PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pyyaml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 cmake pyfftw pillow pshmem tomlkit traitlets>=5.0 # Packages to install with conda CONDA_PKGS = diff --git a/configs/ubuntu-20.04-venv b/configs/ubuntu-20.04-venv index c44269e..6552192 100644 --- a/configs/ubuntu-20.04-venv +++ b/configs/ubuntu-20.04-venv @@ -9,7 +9,7 @@ TEMPLATE = linux PYVERSION = auto # Packages to install with pip -PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pyyaml pkgconfig pytz psutil numpy scipy matplotlib pandas memory_profiler astropy ephem ipython numba cmake nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow +PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pyyaml pkgconfig pytz psutil numpy scipy matplotlib pandas memory_profiler astropy ephem ipython numba cmake nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets>=5.0 # Packages to install with conda CONDA_PKGS = diff --git a/pkgs/boost.sh b/pkgs/boost.sh index 1ea7275..2dd3a78 100644 --- a/pkgs/boost.sh +++ b/pkgs/boost.sh @@ -5,10 +5,10 @@ pkgopts=$@ cleanup="" # NOTE: change URL when changing version. -version=1_72_0 +version=1_76_0 pfile=boost_${version}.tar.bz2 -src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://dl.bintray.com/boostorg/release/1.72.0/source/${pfile} ${pfile}) +src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://dl.bintray.com/boostorg/release/1.76.0/source/${pfile} ${pfile}) if [ "x${src}" = "x" ]; then echo "Failed to fetch ${pkg}" >&2 @@ -20,11 +20,14 @@ log="../log_${pkg}" echo "Building ${pkg}..." >&2 +# NOTE: Due to perpetual flakiness of boost compilation with vendor compilers (e.g. Intel), +# we build boost with separate OS compilers (usually gcc or clang), as specified by the +# config BOOSTCHAIN variable. + rm -rf boost_${version} tar xjf ${src} \ && cd boost_${version} \ && cleanup="${cleanup} $(pwd)" \ - && echo "using mpi : @MPICXX@ : \"@MPI_CPPFLAGS@\" \"@MPI_LDFLAGS@\" \"@MPI_CXXLIB@\" \"@MPI_LIB@\" ;" >> tools/build/user-config.jam \ && echo "option jobs : @MAKEJ@ ;" >> tools/build/user-config.jam \ && BOOST_BUILD_USER_CONFIG=tools/build/user-config.jam \ BZIP2_INCLUDE="@AUX_PREFIX@/include" \ diff --git a/pkgs/h5py.sh b/pkgs/h5py.sh index a7981af..ec6e181 100644 --- a/pkgs/h5py.sh +++ b/pkgs/h5py.sh @@ -34,8 +34,7 @@ rm -rf h5py-${version} tar xzf ${src} \ && cd h5py-${version} \ && cleanup="${cleanup} $(pwd)" \ - && python3 setup.py configure ${hdf5pref} > ${log} \ - && CC="@CC@" python3 setup.py install --prefix "@AUX_PREFIX@" >> ${log} 2>&1 + && CC="@CC@" HDF5_DIR="${hdf5pref}" pip install -v --prefix "@AUX_PREFIX@" . >> ${log} 2>&1 if [ $? -ne 0 ]; then echo "Failed to build ${pkg}" >&2 diff --git a/pkgs/libflac.sh b/pkgs/libflac.sh index 6506412..7a4323e 100644 --- a/pkgs/libflac.sh +++ b/pkgs/libflac.sh @@ -22,7 +22,7 @@ rm -rf flac-${version} tar xJf ${src} \ && cd flac-${version} \ && cleanup="${cleanup} $(pwd)" \ - && CC="@CC@" CFLAGS="@CFLAGS@" \ + && CC="@BUILD_CC@" CFLAGS="-O3 -g -fPIC" \ ./configure @CROSS@ --disable-cpplibs \ --disable-ogg --disable-xmms-plugin \ --prefix="@AUX_PREFIX@" > ${log} 2>&1 \ diff --git a/pkgs/spt3g.sh b/pkgs/spt3g.sh index 0e2a442..36e88c3 100644 --- a/pkgs/spt3g.sh +++ b/pkgs/spt3g.sh @@ -4,7 +4,7 @@ pkg="spt3g" pkgopts=$@ cleanup="" -version=c2c60207458869faacecd4157baba5d65d715760 +version=56393887cf57785690c3746917382f1ec3cc08fb pfile=spt3g_software-${version}.tar.gz src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://github.com/CMB-S4/spt3g_software/archive/${version}.tar.gz ${pfile}) @@ -37,6 +37,10 @@ done # Short version of python to use pyshort=$(echo @PYVERSION@ | sed -e "s/\.//") +# The spt3g package uses Cereal and Boost, which have issues building with some vendor +# compilers (e.g. Intel). Instead, we use the OS compilers (typically gcc or clang) +# for compilation. + rm -rf spt3g_software-${version} tar xzf ${src} \ && cd spt3g_software-${version} \ @@ -51,10 +55,10 @@ tar xzf ${src} \ && LDFLAGS="-Wl,-z,muldefs" \ cmake \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER="@CC@" \ - -DCMAKE_CXX_COMPILER="@CXX@" \ - -DCMAKE_C_FLAGS="@CFLAGS@" \ - -DCMAKE_CXX_FLAGS="@CXXFLAGS@" \ + -DCMAKE_C_COMPILER="@BUILD_CC@" \ + -DCMAKE_CXX_COMPILER="@BUILD_CXX@" \ + -DCMAKE_C_FLAGS="-O3 -g -fPIC" \ + -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -std=c++11" \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ -DBOOST_ROOT="${boost}" \ -DBoost_PYTHON_TYPE="python${pyshort}" \ diff --git a/templates/packages.txt b/templates/packages.txt index ba5d84a..ffbc2d2 100644 --- a/templates/packages.txt +++ b/templates/packages.txt @@ -62,6 +62,9 @@ h5py # Install healpy healpy # +# Install Pixell +pixell +# # Install fitsio fitsio # From b9469e507ed8994f8635d9535a263ea90de530af Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Thu, 13 May 2021 13:35:14 -0700 Subject: [PATCH 3/8] Add python 3.9 configs --- .github/workflows/deploy.yml | 49 +++++++------------ .github/workflows/test.yml | 36 +++++--------- configs/docker-py3.9-debian | 77 ++++++++++++++++++++++++++++++ configs/docker-py3.9-debian.pkgs | 1 + configs/docker-toastdeps-py39 | 1 + configs/docker-toastdeps-py39.pkgs | 1 + 6 files changed, 109 insertions(+), 56 deletions(-) create mode 100644 configs/docker-py3.9-debian create mode 120000 configs/docker-py3.9-debian.pkgs create mode 120000 configs/docker-toastdeps-py39 create mode 120000 configs/docker-toastdeps-py39.pkgs diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5007b5d..b128c0d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -32,38 +32,6 @@ jobs: run: docker tag hpc4cmb/cmbenv:${TAGNAME} hpc4cmb/cmbenv:latest - name: Push Docker Image run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/cmbenv:${TAGNAME} && docker push hpc4cmb/cmbenv:latest - toastdeps-py35: - name: TOAST Dependencies Python 3.5 - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Create Dockerfile - run: ./cmbenv -c docker-toastdeps-py35 -p /usr/local - - name: Set tag name - run: tag=$(echo "${GITHUB_REF}" | sed -e 's/.*\/\(v.*\)/\1/') && echo "::set-env name=TAGNAME::${tag}" - - name: Build Docker Image - run: docker build -t hpc4cmb/toast-deps-py35:${TAGNAME} -f Dockerfile_docker-toastdeps-py35 . - - name: Tag Latest - run: docker tag hpc4cmb/toast-deps-py35:${TAGNAME} hpc4cmb/toast-deps-py35:latest - - name: Push Docker Image - run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/toast-deps-py35:${TAGNAME} && docker push hpc4cmb/toast-deps-py35:latest - toastdeps-py36: - name: TOAST Dependencies Python 3.6 - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Create Dockerfile - run: ./cmbenv -c docker-toastdeps-py36 -p /usr/local - - name: Set tag name - run: tag=$(echo "${GITHUB_REF}" | sed -e 's/.*\/\(v.*\)/\1/') && echo "::set-env name=TAGNAME::${tag}" - - name: Build Docker Image - run: docker build -t hpc4cmb/toast-deps-py36:${TAGNAME} -f Dockerfile_docker-toastdeps-py36 . - - name: Tag Latest - run: docker tag hpc4cmb/toast-deps-py36:${TAGNAME} hpc4cmb/toast-deps-py36:latest - - name: Push Docker Image - run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/toast-deps-py36:${TAGNAME} && docker push hpc4cmb/toast-deps-py36:latest toastdeps-py37: name: TOAST Dependencies Python 3.7 runs-on: ubuntu-latest @@ -96,3 +64,20 @@ jobs: run: docker tag hpc4cmb/toast-deps-py38:${TAGNAME} hpc4cmb/toast-deps-py38:latest - name: Push Docker Image run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/toast-deps-py38:${TAGNAME} && docker push hpc4cmb/toast-deps-py38:latest + toastdeps-py39: + name: TOAST Dependencies Python 3.9 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Create Dockerfile + run: ./cmbenv -c docker-toastdeps-py39 -p /usr/local + - name: Set tag name + run: tag=$(echo "${GITHUB_REF}" | sed -e 's/.*\/\(v.*\)/\1/') && echo "::set-env name=TAGNAME::${tag}" + - name: Build Docker Image + run: docker build -t hpc4cmb/toast-deps-py39:${TAGNAME} -f Dockerfile_docker-toastdeps-py39 . + - name: Tag Latest + run: docker tag hpc4cmb/toast-deps-py39:${TAGNAME} hpc4cmb/toast-deps-py39:latest + - name: Push Docker Image + run: echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin && docker push hpc4cmb/toast-deps-py39:${TAGNAME} && docker push hpc4cmb/toast-deps-py39:latest + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5ee66fe..67d00d1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,30 +9,6 @@ on: branches: [ master ] jobs: - py35: - name: Docker Py3.5 Debian - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Create Dockerfile - run: ./cmbenv -c docker-py3.5-debian -p /usr/local - - name: Build Docker Image - run: docker build -t hpc4cmb/cmbenv:temp_py35 -f Dockerfile_docker-py3.5-debian . - - name: Test Docker Image - run: docker run hpc4cmb/cmbenv:temp_py35 python -c 'import toast.tests; toast.tests.run()' && docker run hpc4cmb/cmbenv:temp_py35 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()' - py36: - name: Docker Py3.6 Debian - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Create Dockerfile - run: ./cmbenv -c docker-py3.6-debian -p /usr/local - - name: Build Docker Image - run: docker build -t hpc4cmb/cmbenv:temp_py36 -f Dockerfile_docker-py3.6-debian . - - name: Test Docker Image - run: docker run hpc4cmb/cmbenv:temp_py36 python -c 'import toast.tests; toast.tests.run()' && docker run hpc4cmb/cmbenv:temp_py36 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()' py37: name: Docker Py3.7 Debian runs-on: ubuntu-latest @@ -57,3 +33,15 @@ jobs: run: docker build -t hpc4cmb/cmbenv:temp_py38 -f Dockerfile_docker-py3.8-debian . - name: Test Docker Image run: docker run hpc4cmb/cmbenv:temp_py38 python -c 'import toast.tests; toast.tests.run()' && docker run hpc4cmb/cmbenv:temp_py38 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()' + py39: + name: Docker Py3.9 Debian + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Create Dockerfile + run: ./cmbenv -c docker-py3.9-debian -p /usr/local + - name: Build Docker Image + run: docker build -t hpc4cmb/cmbenv:temp_py39 -f Dockerfile_docker-py3.9-debian . + - name: Test Docker Image + run: docker run hpc4cmb/cmbenv:temp_py39 python -c 'import toast.tests; toast.tests.run()' && docker run hpc4cmb/cmbenv:temp_py39 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()' diff --git a/configs/docker-py3.9-debian b/configs/docker-py3.9-debian new file mode 100644 index 0000000..2b886e2 --- /dev/null +++ b/configs/docker-py3.9-debian @@ -0,0 +1,77 @@ + +# OS type (linux OR osx) +OSTYPE = linux + +# Specify the template to use +TEMPLATE = docker-debian + +# Python version +PYVERSION = 3.9 + +# Specify the docker base image +DOCKER_BASE = python:3.9-slim-buster + +# Specify additional packages to install for docker builds +DOCKER_SYS_PKGS = m4 libtool autoconf automake llvm libsuitesparse-dev libfftw3-dev libcfitsio-dev libflac-dev zlib1g-dev libbz2-dev libopenblas-dev liblapack-dev libhdf5-dev + +# Packages to install with pip +PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil numpy scipy matplotlib pandas memory_profiler astropy ephem ipython numba tbb cmake sphinx sphinx-rtd-theme pyfftw pshmem tomlkit traitlets>=5.0 + +# Packages to install with conda +CONDA_PKGS = + +# Serial compilers + +CC = gcc +CXX = g++ +FC = gfortran + +# Compiler to use for packages needed on the build machine + +BUILD_CC = gcc +BUILD_CXX = g++ + +# MPI compilers + +MPICC = mpicc +MPICXX = mpicxx +MPIFC = mpifort +MPI_CPPFLAGS = @AUX_PREFIX@/include +MPI_LDFLAGS = @AUX_PREFIX@/lib +MPI_CXXLIB = mpichcxx +MPI_LIB = mpich +MPI_EXTRA_COMP = +MPI_EXTRA_LINK = + +# Compile flags + +CFLAGS = -O2 -g -fPIC -pthread +CXXFLAGS = -O2 -g -fPIC -pthread -std=c++11 +FCFLAGS = -O2 -g -fPIC -pthread + +OPENMP_CFLAGS = -fopenmp +OPENMP_CXXFLAGS = -fopenmp +LDFLAGS = -lpthread -fopenmp + +# Are we doing a cross-compile? +CROSS = + +# Parallel builds +MAKEJ = 1 + +# Define MKL location +MKL = + +# For BLAS/LAPACK, we use openblas + +BLAS = -lopenblas -fopenmp -lm +LAPACK = + +# Boost toolchain name + +BOOSTCHAIN = gcc + +# Group and permissions to set + +CHGRP = +CHMOD = diff --git a/configs/docker-py3.9-debian.pkgs b/configs/docker-py3.9-debian.pkgs new file mode 120000 index 0000000..70e64f4 --- /dev/null +++ b/configs/docker-py3.9-debian.pkgs @@ -0,0 +1 @@ +docker-py3.7-debian.pkgs \ No newline at end of file diff --git a/configs/docker-toastdeps-py39 b/configs/docker-toastdeps-py39 new file mode 120000 index 0000000..ba2a767 --- /dev/null +++ b/configs/docker-toastdeps-py39 @@ -0,0 +1 @@ +docker-py3.9-debian \ No newline at end of file diff --git a/configs/docker-toastdeps-py39.pkgs b/configs/docker-toastdeps-py39.pkgs new file mode 120000 index 0000000..c01b8fa --- /dev/null +++ b/configs/docker-toastdeps-py39.pkgs @@ -0,0 +1 @@ +docker-toastdeps-py37.pkgs \ No newline at end of file From 0dcde3187e8b402475776c53fd29a7780fc703be Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Thu, 13 May 2021 13:44:22 -0700 Subject: [PATCH 4/8] fix typo --- configs/docker-py3.7-debian | 2 +- configs/docker-py3.8-debian | 2 +- configs/docker-py3.9-debian | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configs/docker-py3.7-debian b/configs/docker-py3.7-debian index 706fb44..fffc16e 100644 --- a/configs/docker-py3.7-debian +++ b/configs/docker-py3.7-debian @@ -15,7 +15,7 @@ DOCKER_BASE = python:3.7-slim-buster DOCKER_SYS_PKGS = m4 libtool autoconf automake llvm libsuitesparse-dev libfftw3-dev libcfitsio-dev libflac-dev zlib1g-dev libbz2-dev libopenblas-dev liblapack-dev libhdf5-dev # Packages to install with pip -PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 tbb cmake sphinx sphinx-rtd-theme pyfftw pshmem tomlkit traitlets>=5.0 +PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 tbb cmake sphinx sphinx-rtd-theme pyfftw pshmem tomlkit traitlets # Packages to install with conda CONDA_PKGS = diff --git a/configs/docker-py3.8-debian b/configs/docker-py3.8-debian index 68c4c7f..c61422d 100644 --- a/configs/docker-py3.8-debian +++ b/configs/docker-py3.8-debian @@ -15,7 +15,7 @@ DOCKER_BASE = python:3.8-slim-buster DOCKER_SYS_PKGS = m4 libtool autoconf automake llvm libsuitesparse-dev libfftw3-dev libcfitsio-dev libflac-dev zlib1g-dev libbz2-dev libopenblas-dev liblapack-dev libhdf5-dev # Packages to install with pip -PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 tbb cmake sphinx sphinx-rtd-theme pyfftw pshmem tomlkit traitlets>=5.0 +PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 tbb cmake sphinx sphinx-rtd-theme pyfftw pshmem tomlkit traitlets # Packages to install with conda CONDA_PKGS = diff --git a/configs/docker-py3.9-debian b/configs/docker-py3.9-debian index 2b886e2..5543d56 100644 --- a/configs/docker-py3.9-debian +++ b/configs/docker-py3.9-debian @@ -15,7 +15,7 @@ DOCKER_BASE = python:3.9-slim-buster DOCKER_SYS_PKGS = m4 libtool autoconf automake llvm libsuitesparse-dev libfftw3-dev libcfitsio-dev libflac-dev zlib1g-dev libbz2-dev libopenblas-dev liblapack-dev libhdf5-dev # Packages to install with pip -PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil numpy scipy matplotlib pandas memory_profiler astropy ephem ipython numba tbb cmake sphinx sphinx-rtd-theme pyfftw pshmem tomlkit traitlets>=5.0 +PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pkgconfig pytz psutil numpy scipy matplotlib pandas memory_profiler astropy ephem ipython numba tbb cmake sphinx sphinx-rtd-theme pyfftw pshmem tomlkit traitlets # Packages to install with conda CONDA_PKGS = From 5a3512d7984fa4305289c2319aedb0b0b836df09 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Thu, 13 May 2021 18:53:32 -0700 Subject: [PATCH 5/8] Fix cori-intel config --- configs/cori-intel | 2 +- pkgs/boost.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/configs/cori-intel b/configs/cori-intel index f1f77dc..1315696 100644 --- a/configs/cori-intel +++ b/configs/cori-intel @@ -64,7 +64,7 @@ LAPACK = # Boost toolchain name -BOOSTCHAIN = intel-linux +BOOSTCHAIN = gcc # Group and permissions to set diff --git a/pkgs/boost.sh b/pkgs/boost.sh index 2dd3a78..7a1cb1a 100644 --- a/pkgs/boost.sh +++ b/pkgs/boost.sh @@ -8,7 +8,7 @@ cleanup="" version=1_76_0 pfile=boost_${version}.tar.bz2 -src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://dl.bintray.com/boostorg/release/1.76.0/source/${pfile} ${pfile}) +src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://boostorg.jfrog.io/artifactory/main/release/1.76.0/source/${pfile} ${pfile}) if [ "x${src}" = "x" ]; then echo "Failed to fetch ${pkg}" >&2 @@ -20,9 +20,9 @@ log="../log_${pkg}" echo "Building ${pkg}..." >&2 -# NOTE: Due to perpetual flakiness of boost compilation with vendor compilers (e.g. Intel), -# we build boost with separate OS compilers (usually gcc or clang), as specified by the -# config BOOSTCHAIN variable. +# NOTE: Due to perpetual flakiness of boost compilation with vendor compilers +# (e.g. Intel), we build boost with separate OS compilers (usually gcc or clang), +# as specified by the config BOOSTCHAIN variable. rm -rf boost_${version} tar xjf ${src} \ From 31fdd6180ee1d5696c9d1d0ab14c248686e9302f Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Thu, 13 May 2021 18:59:27 -0700 Subject: [PATCH 6/8] Remove version limit --- configs/cori-gcc | 2 +- configs/cori-gcc-gpu | 2 +- configs/cori-intel | 2 +- configs/docker-ubuntu_20.04_gcc9_py38 | 2 +- configs/linux-conda | 2 +- configs/linux-conda-nomkl | 2 +- configs/linux-venv | 2 +- configs/ubuntu-18.04-venv | 2 +- configs/ubuntu-20.04-venv | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/configs/cori-gcc b/configs/cori-gcc index d9ed6a5..f53ac92 100644 --- a/configs/cori-gcc +++ b/configs/cori-gcc @@ -12,7 +12,7 @@ PYVERSION = 3.7 PIP_PKGS = # Packages to install with conda -CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets>=5.0 +CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets # Serial compilers diff --git a/configs/cori-gcc-gpu b/configs/cori-gcc-gpu index f936837..69b9fca 100644 --- a/configs/cori-gcc-gpu +++ b/configs/cori-gcc-gpu @@ -12,7 +12,7 @@ PYVERSION = 3.7 PIP_PKGS = # Packages to install with conda -CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets>=5.0 +CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets # Serial compilers diff --git a/configs/cori-intel b/configs/cori-intel index 1315696..97696ad 100644 --- a/configs/cori-intel +++ b/configs/cori-intel @@ -12,7 +12,7 @@ PYVERSION = 3.7 PIP_PKGS = # Packages to install with conda -CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets>=5.0 +CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets # Serial compilers diff --git a/configs/docker-ubuntu_20.04_gcc9_py38 b/configs/docker-ubuntu_20.04_gcc9_py38 index bf7addc..0f1ecb3 100644 --- a/configs/docker-ubuntu_20.04_gcc9_py38 +++ b/configs/docker-ubuntu_20.04_gcc9_py38 @@ -12,7 +12,7 @@ DOCKER_BASE = ubuntu:20.04 DOCKER_SYS_PKGS = m4 libtool autoconf automake cmake libcfitsio-dev libfftw3-dev libopenblas-dev liblapack-dev libhdf5-dev libboost-all-dev libopenmpi-dev python3 python3-dev python3-setuptools python3-pkg-resources python3-pip python3-tk python3-numpy python3-scipy python3-matplotlib python3-yaml python3-astropy python3-six python3-psutil python3-ephem python3-pandas python3-memory-profiler python3-ipython cython3 python3-cycler python3-kiwisolver python3-dateutil python3-toml python3-numba black python3-jupyter-client python3-h5py python3-mpi4py libsuitesparse-dev # Packages to install with pip -PIP_PKGS = pshmem tomlkit traitlets>=5.0 +PIP_PKGS = pshmem tomlkit traitlets # Packages to install with conda CONDA_PKGS = diff --git a/configs/linux-conda b/configs/linux-conda index 6754b93..a4d5f54 100644 --- a/configs/linux-conda +++ b/configs/linux-conda @@ -12,7 +12,7 @@ PYVERSION = 3.7 PIP_PKGS = # Packages to install with conda -CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets>=5.0 +CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets # Serial compilers diff --git a/configs/linux-conda-nomkl b/configs/linux-conda-nomkl index 6754b93..a4d5f54 100644 --- a/configs/linux-conda-nomkl +++ b/configs/linux-conda-nomkl @@ -12,7 +12,7 @@ PYVERSION = 3.7 PIP_PKGS = # Packages to install with conda -CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets>=5.0 +CONDA_PKGS = future nose cython numpy scipy matplotlib pyyaml astropy six psutil ephem virtualenv pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets # Serial compilers diff --git a/configs/linux-venv b/configs/linux-venv index 24e82f8..f3dbbca 100644 --- a/configs/linux-venv +++ b/configs/linux-venv @@ -9,7 +9,7 @@ TEMPLATE = linux PYVERSION = auto # Packages to install with pip -PIP_PKGS = future nose Cython numpy scipy matplotlib pyyaml astropy six psutil ephem pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets>=5.0 +PIP_PKGS = future nose Cython numpy scipy matplotlib pyyaml astropy six psutil ephem pandas memory_profiler ipython cycler kiwisolver python-dateutil toml numba tbb nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets # Packages to install with conda CONDA_PKGS = diff --git a/configs/ubuntu-18.04-venv b/configs/ubuntu-18.04-venv index 98b9d31..c14622c 100644 --- a/configs/ubuntu-18.04-venv +++ b/configs/ubuntu-18.04-venv @@ -9,7 +9,7 @@ TEMPLATE = linux PYVERSION = auto # Packages to install with pip -PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pyyaml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 cmake pyfftw pillow pshmem tomlkit traitlets>=5.0 +PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pyyaml pkgconfig pytz psutil==5.7.0 numpy==1.18.4 scipy==1.4.1 matplotlib==3.2.1 pandas==1.0.3 memory_profiler==0.57.0 astropy==4.0.1.post1 ephem==3.7.7.1 ipython==7.14.0 numba==0.49.1 cmake pyfftw pillow pshmem tomlkit traitlets # Packages to install with conda CONDA_PKGS = diff --git a/configs/ubuntu-20.04-venv b/configs/ubuntu-20.04-venv index 6552192..ad47ac0 100644 --- a/configs/ubuntu-20.04-venv +++ b/configs/ubuntu-20.04-venv @@ -9,7 +9,7 @@ TEMPLATE = linux PYVERSION = auto # Packages to install with pip -PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pyyaml pkgconfig pytz psutil numpy scipy matplotlib pandas memory_profiler astropy ephem ipython numba cmake nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets>=5.0 +PIP_PKGS = setuptools future six Cython cycler kiwisolver pyparsing python-dateutil toml pyyaml pkgconfig pytz psutil numpy scipy matplotlib pandas memory_profiler astropy ephem ipython numba cmake nbstripout black wurlitzer ipympl ipykernel jupyterlab pyfftw pillow pshmem tomlkit traitlets # Packages to install with conda CONDA_PKGS = From c7d09e4b796da685ac78c228b67be6dd6676d2b9 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Thu, 13 May 2021 19:31:54 -0700 Subject: [PATCH 7/8] Restore spt3g to cori intel build, since we are building with gcc now. --- configs/cori-intel.pkgs | 2 +- pkgs/patch_spt3g | 25 ------------------------- pkgs/spt3g.sh | 1 - 3 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 pkgs/patch_spt3g diff --git a/configs/cori-intel.pkgs b/configs/cori-intel.pkgs index ad74cfc..72a411f 100644 --- a/configs/cori-intel.pkgs +++ b/configs/cori-intel.pkgs @@ -90,7 +90,7 @@ madam libflac # # Install SPT3G. Frame based data format. -#spt3g +spt3g # # Install TIDAS. HDF5-based timestream data format. tidas diff --git a/pkgs/patch_spt3g b/pkgs/patch_spt3g deleted file mode 100644 index 40746f5..0000000 --- a/pkgs/patch_spt3g +++ /dev/null @@ -1,25 +0,0 @@ -diff -urN spt3g_software_orig/CMakeLists.txt spt3g_software/CMakeLists.txt ---- spt3g_software_orig/CMakeLists.txt 2020-02-14 17:30:40.000000000 -0800 -+++ spt3g_software/CMakeLists.txt 2020-03-03 13:33:55.042779031 -0800 -@@ -37,7 +37,7 @@ - - # Raise errors on every warning by default - # (use target-specific options to disable particular warnings) --set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror") -+#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror") - - # Interface library for flags and library dependencies - add_library(spt3g INTERFACE) -@@ -51,10 +51,10 @@ - endif() - - # Ignore known warnings --target_compile_options(spt3g INTERFACE -Wno-unknown-warning-option -Wno-unused -Wno-unused-result -Wno-deprecated-register -Wno-sign-compare -Wno-constant-conversion -Wno-self-assign-overloaded -Wno-deprecated-declarations) -+#target_compile_options(spt3g INTERFACE -Wno-unknown-warning-option -Wno-unused -Wno-unused-result -Wno-deprecated-register -Wno-sign-compare -Wno-constant-conversion -Wno-self-assign-overloaded -Wno-deprecated-declarations) - - # Fix bugs in GCC 4.4's strict aliasing code by turning it off --target_compile_options(spt3g INTERFACE -fno-strict-aliasing) -+#target_compile_options(spt3g INTERFACE -fno-strict-aliasing) - - # Locate Python - if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.12) diff --git a/pkgs/spt3g.sh b/pkgs/spt3g.sh index 36e88c3..fe34621 100644 --- a/pkgs/spt3g.sh +++ b/pkgs/spt3g.sh @@ -45,7 +45,6 @@ rm -rf spt3g_software-${version} tar xzf ${src} \ && cd spt3g_software-${version} \ && cleanup="${cleanup} $(pwd)" \ - && patch -p1 < "@TOP_DIR@/pkgs/patch_spt3g" > ${log} 2>&1 \ && cd .. \ && rm -rf "@AUX_PREFIX@/spt3g" \ && cp -a spt3g_software-${version} "@AUX_PREFIX@/spt3g" \ From d157ef8be148adbaceead54bf3b88d9bf223d23e Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Thu, 13 May 2021 20:04:20 -0700 Subject: [PATCH 8/8] Add new package for latest uncrustify release. --- configs/cori-gcc-gpu.pkgs | 3 ++ configs/cori-gcc.pkgs | 3 ++ configs/cori-intel.pkgs | 3 ++ configs/docker-py3.7-debian.pkgs | 3 ++ configs/docker-toastdeps-py37.pkgs | 3 ++ configs/docker-ubuntu_20.04_gcc9_py38.pkgs | 3 ++ configs/linux-conda-nomkl.pkgs | 3 ++ configs/linux-conda.pkgs | 3 ++ configs/linux-venv.pkgs | 3 ++ configs/osx-homebrew-clang.pkgs | 3 ++ configs/osx-macports-gcc.pkgs | 3 ++ configs/simons-intel.pkgs | 3 ++ configs/ubuntu-18.04-venv.pkgs | 3 ++ configs/ubuntu-20.04-venv.pkgs | 3 ++ pkgs/uncrustify.sh | 47 ++++++++++++++++++++++ templates/packages.txt | 3 ++ 16 files changed, 92 insertions(+) create mode 100644 pkgs/uncrustify.sh diff --git a/configs/cori-gcc-gpu.pkgs b/configs/cori-gcc-gpu.pkgs index bac468e..f5f4906 100644 --- a/configs/cori-gcc-gpu.pkgs +++ b/configs/cori-gcc-gpu.pkgs @@ -88,6 +88,9 @@ madam # Install SPT3G. Frame based data format. #spt3g # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. #tidas # diff --git a/configs/cori-gcc.pkgs b/configs/cori-gcc.pkgs index cd0597a..710920d 100644 --- a/configs/cori-gcc.pkgs +++ b/configs/cori-gcc.pkgs @@ -92,6 +92,9 @@ libflac # Install SPT3G. Frame based data format. spt3g # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. tidas # diff --git a/configs/cori-intel.pkgs b/configs/cori-intel.pkgs index 72a411f..23e43e4 100644 --- a/configs/cori-intel.pkgs +++ b/configs/cori-intel.pkgs @@ -92,6 +92,9 @@ libflac # Install SPT3G. Frame based data format. spt3g # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. tidas # diff --git a/configs/docker-py3.7-debian.pkgs b/configs/docker-py3.7-debian.pkgs index c719166..e9a835f 100644 --- a/configs/docker-py3.7-debian.pkgs +++ b/configs/docker-py3.7-debian.pkgs @@ -89,6 +89,9 @@ madam:--with-cfitsio=/usr --with-fftw=/usr # Install SPT3G. Frame based data format. #spt3g:boost=/usr/local flaclib=/usr/lib/x86_64-linux-gnu/libFLAC.so # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. tidas # diff --git a/configs/docker-toastdeps-py37.pkgs b/configs/docker-toastdeps-py37.pkgs index 560687a..d580753 100644 --- a/configs/docker-toastdeps-py37.pkgs +++ b/configs/docker-toastdeps-py37.pkgs @@ -89,6 +89,9 @@ madam:--with-cfitsio=/usr --with-fftw=/usr # Install SPT3G. Frame based data format. #spt3g:boost=/usr/local flaclib=/usr/lib/x86_64-linux-gnu/libFLAC.so # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. tidas # diff --git a/configs/docker-ubuntu_20.04_gcc9_py38.pkgs b/configs/docker-ubuntu_20.04_gcc9_py38.pkgs index 21c89bc..4bbd09e 100644 --- a/configs/docker-ubuntu_20.04_gcc9_py38.pkgs +++ b/configs/docker-ubuntu_20.04_gcc9_py38.pkgs @@ -89,6 +89,9 @@ libflac # Install SPT3G. Frame based data format. spt3g # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. tidas # diff --git a/configs/linux-conda-nomkl.pkgs b/configs/linux-conda-nomkl.pkgs index 5465fb0..56776e6 100644 --- a/configs/linux-conda-nomkl.pkgs +++ b/configs/linux-conda-nomkl.pkgs @@ -89,6 +89,9 @@ libflac # Install SPT3G. Frame based data format. spt3g # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. tidas # diff --git a/configs/linux-conda.pkgs b/configs/linux-conda.pkgs index 6cb7917..074158b 100644 --- a/configs/linux-conda.pkgs +++ b/configs/linux-conda.pkgs @@ -89,6 +89,9 @@ libflac # Install SPT3G. Frame based data format. spt3g # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. tidas # diff --git a/configs/linux-venv.pkgs b/configs/linux-venv.pkgs index 3a32805..aca372f 100644 --- a/configs/linux-venv.pkgs +++ b/configs/linux-venv.pkgs @@ -89,6 +89,9 @@ libflac # Install SPT3G. Frame based data format. spt3g # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. tidas # diff --git a/configs/osx-homebrew-clang.pkgs b/configs/osx-homebrew-clang.pkgs index ca7ab90..7eb5e00 100644 --- a/configs/osx-homebrew-clang.pkgs +++ b/configs/osx-homebrew-clang.pkgs @@ -89,6 +89,9 @@ libflac # Install SPT3G. Frame based data format. spt3g # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. tidas # diff --git a/configs/osx-macports-gcc.pkgs b/configs/osx-macports-gcc.pkgs index c1803c3..f27980b 100644 --- a/configs/osx-macports-gcc.pkgs +++ b/configs/osx-macports-gcc.pkgs @@ -89,6 +89,9 @@ libflac # Install SPT3G. Frame based data format. spt3g # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. tidas # diff --git a/configs/simons-intel.pkgs b/configs/simons-intel.pkgs index 249606c..2c5052e 100644 --- a/configs/simons-intel.pkgs +++ b/configs/simons-intel.pkgs @@ -89,6 +89,9 @@ libflac # Install SPT3G. Frame based data format. spt3g # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. #tidas # diff --git a/configs/ubuntu-18.04-venv.pkgs b/configs/ubuntu-18.04-venv.pkgs index 61b088a..9f7bb8e 100644 --- a/configs/ubuntu-18.04-venv.pkgs +++ b/configs/ubuntu-18.04-venv.pkgs @@ -89,6 +89,9 @@ madam:--with-cfitsio=/usr --with-fftw=/usr # Install SPT3G. Frame based data format. spt3g:boost=/usr flaclib=/usr/lib/x86_64-linux-gnu/libFLAC.so # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. tidas # diff --git a/configs/ubuntu-20.04-venv.pkgs b/configs/ubuntu-20.04-venv.pkgs index 18e3d34..3ffcc80 100644 --- a/configs/ubuntu-20.04-venv.pkgs +++ b/configs/ubuntu-20.04-venv.pkgs @@ -89,6 +89,9 @@ madam:--with-cfitsio=/usr --with-fftw=/usr # Install SPT3G. Frame based data format. spt3g:boost=/usr flaclib=/usr/lib/x86_64-linux-gnu/libFLAC.so # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. tidas # diff --git a/pkgs/uncrustify.sh b/pkgs/uncrustify.sh new file mode 100644 index 0000000..5615321 --- /dev/null +++ b/pkgs/uncrustify.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +pkg="uncrustify" +pkgopts=$@ +cleanup="" + +version=0.73.0 +pfile=uncrustify-${version}.tar.gz +src=$(eval "@TOP_DIR@/tools/fetch_check.sh" https://github.com/uncrustify/uncrustify/archive/refs/tags/${pfile} ${pfile}) + +# Yes, this is silly +srcdir=uncrustify-uncrustify-${version} + +if [ "x${src}" = "x" ]; then + echo "Failed to fetch ${pkg}" >&2 + exit 1 +fi +cleanup="${src}" + +log="../../log_${pkg}" + +echo "Building ${pkg}..." >&2 + +rm -rf ${srcdir} +tar xzf ${src} \ + && cd ${srcdir} \ + && cleanup="${cleanup} $(pwd)" \ + && mkdir -p build \ + && cd build \ + && cmake \ + -DCMAKE_C_COMPILER="@BUILD_CC@" \ + -DCMAKE_CXX_COMPILER="@BUILD_CXX@" \ + -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ + -DCMAKE_INSTALL_PREFIX="@AUX_PREFIX@" \ + .. > "${log}" 2>&1 \ + && make -j @MAKEJ@ >> "${log}" 2>&1 \ + && make install >> "${log}" 2>&1 + +if [ $? -ne 0 ]; then + echo "Failed to build ${pkg}" >&2 + exit 1 +fi + +echo "Finished building ${pkg}" >&2 +echo "${cleanup}" +exit 0 diff --git a/templates/packages.txt b/templates/packages.txt index ffbc2d2..9c96d3e 100644 --- a/templates/packages.txt +++ b/templates/packages.txt @@ -83,6 +83,9 @@ madam # Install SPT3G. Frame based data format. spt3g # +# Install uncrustify +uncrustify +# # Install TIDAS. HDF5-based timestream data format. tidas #