From 90f124605b39fb79339dd2efe492ec1437b20b79 Mon Sep 17 00:00:00 2001 From: Marty Kandes Date: Mon, 10 Jul 2023 16:18:34 -0700 Subject: [PATCH] Add spark@3.4.0 % gcc@10.2.0 to expanse/0.17.3/cpu/b --- .../cpu/b/specs/gcc@10.2.0/py-htseq@0.11.2.sh | 2 +- .../spark@3.4.0.o22902088.exp-15-02 | 649 ++++++++++++++++++ .../cpu/b/specs/gcc@10.2.0/spark@3.4.0.sh | 76 ++ 3 files changed, 726 insertions(+), 1 deletion(-) create mode 100644 etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/spark@3.4.0.o22902088.exp-15-02 create mode 100644 etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/spark@3.4.0.sh diff --git a/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/py-htseq@0.11.2.sh b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/py-htseq@0.11.2.sh index 6bf52c38e1916a..fbd2ebe511c2b5 100644 --- a/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/py-htseq@0.11.2.sh +++ b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/py-htseq@0.11.2.sh @@ -70,6 +70,6 @@ fi #spack module lmod refresh --delete-tree -y -#sbatch --dependency="afterok:${SLURM_JOB_ID}" '' +sbatch --dependency="afterok:${SLURM_JOB_ID}" 'spark@3.4.0.sh' sleep 30 diff --git a/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/spark@3.4.0.o22902088.exp-15-02 b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/spark@3.4.0.o22902088.exp-15-02 new file mode 100644 index 00000000000000..56b46a4a40553d --- /dev/null +++ b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/spark@3.4.0.o22902088.exp-15-02 @@ -0,0 +1,649 @@ +1686323436 22902088 1511c7944d387d7e35502660c80f7377 /home/mkandes/.spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/spark@3.4.0.sh + +#!/usr/bin/env bash + +#SBATCH --job-name=spark@3.4.0 +#SBATCH --account=use300 +##SBATCH --reservation=rocky8u7_testing +#SBATCH --partition=ind-shared +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=16 +#SBATCH --mem=32G +#SBATCH --time=01:00:00 +#SBATCH --output=%x.o%j.%N + +declare -xr LOCAL_TIME="$(date +'%Y%m%dT%H%M%S%z')" +declare -xir UNIX_TIME="$(date +'%s')" + +declare -xr LOCAL_SCRATCH_DIR="/scratch/${USER}/job_${SLURM_JOB_ID}" +declare -xr TMPDIR="${LOCAL_SCRATCH_DIR}" + +declare -xr SYSTEM_NAME='expanse' + +declare -xr SPACK_VERSION='0.17.3' +declare -xr SPACK_INSTANCE_NAME='cpu' +declare -xr SPACK_INSTANCE_VERSION='b' +declare -xr SPACK_INSTANCE_DIR="/cm/shared/apps/spack/${SPACK_VERSION}/${SPACK_INSTANCE_NAME}/${SPACK_INSTANCE_VERSION}" + +declare -xr SLURM_JOB_SCRIPT="$(scontrol show job ${SLURM_JOB_ID} | awk -F= '/Command=/{print $2}')" +declare -xr SLURM_JOB_MD5SUM="$(md5sum ${SLURM_JOB_SCRIPT})" + +declare -xr SCHEDULER_MODULE='slurm' + +echo "${UNIX_TIME} ${SLURM_JOB_ID} ${SLURM_JOB_MD5SUM} ${SLURM_JOB_DEPENDENCY}" +echo "" + +cat "${SLURM_JOB_SCRIPT}" + +module purge +module load "${SCHEDULER_MODULE}" +module list +. "${SPACK_INSTANCE_DIR}/share/spack/setup-env.sh" +shopt -s expand_aliases +source ~/.bashrc + +declare -xr SPACK_PACKAGE='spark@3.4.0' +declare -xr SPACK_COMPILER='gcc@10.2.0' +declare -xr SPACK_VARIANTS='+hadoop' +declare -xr SPACK_DEPENDENCIES="^openjdk@11.0.12_7/$(spack find --format '{hash:7}' openjdk@11.0.12_7 % ${SPACK_COMPILER})" +declare -xr SPACK_SPEC="${SPACK_PACKAGE} % ${SPACK_COMPILER} ${SPACK_VARIANTS} ${SPACK_DEPENDENCIES}" + +printenv + +spack config get compilers +spack config get config +spack config get mirrors +spack config get modules +spack config get packages +spack config get repos +spack config get upstreams + +time -p spack spec --long --namespaces --types --reuse "${SPACK_SPEC}" +if [[ "${?}" -ne 0 ]]; then + echo 'ERROR: spack concretization failed.' + exit 1 +fi + +time -p spack install --jobs "${SLURM_CPUS_PER_TASK}" --fail-fast --yes-to-all --reuse "${SPACK_SPEC}" +if [[ "${?}" -ne 0 ]]; then + echo 'ERROR: spack install failed.' + exit 1 +fi + +spack module lmod refresh -y + +#sbatch --dependency="afterok:${SLURM_JOB_ID}" '' + +sleep 30 + +Currently Loaded Modules: + 1) slurm/expanse/21.08.8 + + + +CONDA_SHLVL=0 +LD_LIBRARY_PATH=/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64 +LS_COLORS=rs=0:di=38;5;33:ln=38;5;51:mh=00:pi=40;38;5;11:so=38;5;13:do=38;5;5:bd=48;5;232;38;5;11:cd=48;5;232;38;5;3:or=48;5;232;38;5;9:mi=01;05;37;41:su=48;5;196;38;5;15:sg=48;5;11;38;5;16:ca=48;5;196;38;5;226:tw=48;5;10;38;5;16:ow=48;5;10;38;5;21:st=48;5;21;38;5;15:ex=38;5;40:*.tar=38;5;9:*.tgz=38;5;9:*.arc=38;5;9:*.arj=38;5;9:*.taz=38;5;9:*.lha=38;5;9:*.lz4=38;5;9:*.lzh=38;5;9:*.lzma=38;5;9:*.tlz=38;5;9:*.txz=38;5;9:*.tzo=38;5;9:*.t7z=38;5;9:*.zip=38;5;9:*.z=38;5;9:*.dz=38;5;9:*.gz=38;5;9:*.lrz=38;5;9:*.lz=38;5;9:*.lzo=38;5;9:*.xz=38;5;9:*.zst=38;5;9:*.tzst=38;5;9:*.bz2=38;5;9:*.bz=38;5;9:*.tbz=38;5;9:*.tbz2=38;5;9:*.tz=38;5;9:*.deb=38;5;9:*.rpm=38;5;9:*.jar=38;5;9:*.war=38;5;9:*.ear=38;5;9:*.sar=38;5;9:*.rar=38;5;9:*.alz=38;5;9:*.ace=38;5;9:*.zoo=38;5;9:*.cpio=38;5;9:*.7z=38;5;9:*.rz=38;5;9:*.cab=38;5;9:*.wim=38;5;9:*.swm=38;5;9:*.dwm=38;5;9:*.esd=38;5;9:*.jpg=38;5;13:*.jpeg=38;5;13:*.mjpg=38;5;13:*.mjpeg=38;5;13:*.gif=38;5;13:*.bmp=38;5;13:*.pbm=38;5;13:*.pgm=38;5;13:*.ppm=38;5;13:*.tga=38;5;13:*.xbm=38;5;13:*.xpm=38;5;13:*.tif=38;5;13:*.tiff=38;5;13:*.png=38;5;13:*.svg=38;5;13:*.svgz=38;5;13:*.mng=38;5;13:*.pcx=38;5;13:*.mov=38;5;13:*.mpg=38;5;13:*.mpeg=38;5;13:*.m2v=38;5;13:*.mkv=38;5;13:*.webm=38;5;13:*.ogm=38;5;13:*.mp4=38;5;13:*.m4v=38;5;13:*.mp4v=38;5;13:*.vob=38;5;13:*.qt=38;5;13:*.nuv=38;5;13:*.wmv=38;5;13:*.asf=38;5;13:*.rm=38;5;13:*.rmvb=38;5;13:*.flc=38;5;13:*.avi=38;5;13:*.fli=38;5;13:*.flv=38;5;13:*.gl=38;5;13:*.dl=38;5;13:*.xcf=38;5;13:*.xwd=38;5;13:*.yuv=38;5;13:*.cgm=38;5;13:*.emf=38;5;13:*.ogv=38;5;13:*.ogx=38;5;13:*.aac=38;5;45:*.au=38;5;45:*.flac=38;5;45:*.m4a=38;5;45:*.mid=38;5;45:*.midi=38;5;45:*.mka=38;5;45:*.mp3=38;5;45:*.mpc=38;5;45:*.ogg=38;5;45:*.ra=38;5;45:*.wav=38;5;45:*.oga=38;5;45:*.opus=38;5;45:*.spx=38;5;45:*.xspf=38;5;45: +CONDA_EXE=/home/mkandes/miniconda3/bin/conda +LOCAL_SCRATCH_DIR=/scratch/mkandes/job_22902088 +SLURM_NODEID=0 +LOCAL_SPACK_TMPDIR=/tmp +SLURM_TASK_PID=482756 +__LMOD_REF_COUNT_PATH=/cm/shared/apps/slurm/current/sbin:1;/cm/shared/apps/slurm/current/bin:1;/cm/shared/apps/spack/0.17.3/cpu/b/bin:1;/home/mkandes/miniconda3/condabin:1;/home/mkandes/.local/bin:1;/home/mkandes/bin:1;/usr/local/bin:1;/usr/bin:1;/usr/local/sbin:1;/usr/sbin:1 +_ModuleTable002_=b2NhbC9tb2R1bGVmaWxlcyIsIi9jbS9zaGFyZWQvYXBwcy9hY2Nlc3MvbW9kdWxlZmlsZXMiLCIvZXRjL21vZHVsZWZpbGVzIiwiL3Vzci9zaGFyZS9tb2R1bGVmaWxlcyIsIi91c3Ivc2hhcmUvTW9kdWxlcy9tb2R1bGVmaWxlcyIsIi9jbS9zaGFyZWQvbW9kdWxlZmlsZXMiLH0sWyJzeXN0ZW1CYXNlTVBBVEgiXT0iL2NtL2xvY2FsL21vZHVsZWZpbGVzOi9jbS9zaGFyZWQvYXBwcy9hY2Nlc3MvbW9kdWxlZmlsZXM6L2NtL3NoYXJlZC9tb2R1bGVmaWxlczovZXRjL21vZHVsZWZpbGVzOi91c3Ivc2hhcmUvbW9kdWxlZmlsZXM6L3Vzci9zaGFyZS9Nb2R1bGVzL21vZHVsZWZpbGVzIix9 +SSH_CONNECTION=208.58.214.56 42828 198.202.100.14 22 +SPACK_PYTHON=/usr/bin/python3 +SLURM_PRIO_PROCESS=0 +LANG=en_US.UTF-8 +SLURM_SUBMIT_DIR=/home/mkandes/.spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs +HISTCONTROL=ignoredups +SHARED_SPACK_INSTANCE_VERSION=b +SHARED_SPACK_ROOT=/cm/shared/apps/spack/0.17.3/cpu/b +HOSTNAME=exp-15-02 +LMOD_SYSTEM_DEFAULT_MODULES=DefaultModules +OLDPWD=/home/mkandes/.spack/0.17.3/cpu/b/etc/spack +__LMOD_REF_COUNT__LMFILES_=/cm/local/modulefiles/slurm/expanse/21.08.8:1 +SLURM_CPUS_PER_TASK=16 +SPACK_DEPENDENCIES=^openjdk@11.0.12_7/27cv2ps +ENVIRONMENT=BATCH +SPACK_VERSION=0.17.3 +SLURM_PROCID=0 +SPACK_INSTANCE_VERSION=b +SLURM_JOB_GID=300 +SPACK_VARIANTS=+hadoop +SPACK_INSTANCE_DIR=/cm/shared/apps/spack/0.17.3/cpu/b +__LMOD_REF_COUNT_LD_LIBRARY_PATH=/cm/shared/apps/slurm/current/lib64/slurm:1;/cm/shared/apps/slurm/current/lib64:1 +SLURM_JOB_SCRIPT=/home/mkandes/.spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/spark@3.4.0.sh +SLURMD_NODENAME=exp-15-02 +SLURM_TASKS_PER_NODE=1 +S_COLORS=auto +_CE_M= +which_declare=declare -f +SPACK_COMPILER=gcc@10.2.0 +XDG_SESSION_ID=179391 +LOCAL_SPACK_ROOT=/home/mkandes/.spack/0.17.3/cpu/b/ +SLURM_NNODES=1 +USER=mkandes +SHARED_SPACK_VERSION=0.17.3 +SLURM_NTASKS_PER_NODE=1 +__LMOD_REF_COUNT_MODULEPATH=/home/mkandes/.spack/0.17.3/cpu/b/share/spack/lmod/linux-rocky8-x86_64:2;/cm/local/modulefiles:1;/cm/shared/apps/access/modulefiles:1;/etc/modulefiles:1;/usr/share/modulefiles:1;/usr/share/Modules/modulefiles:1;/cm/shared/modulefiles:1 +__LMOD_REF_COUNT_LOADEDMODULES=slurm/expanse/21.08.8:1 +PWD=/home/mkandes/.spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs +ENABLE_LMOD=1 +SLURM_JOB_NODELIST=exp-15-02 +HOME=/home/mkandes +SLURM_CLUSTER_NAME=expanse +CONDA_PYTHON_EXE=/home/mkandes/miniconda3/bin/python +LMOD_COLORIZE=yes +LOCAL_TIME=20230609T081036-0700 +SLURM_NODELIST=exp-15-02 +SSH_CLIENT=208.58.214.56 42828 22 +LMOD_VERSION=8.2.4 +CPATH=/cm/shared/apps/slurm/current/include +SLURM_NTASKS=1 +LMOD_SETTARG_CMD=: +SLURM_JOB_CPUS_PER_NODE=16 +BASH_ENV=/usr/share/lmod/lmod/init/bash +SLURM_TOPOLOGY_ADDR=exp-15-02 +_CE_CONDA= +SLURM_WORKING_CLUSTER=expanse:mgr1:6817:9472:109 +SLURM_JOB_MD5SUM=1511c7944d387d7e35502660c80f7377 /home/mkandes/.spack/0.17.3/cpu/b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/spark@3.4.0.sh +SPACK_PACKAGE=spark@3.4.0 +__LMOD_REF_COUNT_LIBRARY_PATH=/cm/shared/apps/slurm/current/lib64/slurm:1;/cm/shared/apps/slurm/current/lib64:1 +SLURM_JOB_NAME=spark@3.4.0 +TMPDIR=/scratch/mkandes/job_22902088 +LIBRARY_PATH=/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64 +LMOD_sys=Linux +SLURM_JOBID=22902088 +SYSTEM_NAME=expanse +_ModuleTable001_=X01vZHVsZVRhYmxlXz17WyJNVHZlcnNpb24iXT0zLFsiY19yZWJ1aWxkVGltZSJdPTcyMDAuMCxbImNfc2hvcnRUaW1lIl09MC43NjI2NjA5ODAyMjQ2MSxkZXB0aFQ9e30sZmFtaWx5PXt9LG1UPXtzbHVybT17WyJmbiJdPSIvY20vbG9jYWwvbW9kdWxlZmlsZXMvc2x1cm0vZXhwYW5zZS8yMS4wOC44IixbImZ1bGxOYW1lIl09InNsdXJtL2V4cGFuc2UvMjEuMDguOCIsWyJsb2FkT3JkZXIiXT0xLHByb3BUPXt9LFsic3RhY2tEZXB0aCJdPTAsWyJzdGF0dXMiXT0iYWN0aXZlIixbInVzZXJOYW1lIl09InNsdXJtIix9LH0sbXBhdGhBPXsiL2hvbWUvbWthbmRlcy8uc3BhY2svMC4xNy4zL2NwdS9iL3NoYXJlL3NwYWNrL2xtb2QvbGludXgtcm9ja3k4LXg4Nl82NCIsIi9jbS9s +SLURM_CONF=/cm/shared/apps/slurm/var/etc/expanse/slurm.conf +LOADEDMODULES=slurm/expanse/21.08.8 +__LMOD_REF_COUNT_MANPATH=/cm/shared/apps/slurm/current/man:2;/usr/share/lmod/lmod/share/man:1;/usr/local/share/man:1;/usr/share/man:1;/cm/local/apps/environment-modules/current/share/man:1 +LOCAL_SPACK_NAMESPACE=mkandes +_ModuleTable003_=ay8xLjguMF8yNjUtYjAxIl09e1siZm4iXT0iL2NtL3NoYXJlZC9hcHBzL3NwYWNrLzAuMTcuMy9jcHUvYi9zaGFyZS9zcGFjay9sbW9kL2xpbnV4LXJvY2t5OC14ODZfNjQvZ2NjLzEwLjIuMC9vcGVuamRrLzEuOC4wXzI2NS1iMDEvY2dpeXhjYi5sdWEiLFsiZnVsbE5hbWUiXT0ib3Blbmpkay8xLjguMF8yNjUtYjAxL2NnaXl4Y2IiLFsibG9hZE9yZGVyIl09Nyxwcm9wVD17fSxbInN0YWNrRGVwdGgiXT0wLFsic3RhdHVzIl09ImFjdGl2ZSIsWyJ1c2VyTmFtZSJdPSJvcGVuamRrLzEuOC4wXzI2NS1iMDEiLH0sc2RzYz17WyJmbiJdPSIvY20vc2hhcmVkL21vZHVsZWZpbGVzL3Nkc2MvMS4wLmx1YSIsWyJmdWxsTmFtZSJdPSJzZHNjLzEuMCIsWyJsb2FkT3JkZXIiXT00LHBy +SLURM_NODE_ALIASES=(null) +SLURM_JOB_QOS=ind-shared-normal +LMOD_ROOT=/usr/share/lmod +SLURM_TOPOLOGY_ADDR_PATTERN=node +SSH_TTY=/dev/pts/199 +MAIL=/var/spool/mail/mkandes +SLURM_CPUS_ON_NODE=16 +LMOD_arch=x86_64 +SLURM_JOB_NUM_NODES=1 +__Init_Default_Modules=1 +CMD_WLM_CLUSTER_NAME=expanse +SLURM_MEM_PER_NODE=32768 +SPACK_ROOT=/cm/shared/apps/spack/0.17.3/cpu/b +SHELL=/bin/bash +TERM=xterm-256color +SLURM_JOB_UID=501506 +_ModuleTable_Sz_=2 +__LMOD_REF_COUNT_CPATH=/cm/shared/apps/slurm/current/include:1 +SLURM_JOB_PARTITION=ind-shared +SHARED_SPACK_INSTANCE_NAME=cpu +SPACK_INSTANCE_NAME=cpu +SLURM_JOB_USER=mkandes +SLURM_NPROCS=1 +SHLVL=2 +SLURM_SUBMIT_HOST=login02 +UNIX_TIME=1686323436 +SLURM_JOB_ACCOUNT=use300 +MANPATH=/cm/shared/apps/slurm/current/man:/usr/share/lmod/lmod/share/man:/cm/shared/apps/slurm/current/man:/home/mkandes/miniconda3/man:/usr/local/share/man:/usr/share/man:/usr/local/share/man:/usr/share/man:/cm/local/apps/environment-modules/current/share/man:/cm/local/apps/environment-modules/current/share/man +SPACK_LD_LIBRARY_PATH=/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/gcc-10.2.0/openjdk-1.8.0_265-b01-cgiyxcb6jjt2m35gpsay4gciaivyoyrv/lib:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/gcc-10.2.0-npcyll4gxjhf4tejksmdzlsl3d3usqpd/lib64:/cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/gcc-10.2.0-npcyll4gxjhf4tejksmdzlsl3d3usqpd/lib:/cm/shared/apps/slurm/current/lib64/slurm:/cm/shared/apps/slurm/current/lib64 +LMOD_PREPEND_BLOCK=normal +MODULEPATH=/home/mkandes/.spack/0.17.3/cpu/b/share/spack/lmod/linux-rocky8-x86_64:/cm/local/modulefiles:/cm/shared/apps/access/modulefiles:/etc/modulefiles:/usr/share/modulefiles:/usr/share/Modules/modulefiles:/cm/shared/modulefiles +SLURM_GTIDS=0 +LOGNAME=mkandes +DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/501506/bus +XDG_RUNTIME_DIR=/run/user/501506 +MODULEPATH_ROOT=/usr/share/modulefiles +PATH=/cm/shared/apps/slurm/current/sbin:/cm/shared/apps/slurm/current/bin:/cm/shared/apps/spack/0.17.3/cpu/b/bin:/home/mkandes/miniconda3/condabin:/home/mkandes/.local/bin:/home/mkandes/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin +SLURM_JOB_ID=22902088 +_LMFILES_=/cm/local/modulefiles/slurm/expanse/21.08.8 +DEBUGINFOD_URLS=https://debuginfod.centos.org/ +SCHEDULER_MODULE=slurm +MODULESHOME=/usr/share/lmod/lmod +LMOD_SETTARG_FULL_SUPPORT=no +HISTSIZE=1000 +LMOD_PKG=/usr/share/lmod/lmod +SPACK_SPEC=spark@3.4.0 % gcc@10.2.0 +hadoop ^openjdk@11.0.12_7/27cv2ps +LMOD_CMD=/usr/share/lmod/lmod/libexec/lmod +SLURM_LOCALID=0 +LESSOPEN=||/usr/bin/lesspipe.sh %s +LMOD_FULL_SETTARG_SUPPORT=no +LMOD_DIR=/usr/share/lmod/lmod/libexec +BASH_FUNC_which%%=() { ( alias; + eval ${which_declare} ) | /usr/bin/which --tty-only --read-alias --read-functions --show-tilde --show-dot $@ +} +BASH_FUNC_module%%=() { eval $($LMOD_CMD bash "$@") && eval $(${LMOD_SETTARG_CMD:-:} -s sh) +} +BASH_FUNC_spack%%=() { : this is a shell function from: /cm/shared/apps/spack/0.17.3/cpu/b/share/spack/setup-env.sh; + : the real spack script is here: /cm/shared/apps/spack/0.17.3/cpu/b/bin/spack; + _spack_shell_wrapper "$@"; + return $? +} +BASH_FUNC__spack_shell_wrapper%%=() { for var in LD_LIBRARY_PATH DYLD_LIBRARY_PATH DYLD_FALLBACK_LIBRARY_PATH; + do + eval "if [ -n \"\${${var}-}\" ]; then export SPACK_$var=\${${var}}; fi"; + done; + if [ -n "${ZSH_VERSION:-}" ]; then + emulate -L sh; + fi; + _sp_flags=""; + while [ ! -z ${1+x} ] && [ "${1#-}" != "${1}" ]; do + _sp_flags="$_sp_flags $1"; + shift; + done; + if [ -n "$_sp_flags" ] && [ "${_sp_flags#*h}" != "${_sp_flags}" ] || [ "${_sp_flags#*V}" != "${_sp_flags}" ]; then + command spack $_sp_flags "$@"; + return; + fi; + _sp_subcommand=""; + if [ ! -z ${1+x} ]; then + _sp_subcommand="$1"; + shift; + fi; + case $_sp_subcommand in + "cd") + _sp_arg=""; + if [ -n "$1" ]; then + _sp_arg="$1"; + shift; + fi; + if [ "$_sp_arg" = "-h" ] || [ "$_sp_arg" = "--help" ]; then + command spack cd -h; + else + LOC="$(spack location $_sp_arg "$@")"; + if [ -d "$LOC" ]; then + cd "$LOC"; + else + return 1; + fi; + fi; + return + ;; + "env") + _sp_arg=""; + if [ -n "$1" ]; then + _sp_arg="$1"; + shift; + fi; + if [ "$_sp_arg" = "-h" ] || [ "$_sp_arg" = "--help" ]; then + command spack env -h; + else + case $_sp_arg in + activate) + _a=" $@"; + if [ -z ${1+x} ] || [ "${_a#* --sh}" != "$_a" ] || [ "${_a#* --csh}" != "$_a" ] || [ "${_a#* -h}" != "$_a" ] || [ "${_a#* --help}" != "$_a" ]; then + command spack env activate "$@"; + else + stdout="$(command spack $_sp_flags env activate --sh "$@")" || return; + eval "$stdout"; + fi + ;; + deactivate) + _a=" $@"; + if [ "${_a#* --sh}" != "$_a" ] || [ "${_a#* --csh}" != "$_a" ]; then + command spack env deactivate "$@"; + else + if [ -n "$*" ]; then + command spack env deactivate -h; + else + stdout="$(command spack $_sp_flags env deactivate --sh)" || return; + eval "$stdout"; + fi; + fi + ;; + *) + command spack env $_sp_arg "$@" + ;; + esac; + fi; + return + ;; + "load" | "unload") + _a=" $@"; + if [ "${_a#* --sh}" != "$_a" ] || [ "${_a#* --csh}" != "$_a" ] || [ "${_a#* -h}" != "$_a" ] || [ "${_a#* --list}" != "$_a" ] || [ "${_a#* --help}" != "$_a" ]; then + command spack $_sp_flags $_sp_subcommand "$@"; + else + stdout="$(command spack $_sp_flags $_sp_subcommand --sh "$@")" || return; + eval "$stdout"; + fi + ;; + *) + command spack $_sp_flags $_sp_subcommand "$@" + ;; + esac +} +BASH_FUNC_ml%%=() { eval $($LMOD_DIR/ml_cmd "$@") +} +_=/usr/bin/printenv +compilers: +- compiler: + spec: gcc@8.5.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: + cflags: -O2 -march=native + cxxflags: -O2 -march=native + fflags: -O2 -march=native + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@10.2.0 + paths: + cc: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/gcc-10.2.0-npcyll4gxjhf4tejksmdzlsl3d3usqpd/bin/gcc + cxx: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/gcc-10.2.0-npcyll4gxjhf4tejksmdzlsl3d3usqpd/bin/g++ + f77: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/gcc-10.2.0-npcyll4gxjhf4tejksmdzlsl3d3usqpd/bin/gfortran + fc: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/gcc-10.2.0-npcyll4gxjhf4tejksmdzlsl3d3usqpd/bin/gfortran + flags: + cflags: -O2 -march=native + cxxflags: -O2 -march=native + fflags: -O2 -march=native + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: aocc@3.2.0 + paths: + cc: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/aocc-3.2.0-io3s466wsnnichqc2o2rikbuloev5bmq/bin/clang + cxx: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/aocc-3.2.0-io3s466wsnnichqc2o2rikbuloev5bmq/bin/clang++ + f77: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/aocc-3.2.0-io3s466wsnnichqc2o2rikbuloev5bmq/bin/flang + fc: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/aocc-3.2.0-io3s466wsnnichqc2o2rikbuloev5bmq/bin/flang + flags: + cflags: -O2 -march=native + cxxflags: -O2 -march=native + fflags: -O2 -march=native + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: intel@19.1.3.304 + paths: + cc: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin/icc + cxx: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin/icpc + f77: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin/ifort + fc: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen/gcc-8.5.0/intel-19.1.3.304-6pv46sozip5o35ezrswc4p77gsiawktn/bin/ifort + flags: + cflags: -O2 -march=native + cxxflags: -O2 -march=native + fflags: -O2 -march=native + operating_system: rocky8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +config: + install_tree: + root: /home/mkandes/.spack/0.17.3/cpu/b//opt/spack + projections: + all: ${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH} + template_dirs: + - /cm/shared/apps/spack/0.17.3/cpu/b/share/spack/templates + + - $spack/share/spack/templates + # Temporary locations Spack can try to use for builds. + # + # Recommended options are given below. + # + # Builds can be faster in temporary directories on some (e.g., HPC) systems. + # Specifying `$tempdir` will ensure use of the default temporary directory + # (i.e., ``$TMP` or ``$TMPDIR``). + # + # Another option that prevents conflicts and potential permission issues is + # to specify `$user_cache_path/stage`, which ensures each user builds in their + # home directory. + # + # A more traditional path uses the value of `$spack/var/spack/stage`, which + # builds directly inside Spack's instance without staging them in a + # temporary space. Problems with specifying a path inside a Spack instance + # are that it precludes its use as a system package and its ability to be + # pip installable. + # + # In any case, if the username is not already in the path, Spack will append + # the value of `$user` in an attempt to avoid potential conflicts between + # users in shared temporary spaces. + # + # The build stage can be purged with `spack clean --stage` and + # `spack clean -a`, so it is important that the specified directory uniquely + # identifies Spack staging to avoid accidentally wiping out non-Spack work. + module_roots: + tcl: /home/mkandes/.spack/0.17.3/cpu/b//share/spack/modules + lmod: /home/mkandes/.spack/0.17.3/cpu/b//share/spack/lmod + build_stage: + - /home/mkandes/.spack/0.17.3/cpu/b//var/spack/stage + - /tmp/mkandes/spack-stage + - $tempdir/$user/spack-stage + - $user_cache_path/stage + # - $spack/var/spack/stage + + # Directory in which to run tests and store test results. + # Tests will be stored in directories named by date/time and package + # name/hash. + source_cache: /home/mkandes/.spack/0.17.3/cpu/b//var/spack/cache + misc_cache: ~/.spack/cache + connect_timeout: 10 + verify_ssl: true + suppress_gpg_warnings: false + install_missing_compilers: false + checksum: true + dirty: false + build_language: C + locks: true + build_jobs: 1 + ccache: false + db_lock_timeout: 3 + package_lock_timeout: null + shared_linking: rpath + allow_sgid: true + test_stage: $user_cache_path/test + deprecated: false + url_fetch_method: urllib + concretizer: clingo + terminal_title: false + debug: false +mirrors: + spack-public: https://mirror.spack.io +modules: + default: + 'enable:': + - lmod + lmod: + core_compilers: + - gcc@8.5.0 + hierarchy: + - mpi + hash_length: 0 + blacklist_implicits: true + whitelist: [py-jupyter-core] + naming_scheme: '{name}/{version}/{hash:7}' + projections: + all: '{name}/{version}/{hash:7}' + all: + suffixes: + +openmp: omp + threads=openmp: omp + +ipl64: i64 + environment: + set: + '{name}HOME': '{prefix}' + intel: + environment: + set: + INTEL_LICENSE_FILE: 40000@elprado.sdsc.edu:40200@elprado.sdsc.edu + petsc: + suffixes: + +complex: cmplx + slepc: + suffixes: + ^petsc +complex: cmplx + ^python: + autoload: direct + ^ucx: + autoload: direct + prefix_inspections: + lib: + - LD_LIBRARY_PATH + lib64: + - LD_LIBRARY_PATH + bin: + - PATH + man: + - MANPATH + share/man: + - MANPATH + share/aclocal: + - ACLOCAL_PATH + lib/pkgconfig: + - PKG_CONFIG_PATH + lib64/pkgconfig: + - PKG_CONFIG_PATH + share/pkgconfig: + - PKG_CONFIG_PATH + ? '' + : - CMAKE_PREFIX_PATH + + # These are configurations for the module set named "default" +packages: + lmod: + externals: + - spec: lmod@8.2.4 + prefix: /usr + buildable: false + lustre: + externals: + - spec: lustre@2.15.2 + prefix: /usr + buildable: false + openssl: + externals: + - spec: openssl@1.1.1k + prefix: /usr + buildable: false + rdma-core: + externals: + - spec: rdma-core@43.0 + prefix: /usr + buildable: false + slurm: + externals: + - spec: slurm@21.08.8 + prefix: /cm/shared/apps/slurm/21.08.8 + buildable: false + librsvg: + externals: + - spec: librsvg@2.42.7 + prefix: /usr + buildable: false + ghostscript: + externals: + - spec: ghostscript@9.27 + prefix: /usr + buildable: false + gaussian: + permissions: + read: group + group: gaussian + vasp6: + permissions: + read: group + group: vasp-6 + all: + compiler: [gcc, intel, pgi, clang, xl, nag, fj, aocc] + providers: + awk: [gawk] + blas: [openblas, amdblis] + D: [ldc] + daal: [intel-daal] + elf: [elfutils] + fftw-api: [fftw, amdfftw] + flame: [libflame, amdlibflame] + fuse: [libfuse] + gl: [mesa+opengl, mesa18, opengl] + glu: [mesa-glu, openglu] + glx: [mesa+glx, mesa18+glx, opengl] + golang: [gcc] + iconv: [libiconv] + ipp: [intel-ipp] + java: [openjdk, jdk, ibm-java] + jpeg: [libjpeg-turbo, libjpeg] + lapack: [openblas, amdlibflame] + lua-lang: [lua, lua-luajit] + mariadb-client: [mariadb-c-client, mariadb] + mkl: [intel-mkl] + mpe: [mpe2] + mpi: [openmpi, mpich] + mysql-client: [mysql, mariadb-c-client] + opencl: [pocl] + onedal: [intel-oneapi-dal] + osmesa: [mesa+osmesa, mesa18+osmesa] + pbs: [openpbs, torque] + pil: [py-pillow] + pkgconfig: [pkgconf, pkg-config] + rpc: [libtirpc] + scalapack: [netlib-scalapack, amdscalapack] + sycl: [hipsycl] + szip: [libaec, libszip] + tbb: [intel-tbb] + unwind: [libunwind] + uuid: [util-linux-uuid, libuuid] + xxd: [xxd-standalone, vim] + yacc: [bison, byacc] + ziglang: [zig] + permissions: + read: world + write: user +repos: +- /home/mkandes/.spack/0.17.3/cpu/b//var/spack/repos/mkandes +- $spack/var/spack/repos/sdsc +- $spack/var/spack/repos/builtin +upstreams: + spack-instance-1: + install_tree: /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack +Input spec +-------------------------------- +[ ] .spark@3.4.0%gcc@10.2.0+hadoop +[ ] ^builtin.openjdk@11.0.12_7%gcc@10.2.0 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" arch=linux-rocky8-zen2 + +Concretized +-------------------------------- +po6mvtn [ ] mkandes.spark@3.4.0%gcc@10.2.0 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" +hadoop arch=linux-rocky8-zen2 +bfyu354 [b r ] ^builtin.hadoop@3.3.0%gcc@10.2.0 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" arch=linux-rocky8-zen2 +27cv2ps [b r ] ^builtin.openjdk@11.0.12_7%gcc@10.2.0 cflags="-O2 -march=native" cxxflags="-O2 -march=native" fflags="-O2 -march=native" arch=linux-rocky8-zen2 + +real 12.53 +user 10.94 +sys 0.31 +[+] /cm/shared/apps/spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/gcc-10.2.0/openjdk-11.0.12_7-27cv2psb6xthjbwmjlx53otdbjxch6wg +==> Installing hadoop-3.3.0-bfyu354o4z5trshyxr6zkg463zdoat2f +==> No binary for hadoop-3.3.0-bfyu354o4z5trshyxr6zkg463zdoat2f found: installing from source +==> Using cached archive: /home/mkandes/.spack/0.17.3/cpu/b/var/spack/cache/_source-cache/archive/ea/ea1a0f0afcdfb9b6b9d261cdce5a99023d7e8f72d26409e87f69bda65c663688.tar.gz +==> No patches needed for hadoop +==> hadoop: Executing phase: 'install' +==> hadoop: Successfully installed hadoop-3.3.0-bfyu354o4z5trshyxr6zkg463zdoat2f + Fetch: 1.00s. Build: 33m 55.12s. Total: 33m 56.12s. +[+] /home/mkandes/.spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/gcc-10.2.0/hadoop-3.3.0-bfyu354o4z5trshyxr6zkg463zdoat2f +==> Installing spark-3.4.0-po6mvtnx2dirat72biironfv4ikcw5m2 +==> No binary for spark-3.4.0-po6mvtnx2dirat72biironfv4ikcw5m2 found: installing from source +==> Using cached archive: /home/mkandes/.spack/0.17.3/cpu/b/var/spack/cache/_source-cache/archive/6c/6c0366139dab03f9bfb788b27918e7e3d411adcfa5631de75ac91afac87fcf5f.tgz +==> No patches needed for spark +==> spark: Executing phase: 'install' +==> spark: Successfully installed spark-3.4.0-po6mvtnx2dirat72biironfv4ikcw5m2 + Fetch: 0.57s. Build: 1m 43.23s. Total: 1m 43.80s. +[+] /home/mkandes/.spack/0.17.3/cpu/b/opt/spack/linux-rocky8-zen2/gcc-10.2.0/spark-3.4.0-po6mvtnx2dirat72biironfv4ikcw5m2 +real 2787.18 +user 24.57 +sys 13.98 +==> Regenerating lmod module files diff --git a/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/spark@3.4.0.sh b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/spark@3.4.0.sh new file mode 100644 index 00000000000000..52343294133adf --- /dev/null +++ b/etc/spack/sdsc/expanse/0.17.3/cpu/b/specs/gcc@10.2.0/spark@3.4.0.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +#SBATCH --job-name=spark@3.4.0 +#SBATCH --account=use300 +##SBATCH --reservation=rocky8u7_testing +#SBATCH --partition=ind-shared +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=16 +#SBATCH --mem=32G +#SBATCH --time=01:00:00 +#SBATCH --output=%x.o%j.%N + +declare -xr LOCAL_TIME="$(date +'%Y%m%dT%H%M%S%z')" +declare -xir UNIX_TIME="$(date +'%s')" + +declare -xr LOCAL_SCRATCH_DIR="/scratch/${USER}/job_${SLURM_JOB_ID}" +declare -xr TMPDIR="${LOCAL_SCRATCH_DIR}" + +declare -xr SYSTEM_NAME='expanse' + +declare -xr SPACK_VERSION='0.17.3' +declare -xr SPACK_INSTANCE_NAME='cpu' +declare -xr SPACK_INSTANCE_VERSION='b' +declare -xr SPACK_INSTANCE_DIR="/cm/shared/apps/spack/${SPACK_VERSION}/${SPACK_INSTANCE_NAME}/${SPACK_INSTANCE_VERSION}" + +declare -xr SLURM_JOB_SCRIPT="$(scontrol show job ${SLURM_JOB_ID} | awk -F= '/Command=/{print $2}')" +declare -xr SLURM_JOB_MD5SUM="$(md5sum ${SLURM_JOB_SCRIPT})" + +declare -xr SCHEDULER_MODULE='slurm' + +echo "${UNIX_TIME} ${SLURM_JOB_ID} ${SLURM_JOB_MD5SUM} ${SLURM_JOB_DEPENDENCY}" +echo "" + +cat "${SLURM_JOB_SCRIPT}" + +module purge +module load "${SCHEDULER_MODULE}" +module list +. "${SPACK_INSTANCE_DIR}/share/spack/setup-env.sh" +shopt -s expand_aliases +source ~/.bashrc + +declare -xr SPACK_PACKAGE='spark@3.4.0' +declare -xr SPACK_COMPILER='gcc@10.2.0' +declare -xr SPACK_VARIANTS='+hadoop' +declare -xr SPACK_DEPENDENCIES="^openjdk@11.0.12_7/$(spack find --format '{hash:7}' openjdk@11.0.12_7 % ${SPACK_COMPILER})" +declare -xr SPACK_SPEC="${SPACK_PACKAGE} % ${SPACK_COMPILER} ${SPACK_VARIANTS} ${SPACK_DEPENDENCIES}" + +printenv + +spack config get compilers +spack config get config +spack config get mirrors +spack config get modules +spack config get packages +spack config get repos +spack config get upstreams + +time -p spack spec --long --namespaces --types --reuse "${SPACK_SPEC}" +if [[ "${?}" -ne 0 ]]; then + echo 'ERROR: spack concretization failed.' + exit 1 +fi + +time -p spack install --jobs "${SLURM_CPUS_PER_TASK}" --fail-fast --yes-to-all --reuse "${SPACK_SPEC}" +if [[ "${?}" -ne 0 ]]; then + echo 'ERROR: spack install failed.' + exit 1 +fi + +spack module lmod refresh -y + +#sbatch --dependency="afterok:${SLURM_JOB_ID}" '' + +sleep 30