From 58a599349f130cfafd38ff6ffe9c7a6c2bdd11ca Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Mon, 6 Jan 2025 20:47:35 -0500 Subject: [PATCH 1/5] Changes to build UFS-WM on MacOS platform with clang@15/gfortran@13.3 --- build.sh | 4 +++ modulefiles/ufs_macosx.gnu | 24 ------------- modulefiles/ufs_macosx.gnu.lua | 65 ++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 24 deletions(-) delete mode 100644 modulefiles/ufs_macosx.gnu create mode 100644 modulefiles/ufs_macosx.gnu.lua diff --git a/build.sh b/build.sh index 3f82a882da..f9f5189767 100755 --- a/build.sh +++ b/build.sh @@ -13,6 +13,10 @@ export FC=${FC:-mpif90} BUILD_DIR=${BUILD_DIR:-${UFS_MODEL_DIR}/build} mkdir -p "${BUILD_DIR}" +if [[ "$(uname -s)" == "Darwin" ]]; then + gsed -i'.backup' "s:LINKER_LANGUAGE Fortran:LINKER_LANGUAGE CXX:" ./CMakeLists.txt +fi + cd "${BUILD_DIR}" ARR_CMAKE_FLAGS=() for i in ${CMAKE_FLAGS}; do ARR_CMAKE_FLAGS+=("${i}") ; done diff --git a/modulefiles/ufs_macosx.gnu b/modulefiles/ufs_macosx.gnu deleted file mode 100644 index 961d673b67..0000000000 --- a/modulefiles/ufs_macosx.gnu +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -#%Module###################################################################### -## -## UFS prerequisites: macOS with gcc/gfortran or clang/gfortran compilers - -echo "Setting environment variables for UFS Model on macOS with gcc/gfortran or clang/gfortran" - -## -## load programming environment: compiler, flags, paths -## -export CC=${MPICC:-mpicc} -export CXX=${MPICXX:-mpicxx} -export F77=${MPIF77:-mpif77} -export F90=${MPIF90:-mpif90} -export FC=${MPIFORT:-mpifort} -export CPP=${CPP:-"${F90} -E -x f95-cpp-input"} -export MPICC=${MPICC:-mpicc} -export MPIF90=${MPIF90:-mpif90} - -## -## load cmake -## -export CMAKE_Platform=macosx.gnu diff --git a/modulefiles/ufs_macosx.gnu.lua b/modulefiles/ufs_macosx.gnu.lua new file mode 100644 index 0000000000..62a39b0a9c --- /dev/null +++ b/modulefiles/ufs_macosx.gnu.lua @@ -0,0 +1,65 @@ +help([[ +loads UFS Model prerequisites for MacOS clang/gcc ("gnu") +]]) + +prepend_path("MODULEPATH", "/Users/username/spack-stack/spack-stack-1.8.0/envs/ufs-srw-env/install/modulefiles/Core") + +stack_gnu_ver=os.getenv("stack_apple_clang_ver") or "15.0.0" +load(pathJoin("stack-apple-clang", stack_gnu_ver)) + +stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "5.0.3" +load(pathJoin("stack-openmpi", stack_openmpi_ver)) + +cmake_ver=os.getenv("cmake_ver") or "3.27.9" +load(pathJoin("cmake", cmake_ver)) + +local ufs_modules = { + {["jasper"] = "2.0.32"}, + {["zlib"] = "1.2.13"}, + {["libpng"] = "1.6.37"}, + {["hdf5"] = "1.14.0"}, + {["netcdf-c"] = "4.9.2"}, + {["netcdf-fortran"] = "4.6.1"}, + {["parallelio"] = "2.6.2"}, + {["esmf"] = "8.6.0"}, + {["fms"] = "2024.02"}, + {["bacio"] = "2.4.1"}, + {["crtm"] = "2.4.0.1"}, + {["g2"] = "3.5.1"}, + {["g2tmpl"] = "1.13.0"}, + {["ip"] = "5.0.0"}, + {["sp"] = "2.5.0"}, + {["w3emc"] = "2.10.0"}, + {["gftl-shared"] = "1.9.0"}, + {["mapl"] = "2.40.3-esmf-8.6.0"}, + {["scotch"] = "7.0.4"}, +} + +for i = 1, #ufs_modules do + for name, default_version in pairs(ufs_modules[i]) do + local env_version_name = string.gsub(name, "-", "_") .. "_ver" + load(pathJoin(name, os.getenv(env_version_name) or default_version)) + end +end + +nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" +load(pathJoin("nccmp", nccmp_ver)) + +setenv("CC", "mpicc") +setenv("CXX", "mpicxx") +setenv("FC", "mpifort") +setenv("CMAKE_Platform", "macosx.gnu") +setenv("CMAKE_Fortran_COMPILER_ID", "GNU") + +osx_sysroot=os.getenv("OSX_SYSROOT") +setenv("CMAKE_OSX_SYSROOT","OSX_SYSROOT") + +setenv("CFLAGS"," -Wno-implicit-function-declaration ") + +if mode() == "load" then + LmodMsgRaw([===[ + Please export these env. variables after the module is successfully loaded: + > export LDFLAGS+=" -L${libjpeg_turbo_ROOT}/lib -ljpeg -Wl,-rpath,$libjpeg_turbo_ROOT}/lib -L${jasper_ROOT}/lib -ljasper -Wl,-rpath,${jasper_ROOT}/lib -L${libpng_ROOT}/lib -lpng -Wl,-rpath,${libpng_ROOT}/lib " + ]===]) +end +whatis("Description: UFS build environment") From fa6dccc94e6ca1cea46a9bf5ed58e1d9f5c21746 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Tue, 7 Jan 2025 16:14:58 -0500 Subject: [PATCH 2/5] CMakeLists.txt update for macosx.gnu platform no changes from develop branch for the build.sh script --- CMakeLists.txt | 7 ++++++- build.sh | 4 ---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 702cf6a459..2d962d0417 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -377,7 +377,12 @@ target_link_libraries(ufs PUBLIC "${_ufs_libs_public}") add_executable(ufs_model driver/UFS.F90) add_dependencies(ufs_model ufs) target_link_libraries(ufs_model ufs esmf w3emc::w3emc_d) -set_target_properties(ufs_model PROPERTIES LINKER_LANGUAGE Fortran) + +if(CMAKE_Platform MATCHES "macosx.gnu") + set_target_properties(ufs_model PROPERTIES LINKER_LANGUAGE CXX) +else() + set_target_properties(ufs_model PROPERTIES LINKER_LANGUAGE Fortran) +endif() ############################################################################### ### Install diff --git a/build.sh b/build.sh index f9f5189767..3f82a882da 100755 --- a/build.sh +++ b/build.sh @@ -13,10 +13,6 @@ export FC=${FC:-mpif90} BUILD_DIR=${BUILD_DIR:-${UFS_MODEL_DIR}/build} mkdir -p "${BUILD_DIR}" -if [[ "$(uname -s)" == "Darwin" ]]; then - gsed -i'.backup' "s:LINKER_LANGUAGE Fortran:LINKER_LANGUAGE CXX:" ./CMakeLists.txt -fi - cd "${BUILD_DIR}" ARR_CMAKE_FLAGS=() for i in ${CMAKE_FLAGS}; do ARR_CMAKE_FLAGS+=("${i}") ; done From 816a4cb82037a089bcb419fb98f47886c5c304ae Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Mon, 13 Jan 2025 16:19:43 -0500 Subject: [PATCH 3/5] Update ufs_macosx.gnu.lua modulefile and building scripts for MacOS --- modulefiles/ufs_macosx.gnu.lua | 29 ++++++++++++++++++----------- tests/compile.sh | 2 +- tests/default_vars.sh | 25 +++++++++++++++++++++++++ tests/detect_machine.sh | 5 +++++ tests/opnReqTest | 11 +++++++++++ 5 files changed, 60 insertions(+), 12 deletions(-) diff --git a/modulefiles/ufs_macosx.gnu.lua b/modulefiles/ufs_macosx.gnu.lua index 62a39b0a9c..f2f0725807 100644 --- a/modulefiles/ufs_macosx.gnu.lua +++ b/modulefiles/ufs_macosx.gnu.lua @@ -1,13 +1,15 @@ help([[ -loads UFS Model prerequisites for MacOS clang/gcc ("gnu") +loads UFS Model modules for MacOSX ]]) - -prepend_path("MODULEPATH", "/Users/username/spack-stack/spack-stack-1.8.0/envs/ufs-srw-env/install/modulefiles/Core") +-- Replace the stackpath below by the path of the local spack-stack environment build: +local stackpath = "/Users/username/spack-stack/spack-stack-1.8.0/envs/ufs-srw-env" +local modulepath = stackpath .. "/install/modulefiles/Core" +prepend_path("MODULEPATH", modulepath) stack_gnu_ver=os.getenv("stack_apple_clang_ver") or "15.0.0" load(pathJoin("stack-apple-clang", stack_gnu_ver)) -stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "5.0.3" +stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.6" load(pathJoin("stack-openmpi", stack_openmpi_ver)) cmake_ver=os.getenv("cmake_ver") or "3.27.9" @@ -54,12 +56,17 @@ setenv("CMAKE_Fortran_COMPILER_ID", "GNU") osx_sysroot=os.getenv("OSX_SYSROOT") setenv("CMAKE_OSX_SYSROOT","OSX_SYSROOT") -setenv("CFLAGS"," -Wno-implicit-function-declaration ") -if mode() == "load" then - LmodMsgRaw([===[ - Please export these env. variables after the module is successfully loaded: - > export LDFLAGS+=" -L${libjpeg_turbo_ROOT}/lib -ljpeg -Wl,-rpath,$libjpeg_turbo_ROOT}/lib -L${jasper_ROOT}/lib -ljasper -Wl,-rpath,${jasper_ROOT}/lib -L${libpng_ROOT}/lib -lpng -Wl,-rpath,${libpng_ROOT}/lib " - ]===]) +local libjpeg_ROOT = os.getenv("libjpeg_turbo_ROOT") +local jasper_ROOT = os.getenv("jasper_ROOT") +local libpng_ROOT = os.getenv("libpng_ROOT") +local ldflags0 = os.getenv("LDFLAGS") or "" +local ldflags_add = " -Wl,-no_compact_unwind" + +if jasper_ROOT and libpng_ROOT and libjpeg_ROOT then + local ldflags1 = " -L" .. libjpeg_ROOT .. "/lib -ljpeg -Wl,-rpath," .. libjpeg_ROOT .. "/lib" + local ldflags2 = " -L" .. jasper_ROOT .. "/lib -ljasper -Wl,-rpath," .. jasper_ROOT .. "/lib" + local ldflags3 = " -L" .. libpng_ROOT .. "/lib -lpng -Wl,-rpath," .. libpng_ROOT .. "/lib" + local ldflags = ldflags0 .. ldflags_add .. ldflags1 .. ldflags2 .. ldflags3 + setenv("LDFLAGS", ldflags) end -whatis("Description: UFS build environment") diff --git a/tests/compile.sh b/tests/compile.sh index 2c00304e28..0fc9741ab7 100755 --- a/tests/compile.sh +++ b/tests/compile.sh @@ -56,7 +56,7 @@ BUILD_JOBS=${BUILD_JOBS:-8} set +x case ${MACHINE_ID} in - macosx|linux) + linux) source "${PATHTR}/modulefiles/ufs_${MACHINE_ID}.${RT_COMPILER}" ;; *) diff --git a/tests/default_vars.sh b/tests/default_vars.sh index 81dd617d49..27fe8ac627 100644 --- a/tests/default_vars.sh +++ b/tests/default_vars.sh @@ -230,6 +230,31 @@ elif [[ ${MACHINE_ID} = linux ]]; then export ICE_tasks_cpl_thrd=10 export WAV_tasks_cpl_thrd=12 +elif [[ ${MACHINE_ID} = macosx ]]; then + + export TPN=8 + + export INPES_dflt=3 + export JNPES_dflt=8 + export INPES_thrd=3 + export JNPES_thrd=4 + + export THRD_cpl_dflt=1 + export INPES_cpl_dflt=3 + export JNPES_cpl_dflt=8 + export WPG_cpl_dflt=6 + export OCN_tasks_cpl_dflt=20 + export ICE_tasks_cpl_dflt=10 + export WAV_tasks_cpl_dflt=20 + + export THRD_cpl_thrd=2 + export INPES_cpl_thrd=3 + export JNPES_cpl_thrd=4 + export WPG_cpl_thrd=6 + export OCN_tasks_cpl_thrd=20 + export ICE_tasks_cpl_thrd=10 + export WAV_tasks_cpl_thrd=12 + elif [[ ${MACHINE_ID} = jet ]]; then export TPN=24 diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh index 99419f3555..0342d49089 100755 --- a/tests/detect_machine.sh +++ b/tests/detect_machine.sh @@ -59,6 +59,11 @@ case $(hostname -f) in *) MACHINE_ID=UNKNOWN ;; # Unknown platform esac +# Determine if the system is MacOS/Darwin: +case $(uname -s) in + Darwin) MACHINE_ID=macosx ;; ### MacOS +esac + if [[ ${MACHINE_ID} == "UNKNOWN" ]]; then case ${PW_CSP:-} in "aws" | "google" | "azure") MACHINE_ID=noaacloud ;; diff --git a/tests/opnReqTest b/tests/opnReqTest index f23b85b31a..d3f338452b 100755 --- a/tests/opnReqTest +++ b/tests/opnReqTest @@ -397,6 +397,17 @@ elif [[ $MACHINE_ID = linux ]]; then PTMP=${dprefix}/stmp2 SCHEDULER=none +elif [[ $MACHINE_ID = macosx ]]; then + + PARTITION= + QUEUE= + COMPILE_QUEUE= + dprefix=/Users/username + DISKNM=${dprefix}/data + STMP=${dprefix}/stmp4 + PTMP=${dprefix}/stmp2 + SCHEDULER=none + else error "unknown machine ID. edit detect_machine.sh file" fi From 416346f515f714765abcd795aac46b0c2d49d253 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Mon, 13 Jan 2025 16:34:47 -0500 Subject: [PATCH 4/5] correct ulimit command for MacOS --- tests/run_test.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/run_test.sh b/tests/run_test.sh index cd46804607..bce768738c 100755 --- a/tests/run_test.sh +++ b/tests/run_test.sh @@ -428,7 +428,11 @@ fi ################################################################################ export OMP_ENV=${OMP_ENV:-""} if [[ ${SCHEDULER} = 'none' ]]; then - ulimit -s unlimited + if [[ ${MACHINE_ID} == macosx ]]; then + ulimit -s -S unlimited + else + ulimit -s unlimited + fi if [[ ${CI_TEST} = 'true' ]]; then eval "${OMP_ENV}" redirect_out_err mpiexec -n "${TASKS}" ./fv3.exe else From 9206ca12900eefe5b8b2c0a612a595ed8d886faf Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Thu, 16 Jan 2025 15:08:24 +0000 Subject: [PATCH 5/5] updated ufs_macosx.gnu.lua ldflags list --- modulefiles/ufs_macosx.gnu.lua | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modulefiles/ufs_macosx.gnu.lua b/modulefiles/ufs_macosx.gnu.lua index f2f0725807..e05013a03e 100644 --- a/modulefiles/ufs_macosx.gnu.lua +++ b/modulefiles/ufs_macosx.gnu.lua @@ -9,7 +9,7 @@ prepend_path("MODULEPATH", modulepath) stack_gnu_ver=os.getenv("stack_apple_clang_ver") or "15.0.0" load(pathJoin("stack-apple-clang", stack_gnu_ver)) -stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.6" +stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "5.0.3" load(pathJoin("stack-openmpi", stack_openmpi_ver)) cmake_ver=os.getenv("cmake_ver") or "3.27.9" @@ -19,7 +19,7 @@ local ufs_modules = { {["jasper"] = "2.0.32"}, {["zlib"] = "1.2.13"}, {["libpng"] = "1.6.37"}, - {["hdf5"] = "1.14.0"}, + {["hdf5"] = "1.14.3"}, {["netcdf-c"] = "4.9.2"}, {["netcdf-fortran"] = "4.6.1"}, {["parallelio"] = "2.6.2"}, @@ -61,12 +61,11 @@ local libjpeg_ROOT = os.getenv("libjpeg_turbo_ROOT") local jasper_ROOT = os.getenv("jasper_ROOT") local libpng_ROOT = os.getenv("libpng_ROOT") local ldflags0 = os.getenv("LDFLAGS") or "" -local ldflags_add = " -Wl,-no_compact_unwind" if jasper_ROOT and libpng_ROOT and libjpeg_ROOT then local ldflags1 = " -L" .. libjpeg_ROOT .. "/lib -ljpeg -Wl,-rpath," .. libjpeg_ROOT .. "/lib" local ldflags2 = " -L" .. jasper_ROOT .. "/lib -ljasper -Wl,-rpath," .. jasper_ROOT .. "/lib" local ldflags3 = " -L" .. libpng_ROOT .. "/lib -lpng -Wl,-rpath," .. libpng_ROOT .. "/lib" - local ldflags = ldflags0 .. ldflags_add .. ldflags1 .. ldflags2 .. ldflags3 + local ldflags = ldflags0 .. ldflags1 .. ldflags2 .. ldflags3 setenv("LDFLAGS", ldflags) end