Merge remote-tracking branch 'origin/develop' into feature/experiment…

…al_obs_path * origin/develop: Update global atmos upp job to use COMIN/COMOUT (NOAA-EMC#2867) Update config.resources for bufr sounding job postsnd (NOAA-EMC#2917) Cleanup job for GEFS (NOAA-EMC#2919) Build GDASApp and unset memory in Gaea-C5 xml files (NOAA-EMC#2912) add 1 deg ocean/ice info to parm/config/gfs/config.resources (NOAA-EMC#2922) Support gefs C48 on Azure (NOAA-EMC#2881) Disable native grid writes for non-JEDI experiments; update C384 compression options (NOAA-EMC#2914)
KateFriedman-NOAA · Sep 17, 2024 · 3357a0d · 3357a0d
2 parents eb717a1 + 2602eac
commit 3357a0d
Show file tree

Hide file tree

Showing 19 changed files with 198 additions and 75 deletions.
diff --git a/env/AZUREPW.env b/env/AZUREPW.env
@@ -43,6 +43,13 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
     unset nnodes ufs_ntasks
 
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+
+    export CFP_MP="YES"
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd_opt}
+
 elif [[ "${step}" = "post" ]]; then
 
     export NTHREADS_NP=${NTHREADS1}
@@ -52,4 +59,45 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
     export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
 
+elif [[ "${step}" = "atmos_products" ]]; then
+
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    export NTHREADS_OCNICEPOST=${NTHREADS1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
+elif [[ "${step}" = "ecen" ]]; then
+
+    export NTHREADS_ECEN=${NTHREADSmax}
+    export APRUN_ECEN="${APRUN}"
+
+    export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
+    [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
+    export APRUN_CHGRES="time"
+
+    export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
+    export APRUN_CALCINC="${APRUN}"
+
+elif [[ "${step}" = "esfc" ]]; then
+
+    export NTHREADS_ESFC=${NTHREADSmax}
+    export APRUN_ESFC="${APRUN}"
+
+    export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
+    [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
+    export APRUN_CYCLE="${APRUN}"
+
+elif [[ "${step}" = "epos" ]]; then
+
+    export NTHREADS_EPOS=${NTHREADSmax}
+    export APRUN_EPOS="${APRUN}"
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    export NTHREADS_FIT2OBS=${NTHREADS1}
+    export MPIRUN="${APRUN}"
+
 fi
diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -272,12 +272,12 @@ elif [[ "${step}" = "postsnd" ]]; then
     export OMP_NUM_THREADS=1
 
     export NTHREADS_POSTSND=${NTHREADS1}
-    export mpmd_opt="-ppn 21 ${mpmd_opt}"
-
     export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1}
     [[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task}
     export APRUN_POSTSNDCFP="${launcher} -np ${ntasks_postsndcfp} ${mpmd_opt}"
 
+    export mpmd_opt="-ppn ${tasks_per_node} ${mpmd_opt}"
+
 elif [[ "${step}" = "awips" ]]; then
 
     export NTHREADS_AWIPS=${NTHREADS1}

diff --git a/jobs/JGLOBAL_ATMOS_UPP b/jobs/JGLOBAL_ATMOS_UPP
@@ -12,8 +12,11 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "upp" -c "base upp"
 ##############################################
 
 # Construct COM variables from templates
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS COM_ATMOS_HISTORY COM_ATMOS_MASTER
-if [[ ! -d ${COM_ATMOS_MASTER} ]]; then mkdir -m 775 -p "${COM_ATMOS_MASTER}"; fi
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+   COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
+   COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \
+   COMOUT_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL
+if [[ ! -d ${COMOUT_ATMOS_MASTER} ]]; then mkdir -p "${COMOUT_ATMOS_MASTER}"; fi
 
 
 ###############################################################

diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
@@ -345,7 +345,7 @@ export DELETE_COM_IN_ARCHIVE_JOB="YES"   # NO=retain ROTDIR.  YES default in arc
 export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9}
 
 # The tracker, genesis, and METplus jobs are not supported on CSPs yet
-# TODO: we should place these in workflow/hosts/[csp]pw.yaml as part of AWS/AZURE/GOOGLE setup, not for general.
+# TODO: we should place these in workflow/hosts/[aws|azure|google]pw.yaml as part of CSP's setup, not for general.
 if [[ "${machine}" =~ "PW" ]]; then
    export DO_WAVE="NO"
 fi

diff --git a/parm/config/gefs/config.cleanup b/parm/config/gefs/config.cleanup
@@ -0,0 +1 @@
+../gfs/config.cleanup
diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources
@@ -43,6 +43,10 @@ case ${machine} in
     export PARTITION_BATCH="compute"
     max_tasks_per_node=36
     ;;
+  "AZUREPW")
+    export PARTITION_BATCH="compute"
+    max_tasks_per_node=24
+    ;;
   "GOOGLEPW")
     export PARTITION_BATCH="compute"
     max_tasks_per_node=32
@@ -291,6 +295,14 @@ case ${step} in
     export threads_per_task=1
     export memory="4096M"
     ;;
+
+  "cleanup")
+    export walltime="00:15:00"
+    export ntasks=1
+    export tasks_per_node=1
+    export threads_per_task=1
+    export memory="4096M"
+    ;;
   *)
     echo "FATAL ERROR: Invalid job ${step} passed to ${BASH_SOURCE[0]}"
     exit 1

diff --git a/parm/config/gefs/config.resources.AZUREPW b/parm/config/gefs/config.resources.AZUREPW
@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+# AZURE-specific job resources
+
+export is_exclusive="True"
+unset memory
+
+# shellcheck disable=SC2312
+for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
+  unset "${mem_var}"
+done
diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs
@@ -80,8 +80,8 @@ case "${fv3_res}" in
         export nthreads_fv3_gfs=1
         export nthreads_ufs=1
         export nthreads_ufs_gfs=1
-	      export xr_cnvcld=.false. # Do not pass conv. clouds to Xu-Randall cloud fraction
-	      export cdmbgwd="0.071,2.1,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
+        export xr_cnvcld=.false. # Do not pass conv. clouds to Xu-Randall cloud fraction
+        export cdmbgwd="0.071,2.1,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="40.0,1.77,1.0,1.0"   # settings for GSL drag suite
         export k_split=1
         export n_split=4
@@ -104,8 +104,8 @@ case "${fv3_res}" in
         export nthreads_fv3_gfs=1
         export nthreads_ufs=1
         export nthreads_ufs_gfs=1
-	      export xr_cnvcld=".false."  # Do not pass conv. clouds to Xu-Randall cloud fraction
-	      export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
+        export xr_cnvcld=".false."  # Do not pass conv. clouds to Xu-Randall cloud fraction
+        export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="20.0,2.5,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=3.0e-3      # setting for UGWPv1 non-stationary GWD
         export k_split=1
@@ -254,40 +254,33 @@ export ntasks_fv3_gfs
 export ntasks_quilt
 export ntasks_quilt_gfs
 
-# Determine whether to use compression in the write grid component based on resolution
+# Determine whether to use compression in the write grid component
+# and whether to use parallel NetCDF based on resolution
 case ${fv3_res} in
-  "C48" | "C96" | "C192" | "C384")
+  "C48" | "C96" | "C192")
     zstandard_level=0
     ideflate=0
     quantize_nsd=0
+    OUTPUT_FILETYPE_ATM="netcdf"
+    OUTPUT_FILETYPE_SFC="netcdf"
     ;;
-  "C768" | "C1152" | "C3072")
+  "C384" | "C768" | "C1152" | "C3072")
     zstandard_level=0
     ideflate=1
     quantize_nsd=5
-    ;;
-  *)
-    echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}"
-    exit 15
-    ;;
-esac
-export zstandard_level ideflate quantize_nsd
-
-# Determine whether to use parallel NetCDF based on resolution
-case ${fv3_res} in
-  "C48" | "C96" | "C192" | "C384")
-    OUTPUT_FILETYPE_ATM="netcdf"
-    OUTPUT_FILETYPE_SFC="netcdf"
-    ;;
-  "C768" | "C1152" | "C3072")
     OUTPUT_FILETYPE_ATM="netcdf_parallel"
-    OUTPUT_FILETYPE_SFC="netcdf_parallel"
+    if [[ "${fv3_res}" == "C384" ]]; then
+      OUTPUT_FILETYPE_SFC="netcdf"  # For C384, the write grid component is better off with serial netcdf
+    else
+      OUTPUT_FILETYPE_SFC="netcdf_parallel"
+    fi
     ;;
   *)
     echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}"
     exit 15
     ;;
 esac
+export zstandard_level ideflate quantize_nsd
 export OUTPUT_FILETYPE_ATM OUTPUT_FILETYPE_SFC
 
 # cpl defaults

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
@@ -527,6 +527,7 @@ case ${step} in
     case ${OCNRES} in
       "025") ntasks=480;;
       "050")  ntasks=16;;
+      "100")  ntasks=16;;
       "500")  ntasks=16;;
       *)
         echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${OCNRES}"
@@ -550,6 +551,10 @@ case ${step} in
         ntasks=16
         memory="96GB"
         ;;
+      "100")
+        ntasks=16
+        memory="96GB"
+        ;;
       "500")
         ntasks=16
         memory="24GB"
@@ -576,6 +581,10 @@ case ${step} in
         ntasks=16
         memory="96GB"
         ;;
+      "100")
+        ntasks=16
+        memory="96GB"
+        ;;
       "500")
         ntasks=16
         memory="24GB"
@@ -602,6 +611,10 @@ case ${step} in
         ntasks=16
         memory="96GB"
         ;;
+      "100")
+        ntasks=16
+        memory="96GB"
+        ;;
       "500")
         ntasks=16
         memory="24GB"
@@ -630,6 +643,9 @@ case ${step} in
       "050")
         memory="32GB"
         ntasks=16;;
+      "100")
+        memory="32GB"
+        ntasks=16;;
       "500")
         memory="32GB"
         ntasks=8;;
@@ -1192,9 +1208,23 @@ case ${step} in
   "postsnd")
     walltime="02:00:00"
     export ntasks=141
-    threads_per_task=6
-    export tasks_per_node=21 
     export ntasks_postsndcfp=9
+    case ${CASE} in
+      "C768")
+        tasks_per_node=21
+        threads_per_task=6
+        memory="23GB"
+        ;;
+      "C1152")
+        tasks_per_node=9
+        threads_per_task=14
+        memory="50GB"
+        ;;
+      *)
+        tasks_per_node=21
+        threads_per_task=6
+        ;;
+    esac
     export tasks_per_node_postsndcfp=1
     postsnd_req_cores=$(( tasks_per_node * threads_per_task ))
     if (( postsnd_req_cores > max_tasks_per_node )); then

diff --git a/parm/config/gfs/config.resources.GAEA b/parm/config/gfs/config.resources.GAEA
@@ -21,6 +21,7 @@ case ${step} in
 
 esac
 
+unset memory
 # shellcheck disable=SC2312
 for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
   unset "${mem_var}"

diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs
@@ -356,40 +356,33 @@ export ntasks_fv3_gfs
 export ntasks_quilt_gdas
 export ntasks_quilt_gfs
 
-# Determine whether to use compression in the write grid component based on resolution
+# Determine whether to use compression in the write grid component
+# and whether to use parallel NetCDF based on resolution
 case ${fv3_res} in
-  "C48" | "C96" | "C192" | "C384")
+  "C48" | "C96" | "C192")
     zstandard_level=0
     ideflate=0
     quantize_nsd=0
+    OUTPUT_FILETYPE_ATM="netcdf"
+    OUTPUT_FILETYPE_SFC="netcdf"
     ;;
-  "C768" | "C1152" | "C3072")
+  "C384" | "C768" | "C1152" | "C3072")
     zstandard_level=0
     ideflate=1
     quantize_nsd=5
-    ;;
-  *)
-    echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}"
-    exit 15
-    ;;
-esac
-export zstandard_level ideflate quantize_nsd
-
-# Determine whether to use parallel NetCDF based on resolution
-case ${fv3_res} in
-  "C48" | "C96" | "C192" | "C384")
-    OUTPUT_FILETYPE_ATM="netcdf"
-    OUTPUT_FILETYPE_SFC="netcdf"
-    ;;
-  "C768" | "C1152" | "C3072")
     OUTPUT_FILETYPE_ATM="netcdf_parallel"
-    OUTPUT_FILETYPE_SFC="netcdf_parallel"
+    if [[ "${fv3_res}" == "C384" ]]; then
+      OUTPUT_FILETYPE_SFC="netcdf"  # For C384, the write grid component is better off with serial netcdf
+    else
+      OUTPUT_FILETYPE_SFC="netcdf_parallel"
+    fi
     ;;
   *)
     echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}"
     exit 15
     ;;
 esac
+export zstandard_level ideflate quantize_nsd
 export OUTPUT_FILETYPE_ATM OUTPUT_FILETYPE_SFC
 
 # cpl defaults