SFS Runs at C96mx100 (#2960)

Adds the ability to run C96mx100 with SFS options. Included in this PR
is the ability to use the interpolated/coarse grained 1 degree MOM6 ICs.
Staging the ATM and OCN perturbation files without using the +03 replay
ICs.

---------

Co-authored-by: Eric.Sinsky <eric.sinsky@noaa.gov>
Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
Co-authored-by: Rahul Mahajan <aerorahul@users.noreply.github.com>

.gitignore

@@ -69,6 +69,7 @@ parm/post/params_grib2_tbl_new
 parm/post/post_tag_gfs128
 parm/post/gfs
 parm/post/gefs
+parm/post/sfs
 parm/post/ocean.csv
 parm/post/ice.csv
 parm/post/ocnicepost.nml.jinja2

ci/cases/sfs/C96mx100_S2S.yaml

@@ -0,0 +1,19 @@
+experiment:
+  system: gefs
+  mode: forecast-only
+
+arguments:
+  idate: 1994050100 
+  edate: 1994050100
+  pslot: {{ 'pslot' | getenv }}
+  app: S2S
+  resdetatmos: 96
+  resensatmos: 96
+  resdetocean: 1
+  start: 'cold'
+  nens: 10
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'TOPICDIR' | getenv }}/HR4/C96mx100
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/sfs_defaults.yaml
+

ci/cases/yamls/gefs_defaults_ci.yaml

@@ -2,3 +2,11 @@ defaults:
   !INC {{ HOMEgfs }}/parm/config/gefs/yaml/defaults.yaml
 base:
   ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
+  SFS_POST: "NO"
+  FHOUT_GFS: 6
+stage_ic:
+  USE_OCN_ENS_PERTURB_FILES: "NO"
+  USE_ATM_ENS_PERTURB_FILES: "NO"
+ocn:
+  MOM6_INTERP_ICS: "NO"
+

ci/cases/yamls/gefs_replay_ci.yaml

@@ -11,4 +11,8 @@ base:
   FHOUT_OCN_GFS: 24
   FHOUT_ICE_GFS: 24
   HOMEDIR: {{ 'RUNTESTS' | getenv }}/GLOBAL
+  SFS_POST: "NO"
+stage_ic:
+  USE_OCN_ENS_PERTURB_FILES: "YES"
+  USE_ATM_ENS_PERTURB_FILES: "YES"
 

ci/cases/yamls/sfs_defaults.yaml

@@ -0,0 +1,33 @@
+base:
+  DO_JEDIATMVAR: "NO"
+  DO_JEDIATMENS: "NO"
+  DO_JEDIOCNVAR: "NO"
+  DO_JEDISNOWDA: "NO"
+  DO_MERGENSST: "NO"
+  DO_BUFRSND: "NO"
+  DO_GEMPAK: "NO"
+  DO_AWIPS: "NO"
+  KEEPDATA: "YES"
+  DO_EXTRACTVARS: "NO"
+  FHMAX_GFS: 2976
+  FHMAX_HF_GFS: 0
+  FHOUT_HF_GFS: 1
+  FHOUT_GFS: 24
+  FHOUT_OCN_GFS: 24
+  FHOUT_ICE_GFS: 24
+  FCST_BREAKPOINTS: ""
+  REPLAY_ICS: "NO"
+  HPSSARCH: "NO"
+  LOCALARCH: "NO"
+  SFS_POST: "YES"
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
+fcst:
+  TYPE: "hydro"
+  MONO: "mono"
+  reforecast: "YES"
+  FHZER: 24
+stage_ic:
+  USE_OCN_ENS_PERTURB_FILES: "YES"
+  USE_ATM_ENS_PERTURB_FILES: "YES"
+ocn:
+  MOM6_INTERP_ICS: "YES"

parm/config/gefs/config.atmos_products

@@ -25,9 +25,14 @@ fi
 export FLXGF="NO"  # Create interpolated sflux.1p00 file
 
 # paramlist files for the different forecast hours and downsets
-export paramlista="${PARMgfs}/product/gefs.0p25.fFFF.paramlist.a.txt"
-export paramlista_anl="${PARMgfs}/product/gefs.0p25.anl.paramlist.a.txt"
-export paramlista_f000="${PARMgfs}/product/gefs.0p25.f000.paramlist.a.txt"
-export paramlistb="${PARMgfs}/product/gefs.0p25.fFFF.paramlist.b.txt"
+if [[ ${SFS_POST} == "YES" ]]; then
+    export post_prefix='sfs'
+else
+    export post_prefix='gefs'
+fi
+export paramlista="${PARMgfs}/product/${post_prefix}.0p25.fFFF.paramlist.a.txt"
+export paramlista_anl="${PARMgfs}/product/${post_prefix}.0p25.anl.paramlist.a.txt"
+export paramlista_f000="${PARMgfs}/product/${post_prefix}.0p25.f000.paramlist.a.txt"
+export paramlistb="${PARMgfs}/product/${post_prefix}.0p25.fFFF.paramlist.b.txt"
 
 echo "END: config.atmos_products"

parm/config/gefs/config.base

@@ -64,6 +64,7 @@ export REALTIME="YES"
 
 # Experiment mode (cycled or forecast-only)
 export MODE="@MODE@" # cycled/forecast-only
+export SFS_POST="@SFS_POST@" # TODO, place holder until RUN=SFS is developed
 
 ####################################################
 # DO NOT ADD MACHINE DEPENDENT STUFF BELOW THIS LINE
@@ -237,15 +238,14 @@ else
     export OFFSET_START_HOUR=0
 fi
 
-
 # GFS output and frequency
 export FHMIN_GFS=0
 export FHMAX_GFS="@FHMAX_GFS@"
 # Intermediate times to stop forecast when running in segments
 breakpnts="@FCST_BREAKPOINTS@"
 export FCST_SEGMENTS="${FHMIN_GFS},${breakpnts:+${breakpnts},}${FHMAX_GFS}"
 
-export FHOUT_GFS=6
+export FHOUT_GFS=@FHOUT_GFS@
 export FHMAX_HF_GFS=@FHMAX_HF_GFS@
 export FHOUT_HF_GFS=@FHOUT_HF_GFS@
 export FHOUT_OCN_GFS=@FHOUT_OCN_GFS@
@@ -295,6 +295,7 @@ export ENSMEM=${ENSMEM:-"000"}
 export MEMDIR="mem${ENSMEM}"
 
 export DOIAU="NO"  # While we are not doing IAU, we may want to warm start w/ IAU in the future
+
 # Check if cycle is warm starting with IAU
 if [[ "${EXP_WARM_START}" = ".true." ]]; then
   if [[ "${DOIAU}" = "YES" ]]; then

parm/config/gefs/config.efcs

@@ -27,7 +27,11 @@ source "${EXPDIR}/config.ufs" ${string}
 source "${EXPDIR}/config.resources" efcs
 
 # Stochastic physics parameters (only for ensemble forecasts)
-export DO_SKEB="YES"
+if [[ "${CASE}" == "C96" ]] ; then
+    export DO_SKEB="NO" # SKEB turned off for C96
+else
+    export DO_SKEB="YES" # SKEB turned on for all other resolutions
+fi
 export SKEB="0.8,-999,-999,-999,-999"
 export SKEB_TAU="2.16E4,2.592E5,2.592E6,7.776E6,3.1536E7"
 export SKEB_LSCALE="500.E3,1000.E3,2000.E3,2000.E3,2000.E3"

parm/config/gefs/config.fcst

@@ -44,6 +44,7 @@ export FHOUT=${FHOUT_GFS}
 export FHOUT_HF=${FHOUT_HF_GFS}
 export FHOUT_OCN=${FHOUT_OCN_GFS}
 export FHOUT_ICE=${FHOUT_ICE_GFS}
+export FHZER=@FHZER@
 
 # Get task specific resources
 source "${EXPDIR}/config.resources" fcst
@@ -66,8 +67,8 @@ export FCSTEXEC="ufs_model.x"
 
 #######################################################################
 # Model configuration
-export TYPE="nh"
-export MONO="non-mono"
+export TYPE=@TYPE@
+export MONO=@MONO@
 
 # Use stratosphere h2o physics
 export h2o_phys=".true."
@@ -201,6 +202,11 @@ case ${imp_physics} in
         export hord_xx_nh_nonmono=5
         export vtdm4_nh_nonmono=0.02
         export nord=2
+        if [[ "${TYPE}" == "nh"* ]]; then
+            export dddmp=0.1
+        else
+            export dddmp=0.
+	    fi
         export dddmp=0.1
         export d4_bg=0.12
         ;;
@@ -221,7 +227,11 @@ case ${imp_physics} in
         export vtdm4_nh_nonmono=0.02
         export nord=2
         export d4_bg=0.12
-        export dddmp=0.1
+        if [[ "${TYPE}" == "nh"* ]]; then
+            export dddmp=0.1
+        else
+            export dddmp=0.
+	    fi
         ;;
     *) echo "Unknown microphysics option, ABORT!" ;;
 esac
@@ -268,9 +278,7 @@ else
   export io_layout="1,1"
 fi
 
-if (( OFFSET_START_HOUR != 0 )); then
-    export reforecast="YES"
-fi
+export reforecast=@reforecast@
 # Remember config.efcs will over-ride these values for ensemble forecasts
 # if these variables are re-defined there.
 # Otherwise, the ensemble forecast will inherit from config.fcst

parm/config/gefs/config.stage_ic

@@ -30,4 +30,8 @@ if [[ -z "${ICSDIR}" ]] ; then
 
 fi
 
+#use of perturbations files for ensembles
+export USE_OCN_ENS_PERTURB_FILES=@USE_OCN_ENS_PERTURB_FILES@
+export USE_ATM_ENS_PERTURB_FILES=@USE_ATM_ENS_PERTURB_FILES@
+
 echo "END: config.stage_ic"

parm/config/gefs/config.ufs

@@ -80,7 +80,7 @@ case "${fv3_res}" in
         export nthreads_fv3_gfs=1
         export nthreads_ufs=1
         export nthreads_ufs_gfs=1
-        export xr_cnvcld=.false. # Do not pass conv. clouds to Xu-Randall cloud fraction
+        export xr_cnvcld=".true."  # Pass conv. clouds to Xu-Randall cloud fraction
         export cdmbgwd="0.071,2.1,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="40.0,1.77,1.0,1.0"   # settings for GSL drag suite
         export k_split=1
@@ -98,13 +98,13 @@ case "${fv3_res}" in
         export DELTIM=600
         export layout_x=2
         export layout_y=2
-        export layout_x_gfs=2
-        export layout_y_gfs=2
+        export layout_x_gfs=4
+        export layout_y_gfs=4
         export nthreads_fv3=1
         export nthreads_fv3_gfs=1
         export nthreads_ufs=1
         export nthreads_ufs_gfs=1
-        export xr_cnvcld=".false."  # Do not pass conv. clouds to Xu-Randall cloud fraction
+        export xr_cnvcld=".true."  # Pass conv. clouds to Xu-Randall cloud fraction
         export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="20.0,2.5,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=3.0e-3      # setting for UGWPv1 non-stationary GWD
@@ -131,6 +131,7 @@ case "${fv3_res}" in
         export cdmbgwd="0.23,1.5,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="10.0,3.5,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=1.5e-3      # setting for UGWPv1 non-stationary GWD
+        export xr_cnvcld=".true."  # Pass conv. clouds to Xu-Randall cloud fraction
         export k_split=2
         export n_split=4
         export tau=6.0
@@ -337,7 +338,7 @@ if [[ "${skip_mom6}" == "false" ]]; then
       DT_THERM_MOM6='3600'
       FRUNOFF="runoff.daitren.clim.1deg.nc"
       CHLCLIM="seawifs_1998-2006_smoothed_2X.nc"
-      MOM6_RESTART_SETTING='r'
+      MOM6_RESTART_SETTING=${MOM6_RESTART_SETTING:-'r'}
       MOM6_RIVER_RUNOFF='False'
       eps_imesh="2.5e-1"
       TOPOEDITS="ufs.topo_edits_011818.nc"

parm/config/gefs/yaml/defaults.yaml

@@ -15,7 +15,20 @@ base:
   FCST_BREAKPOINTS: "48"
   REPLAY_ICS: "NO"
   USE_OCN_PERTURB_FILES: "false"
+  FHOUT_GFS: 6
   FHOUT_OCN_GFS: 6
   FHOUT_ICE_GFS: 6
   HPSSARCH: "NO"
   LOCALARCH: "NO"
+  SFS_POST: "NO"
+fcst:
+  reforecast: "NO"
+  FHZER: 6
+  TYPE: "nh"
+  MONO: "non-mono"
+stage_ic:
+  USE_OCN_ENS_PERTURB_FILES: "NO"
+  USE_ATM_ENS_PERTURB_FILES: "NO"
+ocn:
+  MOM6_INTERP_ICS: "NO"
+

parm/config/gfs/config.ocn

@@ -24,6 +24,11 @@ else
   export ODA_INCUPD_NHOURS="3.0"
 fi
 
-
-
+MOM6_INTERP_ICS=@MOM6_INTERP_ICS@
+if [[ "${MOM6_INTERP_ICS}" == "YES" ]]; then
+    export MOM6_RESTART_SETTING='n'
+    export MOM6_WARMSTART_FILE="MOM.res.nc"
+    export MOM6_INIT_FROM_Z='False'
+    export MOM6_INIT_UV='file'
+fi
 echo "END: config.ocn"

parm/config/gfs/config.stage_ic

@@ -29,4 +29,8 @@ if [[ -z "${ICSDIR}" ]] ; then
 
 fi
 
+#use of perturbations files for ensembles
+export USE_OCN_ENS_PERTURB_FILES="NO"
+export USE_ATM_ENS_PERTURB_FILES="NO"
+
 echo "END: config.stage_ic"

parm/product/sfs.0p25.f000.paramlist.a.txt

@@ -0,0 +1,39 @@
+:HGT:surface:
+:PRMSL:mean sea level:
+:PRES:surface:
+:TMP:2 m above ground:
+:TMAX:2 m above ground:
+:TMIN:2 m above ground:
+:RH:2 m above ground:
+:DPT:2 m above ground:
+:UGRD:10 m above ground:
+:VGRD:10 m above ground:
+:APCP:surface:
+:CRAIN:surface:
+:CSNOW:surface:
+:CFRZR:surface:
+:CICEP:surface:
+:PWAT:entire atmosphere (considered as a single layer):
+:CAPE:180-0 mb above ground:
+:CAPE:surface:
+:CIN:180-0 mb above ground:
+:CIN:surface:
+:CPOFP:surface:
+:HLCY:3000-0 m above ground:
+:TCDC:entire atmosphere:
+:WEASD:surface:
+:SNOD:surface:
+:ULWRF:top of atmosphere:
+:DSWRF:surface:
+:DLWRF:surface:
+:USWRF:surface:
+:ULWRF:surface:
+:GUST:surface:
+:SHTFL:surface:
+:LHTFL:surface:
+:ICETK:surface:
+:TSOIL:0-0.1
+:SOILW:0-0.1
+:MSLET:mean sea level:
+:VIS:surface:
+:HGT:cloud ceiling:

parm/product/sfs.0p25.fFFF.paramlist.a.txt

@@ -0,0 +1,52 @@
+:PRMSL:mean sea level:
+:PRES:surface:
+:TMP:surface:
+:TMP:2 m above ground:
+:TMP:850 mb:
+:TMP:500 mb:
+:TMP:200 mb:
+:TMAX:2 m above ground:
+:TMIN:2 m above ground:
+:RH:2 m above ground:
+:DPT:2 m above ground:
+:UGRD:10 m above ground:
+:UGRD:850 mb:
+:UGRD:500 mb:
+:UGRD:200 mb:
+:VGRD:10 m above ground:
+:VGRD:850 mb:
+:VGRD:500 mb:
+:VGRD:200 mb:
+:HGT:850 mb:
+:HGT:500 mb:
+:HGT:200 mb:
+:APCP:surface:
+:CRAIN:surface:
+:CSNOW:surface:
+:CFRZR:surface:
+:CICEP:surface:
+:FDNSSTMP:surface:
+:PWAT:entire atmosphere (considered as a single layer):
+:CAPE:180-0 mb above ground:
+:CAPE:surface:
+:CIN:180-0 mb above ground:
+:CIN:surface:
+:HLCY:3000-0 m above ground:
+:TCDC:entire atmosphere (considered as a single layer):
+:WEASD:surface:
+:SNOD:surface:
+:ULWRF:top of atmosphere:
+:DSWRF:surface:
+:CPOFP:surface:
+:DLWRF:surface:
+:USWRF:surface:
+:ULWRF:surface:
+:GUST:surface:
+:SHTFL:surface:
+:LHTFL:surface:
+:ICETK:surface:
+:TSOIL:0-0.1
+:SOILW:0-0.1
+:MSLET:mean sea level:
+:VIS:surface:
+:HGT:cloud ceiling: