Add task to process reforecast variables to save on WCOSS2

jobs/JGLOBAL_EXTRACTVARS

@@ -0,0 +1,47 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "extractvars" -c "base extractvars"
+
+# Set COM Paths
+for grid in '0p25' '0p50' '1p00'; do
+  prod_dir="COMIN_ATMOS_GRIB_${grid}"
+  GRID=${grid} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL"
+  if [[ ! -d "${!prod_dir}" ]]; then mkdir -p "${!prod_dir}"; fi
+done
+
+YMD="${PDY}" HH="${cyc}" declare_from_tmpl -rx \
+    "COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL" \
+    "COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL" \
+    "COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL" \
+    "COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL" \
+    "COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL" \
+    "COMIN_ICE_NETCDF:COM_ICE_NETCDF_TMPL" \
+    "COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL"
+
+if [[ "${DO_ATM}" == "YES" ]]; then
+  if [[ ! -d "${ARC_RFCST_PROD_ATMOS_F2D}" ]]; then mkdir -p "${ARC_RFCST_PROD_ATMOS_F2D}"; fi
+  if [[ ! -d "${ARC_RFCST_PROD_ATMOS_F3D}" ]]; then mkdir -p "${ARC_RFCST_PROD_ATMOS_F3D}"; fi
+fi
+if [[ "${DO_OCN}" == "YES" ]]; then
+  if [[ ! -d "${ARC_RFCST_PROD_OCN}" ]]; then mkdir -p "${ARC_RFCST_PROD_OCN}"; fi
+fi
+if [[ "${DO_ICE}" == "YES" ]]; then
+  if [[ ! -d "${ARC_RFCST_PROD_ICE}" ]]; then mkdir -p "${ARC_RFCST_PROD_ICE}"; fi
+fi
+if [[ "${DO_WAVE}" == "YES" ]]; then
+  if [[ ! -d "${ARC_RFCST_PROD_WAV}" ]]; then mkdir -p "${ARC_RFCST_PROD_WAV}"; fi
+fi
+
+# Execute the Script
+"${SCRgfs}/exglobal_extractvars.sh"
+status=$?
+(( status != 0 )) && exit "${status}"
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || true
+[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"
+
+exit 0

jobs/rocoto/extractvars.sh

@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+echo
+echo "=============== START TO SOURCE FV3GFS WORKFLOW MODULES ==============="
+. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+[[ "${status}" -ne 0 ]] && exit "${status}"
+
+export job="extractvars"
+export jobid="${job}.$$"
+
+###############################################################
+echo
+echo "=============== START TO RUN EXTRACTVARS ==============="
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_EXTRACTVARS"
+status=$?
+[[ "${status}" -ne 0 ]] && exit "${status}"
+
+exit 0

parm/config/gefs/config.base

@@ -137,6 +137,7 @@ export DO_WAVE="NO"
 export DO_OCN="NO"
 export DO_ICE="NO"
 export DO_AERO="NO"
+export DO_EXTRACTVARS="@DO_EXTRACTVARS@" # Option to process and extract a subset of products to save on disk
 export AERO_FCST_CDUMP="" # When to run aerosol forecast: gdas, gfs, or both
 export AERO_ANL_CDUMP="" # When to run aerosol analysis: gdas, gfs, or both
 export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both

parm/config/gefs/config.extractvars

@@ -0,0 +1,41 @@
+#! /usr/bin/env bash                                                                                                                                                                                                                     
+
+########## config.extractvars ##########
+# Extractvars specific
+
+echo "BEGIN: config.extractvars"
+
+. "${EXPDIR}/config.resources" extractvars
+
+export COMPRSCMD=${COMPRSCMD:-bzip2}
+
+export compress_ocn=0 #1: Compress extracted ocean product, 0: Do not compress extracted ocean product
+export compress_ice=0 #1: Compress extracted ice product, 0: Do not compress extracted ice product
+
+export ocnres="5p00" # Resolution of ocean products
+export iceres="5p00" # Resolution of ice products
+export wavres="5p00" # Resolution of wave products
+
+export depthvar_name="z_l" # Name of depth variable in NetCDF ocean products
+export zmin="0." # Minimum depth to extract from NetCDF ocean products
+export zmax="300." # Maximum depth to extract from NetCDF ocean products
+
+export FHOUT_WAV_EXTRACT=6 # Frequency of wave output to be saved on disk
+
+#Paramater Tables used
+export varlist_2d="${PARMgfs}/product/gefs_shortparmlist_2d.parm" # Parameter table for surface variables
+export varlist_3d="${PARMgfs}/product/gefs_shortparmlist_3d_h.parm" # Parameter table for upper air instantaneous variables
+export varlist_3d_d="${PARMgfs}/product/gefs_shortparmlist_3d_d.parm" # Parameter table for upper air daily-averaged variables
+export varlist_wav="${PARMgfs}/product/gefs_wav_shortparmlist.parm" # Parameter table for wave variables
+export varlist_ocn_netcdf="${PARMgfs}/product/gefs_ocn_shortparmlist.parm" # Parameter table for ocean netcdf variables
+export varlist_ice_netcdf="${PARMgfs}/product/gefs_ice_shortparmlist.parm" # Parameter table for ice netcdf variables
+
+#Directory to save extracted variables
+export ARC_RFCST_PROD="${ARCDIR}/rfcst/${PDY:0:4}/${PDY:0:6}/${PDY:0:8}/mem${ENSMEM}"
+export ARC_RFCST_PROD_ATMOS_F2D="${ARC_RFCST_PROD}/atmos/f2d"
+export ARC_RFCST_PROD_ATMOS_F3D="${ARC_RFCST_PROD}/atmos/f3d"
+export ARC_RFCST_PROD_OCN="${ARC_RFCST_PROD}/ocn"
+export ARC_RFCST_PROD_ICE="${ARC_RFCST_PROD}/ice"
+export ARC_RFCST_PROD_WAV="${ARC_RFCST_PROD}/wav"
+
+echo "END: config.extractvars"

parm/config/gefs/config.resources

@@ -293,6 +293,18 @@ case ${step} in
     export NTASKS=${npe_wavepostpnt}
     ;;
 
+  "extractvars")
+    export wtime_extractvars="00:30:00"
+    export npe_extractvars=1
+    export nth_extractvars=1
+    export npe_node_extractvars="${npe_extractvars}"
+    export wtime_extractvars_gfs="${wtime_extractvars}"
+    export npe_extractvars_gfs="${npe_extractvars}"
+    export nth_extractvars_gfs="${nth_extractvars}"
+    export npe_node_extractvars_gfs="${npe_node_extractvars}"
+    export is_exclusive=False
+    ;;
+
   *)
     echo "FATAL ERROR: Invalid job ${step} passed to ${BASH_SOURCE[0]}"
     exit 1

parm/config/gefs/yaml/defaults.yaml

@@ -7,7 +7,8 @@ base:
   DO_BUFRSND: "NO"
   DO_GEMPAK: "NO"
   DO_AWIPS: "NO"
-  KEEPDATA: "NO" 
+  KEEPDATA: "NO"
+  DO_EXTRACTVARS: "NO"
   FHMAX_GFS: 120
   FHMAX_HF_GFS: 0
   REPLAY_ICS: "NO"

parm/product/gefs_ice_shortparmlist.parm

@@ -0,0 +1,10 @@
+aice_h
+hi_h
+Tsfc_h
+uvel_h
+vvel_h
+hs_h
+albsni_h
+melts_h
+meltb_h
+frzmlt_h

parm/product/gefs_ocn_shortparmlist.parm

@@ -0,0 +1,9 @@
+temp
+SST
+SSH
+SSS
+MLD_003
+taux
+tauy
+SSU
+SSV

parm/product/gefs_shortparmlist_2d.parm

@@ -0,0 +1,38 @@
+:PRES:surface:
+:WEASD:surface:
+:TMP:2 m above ground:
+:TMP:surface:
+:RH:2 m above ground:
+:TMAX:2 m above ground:
+:TMIN:2 m above ground:
+:UGRD:10 m above ground:
+:VGRD:10 m above ground:
+:APCP:surface:
+:CSNOW:surface:
+:CICEP:surface:
+:CFRZR:surface:
+:CRAIN:surface:
+:PWAT:entire atmosphere (considered as a single layer):
+:TCDC:entire atmosphere (considered as a single layer):
+:DSWRF:surface:
+:DLWRF:surface:
+:ULWRF:top of atmosphere:
+:HLCY:3000-0 m above ground:
+:CAPE:180-0 mb above ground:
+:CIN:180-0 mb above ground:
+:PRMSL:mean sea level:
+:USWRF:surface:
+:ULWRF:surface:
+:TSOIL:0-0.1 m below ground:
+:TSOIL:0.1-0.4 m below ground:
+:SOILW:0-0.1 m below ground:
+:SOILW:0.1-0.4 m below ground:
+:SOILW:0.4-1 m below ground:
+:SOILW:1-2 m below ground:
+:PEVPR:surface:
+:LHTFL:surface:
+:SHTFL:surface:
+:WATR:surface:
+:TSNOWP:surface:
+:FDNSSTMP:surface:
+:HGT:highest tropospheric freezing level:

parm/product/gefs_shortparmlist_3d_d.parm

@@ -0,0 +1,34 @@
+:UGRD:1 mb:
+:UGRD:2 mb:
+:UGRD:3 mb:
+:UGRD:5 mb:
+:UGRD:7 mb:
+:UGRD:10 mb:
+:UGRD:20 mb:
+:UGRD:30 mb:
+:UGRD:50 mb:
+:UGRD:70 mb:
+:VGRD:1 mb:
+:VGRD:2 mb:
+:VGRD:3 mb:
+:VGRD:5 mb:
+:VGRD:7 mb:
+:VGRD:10 mb:
+:VGRD:20 mb:
+:VGRD:30 mb:
+:VGRD:50 mb:
+:VGRD:70 mb:
+:TMP:1 mb:
+:TMP:2 mb:
+:TMP:3 mb:
+:TMP:5 mb:
+:TMP:7 mb:
+:TMP:10 mb:
+:TMP:20 mb:
+:TMP:30 mb:
+:TMP:50 mb:
+:TMP:70 mb:
+:HGT:10 mb:
+:HGT:50 mb:
+:O3MR:10 mb:
+:O3MR:50 mb:

parm/product/gefs_shortparmlist_3d_h.parm

@@ -0,0 +1,45 @@
+:HGT:100 mb:
+:TMP:100 mb:
+:UGRD:100 mb:
+:VGRD:100 mb:
+:O3MR:100 mb:
+:HGT:200 mb:
+:TMP:200 mb:
+:RH:200 mb:
+:UGRD:200 mb:
+:VGRD:200 mb:
+:HGT:250 mb:
+:TMP:250 mb:
+:RH:250 mb:
+:UGRD:250 mb:
+:VGRD:250 mb:
+:HGT:500 mb:
+:TMP:500 mb:
+:RH:500 mb:
+:UGRD:500 mb:
+:VGRD:500 mb:
+:HGT:700 mb:
+:TMP:700 mb:
+:RH:700 mb:
+:UGRD:700 mb:
+:VGRD:700 mb:
+:HGT:850 mb:
+:TMP:850 mb:
+:RH:850 mb:
+:VVEL:850 mb:
+:UGRD:850 mb:
+:VGRD:850 mb:
+:HGT:925 mb:
+:TMP:925 mb:
+:RH:925 mb:
+:UGRD:925 mb:
+:VGRD:925 mb:
+:TMP:1000 mb:
+:RH:1000 mb:
+:UGRD:1000 mb:
+:VGRD:1000 mb:
+:HGT:1000 mb:
+:TMP:0.995 sigma level:
+:RH:0.995 sigma level:
+:UGRD:0.995 sigma level:
+:VGRD:0.995 sigma level:

parm/product/gefs_wav_shortparmlist.parm

@@ -0,0 +1,3 @@
+:UGRD:surface:
+:VGRD:surface:
+:HTSGW:surface:

scripts/exglobal_extractvars.sh

@@ -0,0 +1,53 @@
+#! /usr/bin/env bash                                                                                                                                                                    
+
+################################################################################
+## UNIX Script Documentation Block
+## Script name:         exglobal_extractvars.sh
+## Script description:  Extracts variables from atmosphere, ocean, ice and wave
+##                      products and saves these variables in arcdir
+#######################
+# Main body starts here
+#######################
+
+source "${USHgfs}/preamble.sh"
+source "${USHgfs}/extractvars_tools.sh"
+
+# Scripts used
+EXTRCTVARA="${USHgfs}/atmos_extractvars.sh"
+EXTRCTVARO="${USHgfs}/ocnice_extractvars.sh"
+EXTRCTVARW="${USHgfs}/wave_extractvars.sh"
+
+# Set FHMAX_HF_GFS equal to FHMAX_GFS if FHMAX_HF_GFS is greater than FHMAX_GFS
+if (( FHMAX_GFS < FHMAX_HF_GFS )); then
+  export FHMAX_HF_GFS=${FHMAX_GFS}
+fi
+
+# Set FHOUT_WAV_EXTRACT equal to FHOUT_WAV if FHOUT_WAV is not a factor of FHOUT_WAV_EXTRACT
+if (( FHOUT_WAV_EXTRACT % FHOUT_WAV != 0 )); then
+  FHOUT_WAV_EXTRACT=${FHOUT_WAV}
+fi
+
+# Extract variables for atmosphere
+if [[ "${DO_ATM}" == "YES" ]]; then
+  ${EXTRCTVARA} "${DATA}/atmos"
+fi
+
+# Extract variables for ocean
+if [[ "${DO_OCN}" == "YES" ]]; then
+  export component_name="ocn"
+  ${EXTRCTVARO} "${DATA}/ocn" "${varlist_ocn_netcdf}" "${ocnres}" "${compress_ocn}" "${FHOUT_OCN_GFS}" "${ARC_RFCST_PROD_OCN}"
+fi
+
+# Extract variables for ice
+if [[ "${DO_ICE}" == "YES" ]]; then
+  export component_name="ice"
+  ${EXTRCTVARO} "${DATA}/ice" "${varlist_ice_netcdf}" "${iceres}" "${compress_ice}" "${FHOUT_ICE_GFS}" "${ARC_RFCST_PROD_ICE}"
+fi
+
+# Extract variables for wave
+if [[ "${DO_WAVE}" == "YES" ]]; then
+  export component_name="wav"
+  ${EXTRCTVARW} "${DATA}/wav"
+fi
+
+exit 0