Add fixes to products for when REPLAY IC's are used

parm/config/gefs/config.fcst

@@ -242,7 +242,11 @@ export FSICS="0"
 
 #---------------------------------------------------------------------
 # Write more variables to output
-export DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table"
+if (( OFFSET_START_HOUR != 0 )); then
+  export DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table_replay"
+else
+  export DIAG_TABLE="${PARMgfs}/ufs/fv3/diag_table"
+fi
 
 # Write gfs restart files to rerun fcst from any break point
 export restart_interval=${restart_interval_gfs:-12}

parm/config/gefs/config.resources

@@ -272,7 +272,7 @@ case ${step} in
     export walltime_gefs="00:30:00"
     export ntasks_gefs=1
     export threads_per_task_gefs=1
-    export tasks_per_node_gefs="${ntasks}"
+    export tasks_per_node_gefs="${ntasks_gefs}"
     export walltime_gfs="${walltime_gefs}"
     export ntasks_gfs="${ntasks_gefs}"
     export threads_per_tasks_gfs="${threads_per_task_gefs}"

ush/atmos_extractvars.sh

@@ -31,7 +31,16 @@ for outtype in "f2d" "f3d"; do
   outdirpre="${subdata}/${outtype}"
   [[ -d "${outdirpre}" ]] || mkdir -p "${outdirpre}"
 
-  nh=${FHMIN}
+  if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
+    if [[ "${outtype}" == "f2d" ]]; then
+      nh=${OFFSET_START_HOUR}
+    elif [[ "${outtype}" == "f3d" ]]; then
+      nh=${FHOUT_GFS}
+    fi
+  else
+    nh=${FHMIN}
+  fi
+
   while (( nh <= FHMAX_GFS )); do
     fnh=$(printf "%3.3d" "${nh}")
 
@@ -45,11 +54,15 @@ for outtype in "f2d" "f3d"; do
       outres="1p00"
     fi
 
-    if (( nh <= FHMAX_HF_GFS )); then
-      outfreq=${FHOUT_HF_GFS}
-    else
-      outfreq=${FHOUT_GFS}
-    fi                                      
+    if [[ "${outtype}" == "f2d" ]]; then
+      if (( nh < FHMAX_HF_GFS )); then
+        outfreq=${FHOUT_HF_GFS}
+      else
+        outfreq=${FHOUT_GFS}
+      fi
+    elif [[ "${outtype}" == "f3d" ]]; then
+     outfreq=${FHOUT_GFS}
+    fi
 
     com_var="COMIN_ATMOS_GRIB_${outres}"
     infile1="${!com_var}/${RUN}.t${cyc}z.pgrb2.${outres}.f${fnh}"

ush/forecast_postdet.sh

@@ -202,13 +202,23 @@ EOF
 
   #============================================================================
   if [[ "${QUILTING}" = ".true." ]] && [[ "${OUTPUT_GRID}" = "gaussian_grid" ]]; then
-    local FH2 FH3
+    local FH2 FH3 iicnt
+    iicnt=0
     for fhr in ${FV3_OUTPUT_FH}; do
       FH3=$(printf %03i "${fhr}")
       FH2=$(printf %02i "${fhr}")
-      ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atmf${FH3}.nc" "atmf${FH3}.nc"
-      ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${FH3}.nc" "sfcf${FH3}.nc"
-      ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf${FH3}.txt" "log.atm.f${FH3}"
+      # When replaying, the time format outputted by model in filename is HH-MM-SS
+      # because first fhr is a decimal number
+      if [[ ${REPLAY_ICS:-NO} == "YES" ]] && (( fhr >= OFFSET_START_HOUR )); then
+        ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atmf${FH3}.nc" "atmf${FV3_OUTPUT_FH_hhmmss[${iicnt}]}.nc"
+        ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${FH3}.nc" "sfcf${FV3_OUTPUT_FH_hhmmss[${iicnt}]}.nc"
+        ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf${FH3}.txt" "log.atm.f${FV3_OUTPUT_FH_hhmmss[${iicnt}]}"
+        iicnt=$(( iicnt + 1 ))
+      else
+        ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atmf${FH3}.nc" "atmf${FH3}.nc"
+        ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${FH3}.nc" "sfcf${FH3}.nc"
+        ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf${FH3}.txt" "log.atm.f${FH3}"
+      fi
       if [[ "${WRITE_DOPOST}" == ".true." ]]; then
         ${NLN} "${COMOUT_ATMOS_MASTER}/${RUN}.t${cyc}z.master.grb2f${FH3}" "GFSPRS.GrbF${FH2}"
         ${NLN} "${COMOUT_ATMOS_MASTER}/${RUN}.t${cyc}z.sfluxgrbf${FH3}.grib2" "GFSFLX.GrbF${FH2}"
@@ -452,10 +462,19 @@ MOM6_postdet() {
       (( midpoint = last_fhr + interval/2 ))
 
       vdate=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${fhr} hours" +%Y%m%d%H)
-      vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${midpoint} hours" +%Y%m%d%H)
+      #If OFFSET_START_HOUR is greater than 0, OFFSET_START_HOUR should be added to the midpoint for first lead time
+      if (( OFFSET_START_HOUR > 0 )) &&  (( fhr == FHOUT_OCN ));then
+        vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + $(( midpoint + OFFSET_START_HOUR )) hours" +%Y%m%d%H)
+      else
+        vdate_mid=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${midpoint} hours" +%Y%m%d%H)
+      fi
 
       # Native model output uses window midpoint in the filename, but we are mapping that to the end of the period for COM
-      source_file="ocn_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc"
+      if (( OFFSET_START_HOUR > 0 )) &&  (( fhr == FHOUT_OCN ));then
+        source_file="ocn_lead1_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc"
+      else
+        source_file="ocn_${vdate_mid:0:4}_${vdate_mid:4:2}_${vdate_mid:6:2}_${vdate_mid:8:2}.nc"
+      fi
       dest_file="${RUN}.ocean.t${cyc}z.${interval}hr_avg.f${fhr3}.nc"
       ${NLN} "${COMOUT_OCEAN_HISTORY}/${dest_file}" "${DATA}/MOM6_OUTPUT/${source_file}"
 

ush/forecast_predet.sh

@@ -145,6 +145,30 @@ FV3_predet(){
   fi
 
   # Convert output settings into an explicit list for FV3
+  # Create an FV3 fhr list to be used in the namelist for when REPLAY_ICS is set to YES
+  # The FV3 fhr list for the namelist and the FV3 fhr list for the filenames
+  # are only different when REPLAY_ICS is set to YES
+  if [[ "${REPLAY_ICS:-NO}" == "YES"  ]]; then
+    FV3_OUTPUT_FH_NML="$(echo "scale=5; ${OFFSET_START_HOUR}+(${DELTIM}/3600)" | bc -l)"
+    FV3_OUTPUT_FH_s=$(( OFFSET_START_HOUR * 3600 + DELTIM ))
+    local FHMIN_REPLAY=$(( FHMIN + FHOUT ))
+    local fhr=${FHMIN_REPLAY}
+    if (( FHOUT_HF > 0 && FHMAX_HF > 0 )); then
+      FV3_OUTPUT_FH_NML="${FV3_OUTPUT_FH_NML} $(seq -s ' ' "${FHMIN_REPLAY}" "${FHOUT_HF}" "${FHMAX_HF}")"
+      FV3_OUTPUT_FH_s="${FV3_OUTPUT_FH_s} $(seq -s ' ' "$(( FHMIN_REPLAY * 3600 ))" "$(( FHOUT_HF * 3600 ))" "$(( FHMAX_HF * 3600 ))")"
+      fhr=${FHMAX_HF}
+    fi
+    FV3_OUTPUT_FH_NML="${FV3_OUTPUT_FH_NML} $(seq -s ' ' "${fhr}" "${FHOUT}" "${FHMAX}")"
+    FV3_OUTPUT_FH_s="${FV3_OUTPUT_FH_s} $(seq -s ' ' "$(( fhr * 3600 ))" "$(( FHOUT * 3600 ))" "$(( FHMAX * 3600 ))")"
+    local hh mm ss s_total
+    for s_total in ${FV3_OUTPUT_FH_s}; do
+      # Convert seconds to HHH:MM:SS
+      (( ss = s_total, mm = ss / 60, ss %= 60, hh = mm / 60, mm %= 60 )) || true
+      FV3_OUTPUT_FH_hhmmss+=("$(printf "%03d-%02d-%02d" "${hh}" "${mm}" "${ss}")")
+    done
+  fi
+
+  # Create an FV3 fhr list to be used in the filenames
   FV3_OUTPUT_FH=""
   local fhr=${FHMIN}
   if (( FHOUT_HF > 0 && FHMAX_HF > 0 )); then
@@ -153,6 +177,12 @@ FV3_predet(){
   fi
   FV3_OUTPUT_FH="${FV3_OUTPUT_FH} $(seq -s ' ' "${fhr}" "${FHOUT}" "${FHMAX}")"
 
+  # The FV3 fhr list for the namelist and the FV3 fhr list for the filenames
+  # are identical when REPLAY_ICS is set to NO
+  if [[ "${REPLAY_ICS:-NO}" == "NO"  ]]; then
+    FV3_OUTPUT_FH_NML="${FV3_OUTPUT_FH}"
+  fi
+
   # Other options
   MEMBER=$(( 10#${ENSMEM:-"-1"} )) # -1: control, 0: ensemble mean, >0: ensemble member $MEMBER
   PREFIX_ATMINC=${PREFIX_ATMINC:-""} # allow ensemble to use recentered increment

ush/parsing_model_configure_FV3.sh

@@ -48,7 +48,7 @@ local JCHUNK3D=$((2*restile))
 local KCHUNK3D=1
 local IMO=${LONB_IMO}
 local JMO=${LATB_JMO}
-local OUTPUT_FH=${FV3_OUTPUT_FH}
+local OUTPUT_FH=${FV3_OUTPUT_FH_NML}
 local IAU_OFFSET=${IAU_OFFSET:-0}
 
 # Ensure the template exists

ush/parsing_namelists_FV3.sh

@@ -42,6 +42,18 @@ local SDAY=${current_cycle:6:2}
 local CHOUR=${current_cycle:8:2}
 local MOM6_OUTPUT_DIR="./MOM6_OUTPUT"
 
+if (( OFFSET_START_HOUR > 0 )); then
+  local current_cycle_p1 
+  current_cycle_p1=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${FHOUT_OCN} hours" +%Y%m%d%H)
+  local current_cycle_offset
+  current_cycle_offset=$(date --utc -d "${current_cycle:0:8} ${current_cycle:8:2} + ${OFFSET_START_HOUR} hours" +%Y%m%d%H)
+  local SYEAR1=${current_cycle_p1:0:4}
+  local SMONTH1=${current_cycle_p1:4:2}
+  local SDAY1=${current_cycle_p1:6:2}
+  local CHOUR1=${current_cycle_p1:8:2}
+  local CHOUR_offset=${current_cycle_offset:8:2}
+fi
+
 atparse < "${template}" >> "diag_table"