Skip to content

Commit 6404892

Browse files
aerorahulaerorahul
authored
Ocean/ice product generation for GFS and GEFS (NOAA-EMC#2286)
This PR does several things: 1. the model output for ocean and ice in the `COM/` directory are now named per EE2 convention for coupled model. E.g `gfs.ocean.t12z.6hr_avg.f120.nc` and `gfs.ocean.t12z.daily.f120.nc` 2. The products are generated using the `ocnicepost.fd` utility developed by @DeniseWorthen in https://github.com/NOAA-EMC/gfs-utils and converted to grib2 using example scripts provided by @GwenChen-NOAA using `wgrib2`. 3. NetCDF products on the native grid are also generated by subsetting variables from the raw model output. This is done with `xarray`. 4. updates the hash of https://github.com/NOAA-EMC/gfs-utils to include fixes in `ocnicepost.fd` 5. removes NCL related scripting that was previously used for ocean/ice interpolation and `reg2grb2` used for converting to grib2. 6. updates archive scripts to accommodate updated file names 7. removes intermediate ocean processed files such as 2D/3D/xsect data- sets 8. separate jobs are added for ocean and ice product generation. 9. removes intermediate restarts for the mediator and only saves the medi- ator restart at the end of the forecast in `COM`. 10. Increases memory for offline UPP when run at C768. The program segfaults with an OOM when memory is self allocated based on PEs by the scheduler on Hera. 11. Enables ocean/ice ensemble product generation for GEFS 12. Some minor clean-ups Fixes NOAA-EMC#935 Fixes NOAA-EMC#1317 Fixes NOAA-EMC#1864
1 parent 3f99f70 commit 6404892

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+1356
-1599
lines changed

‎.gitignore

+4-2
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ fix/gsi
3636
fix/lut
3737
fix/mom6
3838
fix/orog
39-
fix/reg2grb2
4039
fix/sfc_climo
4140
fix/ugwd
4241
fix/verif
@@ -99,6 +98,9 @@ parm/post/postxconfig-NT-GFS-WAFS.txt
9998
parm/post/postxconfig-NT-GFS.txt
10099
parm/post/postxconfig-NT-gefs-aerosol.txt
101100
parm/post/postxconfig-NT-gefs-chem.txt
101+
parm/post/ocean.csv
102+
parm/post/ice.csv
103+
parm/post/ocnicepost.nml.jinja2
102104
parm/ufs/noahmptable.tbl
103105
parm/ufs/model_configure.IN
104106
parm/ufs/MOM_input_*.IN
@@ -137,7 +139,6 @@ sorc/radmon_bcor.fd
137139
sorc/radmon_time.fd
138140
sorc/rdbfmsua.fd
139141
sorc/recentersigp.fd
140-
sorc/reg2grb2.fd
141142
sorc/supvit.fd
142143
sorc/syndat_getjtbul.fd
143144
sorc/syndat_maksynrc.fd
@@ -147,6 +148,7 @@ sorc/tocsbufr.fd
147148
sorc/upp.fd
148149
sorc/vint.fd
149150
sorc/webtitle.fd
151+
sorc/ocnicepost.fd
150152

151153
# Ignore scripts from externals
152154
#------------------------------

‎env/HERA.env

+7
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,13 @@ elif [[ "${step}" = "atmos_products" ]]; then
199199

200200
export USE_CFP="YES" # Use MPMD for downstream product generation on Hera
201201

202+
elif [[ "${step}" = "oceanice_products" ]]; then
203+
204+
nth_max=$((npe_node_max / npe_node_oceanice_products))
205+
206+
export NTHREADS_OCNICEPOST=${nth_oceanice_products:-1}
207+
export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
208+
202209
elif [[ "${step}" = "ecen" ]]; then
203210

204211
nth_max=$((npe_node_max / npe_node_ecen))

‎env/HERCULES.env

+10
Original file line numberDiff line numberDiff line change
@@ -207,10 +207,20 @@ case ${step} in
207207
[[ ${NTHREADS_UPP} -gt ${nth_max} ]] && export NTHREADS_UPP=${nth_max}
208208
export APRUN_UPP="${launcher} -n ${npe_upp} --cpus-per-task=${NTHREADS_UPP}"
209209
;;
210+
210211
"atmos_products")
211212

212213
export USE_CFP="YES" # Use MPMD for downstream product generation
213214
;;
215+
216+
"oceanice_products")
217+
218+
nth_max=$((npe_node_max / npe_node_oceanice_products))
219+
220+
export NTHREADS_OCNICEPOST=${nth_oceanice_products:-1}
221+
export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
222+
;;
223+
214224
"ecen")
215225

216226
nth_max=$((npe_node_max / npe_node_ecen))

‎env/JET.env

+7
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,13 @@ elif [[ "${step}" = "atmos_products" ]]; then
190190

191191
export USE_CFP="YES" # Use MPMD for downstream product generation
192192

193+
elif [[ "${step}" = "oceanice_products" ]]; then
194+
195+
nth_max=$((npe_node_max / npe_node_oceanice_products))
196+
197+
export NTHREADS_OCNICEPOST=${nth_oceanice_products:-1}
198+
export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
199+
193200
elif [[ "${step}" = "ecen" ]]; then
194201

195202
nth_max=$((npe_node_max / npe_node_ecen))

‎env/ORION.env

+7
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,13 @@ elif [[ "${step}" = "atmos_products" ]]; then
209209

210210
export USE_CFP="YES" # Use MPMD for downstream product generation
211211

212+
elif [[ "${step}" = "oceanice_products" ]]; then
213+
214+
nth_max=$((npe_node_max / npe_node_oceanice_products))
215+
216+
export NTHREADS_OCNICEPOST=${nth_oceanice_products:-1}
217+
export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
218+
212219
elif [[ "${step}" = "ecen" ]]; then
213220

214221
nth_max=$((npe_node_max / npe_node_ecen))

‎env/S4.env

+7
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,13 @@ elif [[ "${step}" = "atmos_products" ]]; then
177177

178178
export USE_CFP="YES" # Use MPMD for downstream product generation
179179

180+
elif [[ "${step}" = "oceanice_products" ]]; then
181+
182+
nth_max=$((npe_node_max / npe_node_oceanice_products))
183+
184+
export NTHREADS_OCNICEPOST=${nth_oceanice_products:-1}
185+
export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
186+
180187
elif [[ "${step}" = "ecen" ]]; then
181188

182189
nth_max=$((npe_node_max / npe_node_ecen))

‎env/WCOSS2.env

+7
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,13 @@ elif [[ "${step}" = "atmos_products" ]]; then
193193

194194
export USE_CFP="YES" # Use MPMD for downstream product generation
195195

196+
elif [[ "${step}" = "oceanice_products" ]]; then
197+
198+
nth_max=$((npe_node_max / npe_node_oceanice_products))
199+
200+
export NTHREADS_OCNICEPOST=${nth_oceanice_products:-1}
201+
export APRUN_OCNICEPOST="${launcher} -n 1 -ppn ${npe_node_oceanice_products} --cpu-bind depth --depth ${NTHREADS_OCNICEPOST}"
202+
196203
elif [[ "${step}" = "ecen" ]]; then
197204

198205
nth_max=$((npe_node_max / npe_node_ecen))

‎jobs/JGLOBAL_ARCHIVE

+2-3
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,15 @@ YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_ANALYSIS COM_ATMOS_BUFR COM_ATMO
1414
COM_ATMOS_TRACK COM_ATMOS_WMO \
1515
COM_CHEM_HISTORY COM_CHEM_ANALYSIS\
1616
COM_MED_RESTART \
17-
COM_ICE_HISTORY COM_ICE_INPUT COM_ICE_RESTART \
17+
COM_ICE_HISTORY COM_ICE_INPUT COM_ICE_RESTART COM_ICE_GRIB \
1818
COM_OBS COM_TOP \
19-
COM_OCEAN_HISTORY COM_OCEAN_INPUT COM_OCEAN_RESTART COM_OCEAN_XSECT COM_OCEAN_2D COM_OCEAN_3D \
19+
COM_OCEAN_HISTORY COM_OCEAN_INPUT COM_OCEAN_RESTART COM_OCEAN_GRIB COM_OCEAN_NETCDF \
2020
COM_OCEAN_ANALYSIS \
2121
COM_WAVE_GRID COM_WAVE_HISTORY COM_WAVE_STATION \
2222
COM_ATMOS_OZNMON COM_ATMOS_RADMON COM_ATMOS_MINMON COM_CONF
2323

2424
for grid in "0p25" "0p50" "1p00"; do
2525
YMD=${PDY} HH=${cyc} GRID=${grid} generate_com -rx "COM_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL"
26-
YMD=${PDY} HH=${cyc} GRID=${grid} generate_com -rx "COM_OCEAN_GRIB_${grid}:COM_OCEAN_GRIB_GRID_TMPL"
2726
done
2827

2928
###############################################################

‎jobs/JGLOBAL_OCEANICE_PRODUCTS

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#! /usr/bin/env bash
2+
3+
source "${HOMEgfs}/ush/preamble.sh"
4+
source "${HOMEgfs}/ush/jjob_header.sh" -e "oceanice_products" -c "base oceanice_products"
5+
6+
7+
##############################################
8+
# Begin JOB SPECIFIC work
9+
##############################################
10+
11+
# Construct COM variables from templates
12+
YMD="${PDY}" HH="${cyc}" generate_com -rx "COM_${COMPONENT^^}_HISTORY"
13+
YMD="${PDY}" HH="${cyc}" generate_com -rx "COM_${COMPONENT^^}_GRIB"
14+
YMD="${PDY}" HH="${cyc}" generate_com -rx "COM_${COMPONENT^^}_NETCDF"
15+
16+
###############################################################
17+
# Run exglobal script
18+
"${HOMEgfs}/scripts/exglobal_oceanice_products.py"
19+
status=$?
20+
(( status != 0 )) && exit "${status}"
21+
22+
##############################################
23+
# End JOB SPECIFIC work
24+
##############################################
25+
26+
##############################################
27+
# Final processing
28+
##############################################
29+
if [[ -e "${pgmout}" ]]; then
30+
cat "${pgmout}"
31+
fi
32+
33+
##########################################
34+
# Remove the Temporary working directory
35+
##########################################
36+
cd "${DATAROOT}" || exit 1
37+
[[ "${KEEPDATA:-NO}" == "NO" ]] && rm -rf "${DATA}"
38+
39+
40+
exit 0

‎jobs/rocoto/oceanice_products.sh

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#! /usr/bin/env bash
2+
3+
source "${HOMEgfs}/ush/preamble.sh"
4+
5+
###############################################################
6+
## ocean ice products driver script
7+
## FHRLST : forecast hour list to post-process (e.g. f000, f000_f001_f002, ...)
8+
###############################################################
9+
10+
# Source FV3GFS workflow modules
11+
. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
12+
status=$?
13+
if (( status != 0 )); then exit "${status}"; fi
14+
15+
###############################################################
16+
# setup python path for workflow utilities and tasks
17+
wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src"
18+
PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}"
19+
export PYTHONPATH
20+
21+
export job="oceanice_products"
22+
export jobid="${job}.$$"
23+
24+
###############################################################
25+
# shellcheck disable=SC2153,SC2001
26+
IFS='_' read -ra fhrs <<< "${FHRLST//f}" # strip off the 'f's and convert to array
27+
28+
#---------------------------------------------------------------
29+
# Execute the JJOB
30+
for fhr in "${fhrs[@]}"; do
31+
export FORECAST_HOUR=$(( 10#${fhr} ))
32+
"${HOMEgfs}/jobs/JGLOBAL_OCEANICE_PRODUCTS"
33+
status=$?
34+
if (( status != 0 )); then exit "${status}"; fi
35+
done
36+
37+
exit 0

‎jobs/rocoto/ocnpost.sh

-119
This file was deleted.

‎modulefiles/module_base.hera.lua

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
3333
load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
3434
load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
3535
load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
36+
load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
3637

3738
-- MET/METplus are not available for use with spack-stack, yet
3839
--load(pathJoin("met", (os.getenv("met_ver") or "None")))

‎modulefiles/module_base.hercules.lua

+1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
3535
load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
3636
load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
3737
load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
38+
load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
3839

3940
setenv("WGRIB2","wgrib2")
4041
setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))

‎modulefiles/module_base.jet.lua

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
3333
load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
3434
load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
3535
load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
36+
load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
3637

3738
setenv("WGRIB2","wgrib2")
3839
setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))

‎modulefiles/module_base.orion.lua

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
3131
load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
3232
load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
3333
load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
34+
load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
3435

3536
-- MET/METplus are not yet supported with spack-stack
3637
--load(pathJoin("met", (os.getenv("met_ver") or "None")))

‎modulefiles/module_base.s4.lua

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
3030
load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
3131
load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
3232
load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
33+
load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
3334

3435
setenv("WGRIB2","wgrib2")
3536
setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))

‎parm/config/gefs/config.oceanice_products

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../gfs/config.oceanice_products

0 commit comments

Comments
 (0)