Skip to content

Commit 43429e2

Browse files
aerorahulaerorahul
authored
Enable AO WCDA test (#1963)
This PR: - adds GSI + SOCA C48 5-deg ocean 3DVar test (courtesy @guillaumevernieres) - adds a toggle to optionally disable ocnanalvrfy job.
1 parent f563528 commit 43429e2

10 files changed

+75
-33
lines changed

‎ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
experiment:
2+
system: gfs
3+
mode: cycled
4+
5+
arguments:
6+
pslot: {{ 'pslot' | getenv }}
7+
app: S2S
8+
resdetatmos: 48
9+
resdetocean: 5.0
10+
comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
11+
expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
12+
icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500
13+
idate: 2021032412
14+
edate: 2021032418
15+
nens: 0
16+
gfs_cyc: 0
17+
start: warm
18+
yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
19+
20+
skip_ci_on_hosts:
21+
- orion
22+
- hercules

‎ci/cases/yamls/soca_gfs_defaults_ci.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
defaults:
2+
!INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
3+
base:
4+
ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
5+
DO_JEDIOCNVAR: "YES"

‎env/HERA.env

+3-15
Original file line numberDiff line numberDiff line change
@@ -93,31 +93,19 @@ elif [[ "${step}" = "ocnanalbmat" ]]; then
9393

9494
export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
9595

96-
nth_max=$((npe_node_max / npe_node_ocnanalbmat))
97-
98-
export NTHREADS_OCNANAL=${nth_ocnanalbmat:-${nth_max}}
99-
[[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
100-
export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat} --cpus-per-task=${NTHREADS_OCNANAL}"
96+
export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat}"
10197

10298
elif [[ "${step}" = "ocnanalrun" ]]; then
10399

104100
export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
105101

106-
nth_max=$((npe_node_max / npe_node_ocnanalrun))
107-
108-
export NTHREADS_OCNANAL=${nth_ocnanalrun:-${nth_max}}
109-
[[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
110-
export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun} --cpus-per-task=${NTHREADS_OCNANAL}"
102+
export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun}"
111103

112104
elif [[ "${step}" = "ocnanalchkpt" ]]; then
113105

114106
export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
115107

116-
nth_max=$((npe_node_max / npe_node_ocnanalchkpt))
117-
118-
export NTHREADS_OCNANAL=${nth_ocnanalchkpt:-${nth_max}}
119-
[[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
120-
export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalchkpt} --cpus-per-task=${NTHREADS_OCNANAL}"
108+
export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalchkpt}"
121109

122110
elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
123111

‎jobs/JGLOBAL_PREP_OCEAN_OBS

+2-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ YMD=${PDY} HH=${cyc} generate_com -rx COMOUT_OBS:COM_OBS_TMPL
1515
##############################################
1616

1717
# Add prep_marine_obs.py to PYTHONPATH
18-
export PYTHONPATH=${HOMEgfs}/sorc/gdas.cd/ush/soca:${PYTHONPATH}
18+
export PYTHONPATH=${HOMEgfs}/sorc/gdas.cd/ush:${PYTHONPATH}
1919

2020
###############################################################
2121
# Run relevant script
@@ -38,7 +38,7 @@ if [[ -e "${pgmout}" ]] ; then
3838
fi
3939

4040
##########################################
41-
# Handle the temporary working directory
41+
# Handle the temporary working directory
4242
##########################################
4343
cd "${DATAROOT}" || (echo "FATAL ERROR: ${DATAROOT} does not exist. ABORT!"; exit 1)
4444
[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"

‎parm/config/gfs/config.base.emc.dyn

+1
Original file line numberDiff line numberDiff line change
@@ -378,6 +378,7 @@ export binary_diag=".false."
378378
# Verification options
379379
export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config.metp; not supported with spack-stack
380380
export DO_FIT2OBS="YES" # Run fit to observations package
381+
export DO_VRFY_OCEANDA="NO" # Run SOCA Ocean DA verification tasks
381382

382383
# Archiving options
383384
export HPSSARCH="@HPSSARCH@" # save data to HPSS archive

‎parm/config/gfs/config.ocn

+9-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,14 @@ if [[ "${DO_JEDIOCNVAR}" == "YES" ]]; then
1616
else
1717
export ODA_INCUPD="False"
1818
fi
19-
export ODA_INCUPD_NHOURS="3.0" # In MOM_input, this is time interval for applying increment
19+
20+
# Time interval for applying the increment
21+
if [[ "${DOIAU}" == "YES" ]]; then
22+
export ODA_INCUPD_NHOURS="6.0"
23+
else
24+
export ODA_INCUPD_NHOURS="3.0"
25+
fi
26+
27+
2028

2129
echo "END: config.ocn"

‎parm/config/gfs/config.resources

+12-5
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,7 @@ case ${step} in
352352
export npe_prepoceanobs=1
353353
export nth_prepoceanobs=1
354354
export npe_node_prepoceanobs=$(( npe_node_max / nth_prepoceanobs ))
355-
export memory_prepoceanobs="24GB"
355+
export memory_prepoceanobs="48GB"
356356
;;
357357

358358
"ocnanalbmat")
@@ -406,13 +406,20 @@ case ${step} in
406406
export nth_ocnanalchkpt=1
407407
export npe_node_ocnanalchkpt=$(( npe_node_max / nth_ocnanalchkpt ))
408408
case ${CASE} in
409-
"C384") memory_ocnanalchkpt="128GB";;
410-
"C96") memory_ocnanalchkpt="32GB";;
411-
"C48") memory_ocnanalchkpt="32GB";;
409+
"C384")
410+
memory_ocnanalchkpt="128GB"
411+
npes=40;;
412+
"C96")
413+
memory_ocnanalchkpt="32GB"
414+
npes=16;;
415+
"C48")
416+
memory_ocnanalchkpt="32GB"
417+
npes=8;;
412418
*)
413419
echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
414420
exit 4
415421
esac
422+
export npe_ocnanalchkpt=${npes}
416423
export memory_ocnanalchkpt
417424
;;
418425

@@ -926,7 +933,7 @@ case ${step} in
926933
export nth_eupd=4
927934
fi
928935
;;
929-
*)
936+
*)
930937
echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
931938
exit 4
932939
;;

‎parm/config/gfs/yaml/defaults.yaml

+7-6
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,14 @@ landanl:
2525
IO_LAYOUT_Y: 1
2626

2727
ocnanal:
28-
SOCA_INPUT_FIX_DIR: "/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca" # TODO: These need to go to glopara fix space. @guillaumevernieres will open an issue
29-
CASE_ANL: "C48"
30-
COMIN_OBS: "/scratch2/NCEPDEV/marineda/r2d2-v2-v3" # TODO: make platform agnostic
31-
SOCA_OBS_LIST: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml"
28+
SOCA_INPUT_FIX_DIR: "/scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/72x35x25/soca" # TODO: These need to go to glopara fix space.
29+
CASE_ANL: "C48" # TODO: Check in gdasapp if used anywhere for SOCA
30+
SOCA_OBS_LIST: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml" # TODO: This is also repeated in oceanprepobs
3231
SOCA_NINNER: 100
33-
R2D2_OBS_SRC: "gdas_marine"
34-
R2D2_OBS_DUMP: "s2s_v1"
3532
SABER_BLOCKS_YAML: ""
3633
NICAS_RESOL: 1
3734
NICAS_GRID_SIZE: 15000
35+
prepoceanobs:
36+
SOCA_OBS_LIST: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml" # TODO: This is also repeated in ocnanal
37+
OBSPREP_YAML: "{{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obsprep/obsprep_config.yaml"
38+
DMPDIR: "/scratch1/NCEPDEV/global/glopara/data/experimental_obs"

‎workflow/applications/gfs_cycled.py

+7-4
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ def __init__(self, conf: Configuration):
1818
self.do_jediocnvar = self._base.get('DO_JEDIOCNVAR', False)
1919
self.do_jedilandda = self._base.get('DO_JEDILANDDA', False)
2020
self.do_mergensst = self._base.get('DO_MERGENSST', False)
21+
self.do_vrfy_oceanda = self._base.get('DO_VRFY_OCEANDA', False)
2122

2223
self.lobsdiag_forenkf = False
2324
self.eupd_cdumps = None
@@ -43,8 +44,9 @@ def _get_app_configs(self):
4344

4445
if self.do_jediocnvar:
4546
configs += ['prepoceanobs', 'ocnanalprep', 'ocnanalbmat',
46-
'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost',
47-
'ocnanalvrfy']
47+
'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost']
48+
if self.do_vrfy_oceanda:
49+
configs += ['ocnanalvrfy']
4850

4951
if self.do_ocean:
5052
configs += ['ocnpost']
@@ -137,8 +139,9 @@ def get_task_names(self):
137139
if self.do_jediocnvar:
138140
gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'ocnanalprep',
139141
'ocnanalbmat', 'ocnanalrun',
140-
'ocnanalchkpt', 'ocnanalpost',
141-
'ocnanalvrfy']
142+
'ocnanalchkpt', 'ocnanalpost']
143+
if self.do_vrfy_oceanda:
144+
gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy']
142145

143146
gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc']
144147

‎workflow/setup_expt.py

+7
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,13 @@ def link_files_from_src_to_dst(src_dir, dst_dir):
224224
src_file = os.path.join(src_dir, fname)
225225
if os.path.exists(src_file):
226226
os.symlink(src_file, os.path.join(dst_dir, fname))
227+
# First 1/2 cycle also needs a atmos increment if doing warm start
228+
if inputs.start in ['warm']:
229+
for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']:
230+
fname = f'{inputs.cdump}.t{idatestr[8:]}z.{ftype}'
231+
src_file = os.path.join(src_dir, fname)
232+
if os.path.exists(src_file):
233+
os.symlink(src_file, os.path.join(dst_dir, fname))
227234

228235
return
229236

0 commit comments

Comments
 (0)