Update scripts to use JEDI skylab v7

.github/pull_request_template.md

@@ -7,7 +7,7 @@ Provide a detailed description of what this PR does. What bug does it fix, or wh
 - [ ] Is baseline change expected ? <!-- Add "Baseline Change" Label -->
 
 ## Subcomponents involved:
-- [ ] DA_update (ufs-community/land-DA)
+- [ ] apply_incr.fd (NOAA-PSL/land-apply_jedi_incr)
 - [ ] ufsLand.fd (NOAA-EPIC/ufs-land-driver-emc-dev)
 - [ ] ufs_model.fd (ufs-community/ufs-weather-model)
 - [ ] vector2tile_converter.fd (NOAA-PSL/land-vector2tile)

.gitmodules

@@ -1,7 +1,3 @@
-[submodule "sorc/DA_update"]
-	path = sorc/DA_update
-	url = https://github.com/ufs-community/land-DA.git
-	ignore = dirty
 [submodule "sorc/ufsLand.fd"]
 	path = sorc/ufsLand.fd
 	url = https://github.com/NOAA-EPIC/ufs-land-driver-emc-dev.git
@@ -14,3 +10,8 @@
 	path = sorc/ufs_model.fd
 	url = https://github.com/ufs-community/ufs-weather-model.git
 	ignore = dirty
+[submodule "sorc/apply_incr.fd"]
+	path = sorc/apply_incr.fd
+	url = https://github.com/NOAA-PSL/land-apply_jedi_incr.git
+        ignore = dirty
+

modulefiles/build_hera_intel.lua

@@ -8,7 +8,7 @@ prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
 prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi"))
 
 load(pathJoin("stack-intel", stack_intel_ver))
-load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
 load(pathJoin("cmake", cmake_ver))
@@ -43,5 +43,4 @@ setenv("FC", "mpiifort")
 setenv("CMAKE_Platform", "hera.intel")
 
 setenv("EPICHOME", "/scratch2/NAGAPE/epic/UFS_Land-DA")
-setenv("JEDI_INSTALL", pathJoin(os.getenv("EPICHOME"),"jedi"))
-
+setenv("JEDI_INSTALL", pathJoin(os.getenv("EPICHOME"),"jedi_skylabv7.0"))

modulefiles/build_orion_intel.lua

@@ -8,7 +8,7 @@ prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack"))
 prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi"))
 
 load(pathJoin("stack-intel", stack_intel_ver))
-load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 load(pathJoin("stack-python", stack_python_ver))
 
 load(pathJoin("cmake", cmake_ver))
@@ -42,5 +42,4 @@ setenv("FC", "mpiifort")
 setenv("CMAKE_Platform", "orion.intel")
 
 setenv("EPICHOME", "/work/noaa/epic/UFS_Land-DA")
-setenv("JEDI_INSTALL", pathJoin(os.getenv("EPICHOME"),"jedi"))
-
+setenv("JEDI_INSTALL", pathJoin(os.getenv("EPICHOME"),"jedi_skylabv7.0"))

modulefiles/build_singularity_intel.lua

@@ -10,8 +10,8 @@ stack_intel_ver=os.getenv("stack_intel_ver") or "2021.8.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
 load("intel-oneapi-mpi/2021.8.0")
-stack_impi_ver=os.getenv("stack_impi_ver") or "2021.8.0"
-load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+stack_intel_oneapi_mpi_ver=os.getenv("stack_intel_oneapi_mpi_ver") or "2021.8.0"
+load(pathJoin("stack-intel-oneapi-mpi", stack_intel_oneapi_mpi_ver))
 
 stack_python_ver=os.getenv("stack_python_ver") or "3.8.10"
 load(pathJoin("stack-python", stack_python_ver))

parm/jedi/GHCN.yaml

@@ -0,0 +1,55 @@
+  - obs space:
+      name: SnowDepthGHCN
+      distribution: 
+        name: Halo
+        halo size: 250e3
+      simulated variables: [totalSnowDepth]
+      obsdatain:
+        engine:
+          type: H5File
+          obsfile: GHCN_XXYYYYXXMMXXDDXXHH.nc
+      obsdataout:
+        engine:
+          type: H5File
+          obsfile: output/DA/hofx/letkf_hofx_ghcn_XXYYYYXXMMXXDDXXHH.nc
+    obs operator:
+      name: Identity
+    obs error:
+      covariance model: diagonal
+    obs localizations:
+    - localization method: Horizontal SOAR
+      lengthscale: 250e3
+      soar horizontal decay: 0.000021
+      max nobs: 50
+    - localization method: Vertical Brasnett
+      vertical lengthscale: 700
+    obs filters:
+    - filter: Bounds Check # negative / missing snow
+      filter variables:
+      - name: totalSnowDepth
+      minvalue: 0.0
+      maxvalue: 10000.0
+    - filter: Domain Check # missing station elevation (-999.9)
+      where:
+      - variable:
+          name: MetaData/stationElevation
+        minvalue: -999.0
+        maxvalue: 10000.0
+    - filter: Domain Check # land only
+      where:
+      - variable:
+          name: GeoVaLs/slmsk
+        minvalue: 0.5
+        maxvalue: 1.5
+    - filter: RejectList  # no land-ice
+      where:
+      - variable:
+          name: GeoVaLs/vtype
+        minvalue: 14.5
+        maxvalue: 15.5
+    - filter: Background Check # gross error check
+      filter variables:
+      - name: totalSnowDepth
+      threshold: 6.25
+      action:
+        name: reject

parm/jedi/gfs-land.yaml

@@ -0,0 +1,10 @@
+field metadata:
+
+- long name: slmsk
+  interpolation type: nearest
+
+- long name: totalSnowDepth
+  interpolation type: nearest
+
+- long name: vtype
+  interpolation type: nearest

parm/jedi/letkfoi_snow.yaml

@@ -0,0 +1,58 @@
+geometry:
+  fms initialization:
+    namelist filename: Data/fv3files/fmsmpp.nml
+    field table filename: Data/fv3files/field_table
+  akbk: Data/fv3files/akbk64.nc4
+  npx: XXREP
+  npy: XXREP
+  npz: 64
+  field metadata override: gfs-land.yaml
+  time invariant fields:
+    state fields:
+      datetime: XXYYYP-XXMP-XXDPTXXHP:00:00Z
+      filetype: fms restart
+      skip coupler file: true
+      state variables: [orog_filt]
+      datapath: XXTPATH
+      filename_orog: XXTSTUB.nc
+    derived fields: [nominal_surface_pressure]
+
+time window:
+  begin: XXYYYP-XXMP-XXDPTXXHP:00:00Z
+  length: PT24H
+
+background:
+ date: &date XXYYYY-XXMM-XXDDTXXHH:00:00Z
+ members:
+   - datetime: XXYYYY-XXMM-XXDDTXXHH:00:00Z
+     filetype: fms restart
+     state variables: [snwdph,vtype,slmsk]
+     datapath: mem_pos/
+     filename_sfcd: XXYYYYXXMMXXDD.XXHH0000.sfc_data.nc
+     filename_cplr: XXYYYYXXMMXXDD.XXHH0000.coupler.res
+   - datetime: XXYYYY-XXMM-XXDDTXXHH:00:00Z
+     filetype: fms restart
+     state variables: [snwdph,vtype,slmsk]
+     datapath: mem_neg/
+     filename_sfcd: XXYYYYXXMMXXDD.XXHH0000.sfc_data.nc
+     filename_cplr: XXYYYYXXMMXXDD.XXHH0000.coupler.res
+
+driver:
+  save posterior mean: false
+  save posterior mean increment: true
+  save posterior ensemble: false
+  run as observer only: XXHOFX
+
+local ensemble DA:
+  solver: LETKF
+  inflation:
+    rtps: 0.0
+    rtpp: 0.0
+    mult: 1.0
+
+output increment:
+  filetype: fms restart
+  filename_sfcd: xainc.sfc_data.nc
+
+observations:
+  observers:

parm/land_analysis_era5_hera.yaml

@@ -12,15 +12,14 @@ workflow:
     ACCOUNT: "nems"
     EXP_NAME: "LETKF"
     EXP_BASEDIR: "/scratch2/NAGAPE/epic/{USER}/landda_test"
-    JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi"
+    JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi_skylabv7.0"
     LANDDA_INPUTS: "/scratch2/NAGAPE/epic/UFS_Land-DA/inputs"
     FORCING: "era5"
     RES: "96"
     FCSTHR: "24"
     NPROCS_ANALYSIS: "6"
     NPROCS_FORECAST: "6"
     OBS_TYPES: "GHCN"
-    fv3bundle_vn: "psl_develop"
     DAtype: "letkfoi_snow"
     SNOWDEPTHVAR: "snwdph"
     TSTUB: "oro_C96.mx100"
@@ -117,7 +116,6 @@ workflow:
         cyc: "&cyc;"
         SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;"
         PTIME: "&PTIME;"
-        fv3bundle_vn: "&fv3bundle_vn;"
         DAtype: "&DAtype;"
         SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
       account: "&ACCOUNT;"
@@ -153,7 +151,6 @@ workflow:
         SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;"
         PTIME: "&PTIME;"
         NTIME: "&NTIME;"
-        fv3bundle_vn: "&fv3bundle_vn;"
         DAtype: "&DAtype;"
         SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
         NPROC_JEDI: "&NPROCS_ANALYSIS;"
@@ -192,7 +189,6 @@ workflow:
         SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;"
         PTIME: "&PTIME;"
         NTIME: "&NTIME;"
-        fv3bundle_vn: "&fv3bundle_vn;"
         DAtype: "&DAtype;"
         SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
         JEDI_INSTALL: "&JEDI_INSTALL;"

parm/land_analysis_era5_orion.yaml

@@ -12,15 +12,14 @@ workflow:
     ACCOUNT: "epic"
     EXP_NAME: "LETKF"
     EXP_BASEDIR: "/work/noaa/epic/{USER}/landda_test"
-    JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi"
+    JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi_skylabv7.0"
     LANDDA_INPUTS: "/work/noaa/epic/UFS_Land-DA/inputs"
     FORCING: "era5"
     RES: "96"
     FCSTHR: "24"
     NPROCS_ANALYSIS: "6"
     NPROCS_FORECAST: "6"
     OBS_TYPES: "GHCN"
-    fv3bundle_vn: "psl_develop"
     DAtype: "letkfoi_snow"
     SNOWDEPTHVAR: "snwdph"
     TSTUB: "oro_C96.mx100"
@@ -117,7 +116,6 @@ workflow:
         cyc: "&cyc;"
         SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;"
         PTIME: "&PTIME;"
-        fv3bundle_vn: "&fv3bundle_vn;"
         DAtype: "&DAtype;"
         SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
       account: "&ACCOUNT;"
@@ -153,7 +151,6 @@ workflow:
         SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;"
         PTIME: "&PTIME;"
         NTIME: "&NTIME;"
-        fv3bundle_vn: "&fv3bundle_vn;"
         DAtype: "&DAtype;"
         SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
         NPROC_JEDI: "&NPROCS_ANALYSIS;"
@@ -192,7 +189,6 @@ workflow:
         SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;"
         PTIME: "&PTIME;"
         NTIME: "&NTIME;"
-        fv3bundle_vn: "&fv3bundle_vn;"
         DAtype: "&DAtype;"
         SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
         JEDI_INSTALL: "&JEDI_INSTALL;"

parm/land_analysis_gswp3_hera.yaml

@@ -12,15 +12,14 @@ workflow:
     ACCOUNT: "nems"
     EXP_NAME: "LETKF"
     EXP_BASEDIR: "/scratch2/NAGAPE/epic/{USER}/landda_test"
-    JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi"
+    JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi_skylabv7.0"
     LANDDA_INPUTS: "/scratch2/NAGAPE/epic/UFS_Land-DA/inputs"
     FORCING: "gswp3"
     RES: "96"
     FCSTHR: "24"
     NPROCS_ANALYSIS: "6"
     NPROCS_FORECAST: "6"
     OBS_TYPES: "GHCN"
-    fv3bundle_vn: "psl_develop"
     DAtype: "letkfoi_snow"
     SNOWDEPTHVAR: "snwdph"
     TSTUB: "oro_C96.mx100"
@@ -117,7 +116,6 @@ workflow:
         cyc: "&cyc;"
         SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;"
         PTIME: "&PTIME;"
-        fv3bundle_vn: "&fv3bundle_vn;"
         DAtype: "&DAtype;"
         SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
       account: "&ACCOUNT;"
@@ -153,7 +151,6 @@ workflow:
         SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;"
         PTIME: "&PTIME;"
         NTIME: "&NTIME;"
-        fv3bundle_vn: "&fv3bundle_vn;"
         DAtype: "&DAtype;"
         SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
         NPROC_JEDI: "&NPROCS_ANALYSIS;"
@@ -192,7 +189,6 @@ workflow:
         SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;"
         PTIME: "&PTIME;"
         NTIME: "&NTIME;"
-        fv3bundle_vn: "&fv3bundle_vn;"
         DAtype: "&DAtype;"
         SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
         JEDI_INSTALL: "&JEDI_INSTALL;"

parm/land_analysis_gswp3_orion.yaml

@@ -12,15 +12,14 @@ workflow:
     ACCOUNT: "epic"
     EXP_NAME: "LETKF"
     EXP_BASEDIR: "/work/noaa/epic/{USER}/landda_test"
-    JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi"
+    JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi_skylabv7.0"
     LANDDA_INPUTS: "/work/noaa/epic/UFS_Land-DA/inputs"
     FORCING: "gswp3"
     RES: "96"
     FCSTHR: "24"
     NPROCS_ANALYSIS: "6"
     NPROCS_FORECAST: "6"
     OBS_TYPES: "GHCN"
-    fv3bundle_vn: "psl_develop"
     DAtype: "letkfoi_snow"
     SNOWDEPTHVAR: "snwdph"
     TSTUB: "oro_C96.mx100"
@@ -117,7 +116,6 @@ workflow:
         cyc: "&cyc;"
         SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;"
         PTIME: "&PTIME;"
-        fv3bundle_vn: "&fv3bundle_vn;"
         DAtype: "&DAtype;"
         SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
       account: "&ACCOUNT;"
@@ -153,7 +151,6 @@ workflow:
         SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;"
         PTIME: "&PTIME;"
         NTIME: "&NTIME;"
-        fv3bundle_vn: "&fv3bundle_vn;"
         DAtype: "&DAtype;"
         SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
         NPROC_JEDI: "&NPROCS_ANALYSIS;"
@@ -192,7 +189,6 @@ workflow:
         SLASH_ENSMEM_SUBDIR: "&SLASH_ENSMEM_SUBDIR;"
         PTIME: "&PTIME;"
         NTIME: "&NTIME;"
-        fv3bundle_vn: "&fv3bundle_vn;"
         DAtype: "&DAtype;"
         SNOWDEPTHVAR: "&SNOWDEPTHVAR;"
         JEDI_INSTALL: "&JEDI_INSTALL;"
@@ -201,7 +197,7 @@ workflow:
       command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "forecast" "&HOMElandda;" "&MACHINE;"'
       jobname: forecast
       nodes: "1:ppn=&NPROCS_FORECAST;"
-      walltime: 00:30:00
+      walltime: 00:40:00
       queue: batch
       join: "&LOGDIR;/forecast.log"
       dependency:

parm/run_without_rocoto.sh

@@ -17,11 +17,11 @@ export FORCING="era5"
 
 if [ "${MACHINE}" = "hera" ]; then
   export EXP_BASEDIR="/scratch2/NAGAPE/epic/{USER}/landda_test"
-  export JEDI_INSTALL="/scratch2/NAGAPE/epic/UFS_Land-DA/jedi"
+  export JEDI_INSTALL="/scratch2/NAGAPE/epic/UFS_Land-DA/jedi_skylabv7.0"
   export LANDDA_INPUTS="/scratch2/NAGAPE/epic/UFS_Land-DA/inputs"
 elif [ "${MACHINE}" = "orion" ]; then
   export EXP_BASEDIR="/work/noaa/epic/chjeon/landda_test"
-  export JEDI_INSTALL="/work/noaa/epic/UFS_Land-DA/jedi"
+  export JEDI_INSTALL="/work/noaa/epic/UFS_Land-DA/jedi_skylabv7.0"
   export LANDDA_INPUTS="/work/noaa/epic/UFS_Land-DA/inputs"
 fi
 
@@ -30,7 +30,6 @@ export FCSTHR="24"
 export NPROCS_ANALYSIS="6"
 export NPROCS_FORECAST="6"
 export OBS_TYPES="GHCN"
-export fv3bundle_vn="psl_develop"
 export DAtype="letkfoi_snow"
 export SNOWDEPTHVAR="snwdph"
 export TSTUB="oro_C96.mx100"

parm/templates/template.coupler.res

@@ -0,0 +1,4 @@
+     2        (Calendar: no_calendar=0, thirty_day_months=1, julian=2, gregorian=3, noleap=4)
+  XXYYYP    XXMP    XXDP   XXHP   0     0        Model start time:   year, month, day, hour, minute, second
+  XXYYYY    XXMM    XXDD   XXHH   0     0        Current model time: year, month, day, hour, minute, second
+