JCSDA · climbfuji · Jul 14, 2023 · Jul 10, 2023 · Jul 11, 2023 · Jul 11, 2023
diff --git a/.gitmodules b/.gitmodules
@@ -3,7 +3,7 @@
   #url = https://github.com/spack/spack
   #branch = develop
   url = https://github.com/jcsda/spack
-  branch = release/1.4.1
+  branch = spack-stack-1.4.1
 [submodule "doc/CMakeModules"]
   path = doc/CMakeModules
   url = https://github.com/noaa-emc/cmakemodules

diff --git a/configs/containers/README.md b/configs/containers/README.md
@@ -25,7 +25,7 @@ To avoid hardcoding specs in the generic container recipes, we keep the specs li
     # py-mysql-connector-python@8.0.32
 ```
 
-### spack-stack-1.4.1 / ufs-weather-model-x.y.z containers for ufs-weather-model as of May 18, 2023
+### spack-stack-1.4.1 / ufs-weather-model-x.y.z containers for ufs-weather-model as of July 5, 2023
 
 **Note. This is not yet working correctly, some libraries are missing. Please do not use yet! Also, if using the clang-mpich container, need to disable openmp for fms, not clear how to do this cleanly.**
 

diff --git a/configs/containers/docker-ubuntu-clang-mpich.yaml b/configs/containers/docker-ubuntu-clang-mpich.yaml
@@ -105,7 +105,7 @@ spack:
       os: ubuntu:20.04
       spack:
         url: https://github.com/jcsda/spack
-        ref: spack-stack-1.4.0
+        ref: spack-stack-1.4.1
         resolve_sha: false
 
     # Whether or not to strip binaries

diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml
@@ -92,7 +92,7 @@ spack:
       os: ubuntu:20.04
       spack:
         url: https://github.com/jcsda/spack
-        ref: spack-stack-1.4.0
+        ref: spack-stack-1.4.1
         resolve_sha: false
 
     # Whether or not to strip binaries

diff --git a/configs/containers/docker-ubuntu-intel-impi.yaml b/configs/containers/docker-ubuntu-intel-impi.yaml
@@ -110,7 +110,7 @@ spack:
       os: ubuntu:20.04
       spack:
         url: https://github.com/jcsda/spack
-        ref: spack-stack-1.4.0
+        ref: spack-stack-1.4.1
         resolve_sha: false
 
     # Whether or not to strip binaries

diff --git a/configs/sites/gaea-c5/packages.yaml b/configs/sites/gaea-c5/packages.yaml
@@ -80,7 +80,7 @@ packages:
     buildable: False
     externals:
     - spec: ecflow@5.8.4+ui+static_boost
-      prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/ecflow-5.8.4-c5
+      prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/ecflow-5.8.4
       modules: [ecflow/5.8.4]
   file:
     externals:
@@ -174,7 +174,7 @@ packages:
     buildable: False
     externals:
     - spec: mysql@8.0.31
-      prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/mysql-8.0.31-c5
+      prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/mysql-8.0.31
       modules: [mysql/8.0.31]
   ncurses:
     externals:
@@ -204,7 +204,7 @@ packages:
   qt:
     externals:
     - spec: qt@5.15.2
-      prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/qt-5.15.2-c5/5.15.2/gcc_64
+      prefix: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/qt-5.15.2/5.15.2/gcc_64
   rdma-core:
     externals:
     - spec: rdma-core@37.0

diff --git a/configs/sites/hercules/mirrors.yaml b/configs/sites/hercules/mirrors.yaml
@@ -1,15 +1,15 @@
 mirrors:
   local-source:
     fetch:
-      url: file:///work/noaa/da/role-da/spack-stack/source-cache
+      url: file:///work/noaa/epic/role-epic/spack-stack/source-cache
       access_pair:
       - null
       - null
       access_token: null
       profile: null
       endpoint_url: null
     push:
-      url: file:///work/noaa/da/role-da/spack-stack/source-cache
+      url: file:///work/noaa/epic/role-epic/spack-stack/source-cache
       access_pair:
       - null
       - null

diff --git a/configs/sites/hercules/packages.yaml b/configs/sites/hercules/packages.yaml
@@ -3,7 +3,7 @@ packages:
     compiler:: [intel@2021.7.1, gcc@11.3.1]
     #compiler:: [oneapi@2022.2.1]
     providers:
-      mpi:: [intel-oneapi-mpi@2021.7.1, openmpi@4.1.4]
+      mpi:: [intel-oneapi-mpi@2021.7.1, openmpi@4.1.5]
 
 ### MPI, Python, MKL
   mpi:
@@ -20,11 +20,11 @@ packages:
     #  - intel-oneapi-mpi/2021.7.1
   openmpi:
     externals:
-    - spec: openmpi@4.1.4%gcc@11.3.1 ~cuda~cxx~cxx_exceptions~java~memchecker+pmi+static~wrapper-rpath
+    - spec: openmpi@4.1.5%gcc@11.3.1~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath
         fabrics=ucx schedulers=slurm
-      prefix: /apps/spack-managed/gcc-11.3.1/openmpi-4.1.4-ruvlmb6yyvzbzbiqaov4zk75ogthczsp
+      prefix: /work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.5/gcc-11.3.1
       modules:
-      - openmpi/4.1.4
+      - openmpi/4.1.5
   python:
     buildable: False
     externals:
@@ -74,9 +74,9 @@ packages:
     buildable: False
     externals:
     - spec: ecflow@5.8.4+ui+static_boost
-      prefix: /work/noaa/epic-ps/role-epic-ps/spack-stack/ecflow-5.8.4-hercules
+      prefix: /work/noaa/epic/role-epic/spack-stack/hercules/ecflow-5.8.4
       modules:
-      - ecflow/5.8.4-hercules
+      - ecflow/5.8.4
   findutils:
     externals:
     - spec: findutils@4.8.0
@@ -119,9 +119,9 @@ packages:
     buildable: False
     externals:
     - spec: mysql@8.0.31
-      prefix: /work/noaa/epic-ps/role-epic-ps/spack-stack/mysql-8.0.31-hercules
+      prefix: /work/noaa/epic/role-epic/spack-stack/hercules/mysql-8.0.31
       modules:
-      - mysql/8.0.31-hercules
+      - mysql/8.0.31
   openssh:
     externals:
     - spec: openssh@8.7p1
@@ -144,7 +144,7 @@ packages:
   qt:
     externals:
     - spec: qt@5.15.8
-      prefix: /apps/spack-managed/gcc-11.3.1/qt-5.15.8-d47tsna6f5dylcpblkfgw4gpn2cucihd
+      prefix: /apps/spack-managed/gcc-12.2.0/qt-5.15.8-gayzhaahclvlybiiykwj2cym4vo33w6x
       modules:
       - qt/5.15.8
   subversion:
@@ -164,3 +164,10 @@ packages:
     externals:
     - spec: wget@1.21.1
       prefix: /usr
+  # Need to use external zlib, because of qt dependence on it (otherwise issues with tar command)
+  zlib:
+    externals:
+    - spec: zlib@1.2.13
+      prefix: /apps/spack-managed/gcc-11.3.1/zlib-1.2.13-ltp4c3zzde3zi3gf7x4b7c7nj5ww4i4g
+      modules:
+      - zlib/1.2.13
diff --git a/configs/sites/orion/mirrors.yaml b/configs/sites/orion/mirrors.yaml
@@ -1,15 +1,15 @@
 mirrors:
   local-source:
     fetch:
-      url: file:///work/noaa/da/role-da/spack-stack/source-cache
+      url: file:///work/noaa/epic/role-epic/spack-stack/source-cache
       access_pair:
       - null
       - null
       access_token: null
       profile: null
       endpoint_url: null
     push:
-      url: file:///work/noaa/da/role-da/spack-stack/source-cache
+      url: file:///work/noaa/epic/role-epic/spack-stack/source-cache
       access_pair:
       - null
       - null

diff --git a/doc/modulefile_templates/openmpi b/doc/modulefile_templates/openmpi
@@ -31,7 +31,7 @@ unsetenv SLURM_EXPORT_ENV
 setenv PSM2_PATH_SELECTION "static_base"
 setenv SLURM_CPU_BIND "none"
 
-# Settings specific for Cheyenne
+# Settings specific for Cheyenne and Hercules
 setenv MPI_ROOT ${OPENMPI_PATH}
 setenv UCX_MAX_RNDV_RAILS "1"
 setenv OMPI_MCA_btl "^openib"
diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst
@@ -63,6 +63,9 @@ Sign into qt, select customized installation, choose qt@5.15.2 only (uncheck all
 .. note::
    On air-gapped systems, the above method may not work (we have not encountered such a system so far).
 
+.. note::
+   If ``./qt-unified-linux-x64-online.run`` fails to start with the error ``qt.qpa.xcb: could not connect to display`` and a role account is being used, follow the procedure described in https://www.thegeekdiary.com/how-to-set-x11-forwarding-export-remote-display-for-users-who-switch-accounts-using-sudo to export the display. A possible warning ``xauth:  file /ncrc/home1/role.epic/.Xauthority does not exist`` can be ignored, since this file gets created by the ``xauth`` command.
+
 ..  _MaintainersSection_ecFlow:
 
 ------------------------------
@@ -243,7 +246,7 @@ MSU Hercules
 ------------------------------
 
 ecflow
-  ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library, using an available ``Qt5`` installation. After loading the following modules, follow the instructions in :numref:`Section %s <MaintainersSection_ecFlow>` to install ``ecflow`` in ``/work/noaa/epic-ps/role-epic-ps/spack-stack/ecflow-5.8.4-hercules``.
+  ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library, using an available ``Qt5`` installation. After loading the following modules, follow the instructions in :numref:`Section %s <MaintainersSection_ecFlow>` to install ``ecflow`` in ``/work/noaa/epic/role-epic/spack-stack/hercules/ecflow-5.8.4``.
 
 .. code-block:: console
 
@@ -253,6 +256,22 @@ ecflow
 mysql
   ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s <MaintainersSection_MySQL>` to install ``mysql`` in ``/work/noaa/epic-ps/role-epic-ps/spack-stack/mysql-8.0.31-hercules``.
 
+openmpi
+  need to load qt so to get consistent zlib (or just load zlib directly, check qt module)
+
+.. code-block:: console
+
+   module purge
+   module load zlib/1.2.13
+   module load ucx/1.13.1
+   ./configure \
+       --prefix=/work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.5/gcc-11.3.1  \
+       --with-ucx=$UCX_ROOT \
+       --with-zlib=$ZLIB_ROOT
+   make VERBOSE=1 -j4
+   make check
+   make install
+
 .. _MaintainersSection_Discover:
 
 ------------------------------
@@ -538,17 +557,8 @@ NOAA RDHPCS Gaea C5
 
 On Gaea C5, ``miniconda``, ``qt``, ``ecflow``, and ``mysql`` need to be installed as a one-off before spack can be used.
 
-miniconda
-   Follow the instructions in :numref:`Section %s <MaintainersSection_Miniconda>` to create a basic ``miniconda`` installation and associated modulefile for working with spack. Don't forget to log off and back on to forget about the conda environment. Use the following workaround to avoid the terminal being spammed by error messages about missing version information (``/usr/bin/lua5.3: /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/miniconda-3.9.12-c5/lib/libtinfo.so.6: no version information available (required by /lib64/libreadline.so.7)``):
-
-.. code-block:: console
-
-   cd /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/miniconda-3.9.12-c5/lib
-   mv libtinfow.so.6.3 libtinfow.so.6.3.conda.original
-   ln -sf /lib64/libtinfo.so.6 libtinfow.so.6.3
-
 qt (qt@5)
-   The default ``qt@5`` in ``/usr`` is incomplete and thus insufficient for building ``ecflow``. After loading/unloading the modules as shown below, refer to :numref:`Section %s <MaintainersSection_Qt5>` to install ``qt@5.15.2`` in ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/qt-5.15.2-c5``. Note that the installation must be done as a regular user due to problems with graphical applications for role accounts.
+   The default ``qt@5`` in ``/usr`` is incomplete and thus insufficient for building ``ecflow``. After loading/unloading the modules as shown below, refer to :numref:`Section %s <MaintainersSection_Qt5>` to install ``qt@5.15.2`` in ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/qt-5.15.2``. :numref:`Section %s <MaintainersSection_Qt5>` describes how to export the X windows environment in order to install ``qt@5`` using the role account.
 
 .. code-block:: console
 
@@ -557,7 +567,7 @@ qt (qt@5)
    module load PrgEnv-gnu/8.3.3
 
 ecflow
-  ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `qt5` and loading the following modules, follow the instructions in :numref:`Section %s <MaintainersSection_ecFlow>`. Because of the dependency on ``miniconda``, that module must be loaded automatically in the ``ecflow`` module (similar to ``qt@5.15.2-c5``).  Ensure to follow the extra instructions in that section for Gaea C5.
+  ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `qt5` and loading the following modules, follow the instructions in :numref:`Section %s <MaintainersSection_ecFlow>`. Because of the dependency on ``miniconda``, that module must be loaded automatically in the ``ecflow`` module (similar to ``qt@5.15.2-c5``).  Ensure to follow the extra instructions in that section for Gaea C5 in ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/ecflow-5.8.4``.
 
    Ensure to follow the extra instructions in that section for Gaea.
 
@@ -568,11 +578,11 @@ ecflow
    module load PrgEnv-gnu/8.3.3
    module load python/3.9.12
 
-   module use /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/modulefiles-c5
+   module use /lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/modulefiles
    module load qt/5.15.2
 
 mysql
-  ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s <MaintainersSection_MySQL>` to install ``mysql`` in ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/mysql-8.0.31-c5``.
+  ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s <MaintainersSection_MySQL>` to install ``mysql`` in ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/mysql-8.0.31``.
 
 .. _MaintainersSection_Hera: