Add oneAPI config for Narwhal, add GNU config for Atlantis, remove Sandy platform; make python variant for esmf configurable (#1318)

- Add a [email protected] config for Narwhal (using icx, icpx, ifort)
- Add a [email protected] config for Atlantis
- Remove Sandy as a supported platform
- Update submodule pointer for spack to bring in [email protected] (official release) and replace [email protected] with [email protected] in all configs/**
- Make python variant for esmf configurable

Author: climbfuji

configs/common/modules_lmod.yaml

@@ -136,6 +136,8 @@ modules:
           ^[email protected]+debug snapshot=b04: 'esmf-8.7.0b04-debug'
           ^[email protected]~debug snapshot=b11: 'esmf-8.7.0b11'
           ^[email protected]+debug snapshot=b11: 'esmf-8.7.0b11-debug'
+          ^[email protected]~debug snapshot=none: 'esmf-8.7.0'
+          ^[email protected]+debug snapshot=none: 'esmf-8.7.0-debug'
       openmpi:
         environment:
           set:

configs/common/modules_tcl.yaml

@@ -138,6 +138,8 @@ modules:
           ^[email protected]+debug snapshot=b04: 'esmf-8.7.0b04-debug'
           ^[email protected]~debug snapshot=b11: 'esmf-8.7.0b11'
           ^[email protected]+debug snapshot=b11: 'esmf-8.7.0b11-debug'
+          ^[email protected]~debug snapshot=none: 'esmf-8.7.0'
+          ^[email protected]+debug snapshot=none: 'esmf-8.7.0-debug'
       openmpi:
         environment:
           set:

configs/common/packages

@@ -54,17 +54,19 @@ packages:
   # Also, check the acorn and derecho site configs which have esmf modifications.
   esmf:
     require:
-      - '~xerces ~pnetcdf +shared +external-parallelio +python'
-      - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11']
-      - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"']
-        when: "%intel"
-        message: "Extra ESMF compile options for Intel"
-      #- any_of: ['']
-      #  when: "%gcc"
-      #  message: "Extra ESMF compile options for GCC"
-      #- any_of: ['']
-      #  when: "%apple-clang"
-      #  message: "Extra ESMF compile options for GCC"
+    - '~xerces ~pnetcdf +shared +external-parallelio'
+    - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0 snapshot=none']
+    - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"']
+      when: "%intel"
+      message: "Extra ESMF compile options for Intel"
+    #- any_of: ['']
+    #  when: "%gcc"
+    #  message: "Extra ESMF compile options for GCC"
+    #- any_of: ['']
+    #  when: "%apple-clang"
+    #  message: "Extra ESMF compile options for GCC"
+    prefer:
+    - '+python'
   fckit:
     require: '@0.11.0 +eckit'
   fftw:

configs/common/packages.yaml

@@ -1,15 +1,15 @@
 mirrors:
   local-source:
     fetch:
-      url: file:///gpfs/fs1/neptune/spack-stack/source-cache
+      url: file:///neptune_diagnostics/spack-stack/source-cache/
       access_pair:
       - null
       - null
       access_token: null
       profile: null
       endpoint_url: null
     push:
-      url: file:///gpfs/fs1/neptune/spack-stack/source-cache
+      url: file:///neptune_diagnostics/spack-stack/source-cache/
       access_pair:
       - null
       - null

configs/sites/tier1/sandy/mirrors.yaml renamed to configs/sites/tier1/atlantis/mirrors.yaml

@@ -0,0 +1,13 @@
+packages:
+  all:
+    compiler:: [[email protected]]
+    providers:
+      mpi:: [[email protected]]
+  mpi:
+    buildable: False
+  openmpi:
+    buildable: False
+    externals:
+    - spec: [email protected]%gcc@=11.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm
+      modules:
+      - openmpi/mlnx/gcc/64/4.1.5a1

configs/sites/tier1/atlantis/packages_gcc.yaml

@@ -15,6 +15,6 @@ packages:
   intel-oneapi-mkl:
     externals:
     - spec: [email protected]
+      prefix: /cm/shared/apps/intel/oneapi
       modules:
       -  mkl/2022.1.0
-      prefix: /cm/shared/apps/intel/oneapi

configs/sites/tier1/atlantis/packages_intel.yaml

@@ -21,6 +21,30 @@ compilers::
         set:
           CRAYPE_LINK_TYPE: 'dynamic'
       extra_rpaths: []
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: cc
+        cxx: CC
+        f77: ftn
+        fc: ftn
+      flags: {}
+      operating_system: sles15
+      modules:
+      - PrgEnv-intel/8.4.0
+      - intel/2024.2
+      - cray-libsci/23.05.1.4
+      - libfabric/1.12.1.2.2.1
+      environment:
+        prepend_path:
+          PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin'
+          CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include'
+          LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64'
+        append_path:
+          CPATH: '/opt/intel/oneapi_2024.2.0.634/compiler/2024.2/opt/compiler/include/intel64'
+        set:
+          CRAYPE_LINK_TYPE: 'dynamic'
+      extra_rpaths: []
   - compiler:
       spec: [email protected]
       paths:

configs/sites/tier1/narwhal/compilers.yaml

@@ -1,15 +1,15 @@
 mirrors:
   local-source:
     fetch:
-      url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache
+      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
       access_pair:
       - null
       - null
       access_token: null
       profile: null
       endpoint_url: null
     push:
-      url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache
+      url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache
       access_pair:
       - null
       - null

configs/sites/tier1/narwhal/mirrors.yaml

@@ -0,0 +1,27 @@
+packages:
+  all:
+    compiler:: [[email protected],[email protected]]
+    providers:
+      mpi:: [[email protected]]
+  mpi:
+    buildable: False
+  cray-mpich:
+    externals:
+    - spec: [email protected]%[email protected] ~wrappers
+      modules:
+      - cray-mpich-ucx/8.1.21
+      - craype-network-ucx
+  intel-oneapi-mkl:
+    externals:
+    - spec: [email protected]%[email protected]
+      prefix: /opt/intel/oneapi_2024.2.0.634
+  intel-oneapi-tbb:
+    externals:
+    - spec: [email protected]%[email protected]
+      prefix: /opt/intel/oneapi_2024.2.0.634
+      modules:
+      - tbb/2021.13
+  intel-oneapi-runtime:
+    externals:
+    - spec: [email protected]%[email protected]
+      prefix: /opt/intel/oneapi_2024.2.0.634