[feature/update_to_spack_v1] Shorten module paths for spack-generated modules and meta-modules

.github/workflows/ubuntu-ci-x86_64-gnu.yaml

@@ -234,9 +234,9 @@ jobs:
 
           export ENVNAME=ue-gcc-11.4.0
           export ENVDIR=$PWD/envs/${ENVNAME}
-          ls -l ${ENVDIR}/install/modulefiles/Core
+          ls -l ${ENVDIR}/modules/Core
 
-          module use ${ENVDIR}/install/modulefiles/Core
+          module use ${ENVDIR}/modules/Core
           module load stack-gcc/11.4.0
           module load stack-openmpi/5.0.8
           module available

.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml

@@ -242,9 +242,9 @@ jobs:
 
           export ENVNAME=ue-oneifx-2024.2.0
           export ENVDIR=$PWD/envs/${ENVNAME}
-          ls -l ${ENVDIR}/install/modulefiles/Core
+          ls -l ${ENVDIR}/modules/Core
 
-          module use ${ENVDIR}/install/modulefiles/Core
+          module use ${ENVDIR}/modules/Core
           module load stack-intel-oneapi-compilers/2024.2.0
           module load stack-intel-oneapi-mpi/2021.13
           module available

.github/workflows/ubuntu-ci-x86_64-oneapi.yaml

@@ -242,9 +242,9 @@ jobs:
 
           export ENVNAME=ue-oneapi-2024.2.0
           export ENVDIR=$PWD/envs/${ENVNAME}
-          ls -l ${ENVDIR}/install/modulefiles/Core
+          ls -l ${ENVDIR}/modules/Core
 
-          module use ${ENVDIR}/install/modulefiles/Core
+          module use ${ENVDIR}/modules/Core
           module load stack-intel-oneapi-compilers/2024.2.0
           module load stack-intel-oneapi-mpi/2021.13
           module available

configs/common/modules_lmod.yaml

@@ -3,14 +3,13 @@
 modules:
   default:
     roots:
-      lmod: $env/install/modulefiles
+      lmod: $env/modules
     lmod:
       # Core compiler is a dummy that is not used to build the stack, do not change.
       # Hopefully nobody is trying to build software with gcc-4.6 in 2023 ...
       core_compilers::
       - [email protected]
       projections:
-        ecmwf-atlas: 'atlas/{version}'
         nlohmann-json: 'json/{version}'
         nlohmann-json-schema-validator: 'json-schema-validator/{version}'
         libjpeg-turbo: 'libjpeg/{version}'

configs/common/modules_tcl.yaml

@@ -3,12 +3,32 @@
 modules:
   default:
     roots:
-      tcl: $env/install/modulefiles
+      tcl: $env/modules
     tcl:
       projections:
-        all: '{compiler.name}/{compiler.version}/{name}/{version}'
+        # Note: These translations of compiler and MPI library names must match the ALIASES
+        # dictionary in spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py
+        # Defaults
+        '^intel-oneapi-compilers-classic^intel-oneapi-mpi': 'impi/{^mpi.version}/intel/{compiler.version}/{name}/{version}'
+        '^intel-oneapi-compilers-classic^mpi': '{^mpi.name}/{^mpi.version}/intel/{compiler.version}/{name}/{version}'
+        '^intel-oneapi-compilers-classic': 'intel/{compiler.version}/{name}/{version}'
+        '^intel-oneapi-compilers^intel-oneapi-mpi': 'impi/{^mpi.version}/oneapi/{compiler.version}/{name}/{version}'
+        '^intel-oneapi-compilers^mpi': '{^mpi.name}/{^mpi.version}/oneapi/{compiler.version}/{name}/{version}'
+        '^intel-oneapi-compilers': 'oneapi/{compiler.version}/{name}/{version}'
+        ^intel-oneapi-mpi: 'impi/{^mpi.version}/{compiler.name}/{compiler.version}/{name}/{version}'
         ^mpi: '{^mpi.name}/{^mpi.version}/{compiler.name}/{compiler.version}/{name}/{version}'
-        ecmwf-atlas: '{compiler.name}/{compiler.version}/atlas/{version}'
+        all: '{compiler.name}/{compiler.version}/{name}/{version}'
+        # The special cases - Intel classic compilers
+        'nlohmann-json ^intel-oneapi-compilers-classic': 'intel/{compiler.version}/json/{version}'
+        'nlohmann-json-schema-validator ^intel-oneapi-compilers-classic': 'intel/{compiler.version}/json-schema-validator/{version}'
+        'libjpeg-turbo ^intel-oneapi-compilers-classic': 'intel/{compiler.version}/libjpeg/{version}'
+        'prod-util ^intel-oneapi-compilers-classic': 'intel/{compiler.version}/prod_util/{version}'
+        # The special cases - Intel LLVM compilers
+        'nlohmann-json ^intel-oneapi-compilers': 'oneapi/{compiler.version}/json/{version}'
+        'nlohmann-json-schema-validator ^intel-oneapi-compilers': 'oneapi/{compiler.version}/json-schema-validator/{version}'
+        'libjpeg-turbo ^intel-oneapi-compilers': 'oneapi/{compiler.version}/libjpeg/{version}'
+        'prod-util ^intel-oneapi-compilers': 'oneapi/{compiler.version}/prod_util/{version}'
+        # The special cases - other compilers
         nlohmann-json: '{compiler.name}/{compiler.version}/json/{version}'
         nlohmann-json-schema-validator: '{compiler.name}/{compiler.version}/json-schema-validator/{version}'
         libjpeg-turbo: '{compiler.name}/{compiler.version}/libjpeg/{version}'

spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py

@@ -52,13 +52,22 @@
     "MPIROOT": "",
 }
 
-
-# DH* TODO: IMPORT THIS FROM SPACK, THEY MAINTAIN THE SAME DICTIONARY
-COMPILER_TRANSLATION_TABLE = {
+# Aliases to shorten module paths for tcl modules. These aliases must match
+# the compiler and MPI name translations in configs/common/modules_tcl.yaml
+ALIASES = {
+    "none" : "none",
+    # Compilers
     "gcc" : "gcc",
-    "clang" : "llvm",
-    "intel" : "intel-oneapi-compilers-classic",
-    "oneapi" : "intel-oneapi-compilers",
+    "intel-oneapi-compilers-classic" : "intel",
+    "intel-oneapi-compilers" : "oneapi",
+    "llvm" : "llvm",
+    # MPI
+    "cray-mpich" : "cray-mpich",
+    # Do we still need intel-mpi, and if yes, use the same impi?
+    "intel-oneapi-mpi" : "impi",
+    "mpich" : "mpich",
+    "mpt" : "mpt",
+    "openmpi" : "openmpi",
 }
 
 
@@ -220,7 +229,7 @@ def get_preferred_compiler():
     return preferred_compiler
 
 
-def remove_compiler_prefices_from_tcl_modulefiles(modulepath, compiler_list, mpi_provider):
+def remove_compiler_prefices_from_tcl_modulefiles(modulepath, compiler_list, mpi_provider, module_choice):
     """Remove compiler and mpi prefices from tcl modulefiles in modulepath"""
     logging.info(f"  ... ... removing compiler/mpi prefices from tcl modulefiles in {modulepath}")
     module_replace_patterns = ["is-loaded", "module load", "depends-on"]
@@ -237,20 +246,30 @@ def remove_compiler_prefices_from_tcl_modulefiles(modulepath, compiler_list, mpi
                 for compiler in compiler_list:
                     # First, compiler-dependent modules
                     (compiler_name, compiler_version) = compiler.split("@")
+                    # Module paths are short names for tcl modules
+                    if module_choice == "lmod":
+                        compiler_alias = compiler_name
+                    else:
+                        compiler_alias = ALIASES[compiler_name]
                     cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format(
-                        pattern, compiler_name, compiler_version, filepath, sed_syntax_fix
+                        pattern, compiler_alias, compiler_version, filepath, sed_syntax_fix
                     )
                     status = os.system(cmd)
                     if not status == 0:
                         raise Exception(f"Error while calling '{cmd}'")
                     # If mpi_provider is not None, also do compiler+mpi-dependent modules
                     if not mpi_provider:
                         continue
+                    # Module paths are short names for tcl modules
+                    if module_choice == "lmod":
+                        mpi_alias = mpi_provider.name
+                    else:
+                        mpi_alias = ALIASES[mpi_provider.name]
                     cmd = "sed -i {6} 's#{0} {1}/{2}/{3}/{4}/#{0} #g' {5}".format(
                         pattern,
-                        mpi_provider.name,
+                        mpi_alias,
                         mpi_provider.version,
-                        compiler_name,
+                        compiler_alias,
                         compiler_version,
                         filepath,
                         sed_syntax_fix,
@@ -381,13 +400,20 @@ def custom_sort_key(entry):
         remove_compiler_prefices_from_tcl_modulefiles(
             modulepath_save,
             compiler_list,
-            mpi_provider = None
+            mpi_provider = None,
+            module_choice = module_choice
         )
 
     for compiler in compilers:
         logging.info(f"  ... configuring compiler {compiler.name}@{compiler.version}")
 
-        modulepath_save = os.path.join(module_dir, compiler.name, str(compiler.version))
+        # Module paths are short names for tcl modules
+        if module_choice == "lmod":
+            compiler_alias = compiler.name
+        else:
+            compiler_alias = ALIASES[compiler.name]
+
+        modulepath_save = os.path.join(module_dir, compiler_alias, str(compiler.version))
         if not os.path.isdir(modulepath_save):
             os.makedirs(modulepath_save)
         logging.info("  ... ... appending {} to MODULEPATHS_SAVE".format(modulepath_save))
@@ -398,7 +424,8 @@ def custom_sort_key(entry):
             remove_compiler_prefices_from_tcl_modulefiles(
                 modulepath_save,
                 compiler_list,
-                mpi_provider = None
+                mpi_provider = None,
+                module_choice = module_choice
             )
 
         # The remainder of the loop is only needed for the preferred compiler
@@ -496,15 +523,20 @@ def custom_sort_key(entry):
         # For tcl, append modulepath for external specs and for specs without
         # compiler dependencies; remove the compiler/mpi prefices from the moduless
         if module_choice == "tcl":
-            modulepath_save = os.path.join(module_dir, mpi_provider.name, str(mpi_provider.version), "none", "none")
+
+            # Module paths are short names for tcl modules
+            mpi_alias = ALIASES[mpi_provider.name]
+
+            modulepath_save = os.path.join(module_dir, mpi_alias, str(mpi_provider.version), "none", "none")
             if not os.path.isdir(modulepath_save):
                 os.makedirs(modulepath_save)
             logging.info("  ... appending {} to MODULEPATHS_SAVE".format(modulepath_save))
             MODULEPATHS_SAVE.append(modulepath_save)
             remove_compiler_prefices_from_tcl_modulefiles(
                 modulepath_save,
                 compiler_list,
-                mpi_provider = mpi_provider
+                mpi_provider = mpi_provider,
+                module_choice = module_choice
             )
 
         for compiler in compilers:
@@ -514,9 +546,15 @@ def custom_sort_key(entry):
                 )
             )
 
+            # Module paths are short names for tcl modules
+            if module_choice == "lmod":
+                mpi_alias = compiler.name
+            else:
+                compiler_alias = ALIASES[compiler.name]
+
             # Spack mpi+compiler module hierarchy
             modulepath_save = os.path.join(
-                module_dir, mpi_provider.name, str(mpi_provider.version), compiler.name, str(compiler.version)
+                module_dir, mpi_alias, str(mpi_provider.version), compiler_alias, str(compiler.version)
             )
             if not os.path.isdir(modulepath_save):
                 os.makedirs(modulepath_save)
@@ -528,7 +566,8 @@ def custom_sort_key(entry):
                 remove_compiler_prefices_from_tcl_modulefiles(
                     modulepath_save,
                     compiler_list,
-                    mpi_provider = mpi_provider
+                    mpi_provider = mpi_provider,
+                    module_choice = module_choice
                 )
 
             # The remainder of the loop is only needed for the preferred compiler
@@ -537,7 +576,7 @@ def custom_sort_key(entry):
 
             # Path and name for mpi module file
             mpi_module_dir = os.path.join(
-                module_dir, compiler.name, str(compiler.version), "stack-" + mpi_provider.name
+                module_dir, compiler_alias, str(compiler.version), "stack-" + mpi_provider.name
             )
             mpi_module_file = os.path.join(
                 mpi_module_dir, str(mpi_provider.version).split("-")[0] + MODULE_FILE_EXTENSION[module_choice]

spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py

@@ -35,7 +35,7 @@ def test_setup_meta_modules():
 
     env_name = "modtest1"
     env_dir = os.path.join(env_root_dir, env_name)
-    module_dir = os.path.join(env_dir, "install", "modulefiles")
+    module_dir = os.path.join(env_dir, "modules")
     if os.path.exists(env_dir):
         shutil.rmtree(env_dir)
 

util/modules_config_check.py

@@ -20,21 +20,12 @@
         yaml_raw = f.read().replace("lmod:", "LMOD_OR_TCL:").replace("tcl:", "LMOD_OR_TCL:")
         modules[lmod_or_tcl] = syaml.load_config(yaml_raw)
 
-# Check subset of projection to ensure keys match
-lmod_projections = modules["lmod"]["modules"]["default"]["LMOD_OR_TCL"]["projections"].keys()
-tcl_projections = modules["tcl"]["modules"]["default"]["LMOD_OR_TCL"]["projections"].keys()
-assert set(lmod_projections) == set(tcl_projections)-{"all", "^mpi"}
-
-for lmod_or_tcl in ("lmod", "tcl"):
-    for key in modules[lmod_or_tcl]["modules"]["default"]["LMOD_OR_TCL"]["projections"].keys():
-        modules[lmod_or_tcl]["modules"]["default"]["LMOD_OR_TCL"]["projections"][key] = "DUMMYVALUE"
-
 # Removing sections we don't want to compare; note this will
-#  affect line numbers in the diff output
+# affect line numbers in the diff output
 del(modules["lmod"]["modules"]["default"]["LMOD_OR_TCL"]["core_compilers"])
 del(modules["lmod"]["modules"]["default"]["LMOD_OR_TCL"]["hierarchy"])
-del(modules["tcl"]["modules"]["default"]["LMOD_OR_TCL"]["projections"]["all"])
-del(modules["tcl"]["modules"]["default"]["LMOD_OR_TCL"]["projections"]["^mpi"])
+del(modules["lmod"]["modules"]["default"]["LMOD_OR_TCL"]["projections"])
+del(modules["tcl"]["modules"]["default"]["LMOD_OR_TCL"]["projections"])
 
 # If sections mismatch, print a diff of the whole configuration
 dump_lmod = syaml.dump_config(modules["lmod"]).split("\n")