Merge pull request #1813 from climbfuji/feature/shorter_modulepaths

climbfuji · web-flow · commit 926b8fa25af7 · 2025-11-12T16:14:41.000-07:00
This PR shortens the module paths for the modules generated by spack module [tcl|lmod] refresh and spack stack setup-meta-modules. It does so by implementing a compiler- and MPI-name translation and shortening the path where the modules are generated. This is done in multiple places:

configs/common/modules_lmod.yaml
(configs/common/modules_tcl.yaml)
spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py
While this solution comes with a certain amount of code duplication, I consider it as the best option because any other solution would have required changes to the spack code itself. Also, we rarely make changes to these names or add new compilers and MPI providers, thus the maintenance effort is low.

While working on this, I realized that the ecmwf-atlas --&gt; atlas translation never worked as intended, because it was missing the MPI module dependency. I checked and the spack-generated modules were named ecmwf-atlas before this PR was made (at least for tcl modules). I therefore removed this mapping, since the module name ecmwf-atlas apparently worked fine until now.
diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml
@@ -234,9 +234,9 @@ jobs:
 
           export ENVNAME=ue-gcc-11.4.0
           export ENVDIR=$PWD/envs/${ENVNAME}
-          ls -l ${ENVDIR}/install/modulefiles/Core
+          ls -l ${ENVDIR}/modules/Core
 
-          module use ${ENVDIR}/install/modulefiles/Core
+          module use ${ENVDIR}/modules/Core
           module load stack-gcc/11.4.0
           module load stack-openmpi/5.0.8
           module available
diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml
@@ -242,9 +242,9 @@ jobs:
 
           export ENVNAME=ue-oneifx-2024.2.0
           export ENVDIR=$PWD/envs/${ENVNAME}
-          ls -l ${ENVDIR}/install/modulefiles/Core
+          ls -l ${ENVDIR}/modules/Core
 
-          module use ${ENVDIR}/install/modulefiles/Core
+          module use ${ENVDIR}/modules/Core
           module load stack-intel-oneapi-compilers/2024.2.0
           module load stack-intel-oneapi-mpi/2021.13
           module available
diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml
@@ -242,9 +242,9 @@ jobs:
 
           export ENVNAME=ue-oneapi-2024.2.0
           export ENVDIR=$PWD/envs/${ENVNAME}
-          ls -l ${ENVDIR}/install/modulefiles/Core
+          ls -l ${ENVDIR}/modules/Core
 
-          module use ${ENVDIR}/install/modulefiles/Core
+          module use ${ENVDIR}/modules/Core
           module load stack-intel-oneapi-compilers/2024.2.0
           module load stack-intel-oneapi-mpi/2021.13
           module available
diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml
@@ -3,14 +3,13 @@
 modules:
   default:
     roots:
-      lmod: $env/install/modulefiles
+      lmod: $env/modules
     lmod:
       # Core compiler is a dummy that is not used to build the stack, do not change.
       # Hopefully nobody is trying to build software with gcc-4.6 in 2023 ...
       core_compilers::
       - gcc@4.6
       projections:
-        ecmwf-atlas: 'atlas/{version}'
         nlohmann-json: 'json/{version}'
         nlohmann-json-schema-validator: 'json-schema-validator/{version}'
         libjpeg-turbo: 'libjpeg/{version}'
diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml
@@ -3,12 +3,32 @@
 modules:
   default:
     roots:
-      tcl: $env/install/modulefiles
+      tcl: $env/modules
     tcl:
       projections:
-        all: '{compiler.name}/{compiler.version}/{name}/{version}'
+        # Note: These translations of compiler and MPI library names must match the ALIASES
+        # dictionary in spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py
+        # Defaults
+        '^intel-oneapi-compilers-classic^intel-oneapi-mpi': 'impi/{^mpi.version}/intel/{compiler.version}/{name}/{version}'
+        '^intel-oneapi-compilers-classic^mpi': '{^mpi.name}/{^mpi.version}/intel/{compiler.version}/{name}/{version}'
+        '^intel-oneapi-compilers-classic': 'intel/{compiler.version}/{name}/{version}'
+        '^intel-oneapi-compilers^intel-oneapi-mpi': 'impi/{^mpi.version}/oneapi/{compiler.version}/{name}/{version}'
+        '^intel-oneapi-compilers^mpi': '{^mpi.name}/{^mpi.version}/oneapi/{compiler.version}/{name}/{version}'
+        '^intel-oneapi-compilers': 'oneapi/{compiler.version}/{name}/{version}'
+        ^intel-oneapi-mpi: 'impi/{^mpi.version}/{compiler.name}/{compiler.version}/{name}/{version}'
         ^mpi: '{^mpi.name}/{^mpi.version}/{compiler.name}/{compiler.version}/{name}/{version}'
-        ecmwf-atlas: '{compiler.name}/{compiler.version}/atlas/{version}'
+        all: '{compiler.name}/{compiler.version}/{name}/{version}'
+        # The special cases - Intel classic compilers
+        'nlohmann-json ^intel-oneapi-compilers-classic': 'intel/{compiler.version}/json/{version}'
+        'nlohmann-json-schema-validator ^intel-oneapi-compilers-classic': 'intel/{compiler.version}/json-schema-validator/{version}'
+        'libjpeg-turbo ^intel-oneapi-compilers-classic': 'intel/{compiler.version}/libjpeg/{version}'
+        'prod-util ^intel-oneapi-compilers-classic': 'intel/{compiler.version}/prod_util/{version}'
+        # The special cases - Intel LLVM compilers
+        'nlohmann-json ^intel-oneapi-compilers': 'oneapi/{compiler.version}/json/{version}'
+        'nlohmann-json-schema-validator ^intel-oneapi-compilers': 'oneapi/{compiler.version}/json-schema-validator/{version}'
+        'libjpeg-turbo ^intel-oneapi-compilers': 'oneapi/{compiler.version}/libjpeg/{version}'
+        'prod-util ^intel-oneapi-compilers': 'oneapi/{compiler.version}/prod_util/{version}'
+        # The special cases - other compilers
         nlohmann-json: '{compiler.name}/{compiler.version}/json/{version}'
         nlohmann-json-schema-validator: '{compiler.name}/{compiler.version}/json-schema-validator/{version}'
         libjpeg-turbo: '{compiler.name}/{compiler.version}/libjpeg/{version}'
diff --git a/repos/builtin b/repos/builtin
@@ -1 +1 @@
-Subproject commit b091bced9a958042cf03f6cd4a2989dfabd10891
+Subproject commit bfc55569ef288de441a42131f58579e8b7fe64a0
diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py
@@ -52,13 +52,22 @@
     "MPIROOT": "",
 }
 
-
-# DH* TODO: IMPORT THIS FROM SPACK, THEY MAINTAIN THE SAME DICTIONARY
-COMPILER_TRANSLATION_TABLE = {
+# Aliases to shorten module paths for tcl modules. These aliases must match
+# the compiler and MPI name translations in configs/common/modules_tcl.yaml
+ALIASES = {
+    "none" : "none",
+    # Compilers
     "gcc" : "gcc",
-    "clang" : "llvm",
-    "intel" : "intel-oneapi-compilers-classic",
-    "oneapi" : "intel-oneapi-compilers",
+    "intel-oneapi-compilers-classic" : "intel",
+    "intel-oneapi-compilers" : "oneapi",
+    "llvm" : "llvm",
+    # MPI
+    "cray-mpich" : "cray-mpich",
+    # Do we still need intel-mpi, and if yes, use the same impi?
+    "intel-oneapi-mpi" : "impi",
+    "mpich" : "mpich",
+    "mpt" : "mpt",
+    "openmpi" : "openmpi",
 }
 
 
@@ -220,7 +229,7 @@ def get_preferred_compiler():
     return preferred_compiler
 
 
-def remove_compiler_prefices_from_tcl_modulefiles(modulepath, compiler_list, mpi_provider):
+def remove_compiler_prefices_from_tcl_modulefiles(modulepath, compiler_list, mpi_provider, module_choice):
     """Remove compiler and mpi prefices from tcl modulefiles in modulepath"""
     logging.info(f"  ... ... removing compiler/mpi prefices from tcl modulefiles in {modulepath}")
     module_replace_patterns = ["is-loaded", "module load", "depends-on"]
@@ -237,20 +246,30 @@ def remove_compiler_prefices_from_tcl_modulefiles(modulepath, compiler_list, mpi
                 for compiler in compiler_list:
                     # First, compiler-dependent modules
                     (compiler_name, compiler_version) = compiler.split("@")
+                    # Module paths are short names for tcl modules
+                    if module_choice == "lmod":
+                        compiler_alias = compiler_name
+                    else:
+                        compiler_alias = ALIASES[compiler_name]
                     cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format(
-                        pattern, compiler_name, compiler_version, filepath, sed_syntax_fix
+                        pattern, compiler_alias, compiler_version, filepath, sed_syntax_fix
                     )
                     status = os.system(cmd)
                     if not status == 0:
                         raise Exception(f"Error while calling '{cmd}'")
                     # If mpi_provider is not None, also do compiler+mpi-dependent modules
                     if not mpi_provider:
                         continue
+                    # Module paths are short names for tcl modules
+                    if module_choice == "lmod":
+                        mpi_alias = mpi_provider.name
+                    else:
+                        mpi_alias = ALIASES[mpi_provider.name]
                     cmd = "sed -i {6} 's#{0} {1}/{2}/{3}/{4}/#{0} #g' {5}".format(
                         pattern,
-                        mpi_provider.name,
+                        mpi_alias,
                         mpi_provider.version,
-                        compiler_name,
+                        compiler_alias,
                         compiler_version,
                         filepath,
                         sed_syntax_fix,
@@ -381,13 +400,20 @@ def custom_sort_key(entry):
         remove_compiler_prefices_from_tcl_modulefiles(
             modulepath_save,
             compiler_list,
-            mpi_provider = None
+            mpi_provider = None,
+            module_choice = module_choice
         )
 
     for compiler in compilers:
         logging.info(f"  ... configuring compiler {compiler.name}@{compiler.version}")
 
-        modulepath_save = os.path.join(module_dir, compiler.name, str(compiler.version))
+        # Module paths are short names for tcl modules
+        if module_choice == "lmod":
+            compiler_alias = compiler.name
+        else:
+            compiler_alias = ALIASES[compiler.name]
+
+        modulepath_save = os.path.join(module_dir, compiler_alias, str(compiler.version))
         if not os.path.isdir(modulepath_save):
             os.makedirs(modulepath_save)
         logging.info("  ... ... appending {} to MODULEPATHS_SAVE".format(modulepath_save))
@@ -398,7 +424,8 @@ def custom_sort_key(entry):
             remove_compiler_prefices_from_tcl_modulefiles(
                 modulepath_save,
                 compiler_list,
-                mpi_provider = None
+                mpi_provider = None,
+                module_choice = module_choice
             )
 
         # The remainder of the loop is only needed for the preferred compiler
@@ -496,15 +523,20 @@ def custom_sort_key(entry):
         # For tcl, append modulepath for external specs and for specs without
         # compiler dependencies; remove the compiler/mpi prefices from the moduless
         if module_choice == "tcl":
-            modulepath_save = os.path.join(module_dir, mpi_provider.name, str(mpi_provider.version), "none", "none")
+
+            # Module paths are short names for tcl modules
+            mpi_alias = ALIASES[mpi_provider.name]
+
+            modulepath_save = os.path.join(module_dir, mpi_alias, str(mpi_provider.version), "none", "none")
             if not os.path.isdir(modulepath_save):
                 os.makedirs(modulepath_save)
             logging.info("  ... appending {} to MODULEPATHS_SAVE".format(modulepath_save))
             MODULEPATHS_SAVE.append(modulepath_save)
             remove_compiler_prefices_from_tcl_modulefiles(
                 modulepath_save,
                 compiler_list,
-                mpi_provider = mpi_provider
+                mpi_provider = mpi_provider,
+                module_choice = module_choice
             )
 
         for compiler in compilers:
@@ -514,9 +546,15 @@ def custom_sort_key(entry):
                 )
             )
 
+            # Module paths are short names for tcl modules
+            if module_choice == "lmod":
+                mpi_alias = compiler.name
+            else:
+                compiler_alias = ALIASES[compiler.name]
+
             # Spack mpi+compiler module hierarchy
             modulepath_save = os.path.join(
-                module_dir, mpi_provider.name, str(mpi_provider.version), compiler.name, str(compiler.version)
+                module_dir, mpi_alias, str(mpi_provider.version), compiler_alias, str(compiler.version)
             )
             if not os.path.isdir(modulepath_save):
                 os.makedirs(modulepath_save)
@@ -528,7 +566,8 @@ def custom_sort_key(entry):
                 remove_compiler_prefices_from_tcl_modulefiles(
                     modulepath_save,
                     compiler_list,
-                    mpi_provider = mpi_provider
+                    mpi_provider = mpi_provider,
+                    module_choice = module_choice
                 )
 
             # The remainder of the loop is only needed for the preferred compiler
@@ -537,7 +576,7 @@ def custom_sort_key(entry):
 
             # Path and name for mpi module file
             mpi_module_dir = os.path.join(
-                module_dir, compiler.name, str(compiler.version), "stack-" + mpi_provider.name
+                module_dir, compiler_alias, str(compiler.version), "stack-" + mpi_provider.name
             )
             mpi_module_file = os.path.join(
                 mpi_module_dir, str(mpi_provider.version).split("-")[0] + MODULE_FILE_EXTENSION[module_choice]
diff --git a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py
@@ -35,7 +35,7 @@ def test_setup_meta_modules():
 
     env_name = "modtest1"
     env_dir = os.path.join(env_root_dir, env_name)
-    module_dir = os.path.join(env_dir, "install", "modulefiles")
+    module_dir = os.path.join(env_dir, "modules")
     if os.path.exists(env_dir):
         shutil.rmtree(env_dir)
 
diff --git a/util/modules_config_check.py b/util/modules_config_check.py
@@ -20,21 +20,12 @@
         yaml_raw = f.read().replace("lmod:", "LMOD_OR_TCL:").replace("tcl:", "LMOD_OR_TCL:")
         modules[lmod_or_tcl] = syaml.load_config(yaml_raw)
 
-# Check subset of projection to ensure keys match
-lmod_projections = modules["lmod"]["modules"]["default"]["LMOD_OR_TCL"]["projections"].keys()
-tcl_projections = modules["tcl"]["modules"]["default"]["LMOD_OR_TCL"]["projections"].keys()
-assert set(lmod_projections) == set(tcl_projections)-{"all", "^mpi"}
-
-for lmod_or_tcl in ("lmod", "tcl"):
-    for key in modules[lmod_or_tcl]["modules"]["default"]["LMOD_OR_TCL"]["projections"].keys():
-        modules[lmod_or_tcl]["modules"]["default"]["LMOD_OR_TCL"]["projections"][key] = "DUMMYVALUE"
-
 # Removing sections we don't want to compare; note this will
-#  affect line numbers in the diff output
+# affect line numbers in the diff output
 del(modules["lmod"]["modules"]["default"]["LMOD_OR_TCL"]["core_compilers"])
 del(modules["lmod"]["modules"]["default"]["LMOD_OR_TCL"]["hierarchy"])
-del(modules["tcl"]["modules"]["default"]["LMOD_OR_TCL"]["projections"]["all"])
-del(modules["tcl"]["modules"]["default"]["LMOD_OR_TCL"]["projections"]["^mpi"])
+del(modules["lmod"]["modules"]["default"]["LMOD_OR_TCL"]["projections"])
+del(modules["tcl"]["modules"]["default"]["LMOD_OR_TCL"]["projections"])
 
 # If sections mismatch, print a diff of the whole configuration
 dump_lmod = syaml.dump_config(modules["lmod"]).split("\n")
