Merge branch 'develop' into bugfix/containers_ip510

Author: climbfuji

.github/workflows/ubuntu-ci-x86_64-gnu.yaml

@@ -196,7 +196,7 @@ jobs:
           echo "# nothing" >> ${SPACK_STACK_DIR}/envs/chaintest/common/packages.yaml
           spack stack create env --name chaintest3 --site linux.default --compiler gcc --upstream ${SPACK_STACK_DIR}/envs/chaintest/install 2>&1 | tee stderr.txt
           cnt=$(grep -c "WARNING.*do not match" stderr.txt || true)
-          if [ $cnt -ne 2 ]; then echo "Missing 'create env' warnings"; exit 1; fi
+          if [ $cnt -lt 3 ]; then echo "Missing 'create env' warnings"; exit 1; fi
 
       - name: test-env
         run: |

configs/common/packages.yaml

@@ -22,7 +22,7 @@ packages:
   bufr:
     require: '@12.1.0 +python'
   bufr-query:
-    require: '@0.0.2 +python'
+    require: '@0.0.4 +python'
   cairo:
     require: '+pic'
   cdo:

configs/containers/specs/jedi-ci.yaml

@@ -8,7 +8,7 @@
     [email protected],
     [email protected],
     [email protected],
-    [email protected].2,
+    [email protected].4,
     [email protected],
     [email protected],
     [email protected],

spack

@@ -207,6 +207,16 @@ def _copy_site_includes(self):
         destination = os.path.join(env_site_dir, "packages.yaml")
         self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination)
 
+    def get_upstream_realpaths(self, upstream_path):
+        spack_yaml_path = os.path.realpath(os.path.join(upstream_path, "../spack.yaml"))
+        with open(spack_yaml_path, "r") as f:
+            spack_yaml = syaml.load_config(f)
+        upstream_paths = [upstream_path]
+        if "upstreams" in spack_yaml["spack"].keys():
+            entries = spack_yaml["spack"]["upstreams"]
+            for entry in entries.items():
+                upstream_paths += self.get_upstream_realpaths(entry[1]["install_tree"])
+        return upstream_paths
 
     def write(self):
         """Write environment out to a spack.yaml in <env_dir>/<name>.
@@ -280,10 +290,10 @@ def write(self):
                 "common": os.path.join(self.env_dir(), "common"),
                 "site": os.path.join(self.env_dir(), "site"),
             }
-            for upstream_path in self.upstreams:
-                upstream_path = upstream_path[0]
-                # spack doesn't handle "~/" correctly, this fixes it:
-                upstream_path = os.path.expanduser(upstream_path)
+            all_upstreams = []
+            for upstream in self.upstreams:
+                all_upstreams.extend(self.get_upstream_realpaths(upstream[0]))
+            for upstream_path in all_upstreams:
                 if not os.path.basename(os.path.normpath(upstream_path)) == "install":
                     logging.warning(
                         "WARNING: Upstream path '%s' is not an 'install' directory!"
@@ -298,7 +308,7 @@ def write(self):
                 if path_parts:
                     name = path_parts["spack_stack_ver"] + "-" + path_parts["env_name"]
                 else:
-                    name = os.path.basename(upstream_path)
+                    name = os.path.realpath(os.path.join(upstream_path, ".."))
                 upstream = "upstreams:%s:install_tree:'%s'" % (name, upstream_path)
                 logging.info("Adding upstream path '%s'" % upstream_path)
                 spack.config.add(upstream, scope=env_scope)
@@ -319,12 +329,20 @@ def write(self):
                              f"'{upstream_path}' do not match! Verify that you are using the same "
                               "version of spack-stack, or really, really know what you are doing.")
                         )
+            new_envrepo = os.path.join(self.env_dir(), "envrepo")
+            for upstream_path in all_upstreams[::-1]:
+                envrepo_path = os.path.realpath(os.path.join(upstream_path, "../envrepo"))
+                if os.path.isdir(envrepo_path):
+                    shutil.copytree(envrepo_path, new_envrepo, dirs_exist_ok=True)
+            if os.path.isdir(new_envrepo):
+                repo_cfg = "repos:[$env/envrepo]"
+                spack.config.add(repo_cfg, scope=env_scope)
 
         if self.modifypkg:
             logging.info("Creating custom repo with packages %s" % ", ".join(self.modifypkg))
             env_repo_path = os.path.join(env_dir, "envrepo")
             env_pkgs_path = os.path.join(env_repo_path, "packages")
-            os.makedirs(env_pkgs_path, exist_ok=False)
+            os.makedirs(env_pkgs_path, exist_ok=True)
             with open(os.path.join(env_repo_path, "repo.yaml"), "w") as f:
                 f.write("repo:\n  namespace: envrepo")
             repo_paths = spack.config.get("repos")
@@ -338,6 +356,7 @@ def write(self):
                             pkg_path,
                             os.path.join(env_pkgs_path, pkg_name),
                             ignore=shutil.ignore_patterns("__pycache__"),
+                            dirs_exist_ok=True,
                         )
                         pkg_found = True
                         # Use the first repo where the package exists:

spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py

@@ -179,29 +179,73 @@ def test_compilers():
 @pytest.mark.extension("stack")
 @pytest.mark.filterwarnings("ignore::UserWarning")
 def test_upstream():
+    base_env = os.path.join(test_dir, "base_env/install/")
+    os.makedirs(os.path.join(base_env, ".spack-db/"), exist_ok=True)
+    base_env_spack_yaml_path = os.path.realpath(os.path.join(base_env, "../spack.yaml"))
+    f_base_env = open(base_env_spack_yaml_path, "w")
+    f_base_env.write("spack:\n  dummytag: dummyvalue")
+    f_base_env.close()
     stack_create(
         "create",
         "env",
         "--site",
         "hera",
         "--name",
-        "upstream_test",
+        "chainedA",
         "--dir",
         test_dir,
         "--compiler",
         "gcc",
         "--upstream",
-        "/test/path/to/upstream/env",
+        base_env,
     )
-    spack_yaml_path = os.path.join(test_dir, "upstream_test", "spack.yaml")
+    spack_yaml_path = os.path.join(test_dir, "chainedA", "spack.yaml")
     with open(spack_yaml_path, "r") as f:
         spack_yaml_txt = f.read()
-    assert "install_tree: /test/path/to/upstream/env" in spack_yaml_txt
+    assert f"install_tree: {base_env}" in spack_yaml_txt
     assert (
         "repos: [$env/envrepo]" not in spack_yaml_txt
     ), "--modify-pkg functionality modified spack.yaml without being called"
 
 
+@pytest.mark.extension("stack")
+@pytest.mark.filterwarnings("ignore::UserWarning")
+def test_layered_upstreams():
+    os.makedirs(os.path.join(test_dir, "chainedA/install/.spack-db/"))
+    os.makedirs(os.path.join(test_dir, "base_env/envrepo/"))
+    os.makedirs(os.path.join(test_dir, "chainedA/envrepo/"))
+    f_base_env_envrepo_file_path = os.path.join(test_dir, "base_env/envrepo/file")
+    f_chainedA_envrepo_file_path = os.path.join(test_dir, "chainedA/envrepo/file")
+    f_base_env_envrepo_file = open(f_base_env_envrepo_file_path, "w")
+    f_chainedA_envrepo_file = open(f_chainedA_envrepo_file_path, "w")
+    f_base_env_envrepo_file.write("bad")
+    f_chainedA_envrepo_file.write("good")
+    f_base_env_envrepo_file.close()
+    f_chainedA_envrepo_file.close()
+    stack_create(
+        "create",
+        "env",
+        "--site",
+        "hera",
+        "--name",
+        "chainedB",
+        "--dir",
+        test_dir,
+        "--compiler",
+        "gcc",
+        "--upstream",
+        os.path.join(test_dir, "chainedA/install/")
+    )
+    spack_yaml_path = os.path.join(test_dir, "chainedB", "spack.yaml")
+    with open(spack_yaml_path, "r") as f:
+        spack_yaml_txt = f.read()
+    assert "/base_env/install/" in spack_yaml_txt
+    assert "/chainedA/install/" in spack_yaml_txt
+    file_path = os.path.join(test_dir, "chainedB/envrepo/file")
+    with open(file_path, "r") as f:
+        assert "good" in f.read()
+
+
 @pytest.mark.extension("stack")
 @pytest.mark.filterwarnings("ignore::UserWarning")
 def test_modifypkg():