When chaining against chained envs, get upstreams+envrepo's (#1396)

This PR makes the env chaining logic in spack stack create env --upstream more robust, specifically by recursively checking to see whether the upstream environments are themselves chained environments and adding those to the new environment's list of upstreams. It also automatically populates an envrepo/ repo directory from those found in the upstream environments.

Author: AlexanderRichert-NOAA

.github/workflows/ubuntu-ci-x86_64-gnu.yaml

@@ -196,7 +196,7 @@ jobs:
           echo "# nothing" >> ${SPACK_STACK_DIR}/envs/chaintest/common/packages.yaml
           spack stack create env --name chaintest3 --site linux.default --compiler gcc --upstream ${SPACK_STACK_DIR}/envs/chaintest/install 2>&1 | tee stderr.txt
           cnt=$(grep -c "WARNING.*do not match" stderr.txt || true)
-          if [ $cnt -ne 2 ]; then echo "Missing 'create env' warnings"; exit 1; fi
+          if [ $cnt -lt 3 ]; then echo "Missing 'create env' warnings"; exit 1; fi
 
       - name: test-env
         run: |

spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py

@@ -207,6 +207,16 @@ def _copy_site_includes(self):
         destination = os.path.join(env_site_dir, "packages.yaml")
         self._copy_or_merge_includes("packages", packages_yaml_path, packages_compiler_yaml_path, destination)
 
+    def get_upstream_realpaths(self, upstream_path):
+        spack_yaml_path = os.path.realpath(os.path.join(upstream_path, "../spack.yaml"))
+        with open(spack_yaml_path, "r") as f:
+            spack_yaml = syaml.load_config(f)
+        upstream_paths = [upstream_path]
+        if "upstreams" in spack_yaml["spack"].keys():
+            entries = spack_yaml["spack"]["upstreams"]
+            for entry in entries.items():
+                upstream_paths += self.get_upstream_realpaths(entry[1]["install_tree"])
+        return upstream_paths
 
     def write(self):
         """Write environment out to a spack.yaml in <env_dir>/<name>.
@@ -280,10 +290,10 @@ def write(self):
                 "common": os.path.join(self.env_dir(), "common"),
                 "site": os.path.join(self.env_dir(), "site"),
             }
-            for upstream_path in self.upstreams:
-                upstream_path = upstream_path[0]
-                # spack doesn't handle "~/" correctly, this fixes it:
-                upstream_path = os.path.expanduser(upstream_path)
+            all_upstreams = []
+            for upstream in self.upstreams:
+                all_upstreams.extend(self.get_upstream_realpaths(upstream[0]))
+            for upstream_path in all_upstreams:
                 if not os.path.basename(os.path.normpath(upstream_path)) == "install":
                     logging.warning(
                         "WARNING: Upstream path '%s' is not an 'install' directory!"
@@ -298,7 +308,7 @@ def write(self):
                 if path_parts:
                     name = path_parts["spack_stack_ver"] + "-" + path_parts["env_name"]
                 else:
-                    name = os.path.basename(upstream_path)
+                    name = os.path.realpath(os.path.join(upstream_path, ".."))
                 upstream = "upstreams:%s:install_tree:'%s'" % (name, upstream_path)
                 logging.info("Adding upstream path '%s'" % upstream_path)
                 spack.config.add(upstream, scope=env_scope)
@@ -319,12 +329,20 @@ def write(self):
                              f"'{upstream_path}' do not match! Verify that you are using the same "
                               "version of spack-stack, or really, really know what you are doing.")
                         )
+            new_envrepo = os.path.join(self.env_dir(), "envrepo")
+            for upstream_path in all_upstreams[::-1]:
+                envrepo_path = os.path.realpath(os.path.join(upstream_path, "../envrepo"))
+                if os.path.isdir(envrepo_path):
+                    shutil.copytree(envrepo_path, new_envrepo, dirs_exist_ok=True)
+            if os.path.isdir(new_envrepo):
+                repo_cfg = "repos:[$env/envrepo]"
+                spack.config.add(repo_cfg, scope=env_scope)
 
         if self.modifypkg:
             logging.info("Creating custom repo with packages %s" % ", ".join(self.modifypkg))
             env_repo_path = os.path.join(env_dir, "envrepo")
             env_pkgs_path = os.path.join(env_repo_path, "packages")
-            os.makedirs(env_pkgs_path, exist_ok=False)
+            os.makedirs(env_pkgs_path, exist_ok=True)
             with open(os.path.join(env_repo_path, "repo.yaml"), "w") as f:
                 f.write("repo:\n  namespace: envrepo")
             repo_paths = spack.config.get("repos")
@@ -338,6 +356,7 @@ def write(self):
                             pkg_path,
                             os.path.join(env_pkgs_path, pkg_name),
                             ignore=shutil.ignore_patterns("__pycache__"),
+                            dirs_exist_ok=True,
                         )
                         pkg_found = True
                         # Use the first repo where the package exists:

spack-ext/lib/jcsda-emc/spack-stack/tests/test_stack_create.py

@@ -179,29 +179,73 @@ def test_compilers():
 @pytest.mark.extension("stack")
 @pytest.mark.filterwarnings("ignore::UserWarning")
 def test_upstream():
+    base_env = os.path.join(test_dir, "base_env/install/")
+    os.makedirs(os.path.join(base_env, ".spack-db/"), exist_ok=True)
+    base_env_spack_yaml_path = os.path.realpath(os.path.join(base_env, "../spack.yaml"))
+    f_base_env = open(base_env_spack_yaml_path, "w")
+    f_base_env.write("spack:\n  dummytag: dummyvalue")
+    f_base_env.close()
     stack_create(
         "create",
         "env",
         "--site",
         "hera",
         "--name",
-        "upstream_test",
+        "chainedA",
         "--dir",
         test_dir,
         "--compiler",
         "gcc",
         "--upstream",
-        "/test/path/to/upstream/env",
+        base_env,
     )
-    spack_yaml_path = os.path.join(test_dir, "upstream_test", "spack.yaml")
+    spack_yaml_path = os.path.join(test_dir, "chainedA", "spack.yaml")
     with open(spack_yaml_path, "r") as f:
         spack_yaml_txt = f.read()
-    assert "install_tree: /test/path/to/upstream/env" in spack_yaml_txt
+    assert f"install_tree: {base_env}" in spack_yaml_txt
     assert (
         "repos: [$env/envrepo]" not in spack_yaml_txt
     ), "--modify-pkg functionality modified spack.yaml without being called"
 
 
+@pytest.mark.extension("stack")
+@pytest.mark.filterwarnings("ignore::UserWarning")
+def test_layered_upstreams():
+    os.makedirs(os.path.join(test_dir, "chainedA/install/.spack-db/"))
+    os.makedirs(os.path.join(test_dir, "base_env/envrepo/"))
+    os.makedirs(os.path.join(test_dir, "chainedA/envrepo/"))
+    f_base_env_envrepo_file_path = os.path.join(test_dir, "base_env/envrepo/file")
+    f_chainedA_envrepo_file_path = os.path.join(test_dir, "chainedA/envrepo/file")
+    f_base_env_envrepo_file = open(f_base_env_envrepo_file_path, "w")
+    f_chainedA_envrepo_file = open(f_chainedA_envrepo_file_path, "w")
+    f_base_env_envrepo_file.write("bad")
+    f_chainedA_envrepo_file.write("good")
+    f_base_env_envrepo_file.close()
+    f_chainedA_envrepo_file.close()
+    stack_create(
+        "create",
+        "env",
+        "--site",
+        "hera",
+        "--name",
+        "chainedB",
+        "--dir",
+        test_dir,
+        "--compiler",
+        "gcc",
+        "--upstream",
+        os.path.join(test_dir, "chainedA/install/")
+    )
+    spack_yaml_path = os.path.join(test_dir, "chainedB", "spack.yaml")
+    with open(spack_yaml_path, "r") as f:
+        spack_yaml_txt = f.read()
+    assert "/base_env/install/" in spack_yaml_txt
+    assert "/chainedA/install/" in spack_yaml_txt
+    file_path = os.path.join(test_dir, "chainedB/envrepo/file")
+    with open(file_path, "r") as f:
+        assert "good" in f.read()
+
+
 @pytest.mark.extension("stack")
 @pytest.mark.filterwarnings("ignore::UserWarning")
 def test_modifypkg():