feat(config): Make hidden file inclusion configurable in OmegaConfigLoader (#5006)

* feat(config): Make hidden file inclusion configurable in OmegaConfigLoader

Signed-off-by: aseemsangalay <[email protected]>

* Add ignore_hidden parameter to docstring

Signed-off-by: Laura Couto <[email protected]>

* Add new parameter to docs

Signed-off-by: Laura Couto <[email protected]>

* Vale suggestion

Signed-off-by: Laura Couto <[email protected]>

* Update docs/configure/advanced_configuration.md

Signed-off-by: Merel Theisen <[email protected]>

* Update RELEASE.md

Signed-off-by: Laura Couto <[email protected]>

* Simplify release note

Signed-off-by: Laura Couto <[email protected]>

---------

Signed-off-by: aseemsangalay <[email protected]>
Signed-off-by: Laura Couto <[email protected]>
Signed-off-by: Merel Theisen <[email protected]>
Co-authored-by: Merel Theisen <[email protected]>
Co-authored-by: L. R. Couto <[email protected]>
Co-authored-by: Laura Couto <[email protected]>

Author: 3 people

RELEASE.md

@@ -1,7 +1,13 @@
 # Upcoming Release
+
 ## Major features and improvements
+* Added the `ignore_hidden` parameter to the `OmegaConfigLoader`.
+
 ## Bug fixes and other changes
+
 ## Community contributions
+Many thanks to the following Kedroids for contributing PRs to this release:
+* [Aseem Sangalay](https://github.com/aseemsangalay)
 
 # Release 1.0.0
 ## Major features and improvements

docs/configure/advanced_configuration.md

@@ -413,3 +413,15 @@ If you run it from the same directory where `parameters.yml` placed it gives the
 ```console
 {'polars_float64': Float64, 'today': datetime.date(2023, 11, 23)}
 ```
+
+### How to ignore hidden files and directories with `OmegaConfigLoader`
+
+The `OmegaConfigLoader` provides an option to ignore hidden files and directories (those starting with a dot, for example, `.hidden_file` or `.hidden_folder`) when loading configuration files. This behaviour is controlled by the `ignore_hidden` parameter, which is set to `True` by default.
+
+If you want to include hidden files and directories in your configuration loading process, you can set `ignore_hidden` to False when instantiating the `OmegaConfigLoader`:
+
+```py
+from kedro.config import OmegaConfigLoader
+
+conf_loader = OmegaConfigLoader(conf_source="conf", ignore_hidden=False)
+```

kedro/config/omegaconf_config.py

@@ -99,6 +99,7 @@ def __init__(  # noqa: PLR0913
         default_run_env: str | None = None,
         custom_resolvers: dict[str, Callable] | None = None,
         merge_strategy: dict[str, str] | None = None,
+        ignore_hidden: bool = True,
     ):
         if isinstance(conf_source, Path):
             conf_source = str(conf_source)
@@ -126,7 +127,10 @@ def __init__(  # noqa: PLR0913
                 see here: https://omegaconf.readthedocs.io/en/2.3_branch/custom_resolvers.html#custom-resolvers
             merge_strategy: A dictionary that specifies the merging strategy for each configuration type.
                 The accepted merging strategies are `soft` and `destructive`. Defaults to `destructive`.
+            ignore_hidden: A boolean flag that determines whether hidden files and directories should be
+                ignored when loading configuration files. If True, ignore hidden files and files in hidden directories.
         """
+        self.ignore_hidden = ignore_hidden
         self.base_env = base_env or ""
         self.default_run_env = default_run_env or ""
         self.merge_strategy = merge_strategy or {}
@@ -332,7 +336,7 @@ def load_and_merge_dir_config(
         paths = []
         for pattern in patterns:
             for each in self._fs.glob(Path(f"{conf_path!s}/{pattern}").as_posix()):
-                if not self._is_hidden(each):
+                if not self.ignore_hidden or not self._is_hidden(each):
                     paths.append(Path(each))
 
         deduplicated_paths = set(paths)

tests/config/test_omegaconf_config.py

@@ -1684,3 +1684,27 @@ def test_remote_dir_not_exists_exception(self, tmp_path, mocker):
             match="Given configuration path either does not exist or is not a valid directory.*Error: No such file or bucket",
         ):
             conf["catalog"]
+
+    def test_hidden_files_ignored_by_default(self, tmp_path):
+        base_dir = tmp_path / "base"
+        base_dir.mkdir()
+        (base_dir / ".hidden.yml").write_text("foo: bar")
+
+        config_loader = OmegaConfigLoader(conf_source=str(tmp_path))
+        config_loader.config_patterns = {"catalog": ["**/*.yml"]}
+
+        result = config_loader["catalog"]
+        assert "foo" not in result
+
+    def test_hidden_files_included_when_not_ignored(self, tmp_path):
+        base_dir = tmp_path / "base"
+        base_dir.mkdir()
+        (base_dir / ".hidden.yml").write_text("foo: bar")
+
+        config_loader = OmegaConfigLoader(
+            conf_source=str(tmp_path), ignore_hidden=False
+        )
+        config_loader.config_patterns = {"catalog": ["**/*.yml"]}
+
+        result = config_loader["catalog"]
+        assert "foo" in result