✨ Merge additional TOML files (#14)

Author: ubmarco

docs/configuration.rst

@@ -239,6 +239,87 @@ The default list excludes:
    written to the output file, potentially creating circular dependencies or
    duplicate configurations.
 
+.. _`config_merge_toml_files`:
+
+needscfg_merge_toml_files
+-------------------------
+
+**Type:** ``list[str]``
+
+**Default:** ``[]`` (empty list)
+
+Specifies a list of TOML file paths to shallow-merge into the final output configuration.
+This allows you to include additional configuration from external TOML files into the
+generated ``ubproject.toml``.
+
+The paths support the same template variables as :ref:`config_outpath`:
+
+- ``${outdir}`` - Replaced with the Sphinx output directory (build directory)
+- ``${srcdir}`` - Replaced with the Sphinx source directory
+
+Relative paths are interpreted relative to the configuration directory (where ``conf.py`` is located).
+
+**Merge behavior:**
+
+- Files are processed in the order they appear in the list
+- Each file is shallow-merged (top-level keys only) into the configuration
+- If a TOML file has a ``[needs]`` table, only that table is merged
+- If no ``[needs]`` table exists, the entire file content is merged
+- Values from merged files **override** values from the Sphinx configuration
+- Later files in the list override earlier files
+
+**Use cases:**
+
+- Add project-specific metadata not available in Sphinx config
+- Include version information from separate TOML files
+- Merge team-wide configuration standards
+- Add deployment-specific settings
+
+**Examples:**
+
+.. code-block:: python
+
+   # Merge a single additional configuration file
+   needscfg_merge_toml_files = ["additional_config.toml"]
+
+   # Merge multiple files (processed in order)
+   needscfg_merge_toml_files = [
+       "${srcdir}/team_defaults.toml",
+       "project_overrides.toml",
+   ]
+
+   # Use build output directory
+   needscfg_merge_toml_files = ["${outdir}/generated_metadata.toml"]
+
+**Example TOML file with needs table:**
+
+.. code-block:: toml
+
+   # additional_config.toml
+   [needs]
+   project_version = "1.2.3"
+   build_date = "2025-10-28"
+
+**Example TOML file without needs table:**
+
+.. code-block:: toml
+
+   # additional_config.toml
+   project_version = "1.2.3"
+   build_date = "2025-10-28"
+
+Both formats work - if a ``[needs]`` table exists, only its contents are merged.
+
+.. note::
+
+   If a merge file doesn't exist, a warning is emitted but the build continues.
+   Failed file loads (e.g., invalid TOML syntax) also emit warnings without stopping the build.
+
+.. tip::
+
+   Use this feature to separate dynamic configuration (like version numbers or build metadata)
+   from static Sphinx-Needs configuration in ``conf.py``.
+
 Examples
 --------
 

needs_config_writer/logging.py

@@ -12,7 +12,12 @@ def get_logger(name: str) -> SphinxLoggerAdapter:
     return logging.getLogger(name)
 
 
-WarningSubTypes = Literal["path_conversion", "unsupported_type", "content_diff"]
+WarningSubTypes = Literal[
+    "path_conversion",
+    "unsupported_type",
+    "content_diff",
+    "merge_failed",
+]
 
 
 def log_warning(

needs_config_writer/main.py

@@ -3,6 +3,7 @@
 
 from sphinx.application import Sphinx
 from sphinx.config import Config
+import tomli
 import tomli_w
 
 from needs_config_writer import __version__
@@ -11,6 +12,28 @@
 LOGGER = get_logger(__name__)
 
 
+def resolve_path_template(path_template: str, app: Sphinx) -> Path:
+    """
+    Resolve a path template with ${outdir} and ${srcdir} variables.
+
+    Args:
+        path_template: Path string that may contain ${outdir} or ${srcdir}
+        app: Sphinx application instance
+
+    Returns:
+        Resolved Path object (absolute if relative to confdir)
+    """
+    path_str = path_template.replace("${outdir}", str(app.outdir))
+    path_str = path_str.replace("${srcdir}", str(app.srcdir))
+    path = Path(path_str)
+
+    # Make relative paths relative to confdir (where conf.py is located)
+    if not path.is_absolute():
+        path = Path(app.confdir) / path
+
+    return path
+
+
 def write_ubproject_file(app: Sphinx, config: Config):
     def get_safe_config(obj: Any, path: str = ""):
         """
@@ -179,21 +202,85 @@ def sort_for_reproducibility(obj: Any, path: str = "") -> Any:
                     if attribute in raw_needs_config:
                         need_attributes[config_name] = safe_value
 
-    # Sort all data structures to ensure reproducible serialization
-    sorted_attributes = sort_for_reproducibility(need_attributes)
+    # Collect additional root-level tables/keys from merged TOML files
+    additional_root_data = {}
+
+    # Merge TOML files if configured
+    if config.needscfg_merge_toml_files:
+        for toml_path_template in config.needscfg_merge_toml_files:
+            toml_path = resolve_path_template(toml_path_template, app)
+
+            if not toml_path.exists():
+                log_warning(
+                    LOGGER,
+                    f"TOML file to merge not found: '{toml_path}' (from template '{toml_path_template}')",
+                    "merge_failed",
+                    location=None,
+                )
+                continue
 
-    # Resolve output path with template substitution
-    output_path_template = config.needscfg_outpath
-    output_path_str = output_path_template.replace("${outdir}", str(app.outdir))
-    output_path_str = output_path_str.replace("${srcdir}", str(app.srcdir))
-    outpath = Path(output_path_str)
+            try:
+                with open(toml_path, "rb") as f:
+                    merge_data = tomli.load(f)
+            except (OSError, PermissionError) as e:
+                log_warning(
+                    LOGGER,
+                    f"Failed to read TOML file '{toml_path}': {e}",
+                    "merge_failed",
+                    location=None,
+                )
+                continue
+            except tomli.TOMLDecodeError as e:
+                log_warning(
+                    LOGGER,
+                    f"Failed to parse TOML file '{toml_path}': {e}",
+                    "merge_failed",
+                    location=None,
+                )
+                continue
 
-    # Make relative paths relative to confdir (where conf.py is located)
-    if not outpath.is_absolute():
-        outpath = Path(app.confdir) / outpath
+            # Shallow merge all root-level keys from the file
+            # If file has a [needs] table, merge it into our needs attributes
+            # All other root-level keys/tables are collected separately
+            for key, value in merge_data.items():
+                if key == "needs":
+                    # Shallow merge into the needs attributes (before sorting)
+                    need_attributes.update(value)
+                else:
+                    # Collect root-level keys and tables
+                    additional_root_data[key] = value
+
+            LOGGER.info(
+                f"Merged TOML configuration from '{toml_path}'",
+                type="ubproject",
+                subtype="merge",
+            )
+
+    # Sort the needs table data with special handling for reproducibility
+    sorted_needs = sort_for_reproducibility(need_attributes)
+
+    # Build the final TOML structure with all root-level keys sorted
+    final_toml_data = {}
+
+    # First add the needs table
+    final_toml_data["needs"] = sorted_needs
+
+    # Add additional root-level data
+    for key, value in additional_root_data.items():
+        # Sort dictionaries in the additional data
+        if isinstance(value, dict):
+            final_toml_data[key] = dict(sorted(value.items()))
+        else:
+            final_toml_data[key] = value
+
+    # Sort all root-level keys (including 'needs') alphabetically
+    final_toml_data = dict(sorted(final_toml_data.items()))
+
+    # Resolve output path with template substitution
+    outpath = resolve_path_template(config.needscfg_outpath, app)
 
     # Generate new content
-    new_content = tomli_w.dumps({"needs": sorted_attributes})
+    new_content = tomli_w.dumps(final_toml_data)
 
     # Add header if configured
     if config.needscfg_add_header:
@@ -290,6 +377,13 @@ def setup(app: Sphinx):
         types=[list],
         description="List of needs_* variable names to exclude from writing (resolved configs).",
     )
+    app.add_config_value(
+        "needscfg_merge_toml_files",
+        [],
+        "html",
+        types=[list],
+        description="List of TOML file paths to shallow-merge into the output configuration.",
+    )
 
     # run this late
     app.connect("config-inited", write_ubproject_file, priority=999)

pyproject.toml

@@ -27,7 +27,12 @@ classifiers = [
   "Topic :: Utilities",
   "Framework :: Sphinx :: Extension",
 ]
-dependencies = ["sphinx>=4.0", "sphinx-needs>4.2.0", "tomli-w>=1.2.0"]
+dependencies = [
+  "sphinx>=4.0",
+  "sphinx-needs>4.2.0",
+  "tomli>=2.3.0",
+  "tomli-w>=1.2.0",
+]
 
 [dependency-groups]
 dev = [