PoC for ragna-base package with just base dependencies

.github/workflows/build_package.yml

@@ -0,0 +1,50 @@
+name: Build Package
+
+on:
+  pull_request:
+  push:
+    branches:
+      - release/*
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        package:
+          - ragna
+          - ragna-base
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install build toml
+
+      - name: Modify pyproject.toml
+        run: python scripts/build_helper.py ${{ matrix.package }}
+
+      - name: Build distribution
+        run: |
+          echo "Building package: ${{ matrix.package }}"
+          python -m build
+          echo "Build complete for ${{ matrix.package }}"
+
+      - name: Verify package
+        run: |
+          echo "Verifying package: ${{ matrix.package }}"
+          pip install dist/*.whl
+          pip check
+
+      - name: Visualize dependency tree of built package
+        run: |
+          pip install pipdeptree
+          pipdeptree -d 1

pyproject.toml

@@ -20,49 +20,16 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
 ]
 requires-python = ">=3.9"
-dependencies = [
-    "aiofiles",
-    "emoji",
-    "fastapi",
-    "httpx",
-    "importlib_metadata>=4.6; python_version<'3.10'",
-    "packaging",
-    "panel==1.4.2",
-    "pydantic>=2",
-    "pydantic-core",
-    "pydantic-settings>=2",
-    "PyJWT",
-    "python-multipart",
-    "redis",
-    "questionary",
-    "rich",
-    "sqlalchemy>=2",
-    "starlette",
-    "tomlkit",
-    "typer",
-    "uvicorn",
-]
-dynamic = ["version"]
+dynamic = ["dependencies", "version"]
 
 [project.urls]
 Homepage = "https://ragna.chat"
 Documentation = "https://ragna.chat"
 Changelog = "https://ragna.chat/en/stable/references/release-notes/"
 Repository = "https://github.com/Quansight/ragna"
 
-[project.optional-dependencies]
-# to update the array below, run scripts/update_optional_dependencies.py
-all = [
-    "chromadb>=0.4.13",
-    "httpx_sse",
-    "ijson",
-    "lancedb>=0.2",
-    "pyarrow",
-    "pymupdf>=1.23.6",
-    "python-docx",
-    "python-pptx",
-    "tiktoken",
-]
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
 
 [tool.setuptools_scm]
 write_to = "ragna/_version.py"

requirements-base.txt

@@ -0,0 +1,20 @@
+aiofiles
+emoji
+fastapi
+httpx
+importlib_metadata>=4.6; python_version<'3.10'
+packaging
+panel==1.4.2
+pydantic>=2
+pydantic-core
+pydantic-settings>=2
+PyJWT
+python-multipart
+redis
+questionary
+rich
+sqlalchemy>=2
+starlette
+tomlkit
+typer
+uvicorn

requirements.txt

@@ -0,0 +1,29 @@
+aiofiles
+emoji
+fastapi
+httpx
+importlib_metadata>=4.6; python_version<'3.10'
+packaging
+panel==1.4.2
+pydantic>=2
+pydantic-core
+pydantic-settings>=2
+PyJWT
+python-multipart
+redis
+questionary
+rich
+sqlalchemy>=2
+starlette
+tomlkit
+typer
+uvicorn
+chromadb>=0.4.13
+httpx_sse
+ijson
+lancedb>=0.2
+pyarrow
+pymupdf>=1.23.6
+python-docx
+python-pptx
+tiktoken

scripts/build_helper.py

@@ -0,0 +1,30 @@
+import toml
+
+
+def modify_pyproject(package_name):
+    with open("pyproject.toml", "r") as f:
+        pyproject_data = toml.load(f)
+
+    pyproject_data["project"]["name"] = package_name
+
+    if package_name == "ragna-base":
+        pyproject_data["tool"]["setuptools"]["dynamic"]["dependencies"] = {
+            "file": ["requirements-base.txt"]
+        }
+    else:
+        pyproject_data["tool"]["setuptools"]["dynamic"]["dependencies"] = {
+            "file": ["requirements.txt"]
+        }
+
+    with open("pyproject.toml", "w") as f:
+        toml.dump(pyproject_data, f)
+
+
+if __name__ == "__main__":
+    import sys
+
+    package_name = sys.argv[1]
+    if package_name not in ["ragna", "ragna-base"]:
+        print("Invalid package name. Must be 'ragna' or 'ragna-base'.")
+        sys.exit(1)
+    modify_pyproject(package_name)

scripts/update_optional_dependencies.py

@@ -2,23 +2,21 @@
 from functools import reduce
 from pathlib import Path
 
-import tomlkit
-import tomlkit.items
 from packaging.requirements import Requirement
 
 import ragna
 from ragna.core import Assistant, SourceStorage
 
 HERE = Path(__file__).parent
-PYPROJECT_TOML = HERE / ".." / "pyproject.toml"
+REQUIREMENTS_TXT = HERE / ".." / "requirements.txt"
 
 
 def main():
     optional_dependencies = make_optional_dependencies(
         extract_builtin_document_handler_requirements(),
         extract_builtin_component_requirements(),
     )
-    update_pyproject_toml(optional_dependencies)
+    update_requirements_txt(optional_dependencies)
 
 
 def make_optional_dependencies(*optional_requirements):
@@ -57,18 +55,27 @@ def extract_builtin_component_requirements():
     return dict(requirements)
 
 
-def update_pyproject_toml(optional_dependencies):
-    with open(PYPROJECT_TOML) as file:
-        document = tomlkit.load(file)
+def update_requirements_txt(optional_dependencies):
+    existing_dependencies = set()
 
-    document["project"]["optional-dependencies"]["all"] = tomlkit.items.Array(
-        list(map(tomlkit.items.String.from_raw, optional_dependencies)),
-        trivia=tomlkit.items.Trivia(),
-        multiline=True,
-    )
+    # Read existing dependencies from requirements.txt
+    if REQUIREMENTS_TXT.exists():
+        with open(REQUIREMENTS_TXT, "r") as file:
+            existing_dependencies = set(line.strip() for line in file)
+
+    new_dependencies = []
+
+    # Check if each optional dependency already exists
+    for dependency in optional_dependencies:
+        if dependency not in existing_dependencies:
+            new_dependencies.append(dependency)
+            existing_dependencies.add(dependency)
 
-    with open(PYPROJECT_TOML, "w") as file:
-        tomlkit.dump(document, file)
+    # Append new dependencies to requirements.txt
+    if new_dependencies:
+        with open(REQUIREMENTS_TXT, "a") as file:
+            file.write("\n".join(new_dependencies))
+            file.write("\n")
 
 
 def append_version_specifiers(version_specifiers, obj):