Merge branch 'main' into feat/infer_schema_for_csv_ndjson

Author: KKould

.github/actions/build_bindings_python/action.yml

@@ -18,7 +18,7 @@ runs:
         raw_version="${{ inputs.version }}"
         # Remove v prefix and suffixes: v1.2.809-nightly -> 1.2.809
         VERSION=$(echo "$raw_version" | sed 's/^v//' | sed 's/-nightly//' | sed 's/-p[0-9]*//')
-        
+
         echo "building version: $raw_version -> $VERSION"
         # Replace any existing version in Cargo.toml and pyproject.toml
         sed "s/^version = \".*\"/version = \"$VERSION\"/" Cargo.toml > Cargo.toml.tmp
@@ -80,9 +80,8 @@ runs:
           # Install cargo-zigbuild manually to avoid musl target issues
           cargo install cargo-zigbuild --target ${{ inputs.target }}
           ../../scripts/setup/dev_setup.sh -yb
-          # Clean up any existing virtual environment
-          rm -rf .venv || true
-          uv venv --python=python3.12
+          # Create virtual environment for build dependencies (overwrite if exists)
+          uv venv .venv --python=python3.12 --clear
           uv sync --all-groups --all-extras
 
     - name: Setup Rust for development builds
@@ -99,11 +98,8 @@ runs:
       run: |
         echo "Building development wheel for testing..."
 
-        # Clean up any existing virtual environment
-        rm -rf .venv || true
-
-        # Create and activate virtual environment
-        uv venv --python python3.12
+        # Create virtual environment (overwrite if exists)
+        uv venv .venv --python python3.12 --clear
         source .venv/bin/activate
 
         # Install development dependencies
@@ -121,25 +117,3 @@ runs:
         python -m pytest tests/ -v --tb=short
 
         echo "All Python binding tests passed!"
-
-    - name: Test built wheels
-      if: inputs.version != ''
-      shell: bash
-      working-directory: src/bendpy
-      run: |
-        echo "Testing built wheels..."
-
-        # Clean up any existing virtual environment
-        rm -rf .venv || true
-
-        # Install from built wheel for release testing
-        uv venv --python python3.12
-        source .venv/bin/activate
-        uv pip install dist/*.whl
-        uv pip install pytest pandas polars pyarrow
-
-        # Run pytest tests
-        echo "Executing pytest tests..."
-        python -m pytest tests/ -v --tb=short
-
-        echo "All Python binding tests passed!"

.github/workflows/bindings.python.yml

@@ -44,12 +44,12 @@ jobs:
             echo "Error: No version provided for workflow_call"
             exit 1
           fi
-          
+
           # Validate version format
           if [[ ! $VERSION =~ ^v[0-9]+\.[0-9]+\.[0-9]+.*$ ]]; then
             echo "Warning: Version $VERSION may not follow semantic versioning (vX.Y.Z)"
           fi
-          
+
           echo "Final version: $VERSION"
           echo "version=$VERSION" >> $GITHUB_OUTPUT
 
@@ -104,29 +104,44 @@ jobs:
           retention-days: 7
 
   macos:
-    name: Build macOS Universal Wheel
+    name: Build macOS Wheels
     if: (github.event_name == 'workflow_call' && inputs.version != '') || github.event_name == 'workflow_dispatch'
     needs: [get-version, test]
     runs-on: macos-latest
+    continue-on-error: true
+    strategy:
+      matrix:
+        target: [x86_64-apple-darwin, aarch64-apple-darwin]
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+
+      - name: Install dependencies
+        run: |
+          # Install OpenSSL and necessary tools
+          brew install openssl@3
+
+          # Use vendored OpenSSL to avoid cross-compilation issues
+          echo "OPENSSL_STATIC=1" >> $GITHUB_ENV
+          echo "OPENSSL_VENDORED=1" >> $GITHUB_ENV
+          echo "PKG_CONFIG_ALLOW_CROSS=1" >> $GITHUB_ENV
+
       - uses: ./.github/actions/build_bindings_python
         with:
-          target: universal2-apple-darwin
+          target: ${{ matrix.target }}
           version: ${{ needs.get-version.outputs.version }}
       - name: Upload macOS wheel
         uses: actions/upload-artifact@v4
         with:
-          name: python-macos-universal
+          name: python-macos-${{ matrix.target }}
           path: src/bendpy/dist/*.whl
           retention-days: 7
 
   publish:
     name: Publish to PyPI
     if: (github.event_name == 'workflow_call' && inputs.version != '') || github.event_name == 'workflow_dispatch'
-    needs: [get-version, test, linux, macos]
+    needs: [get-version, test, linux]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -135,20 +150,26 @@ jobs:
           pattern: python-*
           merge-multiple: true
           path: src/bendpy/dist
+        continue-on-error: true
 
       - name: Show packages to publish
         run: |
           echo "Publishing packages for version: ${{ needs.get-version.outputs.version }}"
           echo "Packages found:"
-          ls -la src/bendpy/dist/
+          ls -la src/bendpy/dist/ || echo "No packages found"
           echo "Total packages: $(ls src/bendpy/dist/*.whl 2>/dev/null | wc -l)"
 
       - name: Publish to PyPI
         timeout-minutes: 10
         run: |
           pip install twine
           echo "Publishing to PyPI..."
-          twine upload --skip-existing --verbose src/bendpy/dist/*.whl
+          if [ -n "$(find src/bendpy/dist -name "*.whl" 2>/dev/null)" ]; then
+            twine upload --skip-existing --verbose src/bendpy/dist/*.whl
+          else
+            echo "No wheel files found to publish"
+            exit 1
+          fi
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}

src/bendpy/README.md

@@ -1,42 +1,91 @@
 # Databend Python Binding
 
-Official Python binding for [Databend](https://databend.com) - The AI-Native Data Warehouse.
+Official Python binding for [Databend](https://databend.com) - The multi-modal data warehouse built for the AI era.
 
-Databend is the open-source alternative to Snowflake with near 100% SQL compatibility and native AI capabilities. Built in Rust with MPP architecture and S3-native storage, Databend unifies structured tables, JSON documents, and vector embeddings in a single platform.
+Databend unifies structured data, JSON documents, and vector embeddings in a single platform with near 100% Snowflake compatibility. Built in Rust with MPP architecture and S3-native storage for cloud-scale analytics.
 
 ## Installation
 
 ```bash
 pip install databend
 ```
 
+To test, run:
+```python
+python3 -c "import databend; ctx = databend.SessionContext(); ctx.sql('SELECT version() AS version').show()"
+```
+
+## API Reference
+
+### Core Operations
+| Method | Description |
+|--------|-------------|
+| `SessionContext()` | Create a new session context |
+| `sql(query)` | Execute SQL query, returns DataFrame |
+
+### File Registration
+| Method | Description |
+|--------|-------------|
+| `register_parquet(name, path, pattern=None, connection=None)` | Register Parquet files as table |
+| `register_csv(name, path, pattern=None, connection=None)` | Register CSV files as table |
+| `register_ndjson(name, path, pattern=None, connection=None)` | Register NDJSON files as table |
+| `register_tsv(name, path, pattern=None, connection=None)` | Register TSV files as table |
+
+### Cloud Storage Connections
+| Method | Description |
+|--------|-------------|
+| `create_s3_connection(name, key, secret, endpoint=None, region=None)` | Create S3 connection |
+| `create_azblob_connection(name, url, account, key)` | Create Azure Blob connection |
+| `create_gcs_connection(name, url, credential)` | Create Google Cloud connection |
+| `list_connections()` | List all connections |
+| `describe_connection(name)` | Show connection details |
+| `drop_connection(name)` | Remove connection |
+
+### Stage Management
+| Method | Description |
+|--------|-------------|
+| `create_stage(name, url, connection)` | Create external stage |
+| `show_stages()` | List all stages |
+| `list_stages(stage_name)` | List files in stage |
+| `describe_stage(name)` | Show stage details |
+| `drop_stage(name)` | Remove stage |
+
+### DataFrame Operations
+| Method | Description |
+|--------|-------------|
+| `collect()` | Execute and collect results |
+| `show(num=20)` | Display results in console |
+| `to_pandas()` | Convert to pandas DataFrame |
+| `to_polars()` | Convert to polars DataFrame |
+| `to_arrow_table()` | Convert to PyArrow Table |
+
 ## Examples
 
-### Local Files
+### Local Tables
 
 ```python
 import databend
 ctx = databend.SessionContext()
 
-# Query local Parquet files
-ctx.register_parquet("orders", "/path/to/orders/")
-ctx.register_parquet("customers", "/path/to/customers/")
-df = ctx.sql("SELECT * FROM orders JOIN customers ON orders.customer_id = customers.id").to_pandas()
+# Create and query in-memory tables
+ctx.sql("CREATE TABLE users (id INT, name STRING, age INT)").collect()
+ctx.sql("INSERT INTO users VALUES (1, 'Alice', 25), (2, 'Bob', 30)").collect()
+df = ctx.sql("SELECT * FROM users WHERE age > 25").to_pandas()
 ```
 
-### Local Tables
+### Working with Local Files
 
 ```python
 import databend
 ctx = databend.SessionContext()
 
-# Create and query local tables
-ctx.sql("CREATE TABLE users (id INT, name STRING, age INT)").collect()
-ctx.sql("INSERT INTO users VALUES (1, 'Alice', 25), (2, 'Bob', 30)").collect()
-df = ctx.sql("SELECT * FROM users WHERE age > 25").to_pandas()
+# Query local Parquet files
+ctx.register_parquet("orders", "/path/to/orders/")
+ctx.register_parquet("customers", "/path/to/customers/")
+df = ctx.sql("SELECT * FROM orders JOIN customers ON orders.customer_id = customers.id").to_pandas()
 ```
 
-### S3 Remote Files
+### Cloud Storage - S3 Files
 
 ```python
 import databend
@@ -49,14 +98,14 @@ ctx.register_parquet("trips", "s3://bucket/trips/", connection="s3")
 df = ctx.sql("SELECT COUNT(*) FROM trips").to_pandas()
 ```
 
-### Remote Tables
+### Cloud Storage - S3 Tables
 
 ```python
 import databend
 import os
 ctx = databend.SessionContext()
 
-# Create S3 connection and table
+# Create S3 connection and persistent table
 ctx.create_s3_connection("s3", os.getenv("AWS_ACCESS_KEY_ID"), os.getenv("AWS_SECRET_ACCESS_KEY"))
 ctx.sql("CREATE TABLE s3_table (id INT, name STRING) 's3://bucket/table/' CONNECTION=(CONNECTION_NAME='s3')").collect()
 df = ctx.sql("SELECT * FROM s3_table").to_pandas()

src/bendpy/pyproject.toml

@@ -7,7 +7,8 @@ build-backend = "maturin"
 version = "0.1.0"
 name = "databend"
 requires-python = ">=3.10"
-description = "Official Python binding for Databend - The AI-Native Data Warehouse. Open-source alternative to Snowflake with near 100% SQL compatibility, native AI capabilities, and unified support for structured, semi-structured, and vector data."
+description = "The multi-modal data warehouse built for the AI era. Unified analytics for structured data, JSON, and vector embeddings with near 100% Snowflake compatibility."
+readme = "README.md"
 license = {text = "Apache-2.0"}
 authors = [{name = "Databend Labs", email = "[email protected]"}]
 maintainers = [{name = "Databend Community", email = "[email protected]"}]
@@ -38,7 +39,7 @@ test = ["pytest", "pandas", "polars", "pyarrow"]
 [project.urls]
 Documentation = "https://pypi.org/project/databend/"
 Homepage =  "https://databend.com"
-Repository = "https://github.com/databendlabs/databend"
+Repository = "https://github.com/databendlabs/databend/tree/main/src/bendpy"
 
 [dependency-groups]
 dev = ["maturin>=1.8.2"]

src/bendpy/src/lib.rs

@@ -89,6 +89,7 @@ fn create_embedded_config(
 
     // Query configuration
     conf.query.tenant_id = Tenant::new_literal("python_binding");
+    conf.query.embedded_mode = true;
     conf.query.cluster_id = "".to_string();
     conf.query.warehouse_id = "".to_string();
     conf.query.node_id = "embedded_node".to_string();

src/query/config/src/config.rs

@@ -1791,6 +1791,8 @@ pub struct QueryConfig {
     /// If in management mode, only can do some meta level operations(database/table/user/stage etc.) with metasrv.
     #[clap(long)]
     pub management_mode: bool,
+    #[clap(long)]
+    pub embedded_mode: bool,
 
     /// Deprecated: jwt_key_file is deprecated, use jwt_key_files to add a list of available jwks url
     #[clap(long, value_name = "VALUE", default_value_t)]
@@ -2017,6 +2019,7 @@ impl TryInto<InnerQueryConfig> for QueryConfig {
             max_query_log_size: self.max_query_log_size,
             databend_enterprise_license: self.databend_enterprise_license,
             management_mode: self.management_mode,
+            embedded_mode: self.embedded_mode,
             parquet_fast_read_bytes: self.parquet_fast_read_bytes,
             max_storage_io_requests: self.max_storage_io_requests,
             jwt_key_file: self.jwt_key_file,
@@ -2159,6 +2162,7 @@ impl From<InnerQueryConfig> for QueryConfig {
             settings: HashMap::new(),
             resources_management: None,
             enable_queries_executor: inner.enable_queries_executor,
+            embedded_mode: inner.embedded_mode,
         }
     }
 }

src/query/config/src/global.rs

@@ -48,6 +48,8 @@ impl GlobalConfig {
     /// Embedded mode is determined by empty cluster_id and warehouse_id
     pub fn is_embedded_mode() -> bool {
         let config = Self::instance();
-        config.query.cluster_id.is_empty() && config.query.warehouse_id.is_empty()
+        config.query.embedded_mode
+            && config.query.cluster_id.is_empty()
+            && config.query.warehouse_id.is_empty()
     }
 }

src/query/config/src/inner.rs

@@ -209,6 +209,7 @@ pub struct QueryConfig {
     pub databend_enterprise_license: Option<String>,
     /// If in management mode, only can do some meta level operations(database/table/user/stage etc.) with metasrv.
     pub management_mode: bool,
+    pub embedded_mode: bool,
 
     pub parquet_fast_read_bytes: Option<u64>,
     pub max_storage_io_requests: Option<u64>,
@@ -302,6 +303,7 @@ impl Default for QueryConfig {
             max_query_log_size: 10_000,
             databend_enterprise_license: None,
             management_mode: false,
+            embedded_mode: false,
             parquet_fast_read_bytes: None,
             max_storage_io_requests: None,
             jwt_key_file: "".to_string(),

src/query/service/tests/it/storages/testdata/configs_table_basic.txt

@@ -163,6 +163,7 @@ DB.Table: 'system'.'configs', Table: configs-table_id:1, ver:0, Engine: SystemCo
 | 'query'   | 'default_storage_format'                             | 'auto'                                                                                                                                                                                                    | ''       |
 | 'query'   | 'disable_system_table_load'                          | 'false'                                                                                                                                                                                                   | ''       |
 | 'query'   | 'discovery_address'                                  | ''                                                                                                                                                                                                        | ''       |
+| 'query'   | 'embedded_mode'                                      | 'false'                                                                                                                                                                                                   | ''       |
 | 'query'   | 'enable_meta_data_upgrade_json_to_pb_from_v307'      | 'false'                                                                                                                                                                                                   | ''       |
 | 'query'   | 'enable_queries_executor'                            | 'false'                                                                                                                                                                                                   | ''       |
 | 'query'   | 'enable_udf_js_script'                               | 'true'                                                                                                                                                                                                    | ''       |