Merge branch 'main' into separate-key-expiration-from-budget-reset

Author: JVenberg

.circleci/config.yml

@@ -616,6 +616,24 @@ jobs:
             wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
             sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
             rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Start PostgreSQL Database
+          command: |
+            docker run -d \
+              --name postgres-db \
+              -e POSTGRES_USER=postgres \
+              -e POSTGRES_PASSWORD=postgres \
+              -e POSTGRES_DB=circle_test \
+              -p 5432:5432 \
+              postgres:14
+      - run:
+          name: Wait for PostgreSQL to be ready
+          command: dockerize -wait tcp://localhost:5432 -timeout 1m
+      - run:
+          name: Set DATABASE_URL environment variable
+          command: |
+            echo 'export DATABASE_URL="postgresql://postgres:postgres@localhost:5432/circle_test"' >> $BASH_ENV
+            source $BASH_ENV
       - run:
           name: Run Security Scans
           command: |
@@ -658,18 +676,16 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
-      - run:
-          name: Install PostgreSQL
-          command: |
-            sudo apt-get update
-            sudo apt-get install postgresql postgresql-contrib
-            echo 'export PATH=/usr/lib/postgresql/*/bin:$PATH' >> $BASH_ENV
       - setup_google_dns
       - run:
           name: Show git commit hash
           command: |
             echo "Git commit hash: $CIRCLE_SHA1"
-
+      - run:
+          name: Install PostgreSQL
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y postgresql-14 postgresql-contrib-14
       - restore_cache:
           keys:
             - v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
@@ -1025,6 +1041,92 @@ jobs:
           paths:
             - llm_responses_api_coverage.xml
             - llm_responses_api_coverage
+  ocr_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - setup_google_dns
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-cov==5.0.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "respx==0.22.0"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/ocr_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml ocr_coverage.xml
+            mv .coverage ocr_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - ocr_coverage.xml
+            - ocr_coverage
+  search_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - setup_google_dns
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-cov==5.0.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "respx==0.22.0"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/search_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml search_coverage.xml
+            mv .coverage search_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - search_coverage.xml
+            - search_coverage
   litellm_mapped_tests:
     docker:
       - image: cimg/python:3.11
@@ -1360,6 +1462,49 @@ jobs:
           paths:
             - logging_coverage.xml
             - logging_coverage
+  audio_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - setup_google_dns
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-cov==5.0.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "respx==0.22.0"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/audio_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml audio_coverage.xml
+            mv .coverage audio_coverage
+
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - audio_coverage.xml
+            - audio_coverage
   installing_litellm_on_python:
     docker:
       - image: circleci/python:3.8
@@ -1564,6 +1709,7 @@ jobs:
             pip install "pytest==7.3.1"
             pip install "pytest-asyncio==0.21.1"
             pip install aiohttp
+            pip install apscheduler
       - run:
           name: Build Docker image
           command: |
@@ -2357,6 +2503,25 @@ jobs:
             pip install "pytest-mock==3.12.0"
             pip install "pytest-asyncio==0.21.1"
             pip install "assemblyai==0.37.0"
+      - run:
+          name: Install dockerize
+          command: |
+            wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Start PostgreSQL Database
+          command: |
+            docker run -d \
+              --name postgres-db \
+              -e POSTGRES_USER=postgres \
+              -e POSTGRES_PASSWORD=postgres \
+              -e POSTGRES_DB=circle_test \
+              -p 5432:5432 \
+              postgres:14
+      - run:
+          name: Wait for PostgreSQL to be ready
+          command: dockerize -wait tcp://localhost:5432 -timeout 1m
       - run:
           name: Build Docker image
           command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@@ -2367,10 +2532,11 @@ jobs:
           command: |
             docker run -d \
               -p 4000:4000 \
-              -e DATABASE_URL=$CLEAN_STORE_MODEL_IN_DB_DATABASE_URL \
+              -e DATABASE_URL=postgresql://postgres:[email protected]:5432/circle_test \
               -e STORE_MODEL_IN_DB="True" \
               -e LITELLM_MASTER_KEY="sk-1234" \
               -e LITELLM_LICENSE=$LITELLM_LICENSE \
+              --add-host host.docker.internal:host-gateway \
               --name my-app \
               -v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
               my-app:latest \
@@ -2400,7 +2566,16 @@ jobs:
             python -m pytest -vv tests/store_model_in_db_tests -x --junitxml=test-results/junit.xml --durations=5
           no_output_timeout:
             120m
-            # Clean up first container
+      - run:
+          name: Stop and remove containers
+          command: |
+            docker stop my-app || true
+            docker rm my-app || true
+            docker stop postgres-db || true
+            docker rm postgres-db || true
+          when: always
+      - store_test_results:
+          path: test-results
 
   proxy_build_from_pip_tests:
     # Change from docker to machine executor
@@ -2435,6 +2610,7 @@ jobs:
             pip install "pytest-mock==3.12.0"
             pip install "pytest-asyncio==0.21.1"
             pip install "mypy==1.18.2"
+            pip install apscheduler
       - run:
           name: Build Docker image
           command: |
@@ -2588,6 +2764,8 @@ jobs:
               -e GEMINI_API_KEY=$GEMINI_API_KEY \
               -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
               -e ASSEMBLYAI_API_KEY=$ASSEMBLYAI_API_KEY \
+              -e AZURE_API_KEY=$AZURE_API_KEY \
+              -e AZURE_API_BASE=$AZURE_API_BASE \
               -e USE_DDTRACE=True \
               -e DD_API_KEY=$DD_API_KEY \
               -e DD_SITE=$DD_SITE \
@@ -2694,7 +2872,7 @@ jobs:
             python -m venv venv
             . venv/bin/activate
             pip install coverage
-            coverage combine llm_translation_coverage llm_responses_api_coverage mcp_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage
+            coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage
             coverage xml
       - codecov/upload:
           file: ./coverage.xml
@@ -3242,6 +3420,18 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - ocr_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
+      - search_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
       - litellm_mapped_enterprise_tests:
           filters:
             branches:
@@ -3284,20 +3474,29 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - audio_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
       - upload-coverage:
           requires:
             - llm_translation_testing
             - mcp_testing
             - google_generate_content_endpoint_testing
             - guardrails_testing
             - llm_responses_api_testing
+            - ocr_testing
+            - search_testing
             - litellm_mapped_tests
             - litellm_mapped_enterprise_tests
             - batches_testing
             - litellm_utils_testing
             - pass_through_unit_testing
             - image_gen_testing
             - logging_testing
+            - audio_testing
             - litellm_router_testing
             - litellm_router_unit_testing
             - caching_unit_tests
@@ -3353,13 +3552,16 @@ workflows:
             - mcp_testing
             - google_generate_content_endpoint_testing
             - llm_responses_api_testing
+            - ocr_testing
+            - search_testing
             - litellm_mapped_tests
             - litellm_mapped_enterprise_tests
             - batches_testing
             - litellm_utils_testing
             - pass_through_unit_testing
             - image_gen_testing
             - logging_testing
+            - audio_testing
             - litellm_router_testing
             - litellm_router_unit_testing
             - caching_unit_tests

.github/workflows/interpret_load_test.py

@@ -88,6 +88,7 @@ def get_docker_run_command(release_version):
 
 
 if __name__ == "__main__":
+    return
     csv_file = "load_test_stats.csv"  # Change this to the path of your CSV file
     markdown_table = interpret_results(csv_file)
 

.github/workflows/test-litellm.yml

@@ -33,11 +33,12 @@ jobs:
         poetry run pip install "google-genai==1.22.0"
         poetry run pip install "google-cloud-aiplatform>=1.38"
         poetry run pip install "fastapi-offline==1.7.3"
+        poetry run pip install "python-multipart==0.0.18"
     - name: Setup litellm-enterprise as local package
       run: |
         cd enterprise
         python -m pip install -e .
         cd ..
     - name: Run tests
       run: |
-        poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 
+        poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50

.gitignore

@@ -97,3 +97,6 @@ litellm_config.yaml
 .vscode/launch.json
 litellm/proxy/to_delete_loadtest_work/*
 update_model_cost_map.py
+tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
+litellm/proxy/_experimental/out/guardrails/index.html
+scripts/test_vertex_ai_search.py

Makefile

@@ -45,7 +45,7 @@ install-proxy-dev-ci:
 install-test-deps: install-proxy-dev
 	poetry run pip install "pytest-retry==1.6.3"
 	poetry run pip install pytest-xdist
-	cd enterprise && python -m pip install -e . && cd ..
+	cd enterprise && poetry run pip install -e . && cd ..
 
 install-helm-unittest:
 	helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"