Skip to content

Commit 06eeb28

Browse files
ishaan-jaffishaan-jaff
authored
Litellm ci cd fixes 2 (#16693)
* litellm_proxy_unit_testing_part1 * test proxy unit test * litellm_proxy_unit_testing_key_generation * test_async_call_with_key_over_model_budget * test_aasync_call_with_key_over_model_budget
1 parent e08cd9b commit 06eeb28

File tree

2 files changed

+329
-68
lines changed

2 files changed

+329
-68
lines changed

‎.circleci/config.yml‎

Lines changed: 245 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -701,13 +701,16 @@ jobs:
701701
paths:
702702
- litellm_security_tests_coverage.xml
703703
- litellm_security_tests_coverage
704-
litellm_proxy_unit_testing: # Runs all tests with the "proxy", "key", "jwt" filenames
704+
# Split proxy unit tests into 3 jobs for faster execution and better debugging
705+
# test_key_generate_prisma runs separately without parallel execution to avoid event loop issues with logging worker
706+
litellm_proxy_unit_testing_key_generation:
705707
docker:
706708
- image: cimg/python:3.11
707709
auth:
708710
username: ${DOCKERHUB_USERNAME}
709711
password: ${DOCKERHUB_PASSWORD}
710712
working_directory: ~/project
713+
resource_class: large
711714
steps:
712715
- checkout
713716
- setup_google_dns
@@ -732,6 +735,114 @@ jobs:
732735
pip install "pytest-retry==1.6.3"
733736
pip install "pytest-asyncio==0.21.1"
734737
pip install "pytest-cov==5.0.0"
738+
pip install "pytest-timeout==2.2.0"
739+
pip install "pytest-forked==1.6.0"
740+
pip install "mypy==1.18.2"
741+
pip install "google-generativeai==0.3.2"
742+
pip install "google-cloud-aiplatform==1.43.0"
743+
pip install "google-genai==1.22.0"
744+
pip install pyarrow
745+
pip install "boto3==1.36.0"
746+
pip install "aioboto3==13.4.0"
747+
pip install langchain
748+
pip install lunary==0.2.5
749+
pip install "azure-identity==1.16.1"
750+
pip install "langfuse==2.59.7"
751+
pip install "logfire==0.29.0"
752+
pip install numpydoc
753+
pip install traceloop-sdk==0.21.1
754+
pip install opentelemetry-api==1.25.0
755+
pip install opentelemetry-sdk==1.25.0
756+
pip install opentelemetry-exporter-otlp==1.25.0
757+
pip install openai==1.100.1
758+
pip install prisma==0.11.0
759+
pip install "detect_secrets==1.5.0"
760+
pip install "httpx==0.24.1"
761+
pip install "respx==0.22.0"
762+
pip install fastapi
763+
pip install "gunicorn==21.2.0"
764+
pip install "anyio==4.2.0"
765+
pip install "aiodynamo==23.10.1"
766+
pip install "asyncio==3.4.3"
767+
pip install "apscheduler==3.10.4"
768+
pip install "PyGithub==1.59.1"
769+
pip install argon2-cffi
770+
pip install "pytest-mock==3.12.0"
771+
pip install python-multipart
772+
pip install google-cloud-aiplatform
773+
pip install prometheus-client==0.20.0
774+
pip install "pydantic==2.10.2"
775+
pip install "diskcache==5.6.1"
776+
pip install "Pillow==10.3.0"
777+
pip install "jsonschema==4.22.0"
778+
pip install "pytest-postgresql==7.0.1"
779+
pip install "fakeredis==2.28.1"
780+
- setup_litellm_enterprise_pip
781+
- save_cache:
782+
paths:
783+
- ./venv
784+
key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
785+
- run:
786+
name: Run prisma ./docker/entrypoint.sh
787+
command: |
788+
set +e
789+
chmod +x docker/entrypoint.sh
790+
./docker/entrypoint.sh
791+
set -e
792+
- run:
793+
name: Run key generation tests (no parallel execution to avoid event loop issues)
794+
command: |
795+
pwd
796+
ls
797+
# Run without -n flag to avoid pytest-xdist event loop conflicts with logging worker
798+
python -m pytest tests/proxy_unit_tests/test_key_generate_prisma.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-key-generation.xml --durations=10 --timeout=300 -vv --log-cli-level=INFO
799+
no_output_timeout: 120m
800+
- run:
801+
name: Rename the coverage files
802+
command: |
803+
mv coverage.xml litellm_proxy_unit_tests_key_generation_coverage.xml
804+
mv .coverage litellm_proxy_unit_tests_key_generation_coverage
805+
- store_test_results:
806+
path: test-results
807+
- persist_to_workspace:
808+
root: .
809+
paths:
810+
- litellm_proxy_unit_tests_key_generation_coverage.xml
811+
- litellm_proxy_unit_tests_key_generation_coverage
812+
litellm_proxy_unit_testing_part1:
813+
docker:
814+
- image: cimg/python:3.11
815+
auth:
816+
username: ${DOCKERHUB_USERNAME}
817+
password: ${DOCKERHUB_PASSWORD}
818+
working_directory: ~/project
819+
resource_class: large
820+
steps:
821+
- checkout
822+
- setup_google_dns
823+
- run:
824+
name: Show git commit hash
825+
command: |
826+
echo "Git commit hash: $CIRCLE_SHA1"
827+
- run:
828+
name: Install PostgreSQL
829+
command: |
830+
sudo apt-get update
831+
sudo apt-get install -y postgresql-14 postgresql-contrib-14
832+
- restore_cache:
833+
keys:
834+
- v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
835+
- run:
836+
name: Install Dependencies
837+
command: |
838+
python -m pip install --upgrade pip
839+
python -m pip install -r .circleci/requirements.txt
840+
pip install "pytest==7.3.1"
841+
pip install "pytest-retry==1.6.3"
842+
pip install "pytest-asyncio==0.21.1"
843+
pip install "pytest-cov==5.0.0"
844+
pip install "pytest-timeout==2.2.0"
845+
pip install "pytest-forked==1.6.0"
735846
pip install "mypy==1.18.2"
736847
pip install "google-generativeai==0.3.2"
737848
pip install "google-cloud-aiplatform==1.43.0"
@@ -785,28 +896,132 @@ jobs:
785896
chmod +x docker/entrypoint.sh
786897
./docker/entrypoint.sh
787898
set -e
788-
# Run pytest and generate JUnit XML report
789899
- run:
790-
name: Run tests
900+
name: Run proxy unit tests (part 1 - auth checks only, key generation in separate job)
791901
command: |
792902
pwd
793903
ls
794-
python -m pytest tests/proxy_unit_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
904+
# Run auth tests with parallel execution (test_key_generate_prisma moved to separate job to avoid event loop issues)
905+
python -m pytest tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part1.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO
795906
no_output_timeout: 120m
796907
- run:
797908
name: Rename the coverage files
798909
command: |
799-
mv coverage.xml litellm_proxy_unit_tests_coverage.xml
800-
mv .coverage litellm_proxy_unit_tests_coverage
801-
# Store test results
910+
mv coverage.xml litellm_proxy_unit_tests_part1_coverage.xml
911+
mv .coverage litellm_proxy_unit_tests_part1_coverage
912+
- store_test_results:
913+
path: test-results
914+
- persist_to_workspace:
915+
root: .
916+
paths:
917+
- litellm_proxy_unit_tests_part1_coverage.xml
918+
- litellm_proxy_unit_tests_part1_coverage
919+
litellm_proxy_unit_testing_part2:
920+
docker:
921+
- image: cimg/python:3.11
922+
auth:
923+
username: ${DOCKERHUB_USERNAME}
924+
password: ${DOCKERHUB_PASSWORD}
925+
working_directory: ~/project
926+
resource_class: large
927+
steps:
928+
- checkout
929+
- setup_google_dns
930+
- run:
931+
name: Show git commit hash
932+
command: |
933+
echo "Git commit hash: $CIRCLE_SHA1"
934+
- run:
935+
name: Install PostgreSQL
936+
command: |
937+
sudo apt-get update
938+
sudo apt-get install -y postgresql-14 postgresql-contrib-14
939+
- restore_cache:
940+
keys:
941+
- v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
942+
- run:
943+
name: Install Dependencies
944+
command: |
945+
python -m pip install --upgrade pip
946+
python -m pip install -r .circleci/requirements.txt
947+
pip install "pytest==7.3.1"
948+
pip install "pytest-retry==1.6.3"
949+
pip install "pytest-asyncio==0.21.1"
950+
pip install "pytest-cov==5.0.0"
951+
pip install "pytest-timeout==2.2.0"
952+
pip install "pytest-forked==1.6.0"
953+
pip install "mypy==1.18.2"
954+
pip install "google-generativeai==0.3.2"
955+
pip install "google-cloud-aiplatform==1.43.0"
956+
pip install "google-genai==1.22.0"
957+
pip install pyarrow
958+
pip install "boto3==1.36.0"
959+
pip install "aioboto3==13.4.0"
960+
pip install langchain
961+
pip install lunary==0.2.5
962+
pip install "azure-identity==1.16.1"
963+
pip install "langfuse==2.59.7"
964+
pip install "logfire==0.29.0"
965+
pip install numpydoc
966+
pip install traceloop-sdk==0.21.1
967+
pip install opentelemetry-api==1.25.0
968+
pip install opentelemetry-sdk==1.25.0
969+
pip install opentelemetry-exporter-otlp==1.25.0
970+
pip install openai==1.100.1
971+
pip install prisma==0.11.0
972+
pip install "detect_secrets==1.5.0"
973+
pip install "httpx==0.24.1"
974+
pip install "respx==0.22.0"
975+
pip install fastapi
976+
pip install "gunicorn==21.2.0"
977+
pip install "anyio==4.2.0"
978+
pip install "aiodynamo==23.10.1"
979+
pip install "asyncio==3.4.3"
980+
pip install "apscheduler==3.10.4"
981+
pip install "PyGithub==1.59.1"
982+
pip install argon2-cffi
983+
pip install "pytest-mock==3.12.0"
984+
pip install python-multipart
985+
pip install google-cloud-aiplatform
986+
pip install prometheus-client==0.20.0
987+
pip install "pydantic==2.10.2"
988+
pip install "diskcache==5.6.1"
989+
pip install "Pillow==10.3.0"
990+
pip install "jsonschema==4.22.0"
991+
pip install "pytest-postgresql==7.0.1"
992+
pip install "fakeredis==2.28.1"
993+
pip install "pytest-xdist==3.6.1"
994+
- setup_litellm_enterprise_pip
995+
- save_cache:
996+
paths:
997+
- ./venv
998+
key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
999+
- run:
1000+
name: Run prisma ./docker/entrypoint.sh
1001+
command: |
1002+
set +e
1003+
chmod +x docker/entrypoint.sh
1004+
./docker/entrypoint.sh
1005+
set -e
1006+
- run:
1007+
name: Run proxy unit tests (part 2 - remaining tests)
1008+
command: |
1009+
pwd
1010+
ls
1011+
python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part2.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO
1012+
no_output_timeout: 120m
1013+
- run:
1014+
name: Rename the coverage files
1015+
command: |
1016+
mv coverage.xml litellm_proxy_unit_tests_part2_coverage.xml
1017+
mv .coverage litellm_proxy_unit_tests_part2_coverage
8021018
- store_test_results:
8031019
path: test-results
804-
8051020
- persist_to_workspace:
8061021
root: .
8071022
paths:
808-
- litellm_proxy_unit_tests_coverage.xml
809-
- litellm_proxy_unit_tests_coverage
1023+
- litellm_proxy_unit_tests_part2_coverage.xml
1024+
- litellm_proxy_unit_tests_part2_coverage
8101025
litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
8111026
docker:
8121027
- image: cimg/python:3.13.1
@@ -2948,7 +3163,7 @@ jobs:
29483163
python -m venv venv
29493164
. venv/bin/activate
29503165
pip install coverage
2951-
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_proxy_tests_coverage litellm_llms_tests_coverage litellm_core_tests_coverage litellm_mapped_tests_coverage
3166+
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
29523167
coverage xml
29533168
- codecov/upload:
29543169
file: ./coverage.xml
@@ -3370,7 +3585,19 @@ workflows:
33703585
only:
33713586
- main
33723587
- /litellm_.*/
3373-
- litellm_proxy_unit_testing:
3588+
- litellm_proxy_unit_testing_key_generation:
3589+
filters:
3590+
branches:
3591+
only:
3592+
- main
3593+
- /litellm_.*/
3594+
- litellm_proxy_unit_testing_part1:
3595+
filters:
3596+
branches:
3597+
only:
3598+
- main
3599+
- /litellm_.*/
3600+
- litellm_proxy_unit_testing_part2:
33743601
filters:
33753602
branches:
33763603
only:
@@ -3590,7 +3817,9 @@ workflows:
35903817
- litellm_router_testing
35913818
- litellm_router_unit_testing
35923819
- caching_unit_tests
3593-
- litellm_proxy_unit_testing
3820+
- litellm_proxy_unit_testing_key_generation
3821+
- litellm_proxy_unit_testing_part1
3822+
- litellm_proxy_unit_testing_part2
35943823
- litellm_security_tests
35953824
- langfuse_logging_unit_tests
35963825
- local_testing
@@ -3662,7 +3891,9 @@ workflows:
36623891
- auth_ui_unit_tests
36633892
- db_migration_disable_update_check
36643893
- e2e_ui_testing
3665-
- litellm_proxy_unit_testing
3894+
- litellm_proxy_unit_testing_key_generation
3895+
- litellm_proxy_unit_testing_part1
3896+
- litellm_proxy_unit_testing_part2
36663897
- litellm_security_tests
36673898
- installing_litellm_on_python
36683899
- installing_litellm_on_python_3_13

0 commit comments

Comments
 (0)