build(config.yml): add codecov to repo (#6172)

* build(config.yml): add codecov to repo

ensures all commits have testing coverage

* build(config.yml): fix ci config

* build: fix .yml

* build(config.yml): fix ci/cd

* ci(config.yml): specify module to measure code coverage for

* ci(config.yml): update config.yml version

* ci: trigger new run

* ci(config.yml): store combine

* build(config.yml): check files before combine

* ci(config.yml): fix check

* ci(config.yml): add codecov coverage to ci/cd

* ci(config.yml): add codecov to router tests

* ci(config.yml): wait for router testing to complete before running codecov upload

* ci(config.yml): handle multiple coverage.xml's

* fix(router.py): cleanup print stack

* ci(config.yml): fix config

* ci(config.yml): fix config

.circleci/config.yml

@@ -1,4 +1,7 @@
-version: 4.3.4
+version: 2.1
+orbs:
+  codecov: codecov/codecov@4.0.1
+
 jobs:
   local_testing:
     docker:
@@ -27,6 +30,7 @@ jobs:
             pip install "pytest==7.3.1"
             pip install "pytest-retry==1.6.3"
             pip install "pytest-asyncio==0.21.1"
+            pip install "pytest-cov==5.0.0"
             pip install mypy
             pip install "google-generativeai==0.3.2"
             pip install "google-cloud-aiplatform==1.43.0"
@@ -99,7 +103,7 @@ jobs:
           command: |
             pwd
             ls
-            python -m pytest -vv tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not router and not assistants"
+            python -m pytest -vv tests/local_testing --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not router and not assistants"
           no_output_timeout: 120m
 
       # Store test results
@@ -163,6 +167,7 @@ jobs:
             python -m pip install -r requirements.txt
             pip install "pytest==7.3.1"
             pip install "respx==0.21.1"
+            pip install "pytest-cov==5.0.0"
             pip install "pytest-retry==1.6.3"
             pip install "pytest-asyncio==0.21.1"
       # Run pytest and generate JUnit XML report
@@ -171,11 +176,22 @@ jobs:
           command: |
             pwd
             ls
-            python -m pytest tests/local_testing -vv -k "router" -x -s -v --junitxml=test-results/junit.xml --durations=5
+            python -m pytest tests/local_testing/test_router.py --cov=litellm --cov-report=xml -vv -k "router" -x -s -v --junitxml=test-results/junit.xml --durations=5
           no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml litellm_router_coverage.xml
+            mv .coverage litellm_router_coverage
       # Store test results
       - store_test_results:
           path: test-results
+
+      - persist_to_workspace:
+          root: .
+          paths:
+            - litellm_router_coverage.xml
+            - litellm_router_coverage
   litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
     docker:
         - image: cimg/python:3.11
@@ -253,6 +269,7 @@ jobs:
             python -m pip install -r requirements.txt
             pip install "pytest==7.3.1"
             pip install "pytest-retry==1.6.3"
+            pip install "pytest-cov==5.0.0"
             pip install "pytest-asyncio==0.21.1"
             pip install "respx==0.21.1"
       # Run pytest and generate JUnit XML report
@@ -261,13 +278,22 @@ jobs:
           command: |
             pwd
             ls
-            python -m pytest -vv tests/llm_translation -x -s -v --junitxml=test-results/junit.xml --durations=5
+            python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
           no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml llm_translation_coverage.xml
+            mv .coverage llm_translation_coverage
         
       # Store test results
       - store_test_results:
           path: test-results
-
+      - persist_to_workspace:
+          root: .
+          paths:
+            - llm_translation_coverage.xml
+            - llm_translation_coverage
   installing_litellm_on_python:
     docker:
       - image: circleci/python:3.8
@@ -678,6 +704,33 @@ jobs:
       - store_test_results:
           path: test-results
 
+  upload-coverage:
+    docker:
+      - image: cimg/python:3.9
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      # Check file locations
+      - run:
+          name: Check coverage file location
+          command: |
+            echo "Current directory:"
+            ls -la
+            echo "\nContents of tests/llm_translation:"
+            ls -la tests/llm_translation
+      - run:
+          name: Combine Coverage
+          command: |
+            python -m venv venv
+            . venv/bin/activate
+            pip install coverage
+            coverage combine llm_translation_coverage litellm_router_coverage
+            coverage xml
+      - codecov/upload:
+          file: ./coverage.xml
+
+
   publish_to_pypi:
     docker:
       - image: cimg/python:3.8
@@ -908,6 +961,10 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - upload-coverage:
+          requires:
+            - llm_translation_testing
+            - litellm_router_testing
       - db_migration_disable_update_check:
           filters:
             branches:
@@ -945,3 +1002,4 @@ workflows:
             branches:
               only:
                 - main
+      

litellm/router.py

@@ -616,7 +616,6 @@ class Router:
         self, model: str, messages: List[Dict[str, str]], **kwargs
     ) -> Union[ModelResponse, CustomStreamWrapper]:
         model_name = None
-        traceback.print_stack()
         try:
             # pick the one that is available (lowest TPM/RPM)
             deployment = self.get_available_deployment(

tests/local_testing/test_completion.py

@@ -4523,7 +4523,6 @@ async def test_dynamic_azure_params(stream, sync_mode):
         try:
             mock_client.assert_not_called()
         except Exception as e:
-            traceback.print_stack()
             raise e
 
 

tests/local_testing/test_custom_logger.py

@@ -41,7 +41,6 @@ class MyCustomHandler(CustomLogger):
 
     def log_pre_api_call(self, model, messages, kwargs):
         print("Pre-API Call")
-        traceback.print_stack()
         self.data_sent_to_api = kwargs["additional_args"].get("complete_input_dict", {})
 
     def log_post_api_call(self, kwargs, response_obj, start_time, end_time):