phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(route_llm_request.py): move to using common router, even for clie… (#8966)

Browse source 
* fix(route_llm_request.py): move to using common router, even for client-side credentials

ensures fallbacks / cooldown logic still works

* test(test_route_llm_request.py): add unit test for route request

* feat(router.py): generate unique model id when clientside credential passed in

Prevents cooldowns for api key 1 from impacting api key 2

* test(test_router.py): update testing to ensure original litellm params not mutated

* fix(router.py): upsert clientside call into llm router model list

enables cooldown logic to work accurately

* fix: fix linting error

* test(test_router_utils.py): add direct test for new util on router
This commit is contained in:
Krish Dholakia 2025-03-03 22:57:08 -08:00 • committed by GitHub
parent 94563ab1e7
commit 2fc6262675
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 273 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

22
tests/local_testing/test_router_utils.py
View file

@ -396,3 +396,25 @@ def test_router_redis_cache():
router._update_redis_cache(cache=redis_cache)
assert router.cache.redis_cache == redis_cache
def test_router_handle_clientside_credential():
deployment = {
"model_name": "gemini/*",
"litellm_params": {"model": "gemini/*"},
"model_info": {
"id": "1",
},
}
router = Router(model_list=[deployment])
new_deployment = router._handle_clientside_credential(
deployment=deployment,
kwargs={
"api_key": "123",
"metadata": {"model_group": "gemini/gemini-1.5-flash"},
},
)
assert new_deployment.litellm_params.api_key == "123"
assert len(router.get_model_list()) == 2