phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

Litellm dev 10 26 2024 (#6472)

Browse source 
* docs(exception_mapping.md): add missing exception types

Fixes https://github.com/Aider-AI/aider/issues/2120#issuecomment-2438971183

* fix(main.py): register custom model pricing with specific key

Ensure custom model pricing is registered to the specific model+provider key combination

* test: make testing more robust for custom pricing

* fix(redis_cache.py): instrument otel logging for sync redis calls

ensures complete coverage for all redis cache calls
This commit is contained in:
Krish Dholakia 2024-10-28 15:05:43 -07:00 • committed by GitHub
parent f44ab00de2
commit 70111a7abd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 310 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

61
tests/local_testing/test_router_fallbacks.py
View file

@ -1337,3 +1337,64 @@ async def test_anthropic_streaming_fallbacks(sync_mode):
mock_client.assert_called_once()
print(chunks)
assert len(chunks) > 0
def test_router_fallbacks_with_custom_model_costs():
"""
Tests prod use-case where a custom model is registered with a different provider + custom costs.
Goal: make sure custom model doesn't override default model costs.
"""
model_list = [
{
"model_name": "claude-3-5-sonnet-20240620",
"litellm_params": {
"model": "claude-3-5-sonnet-20240620",
"api_key": os.environ["ANTHROPIC_API_KEY"],
"input_cost_per_token": 30,
"output_cost_per_token": 60,
},
},
{
"model_name": "claude-3-5-sonnet-aihubmix",
"litellm_params": {
"model": "openai/claude-3-5-sonnet-20240620",
"input_cost_per_token": 0.000003, # 3$/M
"output_cost_per_token": 0.000015, # 15$/M
"api_base": "https://exampleopenaiendpoint-production.up.railway.app",
"api_key": "my-fake-key",
},
},
]
router = Router(
model_list=model_list,
fallbacks=[{"claude-3-5-sonnet-20240620": ["claude-3-5-sonnet-aihubmix"]}],
)
router.completion(
model="claude-3-5-sonnet-aihubmix",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
)
model_info = litellm.get_model_info(model="claude-3-5-sonnet-20240620")
print(f"key: {model_info['key']}")
assert model_info["litellm_provider"] == "anthropic"
response = router.completion(
model="claude-3-5-sonnet-20240620",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
)
print(f"response_cost: {response._hidden_params['response_cost']}")
assert response._hidden_params["response_cost"] > 10
model_info = litellm.get_model_info(model="claude-3-5-sonnet-20240620")
print(f"key: {model_info['key']}")
assert model_info["input_cost_per_token"] == 30
assert model_info["output_cost_per_token"] == 60