phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

Merge pull request #3534 from BerriAI/litellm_fix_cost_calc_bedrock

Browse source 
[Fix] `litellm.completion_cost(model="bedrock/anthropic.claude-instant-v1"..)`
This commit is contained in:
Ishaan Jaff 2024-05-08 16:59:46 -07:00 • committed by GitHub
commit 41a4a06389
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 29 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

9
litellm/tests/test_completion_cost.py
View file

@ -231,14 +231,17 @@ def test_cost_bedrock_pricing():
assert cost == predicted_cost assert cost == predicted_cost
@pytest.mark.skip(reason="AWS disabled our access")
def test_cost_bedrock_pricing_actual_calls(): def test_cost_bedrock_pricing_actual_calls():
litellm.set_verbose = True litellm.set_verbose = True
model = "anthropic.claude-instant-v1" model = "anthropic.claude-instant-v1"
messages = [{"role": "user", "content": "Hey, how's it going?"}] messages = [{"role": "user", "content": "Hey, how's it going?"}]
response = litellm.completion(model=model, messages=messages) response = litellm.completion(
assert response._hidden_params["region_name"] is not None model=model, messages=messages, mock_response="hello cool one"
)
print("response", response)
cost = litellm.completion_cost( cost = litellm.completion_cost(
model="bedrock/anthropic.claude-instant-v1",
completion_response=response, completion_response=response,
messages=[{"role": "user", "content": "Hey, how's it going?"}], messages=[{"role": "user", "content": "Hey, how's it going?"}],
) )

24
litellm/utils.py
View file

@ -4161,8 +4161,30 @@ def cost_per_token(
model_with_provider_and_region in model_cost_ref model_with_provider_and_region in model_cost_ref
): # use region based pricing, if it's available ): # use region based pricing, if it's available
model_with_provider = model_with_provider_and_region model_with_provider = model_with_provider_and_region
if model_with_provider in model_cost_ref:
model_without_prefix = model
model_parts = model.split("/")
if len(model_parts) > 1:
model_without_prefix = model_parts[1]
else:
model_without_prefix = model
"""
Code block that formats model to lookup in litellm.model_cost
Option1. model = "bedrock/ap-northeast-1/anthropic.claude-instant-v1". This is the most accurate since it is region based. Should always be option 1
Option2. model = "openai/gpt-4" - model = provider/model
Option3. model = "anthropic.claude-3" - model = model
"""
if (
model_with_provider in model_cost_ref
): # Option 2. use model with provider, model = "openai/gpt-4"
model = model_with_provider model = model_with_provider
elif model in model_cost_ref: # Option 1. use model passed, model="gpt-4"
model = model
elif (
model_without_prefix in model_cost_ref
): # Option 3. if user passed model="bedrock/anthropic.claude-3", use model="anthropic.claude-3"
model = model_without_prefix
# see this https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models # see this https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models
print_verbose(f"Looking up model={model} in model_cost_map") print_verbose(f"Looking up model={model} in model_cost_map")
if model in model_cost_ref: if model in model_cost_ref: