From b01f31c41fbe19c16a74fde91bcd837e8f7005ca Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 30 Jan 2024 15:34:38 -0800 Subject: [PATCH 1/2] fix(utils.py): check if delta is none --- litellm/utils.py | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index bbc4e651c..3aaf53514 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2929,32 +2929,10 @@ def cost_per_token( model_with_provider_and_region in model_cost_ref ): # use region based pricing, if it's available model_with_provider = model_with_provider_and_region + if model_with_provider in model_cost_ref: + model = model_with_provider # see this https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models print_verbose(f"Looking up model={model} in model_cost_map") - if model_with_provider in model_cost_ref: - print_verbose( - f"Success: model={model_with_provider} in model_cost_map - {model_cost_ref[model_with_provider]}" - ) - print_verbose( - f"applying cost={model_cost_ref[model_with_provider].get('input_cost_per_token', None)} for prompt_tokens={prompt_tokens}" - ) - prompt_tokens_cost_usd_dollar = ( - model_cost_ref[model_with_provider]["input_cost_per_token"] * prompt_tokens - ) - print_verbose( - f"calculated prompt_tokens_cost_usd_dollar: {prompt_tokens_cost_usd_dollar}" - ) - print_verbose( - f"applying cost={model_cost_ref[model_with_provider].get('output_cost_per_token', None)} for completion_tokens={completion_tokens}" - ) - completion_tokens_cost_usd_dollar = ( - model_cost_ref[model_with_provider]["output_cost_per_token"] - * completion_tokens - ) - print_verbose( - f"calculated completion_tokens_cost_usd_dollar: {completion_tokens_cost_usd_dollar}" - ) - return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar if model in model_cost_ref: print_verbose(f"Success: model={model} in model_cost_map") print_verbose( From 60cc3d6b309c3059474c6bd1483a0399bc1422bf Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 30 Jan 2024 15:35:23 -0800 Subject: [PATCH 2/2] fix(utils.py): fix streaming delta content being none edge-case --- litellm/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 3aaf53514..8f00c115d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7487,7 +7487,10 @@ class CustomStreamWrapper: logprobs = None original_chunk = None # this is used for function/tool calling if len(str_line.choices) > 0: - if str_line.choices[0].delta.content is not None: + if ( + str_line.choices[0].delta is not None + and str_line.choices[0].delta.content is not None + ): text = str_line.choices[0].delta.content else: # function/tool calling chunk - when content is None. in this case we just return the original chunk from openai original_chunk = str_line