From 00b55e43895fbf60969d7d65af91b5564e8d23fe Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 29 Nov 2023 15:19:11 -0800 Subject: [PATCH] fix(main.py): have stream_chunk_builder return successful response even if token_counter fails --- litellm/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 99a3ccf3f9..6625728d8d 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2209,8 +2209,10 @@ def stream_chunk_builder(chunks: list, messages: Optional[list]=None): elif len(combined_arguments) > 0: completion_output = combined_arguments # # Update usage information if needed - if messages: + try: response["usage"]["prompt_tokens"] = token_counter(model=model, messages=messages) + except: # don't allow this failing to block a complete streaming response from being returned + response["usage"]["prompt_tokens"] = 0 response["usage"]["completion_tokens"] = token_counter(model=model, text=completion_output) response["usage"]["total_tokens"] = response["usage"]["prompt_tokens"] + response["usage"]["completion_tokens"] return convert_to_model_response_object(response_object=response, model_response_object=litellm.ModelResponse())