phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(utils.py): include system fingerprint in streaming response object

Browse source
This commit is contained in:
Krrish Dholakia 2023-11-30 08:45:35 -08:00
parent 78421d89aa
commit 01c7e18f31
3 changed files with 7 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
litellm/main.py
View file

@ -2143,6 +2143,7 @@ def stream_chunk_builder(chunks: list, messages: Optional[list]=None):
object = chunks[0]["object"] object = chunks[0]["object"]
created = chunks[0]["created"] created = chunks[0]["created"]
model = chunks[0]["model"] model = chunks[0]["model"]
system_fingerprint = chunks[0].get("system_fingerprint", None)
role = chunks[0]["choices"][0]["delta"]["role"] role = chunks[0]["choices"][0]["delta"]["role"]
finish_reason = chunks[-1]["choices"][0]["finish_reason"] finish_reason = chunks[-1]["choices"][0]["finish_reason"]
@ -2152,6 +2153,7 @@ def stream_chunk_builder(chunks: list, messages: Optional[list]=None):
"object": object, "object": object,
"created": created, "created": created,
"model": model, "model": model,
"system_fingerprint": system_fingerprint,
"choices": [ "choices": [
{ {
"index": 0, "index": 0,

6
litellm/tests/test_stream_chunk_builder.py
View file

@ -95,16 +95,18 @@ def test_stream_chunk_builder_litellm_tool_call():
try: try:
litellm.set_verbose = False litellm.set_verbose = False
response = litellm.completion( response = litellm.completion(
model="azure/chatgpt-functioncalling", model="azure/gpt-4-nov-release",
messages=messages, messages=messages,
tools=tools_schema, tools=tools_schema,
stream=True, stream=True,
api_key="os.environ/AZURE_FRANCE_API_KEY",
api_base="https://openai-france-1234.openai.azure.com",
complete_response = True complete_response = True
) )
print(f"complete response: {response}") print(f"complete response: {response}")
print(f"complete response usage: {response.usage}") print(f"complete response usage: {response.usage}")
assert response.system_fingerprint is not None
except Exception as e: except Exception as e:
pytest.fail(f"An exception occurred - {str(e)}") pytest.fail(f"An exception occurred - {str(e)}")

2
litellm/utils.py
View file

@ -1796,7 +1796,6 @@ def register_model(model_cost: Union[str, dict]):
if key in litellm.model_cost: if key in litellm.model_cost:
for k,v in loaded_model_cost[key].items(): for k,v in loaded_model_cost[key].items():
litellm.model_cost[key][k] = v litellm.model_cost[key][k] = v
# litellm.model_cost[key] = loaded_model_cost[key]
# add new model names to provider lists # add new model names to provider lists
if value.get('litellm_provider') == 'openai': if value.get('litellm_provider') == 'openai':
if key not in litellm.open_ai_chat_completion_models: if key not in litellm.open_ai_chat_completion_models:
@ -5291,6 +5290,7 @@ class CustomStreamWrapper:
if self.sent_first_chunk == False: if self.sent_first_chunk == False:
model_response.choices[0].delta["role"] = "assistant" model_response.choices[0].delta["role"] = "assistant"
self.sent_first_chunk = True self.sent_first_chunk = True
# LOGGING
threading.Thread(target=self.logging_obj.success_handler, args=(model_response,)).start() # log response threading.Thread(target=self.logging_obj.success_handler, args=(model_response,)).start() # log response
return model_response return model_response
else: else: