phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 03:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(main.py): fix order of assembly for streaming chunks

Browse source
This commit is contained in:
Krrish Dholakia 2024-01-25 14:51:08 -08:00
parent 09ec6d6458
commit bbe6a92eb9
3 changed files with 8 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

6
litellm/main.py
View file

@ -3343,6 +3343,12 @@ def stream_chunk_builder(
chunks: list, messages: Optional[list] = None, start_time=None, end_time=None
):
model_response = litellm.ModelResponse()
### SORT CHUNKS BASED ON CREATED ORDER ##
if chunks[0]._hidden_params.get("created_at", None):
# Sort chunks based on created_at in ascending order
chunks = sorted(
chunks, key=lambda x: x._hidden_params.get("created_at", float("inf"))
)
# set hidden params from chunk to model_response
if model_response is not None and hasattr(model_response, "_hidden_params"):
model_response._hidden_params = chunks[0].get("_hidden_params", {})

2
litellm/tests/test_custom_logger.py
View file

@ -211,7 +211,7 @@ def test_azure_completion_stream():
{"role": "system", "content": "You are a helpful assistant."},
{
"role": "user",
"content": "write 1 sentence about litellm being amazing",
"content": f"write 1 sentence about litellm being amazing {time.time()}",
},
]
complete_streaming_response = ""

1
litellm/utils.py
View file

@ -7714,6 +7714,7 @@ class CustomStreamWrapper:
else:
self.response_id = model_response.id
model_response._hidden_params["custom_llm_provider"] = self.custom_llm_provider
model_response._hidden_params["created_at"] = time.time()
model_response.choices = [StreamingChoices()]
model_response.choices[0].finish_reason = None
response_obj = {}