phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 03:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(fix) same response_id across chunk

Browse source
This commit is contained in:
ishaan-jaff 2024-01-23 12:56:03 -08:00
parent 6d81da3f43
commit 39b4f19bd8
1 changed files with 5 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

5
litellm/utils.py
View file

@ -7041,6 +7041,7 @@ class CustomStreamWrapper:
self._hidden_params = { self._hidden_params = {
"model_id": (_model_info.get("id", None)) "model_id": (_model_info.get("id", None))
} # returned as x-litellm-model-id response header in proxy } # returned as x-litellm-model-id response header in proxy
self.response_id = None
def __iter__(self): def __iter__(self):
return self return self
@ -7613,6 +7614,10 @@ class CustomStreamWrapper:
def chunk_creator(self, chunk): def chunk_creator(self, chunk):
model_response = ModelResponse(stream=True, model=self.model) model_response = ModelResponse(stream=True, model=self.model)
if self.response_id is not None:
model_response.id = self.response_id
else:
self.response_id = model_response.id
model_response._hidden_params["custom_llm_provider"] = self.custom_llm_provider model_response._hidden_params["custom_llm_provider"] = self.custom_llm_provider
model_response.choices = [StreamingChoices()] model_response.choices = [StreamingChoices()]
model_response.choices[0].finish_reason = None model_response.choices[0].finish_reason = None