phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(utils.py): fix the response object returned when n>1 for stream=true

Browse source 
Fixes https://github.com/BerriAI/litellm/issues/3276
This commit is contained in:
Krrish Dholakia 2024-04-25 13:27:29 -07:00
parent 5ad91e1277
commit 6c5c7cca3d
3 changed files with 310 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

16
litellm/utils.py
View file

@ -19,6 +19,7 @@ from functools import wraps
import datetime, time
import tiktoken
import uuid
from pydantic import BaseModel
import aiohttp
import textwrap
import logging
@ -10120,12 +10121,15 @@ class CustomStreamWrapper:
model_response.id = original_chunk.id
self.response_id = original_chunk.id
if len(original_chunk.choices) > 0:
try:
delta = dict(original_chunk.choices[0].delta)
print_verbose(f"original delta: {delta}")
model_response.choices[0].delta = Delta(**delta)
except Exception as e:
model_response.choices[0].delta = Delta()
choices = []
for idx, choice in enumerate(original_chunk.choices):
try:
if isinstance(choice, BaseModel):
choice_json = choice.model_dump()
choices.append(StreamingChoices(**choice_json))
except Exception as e:
choices.append(StreamingChoices())
model_response.choices = choices
else:
return
model_response.system_fingerprint = (