phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 19:24:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add fake streaming for petals

Browse source
This commit is contained in:
ishaan-jaff 2023-09-30 10:22:04 -07:00
parent 1680a5fad8
commit 48cad985ad
2 changed files with 16 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

7
litellm/main.py
View file

@ -1099,10 +1099,11 @@ def completion(
encoding=encoding,
logging_obj=logging
)
if inspect.isgenerator(model_response) or (stream == True):
# don't try to access stream object,
if stream==True: ## [BETA]
# Fake streaming for petals
resp_string = model_response["choices"][0]["message"]["content"]
response = CustomStreamWrapper(
model_response, model, custom_llm_provider="petals", logging_obj=logging
resp_string, model, custom_llm_provider="petals", logging_obj=logging
)
return response
response = model_response