phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 19:24:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

palm streaming

Browse source
This commit is contained in:
ishaan-jaff 2023-09-26 12:27:49 -07:00
parent 0d47663b8b
commit d1692e89dd
2 changed files with 24 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

9
litellm/main.py
View file

@ -800,6 +800,7 @@ def completion(
or litellm.api_key
)
# palm does not support streaming as yet :(
model_response = palm.completion(
model=model,
messages=messages,
@ -812,10 +813,12 @@ def completion(
api_key=palm_api_key,
logging_obj=logging
)
if "stream_tokens" in optional_params and optional_params["stream_tokens"] == True:
# don't try to access stream object,
# fake palm streaming
if stream == True:
# fake streaming for palm
resp_string = model_response["choices"][0]["message"]["content"]
response = CustomStreamWrapper(
model_response, model, custom_llm_provider="palm", logging_obj=logging
resp_string, model, custom_llm_provider="palm", logging_obj=logging
)
return response
response = model_response