phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(vertex_ai.py): fix streaming logic

Browse source
This commit is contained in:
Krrish Dholakia 2024-04-22 19:15:20 -07:00
parent 0bb8a4434e
commit ec2c70e362
2 changed files with 8 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

11
litellm/llms/vertex_ai.py
View file

@ -560,7 +560,7 @@ def completion(
stream=True, stream=True,
tools=tools, tools=tools,
) )
optional_params["stream"] = True
return model_response return model_response
request_str += f"response = llm_model.generate_content({content})\n" request_str += f"response = llm_model.generate_content({content})\n"
@ -632,7 +632,7 @@ def completion(
}, },
) )
model_response = chat.send_message_streaming(prompt, **optional_params) model_response = chat.send_message_streaming(prompt, **optional_params)
optional_params["stream"] = True
return model_response return model_response
request_str += f"chat.send_message({prompt}, **{optional_params}).text\n" request_str += f"chat.send_message({prompt}, **{optional_params}).text\n"
@ -664,7 +664,7 @@ def completion(
}, },
) )
model_response = llm_model.predict_streaming(prompt, **optional_params) model_response = llm_model.predict_streaming(prompt, **optional_params)
optional_params["stream"] = True
return model_response return model_response
request_str += f"llm_model.predict({prompt}, **{optional_params}).text\n" request_str += f"llm_model.predict({prompt}, **{optional_params}).text\n"
@ -1045,8 +1045,7 @@ async def async_streaming(
generation_config=optional_params, generation_config=optional_params,
tools=tools, tools=tools,
) )
optional_params["stream"] = True
optional_params["tools"] = tools
elif mode == "chat": elif mode == "chat":
chat = llm_model.start_chat() chat = llm_model.start_chat()
optional_params.pop( optional_params.pop(
@ -1065,7 +1064,7 @@ async def async_streaming(
}, },
) )
response = chat.send_message_streaming_async(prompt, **optional_params) response = chat.send_message_streaming_async(prompt, **optional_params)
optional_params["stream"] = True
elif mode == "text": elif mode == "text":
optional_params.pop( optional_params.pop(
"stream", None "stream", None

5
litellm/main.py
View file

@ -1683,13 +1683,14 @@ def completion(
or optional_params.pop("vertex_ai_credentials", None) or optional_params.pop("vertex_ai_credentials", None)
or get_secret("VERTEXAI_CREDENTIALS") or get_secret("VERTEXAI_CREDENTIALS")
) )
new_params = deepcopy(optional_params)
if "claude-3" in model: if "claude-3" in model:
model_response = vertex_ai_anthropic.completion( model_response = vertex_ai_anthropic.completion(
model=model, model=model,
messages=messages, messages=messages,
model_response=model_response, model_response=model_response,
print_verbose=print_verbose, print_verbose=print_verbose,
optional_params=optional_params, optional_params=new_params,
litellm_params=litellm_params, litellm_params=litellm_params,
logger_fn=logger_fn, logger_fn=logger_fn,
encoding=encoding, encoding=encoding,
@ -1705,7 +1706,7 @@ def completion(
messages=messages, messages=messages,
model_response=model_response, model_response=model_response,
print_verbose=print_verbose, print_verbose=print_verbose,
optional_params=optional_params, optional_params=new_params,
litellm_params=litellm_params, litellm_params=litellm_params,
logger_fn=logger_fn, logger_fn=logger_fn,
encoding=encoding, encoding=encoding,