phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 03:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(utils.py): support openrouter streaming

Browse source 
Fixes https://github.com/BerriAI/litellm/issues/5080
This commit is contained in:
Krrish Dholakia 2024-08-21 08:46:55 -07:00
parent dc261dd07a
commit 1b6db8359a
2 changed files with 14 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

71
litellm/tests/test_streaming.py
View file

@ -645,7 +645,16 @@ def test_completion_ollama_hosted_stream():
# test_completion_ollama_hosted_stream()
def test_completion_claude_stream():
@pytest.mark.parametrize(
"model",
[
# "claude-instant-1.2",
# "claude-2",
# "mistral/mistral-medium",
"openrouter/openai/gpt-4o-mini",
],
)
def test_completion_model_stream(model):
try:
messages = [
{"role": "system", "content": "You are a helpful assistant."},
@ -655,7 +664,7 @@ def test_completion_claude_stream():
},
]
response = completion(
model="claude-instant-1.2", messages=messages, stream=True, max_tokens=50
model=model, messages=messages, stream=True, max_tokens=50
)
complete_response = ""
# Add any assertions here to check the response
@ -671,30 +680,6 @@ def test_completion_claude_stream():
pytest.fail(f"Error occurred: {e}")
# test_completion_claude_stream()
def test_completion_claude_2_stream():
litellm.set_verbose = True
response = completion(
model="claude-2",
messages=[{"role": "user", "content": "hello from litellm"}],
stream=True,
)
complete_response = ""
# Add any assertions here to check the response
idx = 0
for chunk in response:
print(chunk)
# print(chunk.choices[0].delta)
chunk, finished = streaming_format_tests(idx, chunk)
if finished:
break
complete_response += chunk
idx += 1
if complete_response.strip() == "":
raise Exception("Empty response received")
print(f"completion_response: {complete_response}")
@pytest.mark.asyncio
async def test_acompletion_claude_2_stream():
try:
@ -832,40 +817,6 @@ async def test_completion_gemini_stream(sync_mode):
# asyncio.run(test_acompletion_gemini_stream())
def test_completion_mistral_api_stream():
try:
litellm.set_verbose = True
print("Testing streaming mistral api response")
response = completion(
model="mistral/mistral-medium",
messages=[
{
"role": "user",
"content": "Hey, how's it going?",
}
],
max_tokens=10,
stream=True,
)
complete_response = ""
has_finish_reason = False
for idx, chunk in enumerate(response):
chunk, finished = streaming_format_tests(idx, chunk)
if finished:
has_finish_reason = True
break
complete_response += chunk
if has_finish_reason == False:
raise Exception("finish reason not set")
if complete_response.strip() == "":
raise Exception("Empty response received")
print(f"completion_response: {complete_response}")
except litellm.APIError as e:
pass
except Exception as e:
pytest.fail(f"Error occurred: {e}")
def test_completion_mistral_api_mistral_large_function_call_with_streaming():
litellm.set_verbose = True
tools = [

6
litellm/utils.py
View file

@ -2896,7 +2896,7 @@ def get_optional_params(
unsupported_params = {}
for k in non_default_params.keys():
if k not in supported_params:
if k == "user" or k == "stream_options":
if k == "user" or k == "stream_options" or k == "stream":
continue
if k == "n" and n == 1: # langchain sends n=1 as a default value
continue # skip this param
@ -2908,8 +2908,8 @@ def get_optional_params(
else:
unsupported_params[k] = non_default_params[k]
if unsupported_params:
if litellm.drop_params == True or (
drop_params is not None and drop_params == True
if litellm.drop_params is True or (
drop_params is not None and drop_params is True
):
pass
else: