phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

Added function_call support to stream_chunk_builder

Browse source
This commit is contained in:
WilliamEspegren 2023-09-17 19:42:55 +02:00
parent 404af1be0f
commit 63a8614a14
No known key found for this signature in database
GPG key ID: 30E0CDDF1B5262CD
2 changed files with 34 additions and 261 deletions
Download patch file Download diff file Expand all files Collapse all files

46
litellm/main.py
View file

@ -1159,21 +1159,43 @@ def stream_chunk_builder(chunks: list):
content_list = []
if "function_call" in chunks[0]["choices"][0]["delta"]:
function_call_name = chunks[0]["choices"][0]["delta"]["function_call"]["name"]
print(function_call_name)
argument_list = []
delta = chunks[0]["choices"][0]["delta"]
function_call = delta.get("function_call", "")
function_call_name = function_call.get("name", "")
for chunk in chunks:
choices = chunk["choices"]
for choice in choices:
delta = choice.get("delta", {})
content = delta.get("content", "")
content_list.append(content)
message = response["choices"][0]["message"]
message["function_call"] = {}
message["function_call"]["name"] = function_call_name
# Combine the "content" strings into a single string
combined_content = "".join(content_list)
for chunk in chunks:
choices = chunk["choices"]
for choice in choices:
delta = choice.get("delta", {})
function_call = delta.get("function_call", "")
# Check if a function call is present
if function_call:
# Now, function_call is expected to be a dictionary
arguments = function_call.get("arguments", "")
argument_list.append(arguments)
# Update the "content" field within the response dictionary
response["choices"][0]["message"]["content"] = combined_content
combined_arguments = "".join(argument_list)
response["choices"][0]["message"]["content"] = None
response["choices"][0]["message"]["function_call"]["arguments"] = combined_arguments
else:
for chunk in chunks:
choices = chunk["choices"]
for choice in choices:
delta = choice.get("delta", {})
content = delta.get("content", "")
content_list.append(content)
# Combine the "content" strings into a single string
combined_content = "".join(content_list)
# Update the "content" field within the response dictionary
response["choices"][0]["message"]["content"] = combined_content
# # Update usage information if needed

249
litellm/tests/test_response.json
View file

@ -1,249 +0,0 @@
OpenAI non stream response:
{
"id": "chatcmpl-7zoXylnQH7IVUQXpbE5AHZWxQhjCW",
"object": "chat.completion",
"created": 1694966666,
"model": "gpt-3.5-turbo-0613",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": null,
"function_call": {
"name": "get_current_weather",
"arguments": "{\n\"location\": \"Boston\"\n}"
}
},
"finish_reason": "function_call"
}
],
"usage": {
"prompt_tokens": 82,
"completion_tokens": 15,
"total_tokens": 97
}
}
OpenAI stream response:
{
"id": "chatcmpl-7zoaRRfxJErRzGNgfrqXrTHRexvG7",
"object": "chat.completion.chunk",
"created": 1694966819,
"model": "gpt-3.5-turbo-0613",
"choices": [
{
"index": 0,
"delta": {
"role": "assistant",
"content": null,
"function_call": {
"name": "get_current_weather",
"arguments": ""
}
},
"finish_reason": null
}
]
}
{
"id": "chatcmpl-7zoaRRfxJErRzGNgfrqXrTHRexvG7",
"object": "chat.completion.chunk",
"created": 1694966819,
"model": "gpt-3.5-turbo-0613",
"choices": [
{
"index": 0,
"delta": {
"function_call": {
"arguments": "{\n"
}
},
"finish_reason": null
}
]
}
{
"id": "chatcmpl-7zoaRRfxJErRzGNgfrqXrTHRexvG7",
"object": "chat.completion.chunk",
"created": 1694966819,
"model": "gpt-3.5-turbo-0613",
"choices": [
{
"index": 0,
"delta": {
"function_call": {
"arguments": " "
}
},
"finish_reason": null
}
]
}
{
"id": "chatcmpl-7zoaRRfxJErRzGNgfrqXrTHRexvG7",
"object": "chat.completion.chunk",
"created": 1694966819,
"model": "gpt-3.5-turbo-0613",
"choices": [
{
"index": 0,
"delta": {
"function_call": {
"arguments": " \""
}
},
"finish_reason": null
}
]
}
{
"id": "chatcmpl-7zoaRRfxJErRzGNgfrqXrTHRexvG7",
"object": "chat.completion.chunk",
"created": 1694966819,
"model": "gpt-3.5-turbo-0613",
"choices": [
{
"index": 0,
"delta": {
"function_call": {
"arguments": "location"
}
},
"finish_reason": null
}
]
}
{
"id": "chatcmpl-7zoaRRfxJErRzGNgfrqXrTHRexvG7",
"object": "chat.completion.chunk",
"created": 1694966819,
"model": "gpt-3.5-turbo-0613",
"choices": [
{
"index": 0,
"delta": {
"function_call": {
"arguments": "\":"
}
},
"finish_reason": null
}
]
}
{
"id": "chatcmpl-7zoaRRfxJErRzGNgfrqXrTHRexvG7",
"object": "chat.completion.chunk",
"created": 1694966819,
"model": "gpt-3.5-turbo-0613",
"choices": [
{
"index": 0,
"delta": {
"function_call": {
"arguments": " \""
}
},
"finish_reason": null
}
]
}
{
"id": "chatcmpl-7zoaRRfxJErRzGNgfrqXrTHRexvG7",
"object": "chat.completion.chunk",
"created": 1694966819,
"model": "gpt-3.5-turbo-0613",
"choices": [
{
"index": 0,
"delta": {
"function_call": {
"arguments": "Boston"
}
},
"finish_reason": null
}
]
}
{
"id": "chatcmpl-7zoaRRfxJErRzGNgfrqXrTHRexvG7",
"object": "chat.completion.chunk",
"created": 1694966819,
"model": "gpt-3.5-turbo-0613",
"choices": [
{
"index": 0,
"delta": {
"function_call": {
"arguments": ","
}
},
"finish_reason": null
}
]
}
{
"id": "chatcmpl-7zoaRRfxJErRzGNgfrqXrTHRexvG7",
"object": "chat.completion.chunk",
"created": 1694966819,
"model": "gpt-3.5-turbo-0613",
"choices": [
{
"index": 0,
"delta": {
"function_call": {
"arguments": " MA"
}
},
"finish_reason": null
}
]
}
{
"id": "chatcmpl-7zoaRRfxJErRzGNgfrqXrTHRexvG7",
"object": "chat.completion.chunk",
"created": 1694966819,
"model": "gpt-3.5-turbo-0613",
"choices": [
{
"index": 0,
"delta": {
"function_call": {
"arguments": "\"\n"
}
},
"finish_reason": null
}
]
}
{
"id": "chatcmpl-7zoaRRfxJErRzGNgfrqXrTHRexvG7",
"object": "chat.completion.chunk",
"created": 1694966819,
"model": "gpt-3.5-turbo-0613",
"choices": [
{
"index": 0,
"delta": {
"function_call": {
"arguments": "}"
}
},
"finish_reason": null
}
]
}
{
"id": "chatcmpl-7zoaRRfxJErRzGNgfrqXrTHRexvG7",
"object": "chat.completion.chunk",
"created": 1694966819,
"model": "gpt-3.5-turbo-0613",
"choices": [
{
"index": 0,
"delta": {},
"finish_reason": "function_call"
}
]
}