phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

support batch /chat/completions on proxy

Browse source
This commit is contained in:
Ishaan Jaff 2024-05-11 13:24:25 -07:00
parent 6561e0838e
commit b8c7bbcb9f
1 changed files with 4 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

5
litellm/proxy/proxy_server.py
View file

@ -3639,7 +3639,7 @@ async def chat_completion(
### MODEL ALIAS MAPPING ### ### MODEL ALIAS MAPPING ###
# check if model name in model alias map # check if model name in model alias map
# get the actual model name # get the actual model name
if data["model"] in litellm.model_alias_map: if isinstance(data["model"], str) and data["model"] in litellm.model_alias_map:
data["model"] = litellm.model_alias_map[data["model"]] data["model"] = litellm.model_alias_map[data["model"]]
## LOGGING OBJECT ## - initialize logging object for logging success/failure events for call ## LOGGING OBJECT ## - initialize logging object for logging success/failure events for call
@ -3673,6 +3673,9 @@ async def chat_completion(
# skip router if user passed their key # skip router if user passed their key
if "api_key" in data: if "api_key" in data:
tasks.append(litellm.acompletion(**data)) tasks.append(litellm.acompletion(**data))
elif isinstance(data["model"], list) and llm_router is not None:
_models = data.pop("model")
tasks.append(llm_router.abatch_completion(models=_models, **data))
elif "user_config" in data: elif "user_config" in data:
# initialize a new router instance. make request using this Router # initialize a new router instance. make request using this Router
router_config = data.pop("user_config") router_config = data.pop("user_config")