phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

feat: global client for sync + async calls (openai + Azure only)

Browse source
This commit is contained in:
Krrish Dholakia 2023-11-16 14:44:06 -08:00
parent 5fd4376802
commit 51bf637656
4 changed files with 22 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

10
litellm/utils.py
View file

@ -1054,6 +1054,16 @@ def client(original_function):
try:
global callback_list, add_breadcrumb, user_logger_fn, Logging
function_id = kwargs["id"] if "id" in kwargs else None
if litellm.client_session is None:
litellm.client_session = httpx.Client(
limits=httpx.Limits(max_connections=100, max_keepalive_connections=20),
timeout = httpx.Timeout(timeout=600.0, connect=5.0)
)
if litellm.aclient_session is None:
litellm.aclient_session = httpx.AsyncClient(
limits=httpx.Limits(max_connections=100, max_keepalive_connections=20),
timeout = httpx.Timeout(timeout=600.0, connect=5.0)
)
if litellm.use_client or ("use_client" in kwargs and kwargs["use_client"] == True):
print_verbose(f"litedebugger initialized")
if "lite_debugger" not in litellm.input_callback: