phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix - add better debugging on num_callbacks test

Browse source
This commit is contained in:
Ishaan Jaff 2024-05-06 13:42:20 -07:00
parent 863f9c60a2
commit 562ef2d2e1
2 changed files with 71 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

57
litellm/proxy/proxy_server.py
View file

@ -9215,24 +9215,24 @@ async def active_callbacks():
"""
global proxy_logging_obj
_alerting = str(general_settings.get("alerting"))
# get success callback
success_callback_names = []
try:
# this was returning a JSON of the values in some of the callbacks
# all we need is the callback name, hence we do str(callback)
success_callback_names = [str(x) for x in litellm.success_callback]
except:
# don't let this block the /health/readiness response, if we can't convert to str -> return litellm.success_callback
success_callback_names = litellm.success_callback
# get success callbacks
_num_callbacks = (
len(litellm.callbacks)
+ len(litellm.input_callback)
+ len(litellm.failure_callback)
+ len(litellm.success_callback)
+ len(litellm._async_failure_callback)
+ len(litellm._async_success_callback)
+ len(litellm._async_input_callback)
litellm_callbacks = [str(x) for x in litellm.callbacks]
litellm_input_callbacks = [str(x) for x in litellm.input_callback]
litellm_failure_callbacks = [str(x) for x in litellm.failure_callback]
litellm_success_callbacks = [str(x) for x in litellm.success_callback]
litellm_async_success_callbacks = [str(x) for x in litellm._async_success_callback]
litellm_async_failure_callbacks = [str(x) for x in litellm._async_failure_callback]
litellm_async_input_callbacks = [str(x) for x in litellm._async_input_callback]
all_litellm_callbacks = (
litellm_callbacks
+ litellm_input_callbacks
+ litellm_failure_callbacks
+ litellm_success_callbacks
+ litellm_async_success_callbacks
+ litellm_async_failure_callbacks
+ litellm_async_input_callbacks
)
alerting = proxy_logging_obj.alerting
@ -9242,20 +9242,15 @@ async def active_callbacks():
return {
"alerting": _alerting,
"litellm.callbacks": [str(x) for x in litellm.callbacks],
"litellm.input_callback": [str(x) for x in litellm.input_callback],
"litellm.failure_callback": [str(x) for x in litellm.failure_callback],
"litellm.success_callback": [str(x) for x in litellm.success_callback],
"litellm._async_success_callback": [
str(x) for x in litellm._async_success_callback
],
"litellm._async_failure_callback": [
str(x) for x in litellm._async_failure_callback
],
"litellm._async_input_callback": [
str(x) for x in litellm._async_input_callback
],
"num_callbacks": _num_callbacks,
"litellm.callbacks": litellm_callbacks,
"litellm.input_callback": litellm_input_callbacks,
"litellm.failure_callback": litellm_failure_callbacks,
"litellm.success_callback": litellm_success_callbacks,
"litellm._async_success_callback": litellm_async_success_callbacks,
"litellm._async_failure_callback": litellm_async_failure_callbacks,
"litellm._async_input_callback": litellm_async_input_callbacks,
"all_litellm_callbacks": all_litellm_callbacks,
"num_callbacks": len(all_litellm_callbacks),
"num_alerting": _num_alerting,
}

54
tests/test_callbacks_on_proxy.py
View file

@ -65,9 +65,11 @@ async def get_active_callbacks(session):
_num_callbacks = _json_response["num_callbacks"]
_num_alerts = _json_response["num_alerting"]
all_litellm_callbacks = _json_response["all_litellm_callbacks"]
print("current number of callbacks: ", _num_callbacks)
print("current number of alerts: ", _num_alerts)
return _num_callbacks, _num_alerts
return _num_callbacks, _num_alerts, all_litellm_callbacks
async def get_current_routing_strategy(session):
@ -108,18 +110,36 @@ async def test_check_num_callbacks():
import uuid
async with aiohttp.ClientSession() as session:
await asyncio.sleep(30)
num_callbacks_1, _ = await get_active_callbacks(session=session)
# await asyncio.sleep(30)
num_callbacks_1, _, all_litellm_callbacks_1 = await get_active_callbacks(
session=session
)
assert num_callbacks_1 > 0
await asyncio.sleep(30)
# await asyncio.sleep(30)
num_callbacks_2, _ = await get_active_callbacks(session=session)
num_callbacks_2, _, all_litellm_callbacks_2 = await get_active_callbacks(
session=session
)
print("all_litellm_callbacks_1", all_litellm_callbacks_1)
print(
"diff in callbacks=",
set(all_litellm_callbacks_1) - set(all_litellm_callbacks_2),
)
assert num_callbacks_1 == num_callbacks_2
await asyncio.sleep(30)
num_callbacks_3, _ = await get_active_callbacks(session=session)
num_callbacks_3, _, all_litellm_callbacks_3 = await get_active_callbacks(
session=session
)
print(
"diff in callbacks = all_litellm_callbacks3 - all_litellm_callbacks2 ",
set(all_litellm_callbacks_3) - set(all_litellm_callbacks_2),
)
assert num_callbacks_1 == num_callbacks_2 == num_callbacks_3
@ -144,17 +164,33 @@ async def test_check_num_callbacks_on_lowest_latency():
original_routing_strategy = await get_current_routing_strategy(session=session)
await config_update(session=session, routing_strategy="latency-based-routing")
num_callbacks_1, num_alerts_1 = await get_active_callbacks(session=session)
num_callbacks_1, num_alerts_1, all_litellm_callbacks_1 = (
await get_active_callbacks(session=session)
)
await asyncio.sleep(30)
num_callbacks_2, num_alerts_2 = await get_active_callbacks(session=session)
num_callbacks_2, num_alerts_2, all_litellm_callbacks_2 = (
await get_active_callbacks(session=session)
)
print(
"diff in callbacks all_litellm_callbacks_2 - all_litellm_callbacks_1 =",
set(all_litellm_callbacks_2) - set(all_litellm_callbacks_1),
)
assert num_callbacks_1 == num_callbacks_2
await asyncio.sleep(30)
num_callbacks_3, num_alerts_3 = await get_active_callbacks(session=session)
num_callbacks_3, num_alerts_3, all_litellm_callbacks_3 = (
await get_active_callbacks(session=session)
)
print(
"diff in callbacks all_litellm_callbacks_3 - all_litellm_callbacks_2 =",
set(all_litellm_callbacks_3) - set(all_litellm_callbacks_2),
)
assert num_callbacks_1 == num_callbacks_2 == num_callbacks_3