From 562ef2d2e14585767dd9162eb06d529c66147fd5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 6 May 2024 13:42:20 -0700 Subject: [PATCH] fix - add better debugging on num_callbacks test --- litellm/proxy/proxy_server.py | 57 +++++++++++++++----------------- tests/test_callbacks_on_proxy.py | 54 +++++++++++++++++++++++++----- 2 files changed, 71 insertions(+), 40 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4bb8dee7f..b0469fdd3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9215,24 +9215,24 @@ async def active_callbacks(): """ global proxy_logging_obj _alerting = str(general_settings.get("alerting")) - # get success callback - success_callback_names = [] - try: - # this was returning a JSON of the values in some of the callbacks - # all we need is the callback name, hence we do str(callback) - success_callback_names = [str(x) for x in litellm.success_callback] - except: - # don't let this block the /health/readiness response, if we can't convert to str -> return litellm.success_callback - success_callback_names = litellm.success_callback + # get success callbacks - _num_callbacks = ( - len(litellm.callbacks) - + len(litellm.input_callback) - + len(litellm.failure_callback) - + len(litellm.success_callback) - + len(litellm._async_failure_callback) - + len(litellm._async_success_callback) - + len(litellm._async_input_callback) + litellm_callbacks = [str(x) for x in litellm.callbacks] + litellm_input_callbacks = [str(x) for x in litellm.input_callback] + litellm_failure_callbacks = [str(x) for x in litellm.failure_callback] + litellm_success_callbacks = [str(x) for x in litellm.success_callback] + litellm_async_success_callbacks = [str(x) for x in litellm._async_success_callback] + litellm_async_failure_callbacks = [str(x) for x in litellm._async_failure_callback] + litellm_async_input_callbacks = [str(x) for x in litellm._async_input_callback] + + all_litellm_callbacks = ( + litellm_callbacks + + litellm_input_callbacks + + litellm_failure_callbacks + + litellm_success_callbacks + + litellm_async_success_callbacks + + litellm_async_failure_callbacks + + litellm_async_input_callbacks ) alerting = proxy_logging_obj.alerting @@ -9242,20 +9242,15 @@ async def active_callbacks(): return { "alerting": _alerting, - "litellm.callbacks": [str(x) for x in litellm.callbacks], - "litellm.input_callback": [str(x) for x in litellm.input_callback], - "litellm.failure_callback": [str(x) for x in litellm.failure_callback], - "litellm.success_callback": [str(x) for x in litellm.success_callback], - "litellm._async_success_callback": [ - str(x) for x in litellm._async_success_callback - ], - "litellm._async_failure_callback": [ - str(x) for x in litellm._async_failure_callback - ], - "litellm._async_input_callback": [ - str(x) for x in litellm._async_input_callback - ], - "num_callbacks": _num_callbacks, + "litellm.callbacks": litellm_callbacks, + "litellm.input_callback": litellm_input_callbacks, + "litellm.failure_callback": litellm_failure_callbacks, + "litellm.success_callback": litellm_success_callbacks, + "litellm._async_success_callback": litellm_async_success_callbacks, + "litellm._async_failure_callback": litellm_async_failure_callbacks, + "litellm._async_input_callback": litellm_async_input_callbacks, + "all_litellm_callbacks": all_litellm_callbacks, + "num_callbacks": len(all_litellm_callbacks), "num_alerting": _num_alerting, } diff --git a/tests/test_callbacks_on_proxy.py b/tests/test_callbacks_on_proxy.py index 7d48518b9..9fd6427d8 100644 --- a/tests/test_callbacks_on_proxy.py +++ b/tests/test_callbacks_on_proxy.py @@ -65,9 +65,11 @@ async def get_active_callbacks(session): _num_callbacks = _json_response["num_callbacks"] _num_alerts = _json_response["num_alerting"] + all_litellm_callbacks = _json_response["all_litellm_callbacks"] + print("current number of callbacks: ", _num_callbacks) print("current number of alerts: ", _num_alerts) - return _num_callbacks, _num_alerts + return _num_callbacks, _num_alerts, all_litellm_callbacks async def get_current_routing_strategy(session): @@ -108,18 +110,36 @@ async def test_check_num_callbacks(): import uuid async with aiohttp.ClientSession() as session: - await asyncio.sleep(30) - num_callbacks_1, _ = await get_active_callbacks(session=session) + # await asyncio.sleep(30) + num_callbacks_1, _, all_litellm_callbacks_1 = await get_active_callbacks( + session=session + ) assert num_callbacks_1 > 0 - await asyncio.sleep(30) + # await asyncio.sleep(30) - num_callbacks_2, _ = await get_active_callbacks(session=session) + num_callbacks_2, _, all_litellm_callbacks_2 = await get_active_callbacks( + session=session + ) + + print("all_litellm_callbacks_1", all_litellm_callbacks_1) + + print( + "diff in callbacks=", + set(all_litellm_callbacks_1) - set(all_litellm_callbacks_2), + ) assert num_callbacks_1 == num_callbacks_2 await asyncio.sleep(30) - num_callbacks_3, _ = await get_active_callbacks(session=session) + num_callbacks_3, _, all_litellm_callbacks_3 = await get_active_callbacks( + session=session + ) + + print( + "diff in callbacks = all_litellm_callbacks3 - all_litellm_callbacks2 ", + set(all_litellm_callbacks_3) - set(all_litellm_callbacks_2), + ) assert num_callbacks_1 == num_callbacks_2 == num_callbacks_3 @@ -144,17 +164,33 @@ async def test_check_num_callbacks_on_lowest_latency(): original_routing_strategy = await get_current_routing_strategy(session=session) await config_update(session=session, routing_strategy="latency-based-routing") - num_callbacks_1, num_alerts_1 = await get_active_callbacks(session=session) + num_callbacks_1, num_alerts_1, all_litellm_callbacks_1 = ( + await get_active_callbacks(session=session) + ) await asyncio.sleep(30) - num_callbacks_2, num_alerts_2 = await get_active_callbacks(session=session) + num_callbacks_2, num_alerts_2, all_litellm_callbacks_2 = ( + await get_active_callbacks(session=session) + ) + + print( + "diff in callbacks all_litellm_callbacks_2 - all_litellm_callbacks_1 =", + set(all_litellm_callbacks_2) - set(all_litellm_callbacks_1), + ) assert num_callbacks_1 == num_callbacks_2 await asyncio.sleep(30) - num_callbacks_3, num_alerts_3 = await get_active_callbacks(session=session) + num_callbacks_3, num_alerts_3, all_litellm_callbacks_3 = ( + await get_active_callbacks(session=session) + ) + + print( + "diff in callbacks all_litellm_callbacks_3 - all_litellm_callbacks_2 =", + set(all_litellm_callbacks_3) - set(all_litellm_callbacks_2), + ) assert num_callbacks_1 == num_callbacks_2 == num_callbacks_3