From e9eead2f257fd739ec8b7c371ca4994d9780d9a7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 3 May 2024 14:22:15 -0700 Subject: [PATCH 1/5] test - size of callbacks, alerts --- tests/test_callbacks_on_proxy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_callbacks_on_proxy.py b/tests/test_callbacks_on_proxy.py index a4d31587d..29ca62b80 100644 --- a/tests/test_callbacks_on_proxy.py +++ b/tests/test_callbacks_on_proxy.py @@ -39,7 +39,7 @@ async def config_update(session, routing_strategy=None): async def get_active_callbacks(session): - url = "http://0.0.0.0:4000/health/readiness" + url = "http://0.0.0.0:4000/active/callbacks" headers = { "Content-Type": "application/json", } @@ -47,7 +47,7 @@ async def get_active_callbacks(session): async with session.get(url, headers=headers) as response: status = response.status response_text = await response.text() - print("response from /health/readiness") + print("response from /active/callbacks") print(response_text) print() From 3997ea64427c0f1396d6f12782629745242f4be3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 3 May 2024 14:24:01 -0700 Subject: [PATCH 2/5] fix - return num callbacks in /active/callbacks --- litellm/proxy/proxy_server.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f5c234963..c9d4d3247 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9203,6 +9203,10 @@ def _db_health_readiness_check(): dependencies=[Depends(user_api_key_auth)], ) async def active_callbacks(): + """ + Returns a list of active callbacks on litellm.callbacks, litellm.input_callback, litellm.failure_callback, litellm.success_callback + """ + global proxy_logging_obj _alerting = str(general_settings.get("alerting")) # get success callback success_callback_names = [] @@ -9219,12 +9223,27 @@ async def active_callbacks(): + len(litellm.input_callback) + len(litellm.failure_callback) + len(litellm.success_callback) + + len(litellm._async_failure_callback) + + len(litellm._async_success_callback) + + len(litellm._async_input_callback) ) + alerting = proxy_logging_obj.alerting + _num_alerting = 0 + if alerting and isinstance(alerting, list): + _num_alerting = len(alerting) + return { "alerting": _alerting, - "success_callbacks": success_callback_names, + "litellm.callbacks": litellm.callbacks, + "litellm.input_callback": litellm.input_callback, + "litellm.failure_callback": litellm.failure_callback, + "litellm.success_callback": success_callback_names, + "litellm._async_success_callback": litellm._async_success_callback, + "litellm._async_failure_callback": litellm._async_failure_callback, + "litellm._async_input_callback": litellm._async_input_callback, "num_callbacks": _num_callbacks, + "num_alerting": _num_alerting, } @@ -9240,17 +9259,6 @@ async def health_readiness(): global general_settings try: # get success callback - _num_callbacks = 0 - try: - _num_callbacks = ( - len(litellm.callbacks) - + len(litellm.input_callback) - + len(litellm.failure_callback) - + len(litellm.success_callback) - ) - except: - _num_callbacks = 0 - success_callback_names = [] try: @@ -9286,7 +9294,6 @@ async def health_readiness(): "cache": cache_type, "litellm_version": version, "success_callbacks": success_callback_names, - "num_callbacks": _num_callbacks, **db_health_status, } else: @@ -9296,7 +9303,6 @@ async def health_readiness(): "cache": cache_type, "litellm_version": version, "success_callbacks": success_callback_names, - "num_callbacks": _num_callbacks, } except Exception as e: raise HTTPException(status_code=503, detail=f"Service Unhealthy ({str(e)})") From e5311d35f24a172f59a8f1cea438d5b5109cacfd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 3 May 2024 14:32:13 -0700 Subject: [PATCH 3/5] fix test len active callbacks --- tests/test_callbacks_on_proxy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_callbacks_on_proxy.py b/tests/test_callbacks_on_proxy.py index 29ca62b80..b5de5df25 100644 --- a/tests/test_callbacks_on_proxy.py +++ b/tests/test_callbacks_on_proxy.py @@ -42,6 +42,7 @@ async def get_active_callbacks(session): url = "http://0.0.0.0:4000/active/callbacks" headers = { "Content-Type": "application/json", + "Authorization": "Bearer sk-1234", } async with session.get(url, headers=headers) as response: From a369867e0208dbd356a2acdffaa79a110e02a28b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 3 May 2024 14:46:44 -0700 Subject: [PATCH 4/5] test - num alerts on callbacks --- tests/test_callbacks_on_proxy.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/test_callbacks_on_proxy.py b/tests/test_callbacks_on_proxy.py index b5de5df25..c10b18ed1 100644 --- a/tests/test_callbacks_on_proxy.py +++ b/tests/test_callbacks_on_proxy.py @@ -24,6 +24,12 @@ async def config_update(session, routing_strategy=None): "router_settings": { "routing_strategy": routing_strategy, }, + "general_settings": { + "alert_to_webhook_url": { + "llm_exceptions": "https://hooks.slack.com/services/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" + }, + "alert_types": ["llm_exceptions", "db_exceptions"], + }, } async with session.post(url, headers=headers, json=data) as response: @@ -58,8 +64,10 @@ async def get_active_callbacks(session): _json_response = await response.json() _num_callbacks = _json_response["num_callbacks"] + _num_alerts = _json_response["num_alerting"] print("current number of callbacks: ", _num_callbacks) - return _num_callbacks + print("current number of alerts: ", _num_alerts) + return _num_callbacks, _num_alerts async def get_current_routing_strategy(session): @@ -100,20 +108,20 @@ async def test_check_num_callbacks(): import uuid async with aiohttp.ClientSession() as session: - num_callbacks_1 = await get_active_callbacks(session=session) + num_callbacks_1, _ = await get_active_callbacks(session=session) assert ( num_callbacks_1 > 0 ) # /health/readiness returns 0 when some calculation goes wrong await asyncio.sleep(30) - num_callbacks_2 = await get_active_callbacks(session=session) + num_callbacks_2, _ = await get_active_callbacks(session=session) assert num_callbacks_1 == num_callbacks_2 await asyncio.sleep(30) - num_callbacks_3 = await get_active_callbacks(session=session) + num_callbacks_3, _ = await get_active_callbacks(session=session) assert num_callbacks_1 == num_callbacks_2 == num_callbacks_3 @@ -137,21 +145,23 @@ async def test_check_num_callbacks_on_lowest_latency(): original_routing_strategy = await get_current_routing_strategy(session=session) await config_update(session=session, routing_strategy="latency-based-routing") - num_callbacks_1 = await get_active_callbacks(session=session) + num_callbacks_1, num_alerts_1 = await get_active_callbacks(session=session) assert ( num_callbacks_1 > 0 ) # /health/readiness returns 0 when some calculation goes wrong await asyncio.sleep(30) - num_callbacks_2 = await get_active_callbacks(session=session) + num_callbacks_2, num_alerts_2 = await get_active_callbacks(session=session) assert num_callbacks_1 == num_callbacks_2 await asyncio.sleep(30) - num_callbacks_3 = await get_active_callbacks(session=session) + num_callbacks_3, num_alerts_3 = await get_active_callbacks(session=session) assert num_callbacks_1 == num_callbacks_2 == num_callbacks_3 + assert num_alerts_1 == num_alerts_2 == num_alerts_3 + await config_update(session=session, routing_strategy=original_routing_strategy) From ab27866b6a025680188efab51d17aa9274d3f86e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 3 May 2024 14:58:11 -0700 Subject: [PATCH 5/5] fix test slack alerting len --- litellm/proxy/proxy_server.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c9d4d3247..dfc6db6ba 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9235,13 +9235,19 @@ async def active_callbacks(): return { "alerting": _alerting, - "litellm.callbacks": litellm.callbacks, - "litellm.input_callback": litellm.input_callback, - "litellm.failure_callback": litellm.failure_callback, - "litellm.success_callback": success_callback_names, - "litellm._async_success_callback": litellm._async_success_callback, - "litellm._async_failure_callback": litellm._async_failure_callback, - "litellm._async_input_callback": litellm._async_input_callback, + "litellm.callbacks": [str(x) for x in litellm.callbacks], + "litellm.input_callback": [str(x) for x in litellm.input_callback], + "litellm.failure_callback": [str(x) for x in litellm.failure_callback], + "litellm.success_callback": [str(x) for x in litellm.success_callback], + "litellm._async_success_callback": [ + str(x) for x in litellm._async_success_callback + ], + "litellm._async_failure_callback": [ + str(x) for x in litellm._async_failure_callback + ], + "litellm._async_input_callback": [ + str(x) for x in litellm._async_input_callback + ], "num_callbacks": _num_callbacks, "num_alerting": _num_alerting, }