phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(base_routing_strategy.py): fix base to handle no running event loop

Browse source 
run in a separate thread
This commit is contained in:
Krrish Dholakia 2025-03-18 22:20:39 -07:00
parent a3d000baaa
commit 3033c40739
2 changed files with 19 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

15
litellm/router_strategy/base_routing_strategy.py
View file

@ -22,11 +22,18 @@ class BaseRoutingStrategy(ABC):
self.dual_cache = dual_cache self.dual_cache = dual_cache
self.redis_increment_operation_queue: List[RedisPipelineIncrementOperation] = [] self.redis_increment_operation_queue: List[RedisPipelineIncrementOperation] = []
if should_batch_redis_writes: if should_batch_redis_writes:
asyncio.create_task( import threading
self.periodic_sync_in_memory_spend_with_redis(
default_sync_interval=default_sync_interval thread = threading.Thread(
) target=asyncio.run,
args=(
self.periodic_sync_in_memory_spend_with_redis(
default_sync_interval=default_sync_interval
),
),
daemon=True,
) )
thread.start()
self.in_memory_keys_to_update: set[str] = ( self.in_memory_keys_to_update: set[str] = (
set() set()

11
tests/litellm/router_strategy/test_base_routing_strategy.py
View file

@ -102,9 +102,14 @@ async def test_sync_in_memory_spend_with_redis(base_strategy, mock_dual_cache):
await base_strategy._sync_in_memory_spend_with_redis() await base_strategy._sync_in_memory_spend_with_redis()
# Verify Redis batch get was called with sorted list for consistent testing # Verify Redis batch get was called with sorted list for consistent testing
mock_dual_cache.redis_cache.async_batch_get_cache.assert_called_once_with( key_list = mock_dual_cache.redis_cache.async_batch_get_cache.call_args.kwargs[
key_list=sorted(["key1", "key2"]) "key_list"
) ]
sorted(key_list) == sorted(["key1", "key2"])
# mock_dual_cache.redis_cache.async_batch_get_cache.assert_called_once_with(
# key_list=sorted()
# )
# Verify in-memory cache was updated # Verify in-memory cache was updated
assert mock_dual_cache.in_memory_cache.async_set_cache.call_count == 2 assert mock_dual_cache.in_memory_cache.async_set_cache.call_count == 2