phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 10:44:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[Perf Fix] Don't always read from Redis by Default (#5877)

Browse source 
* fix use previous internal usage caching logic

* fix test_dual_cache_uses_redis
This commit is contained in:
Ishaan Jaff 2024-09-24 21:34:18 -07:00 • committed by GitHub
parent 3ccdb42d26
commit 2000e8cde9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 7 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

2
litellm/proxy/utils.py
View file

@ -223,7 +223,7 @@ class ProxyLogging:
self.call_details: dict = {}
self.call_details["user_api_key_cache"] = user_api_key_cache
self.internal_usage_cache = DualCache(
default_in_memory_ttl=1, always_read_redis=litellm.always_read_redis
default_in_memory_ttl=1
) # ping redis cache every 1s
self.max_parallel_request_limiter = _PROXY_MaxParallelRequestsHandler(
self.internal_usage_cache