phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 11:43:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(refactor) caching - use _sync_set_cache (#6224)

Browse source 
* caching - use _sync_set_cache

* add sync _sync_add_streaming_response_to_cache

* use caching class for cache storage
This commit is contained in:
Ishaan Jaff 2024-10-16 10:38:07 +05:30 • committed by GitHub
parent a04fc1a921
commit 4eea0652eb
3 changed files with 89 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

13
litellm/litellm_core_utils/litellm_logging.py
View file

@ -938,19 +938,6 @@ class Logging:
else:
callbacks = litellm.success_callback
## STREAMING CACHING ##
if "cache" in callbacks and litellm.cache is not None:
# this only logs streaming once, complete_streaming_response exists i.e when stream ends
print_verbose("success_callback: reaches cache for logging!")
kwargs = self.model_call_details
if self.stream and _caching_complete_streaming_response is not None:
print_verbose(
"success_callback: reaches cache for logging, there is a complete_streaming_response. Adding to cache"
)
result = _caching_complete_streaming_response
# only add to cache once we have a complete streaming response
litellm.cache.add_cache(result, **kwargs)
## REDACT MESSAGES ##
result = redact_message_input_output_from_logging(
model_call_details=(