phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(utils.py): only return cached streaming object for streaming calls

Browse source
This commit is contained in:
Krrish Dholakia 2024-02-21 21:27:40 -08:00
parent f1742769a2
commit fb2ae3a032
2 changed files with 10 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

4
litellm/caching.py
View file

@ -124,7 +124,9 @@ class RedisCache(BaseCache):
self.redis_client.set(name=key, value=str(value), ex=ttl) self.redis_client.set(name=key, value=str(value), ex=ttl)
except Exception as e: except Exception as e:
# NON blocking - notify users Redis is throwing an exception # NON blocking - notify users Redis is throwing an exception
print_verbose("LiteLLM Caching: set() - Got exception from REDIS : ", e) print_verbose(
f"LiteLLM Caching: set() - Got exception from REDIS : {str(e)}"
)
async def async_set_cache(self, key, value, **kwargs): async def async_set_cache(self, key, value, **kwargs):
_redis_client = self.init_async_client() _redis_client = self.init_async_client()

14
litellm/utils.py
View file

@ -2330,13 +2330,13 @@ def client(original_function):
model_response_object=ModelResponse(), model_response_object=ModelResponse(),
stream=kwargs.get("stream", False), stream=kwargs.get("stream", False),
) )
if kwargs.get("stream", False) == True:
cached_result = CustomStreamWrapper( cached_result = CustomStreamWrapper(
completion_stream=cached_result, completion_stream=cached_result,
model=model, model=model,
custom_llm_provider="cached_response", custom_llm_provider="cached_response",
logging_obj=logging_obj, logging_obj=logging_obj,
) )
elif call_type == CallTypes.embedding.value and isinstance( elif call_type == CallTypes.embedding.value and isinstance(
cached_result, dict cached_result, dict
): ):