(feat) caching: Use seed, max_tokens etc in cache key

This commit is contained in:
ishaan-jaff 2023-11-23 18:16:42 -08:00
parent 1cf85b4eb9
commit 3660fb1f7f

View file

@ -192,17 +192,13 @@ class Cache:
Returns:
str: The cache key generated from the arguments, or None if no cache key could be generated.
"""
prompt = get_prompt(*args, **kwargs)
if prompt is not None:
cache_key = prompt
if "model" in kwargs:
cache_key += kwargs["model"]
elif "input" in kwargs:
cache_key = " ".join(kwargs["input"])
if "model" in kwargs:
cache_key += kwargs["model"]
else:
return None
cache_key =""
for param in kwargs:
# ignore litellm params here
if param in set(["litellm_call_id", "litellm_logging_obj"]):
continue
param_value = kwargs[param]
cache_key+= f"{str(param)}: {str(param_value)}"
return cache_key
def generate_streaming_content(self, content):