forked from phoenix/litellm-mirror
fix(caching.py): support ttl, s-max-age, and no-cache cache controls
https://github.com/BerriAI/litellm/issues/1306
This commit is contained in:
parent
8772d87947
commit
8cee267a5b
5 changed files with 182 additions and 76 deletions
|
@ -1,4 +1,4 @@
|
|||
import sys, os
|
||||
import sys, os, uuid
|
||||
import time
|
||||
import traceback
|
||||
from dotenv import load_dotenv
|
||||
|
@ -81,6 +81,46 @@ def test_caching_with_ttl():
|
|||
pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
def test_caching_with_cache_controls():
|
||||
try:
|
||||
litellm.set_verbose = True
|
||||
litellm.cache = Cache()
|
||||
message = [{"role": "user", "content": f"Hey, how's it going? {uuid.uuid4()}"}]
|
||||
## TTL = 0
|
||||
response1 = completion(
|
||||
model="gpt-3.5-turbo", messages=messages, cache={"ttl": 0}
|
||||
)
|
||||
response2 = completion(
|
||||
model="gpt-3.5-turbo", messages=messages, cache={"s-max-age": 10}
|
||||
)
|
||||
print(f"response1: {response1}")
|
||||
print(f"response2: {response2}")
|
||||
assert (
|
||||
response2["choices"][0]["message"]["content"]
|
||||
!= response1["choices"][0]["message"]["content"]
|
||||
)
|
||||
message = [{"role": "user", "content": f"Hey, how's it going? {uuid.uuid4()}"}]
|
||||
## TTL = 5
|
||||
response1 = completion(
|
||||
model="gpt-3.5-turbo", messages=messages, cache={"ttl": 5}
|
||||
)
|
||||
response2 = completion(
|
||||
model="gpt-3.5-turbo", messages=messages, cache={"s-max-age": 5}
|
||||
)
|
||||
print(f"response1: {response1}")
|
||||
print(f"response2: {response2}")
|
||||
assert (
|
||||
response2["choices"][0]["message"]["content"]
|
||||
== response1["choices"][0]["message"]["content"]
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"error occurred: {traceback.format_exc()}")
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
# test_caching_with_cache_controls()
|
||||
|
||||
|
||||
def test_caching_with_models_v2():
|
||||
messages = [
|
||||
{"role": "user", "content": "who is ishaan CTO of litellm from litellm 2023"}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue