phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 11:14:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(litellm_logging.py): fix async caching for sync streaming calls (don't do it)

Browse source 
Checks if call is async before running async caching for streaming call

Fixes https://github.com/BerriAI/litellm/issues/4511#issuecomment-2233211808
This commit is contained in:
Krrish Dholakia 2024-07-17 11:15:30 -07:00
parent e7f8ee2aba
commit d2e64f21f3
2 changed files with 22 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

9
litellm/litellm_core_utils/litellm_logging.py
View file

@ -1337,7 +1337,14 @@ class Logging:
if kwargs.get("no-log", False) == True: if kwargs.get("no-log", False) == True:
print_verbose("no-log request, skipping logging") print_verbose("no-log request, skipping logging")
continue continue
if callback == "cache" and litellm.cache is not None: if (
callback == "cache"
and litellm.cache is not None
and self.model_call_details.get("litellm_params", {}).get(
"acompletion", False
)
is True
):
# set_cache once complete streaming response is built # set_cache once complete streaming response is built
print_verbose("async success_callback: reaches cache for logging!") print_verbose("async success_callback: reaches cache for logging!")
kwargs = self.model_call_details kwargs = self.model_call_details

20
litellm/tests/test_caching.py
View file

@ -1579,18 +1579,21 @@ async def test_redis_semantic_cache_acompletion():
assert response1.id == response2.id assert response1.id == response2.id
def test_caching_redis_simple(caplog): def test_caching_redis_simple(caplog, capsys):
""" """
Relevant issue - https://github.com/BerriAI/litellm/issues/4511 Relevant issue - https://github.com/BerriAI/litellm/issues/4511
""" """
litellm.set_verbose = True ## REQUIRED FOR TEST.
litellm.cache = Cache( litellm.cache = Cache(
type="redis", url=os.getenv("REDIS_SSL_URL") type="redis", url=os.getenv("REDIS_SSL_URL")
) # passing `supported_call_types = ["completion"]` has no effect ) # passing `supported_call_types = ["completion"]` has no effect
s = time.time() s = time.time()
uuid_str = str(uuid.uuid4())
x = completion( x = completion(
model="gpt-4o", model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello, how are you? Wink"}], messages=[{"role": "user", "content": f"Hello, how are you? Wink {uuid_str}"}],
stream=True, stream=True,
) )
for m in x: for m in x:
@ -1599,8 +1602,8 @@ def test_caching_redis_simple(caplog):
s2 = time.time() s2 = time.time()
x = completion( x = completion(
model="gpt-4o", model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello, how are you? Wink"}], messages=[{"role": "user", "content": f"Hello, how are you? Wink {uuid_str}"}],
stream=True, stream=True,
) )
for m in x: for m in x:
@ -1609,11 +1612,15 @@ def test_caching_redis_simple(caplog):
redis_async_caching_error = False redis_async_caching_error = False
redis_service_logging_error = False redis_service_logging_error = False
captured = capsys.readouterr()
captured_logs = [rec.message for rec in caplog.records] captured_logs = [rec.message for rec in caplog.records]
print(f"captured_logs: {captured_logs}") print(f"captured_logs: {captured_logs}")
for item in captured_logs: for item in captured_logs:
if "Error connecting to Async Redis client" in item: if (
"Error connecting to Async Redis client" in item
or "Set ASYNC Redis Cache" in item
):
redis_async_caching_error = True redis_async_caching_error = True
if "ServiceLogging.async_service_success_hook" in item: if "ServiceLogging.async_service_success_hook" in item:
@ -1621,3 +1628,4 @@ def test_caching_redis_simple(caplog):
assert redis_async_caching_error is False assert redis_async_caching_error is False
assert redis_service_logging_error is False assert redis_service_logging_error is False
assert "async success_callback: reaches cache for logging" not in captured.out