phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

Litellm dev 10 29 2024 (#6502)

Browse source 
* fix(core_helpers.py): return None, instead of raising kwargs is None error

Closes https://github.com/BerriAI/litellm/issues/6500

* docs(cost_tracking.md): cleanup doc

* fix(vertex_and_google_ai_studio.py): handle function call with no params passed in

Closes https://github.com/BerriAI/litellm/issues/6495

* test(test_router_timeout.py): add test for router timeout + retry logic

* test: update test to use module level values

* (fix) Prometheus - Log Postgres DB latency, status on prometheus  (#6484)

* fix logging DB fails on prometheus

* unit testing log to otel wrapper

* unit testing for service logger + prometheus

* use LATENCY buckets for service logging

* fix service logging

* docs clarify vertex vs gemini

* (router_strategy/) ensure all async functions use async cache methods (#6489)

* fix router strat

* use async set / get cache in router_strategy

* add coverage for router strategy

* fix imports

* fix batch_get_cache

* use async methods for least busy

* fix least busy use async methods

* fix test_dual_cache_increment

* test async_get_available_deployment when routing_strategy="least-busy"

* (fix) proxy - fix when `STORE_MODEL_IN_DB` should be set (#6492)

* set store_model_in_db at the top

* correctly use store_model_in_db global

* (fix) `PrometheusServicesLogger` `_get_metric` should return metric in Registry  (#6486)

* fix logging DB fails on prometheus

* unit testing log to otel wrapper

* unit testing for service logger + prometheus

* use LATENCY buckets for service logging

* fix service logging

* fix _get_metric in prom services logger

* add clear doc string

* unit testing for prom service logger

* bump: version 1.51.0 → 1.51.1

* Add `azure/gpt-4o-mini-2024-07-18` to model_prices_and_context_window.json (#6477)

* Update utils.py (#6468)

Fixed missing keys

* (perf) Litellm redis router fix - ~100ms improvement (#6483)

* docs(exception_mapping.md): add missing exception types

Fixes https://github.com/Aider-AI/aider/issues/2120#issuecomment-2438971183

* fix(main.py): register custom model pricing with specific key

Ensure custom model pricing is registered to the specific model+provider key combination

* test: make testing more robust for custom pricing

* fix(redis_cache.py): instrument otel logging for sync redis calls

ensures complete coverage for all redis cache calls

* refactor: pass parent_otel_span for redis caching calls in router

allows for more observability into what calls are causing latency issues

* test: update tests with new params

* refactor: ensure e2e otel tracing for router

* refactor(router.py): add more otel tracing acrosss router

catch all latency issues for router requests

* fix: fix linting error

* fix(router.py): fix linting error

* fix: fix test

* test: fix tests

* fix(dual_cache.py): pass ttl to redis cache

* fix: fix param

* perf(cooldown_cache.py): improve cooldown cache, to store cache results in memory for 5s, prevents redis call from being made on each request

reduces 100ms latency per call with caching enabled on router

* fix: fix test

* fix(cooldown_cache.py): handle if a result is None

* fix(cooldown_cache.py): add debug statements

* refactor(dual_cache.py): move to using an in-memory check for batch get cache, to prevent redis from being hit for every call

* fix(cooldown_cache.py): fix linting erropr

* refactor(prometheus.py): move to using standard logging payload for reading the remaining request / tokens

Ensures prometheus token tracking works for anthropic as well

* fix: fix linting error

* fix(redis_cache.py): make sure ttl is always int (handle float values)

Fixes issue where redis_client.ex was not working correctly due to float ttl

* fix: fix linting error

* test: update test

* fix: fix linting error

---------

Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Co-authored-by: Xingyao Wang <xingyao@all-hands.dev>
Co-authored-by: vibhanshu-ob <115142120+vibhanshu-ob@users.noreply.github.com>
This commit is contained in:
Krish Dholakia 2024-10-29 22:04:16 -07:00 • committed by GitHub
parent 6b9be5092f
commit 1e403a8447
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 286 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

1
tests/llm_translation/test_optional_params.py
View file

@ -786,6 +786,7 @@ def test_unmapped_vertex_anthropic_model():
assert "max_retries" not in optional_params
@pytest.mark.parametrize("provider", ["anthropic", "vertex_ai"])
def test_anthropic_parallel_tool_calls(provider):
optional_params = get_optional_params(

63
tests/llm_translation/test_vertex.py
View file

@ -12,8 +12,9 @@ from unittest.mock import AsyncMock, MagicMock, patch
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
import pytest
import litellm
from litellm import get_optional_params
def test_completion_pydantic_obj_2():
@ -117,3 +118,63 @@ def test_build_vertex_schema():
assert new_schema["type"] == schema["type"]
assert new_schema["properties"] == schema["properties"]
assert "required" in new_schema and new_schema["required"] == schema["required"]
@pytest.mark.parametrize(
"tools, key",
[
([{"googleSearchRetrieval": {}}], "googleSearchRetrieval"),
([{"code_execution": {}}], "code_execution"),
],
)
def test_vertex_tool_params(tools, key):
optional_params = get_optional_params(
model="gemini-1.5-pro",
custom_llm_provider="vertex_ai",
tools=tools,
)
print(optional_params)
assert optional_params["tools"][0][key] == {}
@pytest.mark.parametrize(
"tool, expect_parameters",
[
(
{
"name": "test_function",
"description": "test_function_description",
"parameters": {
"type": "object",
"properties": {"test_param": {"type": "string"}},
},
},
True,
),
(
{
"name": "test_function",
},
False,
),
],
)
def test_vertex_function_translation(tool, expect_parameters):
"""
If param not set, don't set it in the request
"""
tools = [tool]
optional_params = get_optional_params(
model="gemini-1.5-pro",
custom_llm_provider="vertex_ai",
tools=tools,
)
print(optional_params)
if expect_parameters:
assert "parameters" in optional_params["tools"][0]["function_declarations"][0]
else:
assert (
"parameters" not in optional_params["tools"][0]["function_declarations"][0]
)

4
tests/local_testing/test_caching.py
View file

@ -609,7 +609,7 @@ async def test_embedding_caching_redis_ttl():
type="redis",
host="dummy_host",
password="dummy_password",
default_in_redis_ttl=2.5,
default_in_redis_ttl=2,
)
inputs = [
@ -635,7 +635,7 @@ async def test_embedding_caching_redis_ttl():
print(f"redis pipeline set args: {args}")
print(f"redis pipeline set kwargs: {kwargs}")
assert kwargs.get("ex") == datetime.timedelta(
seconds=2.5
seconds=2
) # Check if TTL is set to 2.5 seconds

31
tests/local_testing/test_function_calling.py
View file

@ -612,3 +612,34 @@ def test_passing_tool_result_as_list():
print(resp)
assert resp.usage.prompt_tokens_details.cached_tokens > 0
def test_function_calling_with_gemini():
litellm.set_verbose = True
resp = litellm.completion(
model="gemini/gemini-1.5-pro-002",
messages=[
{
"content": [
{
"type": "text",
"text": "You are a helpful assistant that can interact with a computer to solve tasks.\n<IMPORTANT>\n* If user provides a path, you should NOT assume it's relative to the current working directory. Instead, you should explore the file system to find the file before working on it.\n</IMPORTANT>\n",
}
],
"role": "system",
},
{
"content": [{"type": "text", "text": "Hey, how's it going?"}],
"role": "user",
},
],
tools=[
{
"type": "function",
"function": {
"name": "finish",
"description": "Finish the interaction when the task is complete OR if the assistant cannot proceed further with the task.",
},
},
],
)

50
tests/local_testing/test_router_timeout.py
View file

@ -13,7 +13,7 @@ sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
from unittest.mock import patch, MagicMock, AsyncMock
import os
from dotenv import load_dotenv
@ -139,3 +139,51 @@ async def test_router_timeouts_bedrock():
pytest.fail(
f"Did not raise error `openai.APITimeoutError`. Instead raised error type: {type(e)}, Error: {e}"
)
@pytest.mark.parametrize(
"num_retries, expected_call_count",
[(0, 1), (1, 2), (2, 3), (3, 4)],
)
def test_router_timeout_with_retries_anthropic_model(num_retries, expected_call_count):
"""
If request hits custom timeout, ensure it's retried.
"""
litellm._turn_on_debug()
from litellm.llms.custom_httpx.http_handler import HTTPHandler
import time
litellm.num_retries = num_retries
litellm.request_timeout = 0.000001
router = Router(
model_list=[
{
"model_name": "claude-3-haiku",
"litellm_params": {
"model": "anthropic/claude-3-haiku-20240307",
},
}
],
)
custom_client = HTTPHandler()
with patch.object(custom_client, "post", new=MagicMock()) as mock_client:
try:
def delayed_response(*args, **kwargs):
time.sleep(0.01) # Exceeds the 0.000001 timeout
raise TimeoutError("Request timed out.")
mock_client.side_effect = delayed_response
router.completion(
model="claude-3-haiku",
messages=[{"role": "user", "content": "hello, who are u"}],
client=custom_client,
)
except litellm.Timeout:
pass
assert mock_client.call_count == expected_call_count

9
tests/logging_callback_tests/test_prometheus_unit_tests.py
View file

@ -549,13 +549,14 @@ def test_set_llm_deployment_success_metrics(prometheus_logger):
standard_logging_payload = create_standard_logging_payload()
standard_logging_payload["hidden_params"]["additional_headers"] = {
"x_ratelimit_remaining_requests": 123,
"x_ratelimit_remaining_tokens": 4321,
}
# Create test data
request_kwargs = {
"model": "gpt-3.5-turbo",
"response_headers": {
"x-ratelimit-remaining-requests": 123,
"x-ratelimit-remaining-tokens": 4321,
},
"litellm_params": {
"custom_llm_provider": "openai",
"metadata": {"model_info": {"id": "model-123"}},

39
tests/logging_callback_tests/test_standard_logging_payload.py
View file

@ -65,3 +65,42 @@ def test_get_usage(response_obj, expected_values):
assert usage.prompt_tokens == expected_values[0]
assert usage.completion_tokens == expected_values[1]
assert usage.total_tokens == expected_values[2]
def test_get_additional_headers():
additional_headers = {
"x-ratelimit-limit-requests": "2000",
"x-ratelimit-remaining-requests": "1999",
"x-ratelimit-limit-tokens": "160000",
"x-ratelimit-remaining-tokens": "160000",
"llm_provider-date": "Tue, 29 Oct 2024 23:57:37 GMT",
"llm_provider-content-type": "application/json",
"llm_provider-transfer-encoding": "chunked",
"llm_provider-connection": "keep-alive",
"llm_provider-anthropic-ratelimit-requests-limit": "2000",
"llm_provider-anthropic-ratelimit-requests-remaining": "1999",
"llm_provider-anthropic-ratelimit-requests-reset": "2024-10-29T23:57:40Z",
"llm_provider-anthropic-ratelimit-tokens-limit": "160000",
"llm_provider-anthropic-ratelimit-tokens-remaining": "160000",
"llm_provider-anthropic-ratelimit-tokens-reset": "2024-10-29T23:57:36Z",
"llm_provider-request-id": "req_01F6CycZZPSHKRCCctcS1Vto",
"llm_provider-via": "1.1 google",
"llm_provider-cf-cache-status": "DYNAMIC",
"llm_provider-x-robots-tag": "none",
"llm_provider-server": "cloudflare",
"llm_provider-cf-ray": "8da71bdbc9b57abb-SJC",
"llm_provider-content-encoding": "gzip",
"llm_provider-x-ratelimit-limit-requests": "2000",
"llm_provider-x-ratelimit-remaining-requests": "1999",
"llm_provider-x-ratelimit-limit-tokens": "160000",
"llm_provider-x-ratelimit-remaining-tokens": "160000",
}
additional_logging_headers = StandardLoggingPayloadSetup.get_additional_headers(
additional_headers
)
assert additional_logging_headers == {
"x_ratelimit_limit_requests": 2000,
"x_ratelimit_remaining_requests": 1999,
"x_ratelimit_limit_tokens": 160000,
"x_ratelimit_remaining_tokens": 160000,
}