phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

LiteLLM Minor Fixes & Improvements (10/23/2024) (#6407)

Browse source 
* docs(bedrock.md): clarify bedrock auth in litellm docs

* fix(convert_dict_to_response.py): Fixes https://github.com/BerriAI/litellm/issues/6387

* feat(pattern_match_deployments.py): more robust handling for wildcard routes (model_name: custom_route/* -> openai/*)

Enables user to expose custom routes to users with dynamic handling

* test: add more testing

* docs(custom_pricing.md): add debug tutorial for custom pricing

* test: skip codestral test - unreachable backend

* test: fix test

* fix(pattern_matching_deployments.py): fix typing

* test: cleanup codestral tests - backend api unavailable

* (refactor) prometheus async_log_success_event to be under 100 LOC  (#6416)

* unit testig for prometheus

* unit testing for success metrics

* use 1 helper for _increment_token_metrics

* use helper for _increment_remaining_budget_metrics

* use _increment_remaining_budget_metrics

* use _increment_top_level_request_and_spend_metrics

* use helper for _set_latency_metrics

* remove noqa violation

* fix test prometheus

* test prometheus

* unit testing for all prometheus helper functions

* fix prom unit tests

* fix unit tests prometheus

* fix unit test prom

* (refactor) router - use static methods for client init utils  (#6420)

* use InitalizeOpenAISDKClient

* use InitalizeOpenAISDKClient static method

* fix  # noqa: PLR0915

* (code cleanup) remove unused and undocumented logging integrations - litedebugger, berrispend  (#6406)

* code cleanup remove unused and undocumented code files

* fix unused logging integrations cleanup

* bump: version 1.50.3 → 1.50.4

---------

Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com>
This commit is contained in:
Krish Dholakia 2024-10-24 19:01:41 -07:00 • committed by GitHub
parent c04c4a82f1
commit 1cd1d23fdf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 235 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

38
tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py
View file

@ -695,3 +695,41 @@ def test_convert_to_model_response_object_error():
_response_headers=None,
convert_tool_call_to_json_mode=False,
)
def test_image_generation_openai_with_pydantic_warning(caplog):
try:
import logging
from litellm.types.utils import ImageResponse, ImageObject
convert_response_args = {
"response_object": {
"created": 1729709945,
"data": [
{
"b64_json": None,
"revised_prompt": "Generate an image of a baby sea otter. It should look incredibly cute, with big, soulful eyes and a fluffy, wet fur coat. The sea otter should be on its back, as sea otters often do, with its tiny hands holding onto a shell as if it is its precious toy. The background should be a tranquil sea under a clear sky, with soft sunlight reflecting off the waters. The color palette should be soothing with blues, browns, and white.",
"url": "https://oaidalleapiprodscus.blob.core.windows.net/private/org-ikDc4ex8NB5ZzfTf8m5WYVB7/user-JpwZsbIXubBZvan3Y3GchiiB/img-LL0uoOv4CFJIvNYxoNCKB8oc.png?st=2024-10-23T17%3A59%3A05Z&se=2024-10-23T19%3A59%3A05Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=d505667d-d6c1-4a0a-bac7-5c84a87759f8&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-10-22T19%3A26%3A22Z&ske=2024-10-23T19%3A26%3A22Z&sks=b&skv=2024-08-04&sig=Hl4wczJ3H2vZNdLRt/7JvNi6NvQGDnbNkDy15%2Bl3k5s%3D",
}
],
},
"model_response_object": ImageResponse(
created=1729709929,
data=[],
),
"response_type": "image_generation",
"stream": False,
"start_time": None,
"end_time": None,
"hidden_params": None,
"_response_headers": None,
"convert_tool_call_to_json_mode": None,
}
resp: ImageResponse = convert_to_model_response_object(**convert_response_args)
assert resp is not None
assert resp.data is not None
assert len(resp.data) == 1
assert isinstance(resp.data[0], ImageObject)
except Exception as e:
pytest.fail(f"Test failed with exception: {e}")

8
tests/local_testing/test_router_pattern_matching.py
View file

@ -42,7 +42,7 @@ def test_add_pattern():
)
router.add_pattern("openai/*", deployment.to_json(exclude_none=True))
assert len(router.patterns) == 1
assert list(router.patterns.keys())[0] == "^openai/.*$"
assert list(router.patterns.keys())[0] == "openai/(.*)"
# try getting the pattern
assert router.route(request="openai/gpt-15") == [
@ -64,7 +64,7 @@ def test_add_pattern_vertex_ai():
)
router.add_pattern("vertex_ai/*", deployment.to_json(exclude_none=True))
assert len(router.patterns) == 1
assert list(router.patterns.keys())[0] == "^vertex_ai/.*$"
assert list(router.patterns.keys())[0] == "vertex_ai/(.*)"
# try getting the pattern
assert router.route(request="vertex_ai/gemini-1.5-flash-latest") == [
@ -99,10 +99,10 @@ def test_pattern_to_regex():
Tests that the pattern is converted to a regex
"""
router = PatternMatchRouter()
assert router._pattern_to_regex("openai/*") == "^openai/.*$"
assert router._pattern_to_regex("openai/*") == "openai/(.*)"
assert (
router._pattern_to_regex("openai/fo::*::static::*")
== "^openai/fo::.*::static::.*$"
== "openai/fo::(.*)::static::(.*)"
)

69
tests/router_unit_tests/test_router_helper_utils.py
View file

@ -914,3 +914,72 @@ def test_replace_model_in_jsonl(model_list):
router = Router(model_list=model_list)
deployments = router.pattern_router.get_deployments_by_pattern(model="claude-3")
assert deployments is not None
# def test_pattern_match_deployments(model_list):
# from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
# import re
# patter_router = PatternMatchRouter()
# request = "fo::hi::static::hello"
# model_name = "fo::*:static::*"
# model_name_regex = patter_router._pattern_to_regex(model_name)
# # Match against the request
# match = re.match(model_name_regex, request)
# print(f"match: {match}")
# print(f"match.end: {match.end()}")
# if match is None:
# raise ValueError("Match not found")
# updated_model = patter_router.set_deployment_model_name(
# matched_pattern=match, litellm_deployment_litellm_model="openai/*"
# )
# assert updated_model == "openai/fo::hi:static::hello"
@pytest.mark.parametrize(
"user_request_model, model_name, litellm_model, expected_model",
[
("llmengine/foo", "llmengine/*", "openai/foo", "openai/foo"),
("llmengine/foo", "llmengine/*", "openai/*", "openai/foo"),
(
"fo::hi::static::hello",
"fo::*::static::*",
"openai/fo::*:static::*",
"openai/fo::hi:static::hello",
),
(
"fo::hi::static::hello",
"fo::*::static::*",
"openai/gpt-3.5-turbo",
"openai/gpt-3.5-turbo",
),
],
)
def test_pattern_match_deployment_set_model_name(
user_request_model, model_name, litellm_model, expected_model
):
from re import Match
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
pattern_router = PatternMatchRouter()
import re
# Convert model_name into a proper regex
model_name_regex = pattern_router._pattern_to_regex(model_name)
# Match against the request
match = re.match(model_name_regex, user_request_model)
if match is None:
raise ValueError("Match not found")
# Call the set_deployment_model_name function
updated_model = pattern_router.set_deployment_model_name(match, litellm_model)
print(updated_model) # Expected output: "openai/fo::hi:static::hello"
assert updated_model == expected_model