From f4c49755a044e98d0dca550bbe3280918db3ff8e Mon Sep 17 00:00:00 2001 From: Raymond Huang <1415176717@qq.com> Date: Wed, 5 Jun 2024 23:40:55 -0700 Subject: [PATCH 1/9] fix token counter bug --- litellm/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/utils.py b/litellm/utils.py index 178860094..a1ef3352b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4060,6 +4060,7 @@ def openai_token_counter( for c in value: if c["type"] == "text": text += c["text"] + num_tokens += len(encoding.encode(c["text"], disallowed_special=())) elif c["type"] == "image_url": if isinstance(c["image_url"], dict): image_url_dict = c["image_url"] From a7dcf25722121c19035b09e197fcd114bf899fc3 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 6 Jun 2024 09:46:51 -0700 Subject: [PATCH 2/9] feat(router.py): enable settting 'order' for a deployment in model list Allows user to control which model gets called first in model group --- litellm/router.py | 6 ++++++ litellm/tests/test_router.py | 42 ++++++++++++++++++++++++++++++++++++ litellm/utils.py | 21 ++++++++++++++++++ 3 files changed, 69 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index e3fed496f..c1b36e114 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4048,6 +4048,12 @@ class Router: for idx in reversed(invalid_model_indices): _returned_deployments.pop(idx) + ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) + if len(_returned_deployments) > 0: + _returned_deployments = litellm.utils._get_order_filtered_deployments( + _returned_deployments + ) + return _returned_deployments def _common_checks_available_deployment( diff --git a/litellm/tests/test_router.py b/litellm/tests/test_router.py index d76dec25c..02bf9a16b 100644 --- a/litellm/tests/test_router.py +++ b/litellm/tests/test_router.py @@ -38,6 +38,48 @@ def test_router_sensitive_keys(): assert "special-key" not in str(e) +def test_router_order(): + """ + Asserts for 2 models in a model group, model with order=1 always called first + """ + router = Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-4o", + "api_key": os.getenv("OPENAI_API_KEY"), + "mock_response": "Hello world", + "order": 1, + }, + "model_info": {"id": "1"}, + }, + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad-key", + "mock_response": Exception("this is a bad key"), + "order": 2, + }, + "model_info": {"id": "2"}, + }, + ], + num_retries=0, + allowed_fails=0, + enable_pre_call_checks=True, + ) + + for _ in range(100): + response = router.completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + ) + + assert isinstance(response, litellm.ModelResponse) + assert response._hidden_params["model_id"] == "1" + + @pytest.mark.parametrize("num_retries", [None, 2]) @pytest.mark.parametrize("max_retries", [None, 4]) def test_router_num_retries_init(num_retries, max_retries): diff --git a/litellm/utils.py b/litellm/utils.py index 178860094..45e233fec 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6196,6 +6196,27 @@ def calculate_max_parallel_requests( return None +def _get_order_filtered_deployments(healthy_deployments: List[Dict]) -> List: + min_order = min( + ( + deployment["litellm_params"]["order"] + for deployment in healthy_deployments + if "order" in deployment["litellm_params"] + ), + default=None, + ) + + if min_order is not None: + filtered_deployments = [ + deployment + for deployment in healthy_deployments + if deployment["litellm_params"].get("order") == min_order + ] + + return filtered_deployments + return healthy_deployments + + def _get_model_region( custom_llm_provider: str, litellm_params: LiteLLM_Params ) -> Optional[str]: From 968f54a65c3541df1bbb8ed0a035065e03498af5 Mon Sep 17 00:00:00 2001 From: Raymond Huang <1415176717@qq.com> Date: Thu, 6 Jun 2024 10:11:27 -0700 Subject: [PATCH 3/9] test --- litellm/tests/test_token_counter.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/litellm/tests/test_token_counter.py b/litellm/tests/test_token_counter.py index 194dfb8af..a6f7cd761 100644 --- a/litellm/tests/test_token_counter.py +++ b/litellm/tests/test_token_counter.py @@ -186,3 +186,13 @@ def test_load_test_token_counter(model): total_time = end_time - start_time print("model={}, total test time={}".format(model, total_time)) assert total_time < 10, f"Total encoding time > 10s, {total_time}" + +def test_openai_token_with_image_and_text(): + model = "gpt-4o" + full_request = {'model': 'gpt-4o', 'tools': [{'type': 'function', 'function': {'name': 'json', 'parameters': {'type': 'object', 'required': ['clause'], 'properties': {'clause': {'type': 'string'}}}, 'description': 'Respond with a JSON object.'}}], 'logprobs': False, 'messages': [{'role': 'user', 'content': [{'text': '\n Just some long text, long long text, and you know it will be longer than 7 tokens definetly.', 'type': 'text'}]}], 'tool_choice': {'type': 'function', 'function': {'name': 'json'}}, 'exclude_models': [], 'disable_fallback': False, 'exclude_providers': []} + messages = full_request.get("messages", []) + + token_count = token_counter(model=model, messages=messages) + print(token_count) + +test_openai_token_with_image_and_text() \ No newline at end of file From 636bc0e2ec4c78c7fb715e6a79054d33d4955427 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 6 Jun 2024 16:46:54 -0700 Subject: [PATCH 4/9] build(config.yml): update circle ci schema to run on all branches --- .circleci/config.yml | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 18bfeedb5..e919d37da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -319,24 +319,9 @@ workflows: version: 2 build_and_test: jobs: - - local_testing: - filters: - branches: - only: - - main - - /litellm_.*/ - - build_and_test: - filters: - branches: - only: - - main - - /litellm_.*/ - - installing_litellm_on_python: - filters: - branches: - only: - - main - - /litellm_.*/ + - local_testing + - build_and_test + - installing_litellm_on_python - publish_to_pypi: requires: - local_testing From 6ff67b0df4bb5c88fd3ffa299c4962c1687456de Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 6 Jun 2024 22:19:20 -0700 Subject: [PATCH 5/9] =?UTF-8?q?bump:=20version=201.40.4=20=E2=86=92=201.40?= =?UTF-8?q?.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 648a8b41a..c34664039 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.40.4" +version = "1.40.5" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -80,7 +80,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.40.4" +version = "1.40.5" version_files = [ "pyproject.toml:^version" ] From e81aa8986cdaaceea1d615404db0448df74c2791 Mon Sep 17 00:00:00 2001 From: JakobDylanC Date: Fri, 7 Jun 2024 07:35:53 -0400 Subject: [PATCH 6/9] docs: add llmcord.py to projects --- .../docs/projects/llmcord.py (Discord LLM Chatbot).md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/my-website/docs/projects/llmcord.py (Discord LLM Chatbot).md diff --git a/docs/my-website/docs/projects/llmcord.py (Discord LLM Chatbot).md b/docs/my-website/docs/projects/llmcord.py (Discord LLM Chatbot).md new file mode 100644 index 000000000..f8acb9383 --- /dev/null +++ b/docs/my-website/docs/projects/llmcord.py (Discord LLM Chatbot).md @@ -0,0 +1,3 @@ +llmcord.py lets you and your friends chat with LLMs directly in your Discord server. It works with practically any LLM, remote or locally hosted. + +Github: https://github.com/jakobdylanc/discord-llm-chatbot From dbfdbf37c9df40a03398cb5c602395d201ddd11f Mon Sep 17 00:00:00 2001 From: Takashi Iwamoto Date: Fri, 7 Jun 2024 22:20:16 +0900 Subject: [PATCH 7/9] Fix log message in Custom Callbacks doc --- docs/my-website/docs/observability/custom_callback.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/observability/custom_callback.md b/docs/my-website/docs/observability/custom_callback.md index 316822227..373b4a96c 100644 --- a/docs/my-website/docs/observability/custom_callback.md +++ b/docs/my-website/docs/observability/custom_callback.md @@ -38,7 +38,7 @@ class MyCustomHandler(CustomLogger): print(f"On Async Success") async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): - print(f"On Async Success") + print(f"On Async Failure") customHandler = MyCustomHandler() From ff5136f5ed58f1568bdda6202128768bbc011068 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 7 Jun 2024 07:05:32 -0700 Subject: [PATCH 8/9] test(test_completion.py): fix test to use mock response - prevent hang --- litellm/tests/test_completion.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 47c55ca4f..40bc31eaa 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -2538,6 +2538,7 @@ def test_replicate_custom_prompt_dict(): "content": "what is yc write 1 paragraph", } ], + mock_response="hello world", repetition_penalty=0.1, num_retries=3, ) From 11de954abd87c9b633749196fdb1e7944568207c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 7 Jun 2024 07:58:10 -0700 Subject: [PATCH 9/9] build(config.yml): setup approve and run workflow for circle ci testing on pr's --- .circleci/config.yml | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e919d37da..f806a4546 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -319,9 +319,37 @@ workflows: version: 2 build_and_test: jobs: - - local_testing - - build_and_test - - installing_litellm_on_python + - hold: + type: approval + filters: + branches: + ignore: + - main + - /litellm_.*/ + - local_testing: + requires: + - hold + filters: + branches: + ignore: + - main + - /litellm_.*/ + - build_and_test: + requires: + - hold + filters: + branches: + ignore: + - main + - /litellm_.*/ + - installing_litellm_on_python: + requires: + - hold + filters: + branches: + ignore: + - main + - /litellm_.*/ - publish_to_pypi: requires: - local_testing