From c49c88c8e552064c89611fc4f14cf2d4ec42fccd Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 3 Feb 2024 19:22:48 -0800 Subject: [PATCH] fix(utils.py): route together ai calls to openai client together ai is now openai-compatible n --- litellm/__init__.py | 2 ++ litellm/llms/openai.py | 4 ++-- litellm/llms/together_ai.py | 4 ++++ litellm/main.py | 4 ++++ litellm/tests/test_completion.py | 3 ++- litellm/utils.py | 10 +++++++++- 6 files changed, 23 insertions(+), 4 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 6bdfe5e10..3f2a1e4b4 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -285,6 +285,7 @@ openai_compatible_endpoints: List = [ "api.endpoints.anyscale.com/v1", "api.deepinfra.com/v1/openai", "api.mistral.ai/v1", + "api.together.xyz/v1", ] # this is maintained for Exception Mapping @@ -294,6 +295,7 @@ openai_compatible_providers: List = [ "deepinfra", "perplexity", "xinference", + "together_ai", ] diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index 7121d7bc7..3f151d1a9 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -440,8 +440,8 @@ class OpenAIChatCompletion(BaseLLM): input=data["messages"], api_key=api_key, additional_args={ - "headers": headers, - "api_base": api_base, + "headers": {"Authorization": f"Bearer {openai_client.api_key}"}, + "api_base": openai_client._base_url._uri_reference, "acompletion": False, "complete_input_dict": data, }, diff --git a/litellm/llms/together_ai.py b/litellm/llms/together_ai.py index d4b85e9ca..15ed29916 100644 --- a/litellm/llms/together_ai.py +++ b/litellm/llms/together_ai.py @@ -1,3 +1,7 @@ +""" +Deprecated. We now do together ai calls via the openai client. +Reference: https://docs.together.ai/docs/openai-api-compatibility +""" import os, types import json from enum import Enum diff --git a/litellm/main.py b/litellm/main.py index 2df5de89c..bc33a69e5 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -791,6 +791,7 @@ def completion( or custom_llm_provider == "anyscale" or custom_llm_provider == "mistral" or custom_llm_provider == "openai" + or custom_llm_provider == "together_ai" or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo ): # allow user to make an openai call with a custom base # note: if a user sets a custom base - we should ensure this works @@ -1330,6 +1331,9 @@ def completion( or ("togethercomputer" in model) or (model in litellm.together_ai_models) ): + """ + Deprecated. We now do together ai calls via the openai client - https://docs.together.ai/docs/openai-api-compatibility + """ custom_llm_provider = "together_ai" together_ai_key = ( api_key diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 54640b54b..d98745d0b 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -1994,11 +1994,12 @@ def test_completion_palm_stream(): def test_completion_together_ai_stream(): + litellm.set_verbose = True user_message = "Write 1pg about YC & litellm" messages = [{"content": user_message, "role": "user"}] try: response = completion( - model="together_ai/mistralai/Mistral-7B-Instruct-v0.1", + model="together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1", messages=messages, stream=True, max_tokens=5, diff --git a/litellm/utils.py b/litellm/utils.py index ae8425d09..6aba17f95 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -863,6 +863,7 @@ class Logging: curl_command += additional_args.get("request_str", None) elif api_base == "": curl_command = self.model_call_details + print_verbose(f"\033[92m{curl_command}\033[0m\n") verbose_logger.info(f"\033[92m{curl_command}\033[0m\n") if self.logger_fn and callable(self.logger_fn): try: @@ -4043,7 +4044,7 @@ def get_optional_params( _check_valid_arg(supported_params=supported_params) if stream: - optional_params["stream_tokens"] = stream + optional_params["stream"] = stream if temperature is not None: optional_params["temperature"] = temperature if top_p is not None: @@ -4677,6 +4678,13 @@ def get_llm_provider( # voyage is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.voyageai.com/v1 api_base = "https://api.voyageai.com/v1" dynamic_api_key = get_secret("VOYAGE_API_KEY") + elif custom_llm_provider == "together_ai": + api_base = "https://api.together.xyz/v1" + dynamic_api_key = ( + get_secret("TOGETHER_API_KEY") + or get_secret("TOGETHER_AI_API_KEY") + or get_secret("TOGETHERAI_API_KEY") + ) return model, custom_llm_provider, dynamic_api_key, api_base elif model.split("/", 1)[0] in litellm.provider_list: custom_llm_provider = model.split("/", 1)[0]