phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 11:14:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

formatting fixes

Browse source
This commit is contained in:
ishaan-jaff 2023-08-21 17:09:53 -07:00
parent 2382735f49
commit c67e59cd10
9 changed files with 299 additions and 104 deletions
Download patch file Download diff file Expand all files Collapse all files

22
litellm/__init__.py
View file

@ -1,5 +1,6 @@
import threading import threading
from typing import Callable, List, Optional from typing import Callable, List, Optional
input_callback: List[str] = [] input_callback: List[str] = []
success_callback: List[str] = [] success_callback: List[str] = []
failure_callback: List[str] = [] failure_callback: List[str] = []
@ -20,7 +21,8 @@ vertex_project: Optional[str] = None
vertex_location: Optional[str] = None vertex_location: Optional[str] = None
togetherai_api_key: Optional[str] = None togetherai_api_key: Optional[str] = None
caching = False caching = False
caching_with_models = False # if you want the caching key to be model + prompt caching_with_models = False # if you want the caching key to be model + prompt
debugger = False
model_cost = { model_cost = {
"gpt-3.5-turbo": { "gpt-3.5-turbo": {
"max_tokens": 4000, "max_tokens": 4000,
@ -156,7 +158,7 @@ replicate_models = [
"a16z-infra/llama-2-7b-chat:7b0bfc9aff140d5b75bacbed23e91fd3c34b01a1e958d32132de6e0a19796e2c", "a16z-infra/llama-2-7b-chat:7b0bfc9aff140d5b75bacbed23e91fd3c34b01a1e958d32132de6e0a19796e2c",
"replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b", "replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b",
"daanelson/flan-t5-large:ce962b3f6792a57074a601d3979db5839697add2e4e02696b3ced4c022d4767f", "daanelson/flan-t5-large:ce962b3f6792a57074a601d3979db5839697add2e4e02696b3ced4c022d4767f",
"replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad" "replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad",
] # placeholder, to make sure we accept any replicate model in our model_list ] # placeholder, to make sure we accept any replicate model in our model_list
openrouter_models = [ openrouter_models = [
@ -196,14 +198,10 @@ ai21_models = ["j2-ultra", "j2-mid", "j2-light"]
together_ai_models = [ together_ai_models = [
"togethercomputer/llama-2-70b-chat", "togethercomputer/llama-2-70b-chat",
"togethercomputer/Llama-2-7B-32K-Instruct", "togethercomputer/Llama-2-7B-32K-Instruct",
"togethercomputer/llama-2-7b" "togethercomputer/llama-2-7b",
] ]
baseten_models = [ baseten_models = ["qvv0xeq", "q841o8w", "31dxrj3"] # FALCON 7B # WizardLM # Mosaic ML
"qvv0xeq", # FALCON 7B
"q841o8w", # WizardLM
"31dxrj3" # Mosaic ML
]
model_list = ( model_list = (
open_ai_chat_completion_models open_ai_chat_completion_models
@ -231,12 +229,11 @@ provider_list = [
"openrouter", "openrouter",
"vertex_ai", "vertex_ai",
"ai21", "ai21",
"baseten" "baseten",
] ]
models_by_provider = { models_by_provider = {
"openai": open_ai_chat_completion_models "openai": open_ai_chat_completion_models + open_ai_text_completion_models,
+ open_ai_text_completion_models,
"cohere": cohere_models, "cohere": cohere_models,
"anthropic": anthropic_models, "anthropic": anthropic_models,
"replicate": replicate_models, "replicate": replicate_models,
@ -263,8 +260,11 @@ from .utils import (
completion_cost, completion_cost,
get_litellm_params, get_litellm_params,
Logging, Logging,
<<<<<<< HEAD
acreate, acreate,
get_model_list get_model_list
=======
>>>>>>> 878f1a6 (formatting fixes)
) )
from .main import * # type: ignore from .main import * # type: ignore
from .integrations import * from .integrations import *

65
litellm/integrations/litedebugger.py
View file

@ -1,5 +1,6 @@
import requests, traceback, json, os import requests, traceback, json, os
class LiteDebugger: class LiteDebugger:
user_email = None user_email = None
dashboard_url = None dashboard_url = None
@ -14,43 +15,57 @@ class LiteDebugger:
self.dashboard_url = 'https://admin.litellm.ai/' + self.user_email self.dashboard_url = 'https://admin.litellm.ai/' + self.user_email
print(f"Here's your free Dashboard 👉 {self.dashboard_url}") print(f"Here's your free Dashboard 👉 {self.dashboard_url}")
if self.user_email == None: if self.user_email == None:
raise Exception("[Non-Blocking Error] LiteLLMDebugger: Missing LITELLM_EMAIL. Set it in your environment. Eg.: os.environ['LITELLM_EMAIL']= <your_email>") raise Exception(
"[Non-Blocking Error] LiteLLMDebugger: Missing LITELLM_EMAIL. Set it in your environment. Eg.: os.environ['LITELLM_EMAIL']= <your_email>"
)
except Exception as e: except Exception as e:
raise ValueError("[Non-Blocking Error] LiteLLMDebugger: Missing LITELLM_EMAIL. Set it in your environment. Eg.: os.environ['LITELLM_EMAIL']= <your_email>") raise ValueError(
"[Non-Blocking Error] LiteLLMDebugger: Missing LITELLM_EMAIL. Set it in your environment. Eg.: os.environ['LITELLM_EMAIL']= <your_email>"
)
def input_log_event(
def input_log_event(self, model, messages, end_user, litellm_call_id, print_verbose): self, model, messages, end_user, litellm_call_id, print_verbose
):
try: try:
print_verbose( print_verbose(
f"LiteLLMDebugger: Logging - Enters input logging function for model {model}" f"LiteLLMDebugger: Logging - Enters input logging function for model {model}"
) )
litellm_data_obj = { litellm_data_obj = {
"model": model, "model": model,
"messages": messages, "messages": messages,
"end_user": end_user, "end_user": end_user,
"status": "initiated", "status": "initiated",
"litellm_call_id": litellm_call_id, "litellm_call_id": litellm_call_id,
"user_email": self.user_email "user_email": self.user_email,
} }
response = requests.post(url=self.api_url, headers={"content-type": "application/json"}, data=json.dumps(litellm_data_obj)) response = requests.post(
url=self.api_url,
headers={"content-type": "application/json"},
data=json.dumps(litellm_data_obj),
)
print_verbose(f"LiteDebugger: api response - {response.text}") print_verbose(f"LiteDebugger: api response - {response.text}")
except: except:
print_verbose(f"[Non-Blocking Error] LiteDebugger: Logging Error - {traceback.format_exc()}") print_verbose(
f"[Non-Blocking Error] LiteDebugger: Logging Error - {traceback.format_exc()}"
)
pass pass
def log_event(self, model, def log_event(
self,
model,
messages, messages,
end_user, end_user,
response_obj, response_obj,
start_time, start_time,
end_time, end_time,
litellm_call_id, litellm_call_id,
print_verbose,): print_verbose,
):
try: try:
print_verbose( print_verbose(
f"LiteLLMDebugger: Logging - Enters input logging function for model {model}" f"LiteLLMDebugger: Logging - Enters input logging function for model {model}"
) )
total_cost = 0 # [TODO] implement cost tracking total_cost = 0 # [TODO] implement cost tracking
response_time = (end_time - start_time).total_seconds() response_time = (end_time - start_time).total_seconds()
if "choices" in response_obj: if "choices" in response_obj:
litellm_data_obj = { litellm_data_obj = {
@ -62,12 +77,16 @@ class LiteDebugger:
"end_user": end_user, "end_user": end_user,
"litellm_call_id": litellm_call_id, "litellm_call_id": litellm_call_id,
"status": "success", "status": "success",
"user_email": self.user_email "user_email": self.user_email,
} }
print_verbose( print_verbose(
f"LiteDebugger: Logging - final data object: {litellm_data_obj}" f"LiteDebugger: Logging - final data object: {litellm_data_obj}"
) )
response = requests.post(url=self.api_url, headers={"content-type": "application/json"}, data=json.dumps(litellm_data_obj)) response = requests.post(
url=self.api_url,
headers={"content-type": "application/json"},
data=json.dumps(litellm_data_obj),
)
elif "error" in response_obj: elif "error" in response_obj:
if "Unable to map your input to a model." in response_obj["error"]: if "Unable to map your input to a model." in response_obj["error"]:
total_cost = 0 total_cost = 0
@ -80,13 +99,19 @@ class LiteDebugger:
"end_user": end_user, "end_user": end_user,
"litellm_call_id": litellm_call_id, "litellm_call_id": litellm_call_id,
"status": "failure", "status": "failure",
"user_email": self.user_email "user_email": self.user_email,
} }
print_verbose( print_verbose(
f"LiteDebugger: Logging - final data object: {litellm_data_obj}" f"LiteDebugger: Logging - final data object: {litellm_data_obj}"
) )
response = requests.post(url=self.api_url, headers={"content-type": "application/json"}, data=json.dumps(litellm_data_obj)) response = requests.post(
url=self.api_url,
headers={"content-type": "application/json"},
data=json.dumps(litellm_data_obj),
)
print_verbose(f"LiteDebugger: api response - {response.text}") print_verbose(f"LiteDebugger: api response - {response.text}")
except: except:
print_verbose(f"[Non-Blocking Error] LiteDebugger: Logging Error - {traceback.format_exc()}") print_verbose(
pass f"[Non-Blocking Error] LiteDebugger: Logging Error - {traceback.format_exc()}"
)
pass

24
litellm/integrations/supabase.py
View file

@ -144,23 +144,25 @@ class Supabase:
) )
return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar
def input_log_event(self, model, messages, end_user, litellm_call_id, print_verbose): def input_log_event(
self, model, messages, end_user, litellm_call_id, print_verbose
):
try: try:
print_verbose( print_verbose(
f"Supabase Logging - Enters input logging function for model {model}" f"Supabase Logging - Enters input logging function for model {model}"
) )
supabase_data_obj = { supabase_data_obj = {
"model": model, "model": model,
"messages": messages, "messages": messages,
"end_user": end_user, "end_user": end_user,
"status": "initiated", "status": "initiated",
"litellm_call_id": litellm_call_id "litellm_call_id": litellm_call_id,
} }
data, count = ( data, count = (
self.supabase_client.table(self.supabase_table_name) self.supabase_client.table(self.supabase_table_name)
.insert(supabase_data_obj) .insert(supabase_data_obj)
.execute() .execute()
) )
print(f"data: {data}") print(f"data: {data}")
except: except:
print_verbose(f"Supabase Logging Error - {traceback.format_exc()}") print_verbose(f"Supabase Logging Error - {traceback.format_exc()}")
@ -200,7 +202,7 @@ class Supabase:
"response": response_obj["choices"][0]["message"]["content"], "response": response_obj["choices"][0]["message"]["content"],
"end_user": end_user, "end_user": end_user,
"litellm_call_id": litellm_call_id, "litellm_call_id": litellm_call_id,
"status": "success" "status": "success",
} }
print_verbose( print_verbose(
f"Supabase Logging - final data object: {supabase_data_obj}" f"Supabase Logging - final data object: {supabase_data_obj}"
@ -221,7 +223,7 @@ class Supabase:
"error": response_obj["error"], "error": response_obj["error"],
"end_user": end_user, "end_user": end_user,
"litellm_call_id": litellm_call_id, "litellm_call_id": litellm_call_id,
"status": "failure" "status": "failure",
} }
print_verbose( print_verbose(
f"Supabase Logging - final data object: {supabase_data_obj}" f"Supabase Logging - final data object: {supabase_data_obj}"

17
litellm/llms/anthropic.py
View file

@ -21,7 +21,9 @@ class AnthropicError(Exception):
class AnthropicLLM: class AnthropicLLM:
def __init__(self, encoding, default_max_tokens_to_sample, logging_obj, api_key=None): def __init__(
self, encoding, default_max_tokens_to_sample, logging_obj, api_key=None
):
self.encoding = encoding self.encoding = encoding
self.default_max_tokens_to_sample = default_max_tokens_to_sample self.default_max_tokens_to_sample = default_max_tokens_to_sample
self.completion_url = "https://api.anthropic.com/v1/complete" self.completion_url = "https://api.anthropic.com/v1/complete"
@ -84,7 +86,11 @@ class AnthropicLLM:
} }
## LOGGING ## LOGGING
self.logging_obj.pre_call(input=prompt, api_key=self.api_key, additional_args={"complete_input_dict": data}) self.logging_obj.pre_call(
input=prompt,
api_key=self.api_key,
additional_args={"complete_input_dict": data},
)
## COMPLETION CALL ## COMPLETION CALL
response = requests.post( response = requests.post(
self.completion_url, headers=self.headers, data=json.dumps(data) self.completion_url, headers=self.headers, data=json.dumps(data)
@ -93,7 +99,12 @@ class AnthropicLLM:
return response.iter_lines() return response.iter_lines()
else: else:
## LOGGING ## LOGGING
self.logging_obj.post_call(input=prompt, api_key=self.api_key, original_response=response.text, additional_args={"complete_input_dict": data}) self.logging_obj.post_call(
input=prompt,
api_key=self.api_key,
original_response=response.text,
additional_args={"complete_input_dict": data},
)
print_verbose(f"raw model_response: {response.text}") print_verbose(f"raw model_response: {response.text}")
## RESPONSE OBJECT ## RESPONSE OBJECT
completion_response = response.json() completion_response = response.json()

18
litellm/llms/huggingface_restapi.py
View file

@ -72,12 +72,13 @@ class HuggingfaceRestAPILLM:
if "max_tokens" in optional_params: if "max_tokens" in optional_params:
value = optional_params.pop("max_tokens") value = optional_params.pop("max_tokens")
optional_params["max_new_tokens"] = value optional_params["max_new_tokens"] = value
data = { data = {"inputs": prompt, "parameters": optional_params}
"inputs": prompt,
"parameters": optional_params
}
## LOGGING ## LOGGING
self.logging_obj.pre_call(input=prompt, api_key=self.api_key, additional_args={"complete_input_dict": data}) self.logging_obj.pre_call(
input=prompt,
api_key=self.api_key,
additional_args={"complete_input_dict": data},
)
## COMPLETION CALL ## COMPLETION CALL
response = requests.post( response = requests.post(
completion_url, headers=self.headers, data=json.dumps(data) completion_url, headers=self.headers, data=json.dumps(data)
@ -86,7 +87,12 @@ class HuggingfaceRestAPILLM:
return response.iter_lines() return response.iter_lines()
else: else:
## LOGGING ## LOGGING
self.logging_obj.post_call(input=prompt, api_key=self.api_key, original_response=response.text, additional_args={"complete_input_dict": data}) self.logging_obj.post_call(
input=prompt,
api_key=self.api_key,
original_response=response.text,
additional_args={"complete_input_dict": data},
)
## RESPONSE OBJECT ## RESPONSE OBJECT
completion_response = response.json() completion_response = response.json()
print_verbose(f"response: {completion_response}") print_verbose(f"response: {completion_response}")

188
litellm/main.py
View file

@ -10,7 +10,7 @@ from litellm import ( # type: ignore
timeout, timeout,
get_optional_params, get_optional_params,
get_litellm_params, get_litellm_params,
Logging Logging,
) )
from litellm.utils import ( from litellm.utils import (
get_secret, get_secret,
@ -96,10 +96,14 @@ def completion(
model_response = ModelResponse() model_response = ModelResponse()
if azure: # this flag is deprecated, remove once notebooks are also updated. if azure: # this flag is deprecated, remove once notebooks are also updated.
custom_llm_provider = "azure" custom_llm_provider = "azure"
elif model.split("/", 1)[0] in litellm.provider_list: # allow custom provider to be passed in via the model name "azure/chatgpt-test" elif (
model.split("/", 1)[0] in litellm.provider_list
): # allow custom provider to be passed in via the model name "azure/chatgpt-test"
custom_llm_provider = model.split("/", 1)[0] custom_llm_provider = model.split("/", 1)[0]
model = model.split("/", 1)[1] model = model.split("/", 1)[1]
if "replicate" == custom_llm_provider and "/" not in model: # handle the "replicate/llama2..." edge-case if (
"replicate" == custom_llm_provider and "/" not in model
): # handle the "replicate/llama2..." edge-case
model = custom_llm_provider + "/" + model model = custom_llm_provider + "/" + model
# check if user passed in any of the OpenAI optional params # check if user passed in any of the OpenAI optional params
optional_params = get_optional_params( optional_params = get_optional_params(
@ -130,9 +134,14 @@ def completion(
verbose=verbose, verbose=verbose,
custom_llm_provider=custom_llm_provider, custom_llm_provider=custom_llm_provider,
custom_api_base=custom_api_base, custom_api_base=custom_api_base,
litellm_call_id=litellm_call_id litellm_call_id=litellm_call_id,
)
logging = Logging(
model=model,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
) )
logging = Logging(model=model, messages=messages, optional_params=optional_params, litellm_params=litellm_params)
if custom_llm_provider == "azure": if custom_llm_provider == "azure":
# azure configs # azure configs
openai.api_type = "azure" openai.api_type = "azure"
@ -153,7 +162,15 @@ def completion(
# set key # set key
openai.api_key = api_key openai.api_key = api_key
## LOGGING ## LOGGING
logging.pre_call(input=messages, api_key=openai.api_key, additional_args={"litellm.headers": litellm.headers, "api_version": openai.api_version, "api_base": openai.api_base}) logging.pre_call(
input=messages,
api_key=openai.api_key,
additional_args={
"headers": litellm.headers,
"api_version": openai.api_version,
"api_base": openai.api_base,
},
)
## COMPLETION CALL ## COMPLETION CALL
if litellm.headers: if litellm.headers:
response = openai.ChatCompletion.create( response = openai.ChatCompletion.create(
@ -168,7 +185,16 @@ def completion(
) )
## LOGGING ## LOGGING
logging.post_call(input=messages, api_key=openai.api_key, original_response=response, additional_args={"headers": litellm.headers, "api_version": openai.api_version, "api_base": openai.api_base}) logging.post_call(
input=messages,
api_key=openai.api_key,
original_response=response,
additional_args={
"headers": litellm.headers,
"api_version": openai.api_version,
"api_base": openai.api_base,
},
)
elif ( elif (
model in litellm.open_ai_chat_completion_models model in litellm.open_ai_chat_completion_models
or custom_llm_provider == "custom_openai" or custom_llm_provider == "custom_openai"
@ -193,7 +219,11 @@ def completion(
openai.api_key = api_key openai.api_key = api_key
## LOGGING ## LOGGING
logging.pre_call(input=messages, api_key=api_key, additional_args={"headers": litellm.headers, "api_base": api_base}) logging.pre_call(
input=messages,
api_key=api_key,
additional_args={"headers": litellm.headers, "api_base": api_base},
)
## COMPLETION CALL ## COMPLETION CALL
if litellm.headers: if litellm.headers:
response = openai.ChatCompletion.create( response = openai.ChatCompletion.create(
@ -207,7 +237,12 @@ def completion(
model=model, messages=messages, **optional_params model=model, messages=messages, **optional_params
) )
## LOGGING ## LOGGING
logging.post_call(input=messages, api_key=api_key, original_response=response, additional_args={"headers": litellm.headers}) logging.post_call(
input=messages,
api_key=api_key,
original_response=response,
additional_args={"headers": litellm.headers},
)
elif model in litellm.open_ai_text_completion_models: elif model in litellm.open_ai_text_completion_models:
openai.api_type = "openai" openai.api_type = "openai"
openai.api_base = ( openai.api_base = (
@ -228,7 +263,16 @@ def completion(
openai.organization = litellm.organization openai.organization = litellm.organization
prompt = " ".join([message["content"] for message in messages]) prompt = " ".join([message["content"] for message in messages])
## LOGGING ## LOGGING
logging.pre_call(input=prompt, api_key=api_key, additional_args={"openai_organization": litellm.organization, "headers": litellm.headers, "api_base": openai.api_base, "api_type": openai.api_type}) logging.pre_call(
input=prompt,
api_key=api_key,
additional_args={
"openai_organization": litellm.organization,
"headers": litellm.headers,
"api_base": openai.api_base,
"api_type": openai.api_type,
},
)
## COMPLETION CALL ## COMPLETION CALL
if litellm.headers: if litellm.headers:
response = openai.Completion.create( response = openai.Completion.create(
@ -239,7 +283,17 @@ def completion(
else: else:
response = openai.Completion.create(model=model, prompt=prompt) response = openai.Completion.create(model=model, prompt=prompt)
## LOGGING ## LOGGING
logging.post_call(input=prompt, api_key=api_key, original_response=response, additional_args={"openai_organization": litellm.organization, "headers": litellm.headers, "api_base": openai.api_base, "api_type": openai.api_type}) logging.post_call(
input=prompt,
api_key=api_key,
original_response=response,
additional_args={
"openai_organization": litellm.organization,
"headers": litellm.headers,
"api_base": openai.api_base,
"api_type": openai.api_type,
},
)
## RESPONSE OBJECT ## RESPONSE OBJECT
completion_response = response["choices"][0]["text"] completion_response = response["choices"][0]["text"]
model_response["choices"][0]["message"]["content"] = completion_response model_response["choices"][0]["message"]["content"] = completion_response
@ -270,7 +324,14 @@ def completion(
input["max_length"] = max_tokens # for t5 models input["max_length"] = max_tokens # for t5 models
input["max_new_tokens"] = max_tokens # for llama2 models input["max_new_tokens"] = max_tokens # for llama2 models
## LOGGING ## LOGGING
logging.pre_call(input=prompt, api_key=replicate_key, additional_args={"complete_input_dict": input, "max_tokens": max_tokens}) logging.pre_call(
input=prompt,
api_key=replicate_key,
additional_args={
"complete_input_dict": input,
"max_tokens": max_tokens,
},
)
## COMPLETION CALL ## COMPLETION CALL
output = replicate.run(model, input=input) output = replicate.run(model, input=input)
if "stream" in optional_params and optional_params["stream"] == True: if "stream" in optional_params and optional_params["stream"] == True:
@ -283,7 +344,15 @@ def completion(
response += item response += item
completion_response = response completion_response = response
## LOGGING ## LOGGING
logging.post_call(input=prompt, api_key=replicate_key, original_response=completion_response, additional_args={"complete_input_dict": input, "max_tokens": max_tokens}) logging.post_call(
input=prompt,
api_key=replicate_key,
original_response=completion_response,
additional_args={
"complete_input_dict": input,
"max_tokens": max_tokens,
},
)
## USAGE ## USAGE
prompt_tokens = len(encoding.encode(prompt)) prompt_tokens = len(encoding.encode(prompt))
completion_tokens = len(encoding.encode(completion_response)) completion_tokens = len(encoding.encode(completion_response))
@ -305,7 +374,7 @@ def completion(
encoding=encoding, encoding=encoding,
default_max_tokens_to_sample=litellm.max_tokens, default_max_tokens_to_sample=litellm.max_tokens,
api_key=anthropic_key, api_key=anthropic_key,
logging_obj = logging # model call logging done inside the class as we make need to modify I/O to fit anthropic's requirements logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit anthropic's requirements
) )
model_response = anthropic_client.completion( model_response = anthropic_client.completion(
model=model, model=model,
@ -369,7 +438,9 @@ def completion(
**optional_params, **optional_params,
) )
## LOGGING ## LOGGING
logging.post_call(input=messages, api_key=openai.api_key, original_response=response) logging.post_call(
input=messages, api_key=openai.api_key, original_response=response
)
elif model in litellm.cohere_models: elif model in litellm.cohere_models:
# import cohere/if it fails then pip install cohere # import cohere/if it fails then pip install cohere
install_and_import("cohere") install_and_import("cohere")
@ -392,7 +463,9 @@ def completion(
response = CustomStreamWrapper(response, model) response = CustomStreamWrapper(response, model)
return response return response
## LOGGING ## LOGGING
logging.post_call(input=prompt, api_key=cohere_key, original_response=response) logging.post_call(
input=prompt, api_key=cohere_key, original_response=response
)
## USAGE ## USAGE
completion_response = response[0].text completion_response = response[0].text
prompt_tokens = len(encoding.encode(prompt)) prompt_tokens = len(encoding.encode(prompt))
@ -475,7 +548,9 @@ def completion(
headers=headers, headers=headers,
) )
## LOGGING ## LOGGING
logging.post_call(input=prompt, api_key=TOGETHER_AI_TOKEN, original_response=res.text) logging.post_call(
input=prompt, api_key=TOGETHER_AI_TOKEN, original_response=res.text
)
# make this safe for reading, if output does not exist raise an error # make this safe for reading, if output does not exist raise an error
json_response = res.json() json_response = res.json()
if "output" not in json_response: if "output" not in json_response:
@ -516,7 +591,9 @@ def completion(
completion_response = chat.send_message(prompt, **optional_params) completion_response = chat.send_message(prompt, **optional_params)
## LOGGING ## LOGGING
logging.post_call(input=prompt, api_key=None, original_response=completion_response) logging.post_call(
input=prompt, api_key=None, original_response=completion_response
)
## RESPONSE OBJECT ## RESPONSE OBJECT
model_response["choices"][0]["message"]["content"] = completion_response model_response["choices"][0]["message"]["content"] = completion_response
@ -541,7 +618,9 @@ def completion(
completion_response = vertex_model.predict(prompt, **optional_params) completion_response = vertex_model.predict(prompt, **optional_params)
## LOGGING ## LOGGING
logging.post_call(input=prompt, api_key=None, original_response=completion_response) logging.post_call(
input=prompt, api_key=None, original_response=completion_response
)
## RESPONSE OBJECT ## RESPONSE OBJECT
model_response["choices"][0]["message"]["content"] = completion_response model_response["choices"][0]["message"]["content"] = completion_response
model_response["created"] = time.time() model_response["created"] = time.time()
@ -564,7 +643,11 @@ def completion(
completion_response = ai21_response["completions"][0]["data"]["text"] completion_response = ai21_response["completions"][0]["data"]["text"]
## LOGGING ## LOGGING
logging.post_call(input=prompt, api_key=ai21.api_key, original_response=completion_response) logging.post_call(
input=prompt,
api_key=ai21.api_key,
original_response=completion_response,
)
## RESPONSE OBJECT ## RESPONSE OBJECT
model_response["choices"][0]["message"]["content"] = completion_response model_response["choices"][0]["message"]["content"] = completion_response
@ -578,7 +661,9 @@ def completion(
prompt = " ".join([message["content"] for message in messages]) prompt = " ".join([message["content"] for message in messages])
## LOGGING ## LOGGING
logging.pre_call(input=prompt, api_key=None, additional_args={"endpoint": endpoint}) logging.pre_call(
input=prompt, api_key=None, additional_args={"endpoint": endpoint}
)
generator = get_ollama_response_stream(endpoint, model, prompt) generator = get_ollama_response_stream(endpoint, model, prompt)
# assume all responses are streamed # assume all responses are streamed
@ -605,7 +690,11 @@ def completion(
completion_response = completion_response["generated_text"] completion_response = completion_response["generated_text"]
## LOGGING ## LOGGING
logging.post_call(input=prompt, api_key=base_ten_key, original_response=completion_response) logging.post_call(
input=prompt,
api_key=base_ten_key,
original_response=completion_response,
)
## RESPONSE OBJECT ## RESPONSE OBJECT
model_response["choices"][0]["message"]["content"] = completion_response model_response["choices"][0]["message"]["content"] = completion_response
@ -622,13 +711,22 @@ def completion(
prompt = " ".join([message["content"] for message in messages]) prompt = " ".join([message["content"] for message in messages])
## LOGGING ## LOGGING
logging.pre_call(input=prompt, api_key=None, additional_args={"url": url, "max_new_tokens": 100}) logging.pre_call(
input=prompt,
api_key=None,
additional_args={"url": url, "max_new_tokens": 100},
)
response = requests.post( response = requests.post(
url, data={"inputs": prompt, "max_new_tokens": 100, "model": model} url, data={"inputs": prompt, "max_new_tokens": 100, "model": model}
) )
## LOGGING ## LOGGING
logging.post_call(input=prompt, api_key=None, original_response=response.text, additional_args={"url": url, "max_new_tokens": 100}) logging.post_call(
input=prompt,
api_key=None,
original_response=response.text,
additional_args={"url": url, "max_new_tokens": 100},
)
completion_response = response.json()["outputs"] completion_response = response.json()["outputs"]
@ -676,10 +774,22 @@ def batch_completion(*args, **kwargs):
@timeout( # type: ignore @timeout( # type: ignore
60 60
) ## set timeouts, in case calls hang (e.g. Azure) - default is 60s, override with `force_timeout` ) ## set timeouts, in case calls hang (e.g. Azure) - default is 60s, override with `force_timeout`
def embedding(model, input=[], azure=False, force_timeout=60, litellm_call_id=None, logger_fn=None): def embedding(
model, input=[], azure=False, force_timeout=60, litellm_call_id=None, logger_fn=None
):
try: try:
response = None response = None
logging = Logging(model=model, messages=input, optional_params={}, litellm_params={"azure": azure, "force_timeout": force_timeout, "logger_fn": logger_fn, "litellm_call_id": litellm_call_id}) logging = Logging(
model=model,
messages=input,
optional_params={},
litellm_params={
"azure": azure,
"force_timeout": force_timeout,
"logger_fn": logger_fn,
"litellm_call_id": litellm_call_id,
},
)
if azure == True: if azure == True:
# azure configs # azure configs
openai.api_type = "azure" openai.api_type = "azure"
@ -687,7 +797,15 @@ def embedding(model, input=[], azure=False, force_timeout=60, litellm_call_id=No
openai.api_version = get_secret("AZURE_API_VERSION") openai.api_version = get_secret("AZURE_API_VERSION")
openai.api_key = get_secret("AZURE_API_KEY") openai.api_key = get_secret("AZURE_API_KEY")
## LOGGING ## LOGGING
logging.pre_call(input=input, api_key=openai.api_key, additional_args={"api_type": openai.api_type, "api_base": openai.api_base, "api_version": openai.api_version}) logging.pre_call(
input=input,
api_key=openai.api_key,
additional_args={
"api_type": openai.api_type,
"api_base": openai.api_base,
"api_version": openai.api_version,
},
)
## EMBEDDING CALL ## EMBEDDING CALL
response = openai.Embedding.create(input=input, engine=model) response = openai.Embedding.create(input=input, engine=model)
print_verbose(f"response_value: {str(response)[:50]}") print_verbose(f"response_value: {str(response)[:50]}")
@ -697,7 +815,15 @@ def embedding(model, input=[], azure=False, force_timeout=60, litellm_call_id=No
openai.api_version = None openai.api_version = None
openai.api_key = get_secret("OPENAI_API_KEY") openai.api_key = get_secret("OPENAI_API_KEY")
## LOGGING ## LOGGING
logging.pre_call(input=input, api_key=openai.api_key, additional_args={"api_type": openai.api_type, "api_base": openai.api_base, "api_version": openai.api_version}) logging.pre_call(
input=input,
api_key=openai.api_key,
additional_args={
"api_type": openai.api_type,
"api_base": openai.api_base,
"api_version": openai.api_version,
},
)
## EMBEDDING CALL ## EMBEDDING CALL
response = openai.Embedding.create(input=input, model=model) response = openai.Embedding.create(input=input, model=model)
print_verbose(f"response_value: {str(response)[:50]}") print_verbose(f"response_value: {str(response)[:50]}")
@ -710,7 +836,11 @@ def embedding(model, input=[], azure=False, force_timeout=60, litellm_call_id=No
## LOGGING ## LOGGING
logging.post_call(input=input, api_key=openai.api_key, original_response=e) logging.post_call(input=input, api_key=openai.api_key, original_response=e)
## Map to OpenAI Exception ## Map to OpenAI Exception
raise exception_type(model=model, original_exception=e, custom_llm_provider="azure" if azure==True else None) raise exception_type(
model=model,
original_exception=e,
custom_llm_provider="azure" if azure == True else None,
)
####### HELPER FUNCTIONS ################ ####### HELPER FUNCTIONS ################

4
litellm/tests/test_caching.py
View file

@ -34,7 +34,6 @@ def test_caching():
pytest.fail(f"Error occurred: {e}") pytest.fail(f"Error occurred: {e}")
def test_caching_with_models(): def test_caching_with_models():
litellm.caching_with_models = True litellm.caching_with_models = True
response2 = completion(model="gpt-3.5-turbo", messages=messages) response2 = completion(model="gpt-3.5-turbo", messages=messages)
@ -47,6 +46,3 @@ def test_caching_with_models():
print(f"response2: {response2}") print(f"response2: {response2}")
print(f"response3: {response3}") print(f"response3: {response3}")
pytest.fail(f"Error occurred: {e}") pytest.fail(f"Error occurred: {e}")

12
litellm/tests/test_completion.py
View file

@ -28,15 +28,19 @@ def logger_fn(user_model_dict):
def test_completion_custom_provider_model_name(): def test_completion_custom_provider_model_name():
try: try:
response = completion( response = completion(
model="together_ai/togethercomputer/llama-2-70b-chat", messages=messages, logger_fn=logger_fn model="together_ai/togethercomputer/llama-2-70b-chat",
messages=messages,
logger_fn=logger_fn,
) )
# Add any assertions here to check the response # Add any assertions here to check the response
print(response) print(response)
except Exception as e: except Exception as e:
pytest.fail(f"Error occurred: {e}") pytest.fail(f"Error occurred: {e}")
test_completion_custom_provider_model_name() test_completion_custom_provider_model_name()
def test_completion_claude(): def test_completion_claude():
try: try:
response = completion( response = completion(
@ -89,7 +93,10 @@ def test_completion_claude_stream():
def test_completion_cohere(): def test_completion_cohere():
try: try:
response = completion( response = completion(
model="command-nightly", messages=messages, max_tokens=100, logit_bias={40: 10} model="command-nightly",
messages=messages,
max_tokens=100,
logit_bias={40: 10},
) )
# Add any assertions here to check the response # Add any assertions here to check the response
print(response) print(response)
@ -103,6 +110,7 @@ def test_completion_cohere():
except Exception as e: except Exception as e:
pytest.fail(f"Error occurred: {e}") pytest.fail(f"Error occurred: {e}")
def test_completion_cohere_stream(): def test_completion_cohere_stream():
try: try:
messages = [ messages = [

53
litellm/utils.py
View file

@ -139,6 +139,7 @@ def install_and_import(package: str):
# Logging function -> log the exact model details + what's being sent | Non-Blocking # Logging function -> log the exact model details + what's being sent | Non-Blocking
class Logging: class Logging:
global supabaseClient, liteDebuggerClient global supabaseClient, liteDebuggerClient
def __init__(self, model, messages, optional_params, litellm_params): def __init__(self, model, messages, optional_params, litellm_params):
self.model = model self.model = model
self.messages = messages self.messages = messages
@ -146,19 +147,19 @@ class Logging:
self.litellm_params = litellm_params self.litellm_params = litellm_params
self.logger_fn = litellm_params["logger_fn"] self.logger_fn = litellm_params["logger_fn"]
self.model_call_details = { self.model_call_details = {
"model": model, "model": model,
"messages": messages, "messages": messages,
"optional_params": self.optional_params, "optional_params": self.optional_params,
"litellm_params": self.litellm_params, "litellm_params": self.litellm_params,
} }
def pre_call(self, input, api_key, additional_args={}): def pre_call(self, input, api_key, additional_args={}):
try: try:
print_verbose(f"logging pre call for model: {self.model}") print_verbose(f"logging pre call for model: {self.model}")
self.model_call_details["input"] = input self.model_call_details["input"] = input
self.model_call_details["api_key"] = api_key self.model_call_details["api_key"] = api_key
self.model_call_details["additional_args"] = additional_args self.model_call_details["additional_args"] = additional_args
## User Logging -> if you pass in a custom logging function ## User Logging -> if you pass in a custom logging function
print_verbose( print_verbose(
f"Logging Details: logger_fn - {self.logger_fn} | callable(logger_fn) - {callable(self.logger_fn)}" f"Logging Details: logger_fn - {self.logger_fn} | callable(logger_fn) - {callable(self.logger_fn)}"
@ -173,7 +174,7 @@ class Logging:
f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {traceback.format_exc()}" f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging {traceback.format_exc()}"
) )
## Input Integration Logging -> If you want to log the fact that an attempt to call the model was made ## Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
for callback in litellm.input_callback: for callback in litellm.input_callback:
try: try:
if callback == "supabase": if callback == "supabase":
@ -201,11 +202,13 @@ class Logging:
print_verbose=print_verbose, print_verbose=print_verbose,
) )
except Exception as e: except Exception as e:
print_verbose(f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while input logging with integrations {traceback.format_exc()}") print_verbose(
f"LiteLLM.LoggingError: [Non-Blocking] Exception occurred while input logging with integrations {traceback.format_exc()}"
)
print_verbose( print_verbose(
f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}" f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}"
) )
if capture_exception: # log this error to sentry for debugging if capture_exception: # log this error to sentry for debugging
capture_exception(e) capture_exception(e)
except: except:
print_verbose( print_verbose(
@ -214,9 +217,9 @@ class Logging:
print_verbose( print_verbose(
f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}" f"LiteLLM.Logging: is sentry capture exception initialized {capture_exception}"
) )
if capture_exception: # log this error to sentry for debugging if capture_exception: # log this error to sentry for debugging
capture_exception(e) capture_exception(e)
def post_call(self, input, api_key, original_response, additional_args={}): def post_call(self, input, api_key, original_response, additional_args={}):
# Do something here # Do something here
try: try:
@ -224,7 +227,7 @@ class Logging:
self.model_call_details["api_key"] = api_key self.model_call_details["api_key"] = api_key
self.model_call_details["original_response"] = original_response self.model_call_details["original_response"] = original_response
self.model_call_details["additional_args"] = additional_args self.model_call_details["additional_args"] = additional_args
## User Logging -> if you pass in a custom logging function ## User Logging -> if you pass in a custom logging function
print_verbose( print_verbose(
f"Logging Details: logger_fn - {self.logger_fn} | callable(logger_fn) - {callable(self.logger_fn)}" f"Logging Details: logger_fn - {self.logger_fn} | callable(logger_fn) - {callable(self.logger_fn)}"
@ -244,6 +247,7 @@ class Logging:
) )
pass pass
def exception_logging( def exception_logging(
additional_args={}, additional_args={},
logger_fn=None, logger_fn=None,
@ -278,6 +282,7 @@ def exception_logging(
# make it easy to log if completion/embedding runs succeeded or failed + see what happened | Non-Blocking # make it easy to log if completion/embedding runs succeeded or failed + see what happened | Non-Blocking
def client(original_function): def client(original_function):
global liteDebuggerClient global liteDebuggerClient
def function_setup( def function_setup(
*args, **kwargs *args, **kwargs
): # just run once to check if user wants to send their data anywhere - PostHog/Sentry/Slack/etc. ): # just run once to check if user wants to send their data anywhere - PostHog/Sentry/Slack/etc.
@ -288,10 +293,16 @@ def client(original_function):
litellm.success_callback.append("lite_debugger") litellm.success_callback.append("lite_debugger")
litellm.failure_callback.append("lite_debugger") litellm.failure_callback.append("lite_debugger")
if ( if (
len(litellm.input_callback) > 0 or len(litellm.success_callback) > 0 or len(litellm.failure_callback) > 0 len(litellm.input_callback) > 0
or len(litellm.success_callback) > 0
or len(litellm.failure_callback) > 0
) and len(callback_list) == 0: ) and len(callback_list) == 0:
callback_list = list( callback_list = list(
set(litellm.input_callback + litellm.success_callback + litellm.failure_callback) set(
litellm.input_callback
+ litellm.success_callback
+ litellm.failure_callback
)
) )
set_callbacks( set_callbacks(
callback_list=callback_list, callback_list=callback_list,
@ -413,7 +424,9 @@ def client(original_function):
) # don't interrupt execution of main thread ) # don't interrupt execution of main thread
my_thread.start() my_thread.start()
if hasattr(e, "message"): if hasattr(e, "message"):
if liteDebuggerClient and liteDebuggerClient.dashboard_url != None: # make it easy to get to the debugger logs if you've initialized it if (
liteDebuggerClient and liteDebuggerClient.dashboard_url != None
): # make it easy to get to the debugger logs if you've initialized it
e.message += f"\n Check the log in your dashboard - {liteDebuggerClient.dashboard_url}" e.message += f"\n Check the log in your dashboard - {liteDebuggerClient.dashboard_url}"
raise e raise e
@ -497,7 +510,7 @@ def get_litellm_params(
"verbose": verbose, "verbose": verbose,
"custom_llm_provider": custom_llm_provider, "custom_llm_provider": custom_llm_provider,
"custom_api_base": custom_api_base, "custom_api_base": custom_api_base,
"litellm_call_id": litellm_call_id "litellm_call_id": litellm_call_id,
} }
return litellm_params return litellm_params
@ -1052,14 +1065,18 @@ def prompt_token_calculator(model, messages):
def valid_model(model): def valid_model(model):
try: try:
# for a given model name, check if the user has the right permissions to access the model # for a given model name, check if the user has the right permissions to access the model
if model in litellm.open_ai_chat_completion_models or model in litellm.open_ai_text_completion_models: if (
model in litellm.open_ai_chat_completion_models
or model in litellm.open_ai_text_completion_models
):
openai.Model.retrieve(model) openai.Model.retrieve(model)
else: else:
messages = [{"role": "user", "content": "Hello World"}] messages = [{"role": "user", "content": "Hello World"}]
litellm.completion(model=model, messages=messages) litellm.completion(model=model, messages=messages)
except: except:
raise InvalidRequestError(message="", model=model, llm_provider="") raise InvalidRequestError(message="", model=model, llm_provider="")
# integration helper function # integration helper function
def modify_integration(integration_name, integration_params): def modify_integration(integration_name, integration_params):
@ -1410,7 +1427,7 @@ async def stream_to_string(generator):
return response return response
########## Together AI streaming ############################# [TODO] move together ai to it's own llm class ########## Together AI streaming ############################# [TODO] move together ai to it's own llm class
async def together_ai_completion_streaming(json_data, headers): async def together_ai_completion_streaming(json_data, headers):
session = aiohttp.ClientSession() session = aiohttp.ClientSession()
url = "https://api.together.xyz/inference" url = "https://api.together.xyz/inference"