mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-25 02:34:29 +00:00
236 lines
6.2 KiB
Python
236 lines
6.2 KiB
Python
import threading
|
|
from typing import Callable, List, Optional
|
|
input_callback: List[str] = []
|
|
success_callback: List[str] = []
|
|
failure_callback: List[str] = []
|
|
set_verbose = False
|
|
telemetry = True
|
|
max_tokens = 256 # OpenAI Defaults
|
|
retry = True
|
|
api_key: Optional[str] = None
|
|
openai_key: Optional[str] = None
|
|
azure_key: Optional[str] = None
|
|
anthropic_key: Optional[str] = None
|
|
replicate_key: Optional[str] = None
|
|
cohere_key: Optional[str] = None
|
|
openrouter_key: Optional[str] = None
|
|
huggingface_key: Optional[str] = None
|
|
vertex_project: Optional[str] = None
|
|
vertex_location: Optional[str] = None
|
|
hugging_api_token: Optional[str] = None
|
|
togetherai_api_key: Optional[str] = None
|
|
caching = False
|
|
caching_with_models = False # if you want the caching key to be model + prompt
|
|
model_cost = {
|
|
"gpt-3.5-turbo": {
|
|
"max_tokens": 4000,
|
|
"input_cost_per_token": 0.0000015,
|
|
"output_cost_per_token": 0.000002,
|
|
},
|
|
"gpt-35-turbo": {
|
|
"max_tokens": 4000,
|
|
"input_cost_per_token": 0.0000015,
|
|
"output_cost_per_token": 0.000002,
|
|
}, # azure model name
|
|
"gpt-3.5-turbo-0613": {
|
|
"max_tokens": 4000,
|
|
"input_cost_per_token": 0.0000015,
|
|
"output_cost_per_token": 0.000002,
|
|
},
|
|
"gpt-3.5-turbo-0301": {
|
|
"max_tokens": 4000,
|
|
"input_cost_per_token": 0.0000015,
|
|
"output_cost_per_token": 0.000002,
|
|
},
|
|
"gpt-3.5-turbo-16k": {
|
|
"max_tokens": 16000,
|
|
"input_cost_per_token": 0.000003,
|
|
"output_cost_per_token": 0.000004,
|
|
},
|
|
"gpt-35-turbo-16k": {
|
|
"max_tokens": 16000,
|
|
"input_cost_per_token": 0.000003,
|
|
"output_cost_per_token": 0.000004,
|
|
}, # azure model name
|
|
"gpt-3.5-turbo-16k-0613": {
|
|
"max_tokens": 16000,
|
|
"input_cost_per_token": 0.000003,
|
|
"output_cost_per_token": 0.000004,
|
|
},
|
|
"gpt-4": {
|
|
"max_tokens": 8000,
|
|
"input_cost_per_token": 0.000003,
|
|
"output_cost_per_token": 0.00006,
|
|
},
|
|
"gpt-4-0613": {
|
|
"max_tokens": 8000,
|
|
"input_cost_per_token": 0.000003,
|
|
"output_cost_per_token": 0.00006,
|
|
},
|
|
"gpt-4-32k": {
|
|
"max_tokens": 8000,
|
|
"input_cost_per_token": 0.00006,
|
|
"output_cost_per_token": 0.00012,
|
|
},
|
|
"claude-instant-1": {
|
|
"max_tokens": 100000,
|
|
"input_cost_per_token": 0.00000163,
|
|
"output_cost_per_token": 0.00000551,
|
|
},
|
|
"claude-2": {
|
|
"max_tokens": 100000,
|
|
"input_cost_per_token": 0.00001102,
|
|
"output_cost_per_token": 0.00003268,
|
|
},
|
|
"text-bison-001": {
|
|
"max_tokens": 8192,
|
|
"input_cost_per_token": 0.000004,
|
|
"output_cost_per_token": 0.000004,
|
|
},
|
|
"chat-bison-001": {
|
|
"max_tokens": 4096,
|
|
"input_cost_per_token": 0.000002,
|
|
"output_cost_per_token": 0.000002,
|
|
},
|
|
"command-nightly": {
|
|
"max_tokens": 4096,
|
|
"input_cost_per_token": 0.000015,
|
|
"output_cost_per_token": 0.000015,
|
|
},
|
|
}
|
|
|
|
|
|
####### THREAD-SPECIFIC DATA ###################
|
|
class MyLocal(threading.local):
|
|
def __init__(self):
|
|
self.user = "Hello World"
|
|
|
|
|
|
_thread_context = MyLocal()
|
|
|
|
|
|
def identify(event_details):
|
|
# Store user in thread local data
|
|
if "user" in event_details:
|
|
_thread_context.user = event_details["user"]
|
|
|
|
|
|
####### ADDITIONAL PARAMS ################### configurable params if you use proxy models like Helicone, map spend to org id, etc.
|
|
api_base = None
|
|
headers = None
|
|
api_version = None
|
|
organization = None
|
|
config_path = None
|
|
####### Secret Manager #####################
|
|
secret_manager_client = None
|
|
####### COMPLETION MODELS ###################
|
|
open_ai_chat_completion_models = [
|
|
"gpt-4",
|
|
"gpt-4-0613",
|
|
"gpt-4-32k",
|
|
"gpt-4-32k-0613",
|
|
#################
|
|
"gpt-3.5-turbo",
|
|
"gpt-3.5-turbo-16k",
|
|
"gpt-3.5-turbo-0613",
|
|
"gpt-3.5-turbo-16k-0613",
|
|
]
|
|
open_ai_text_completion_models = ["text-davinci-003"]
|
|
|
|
cohere_models = [
|
|
"command-nightly",
|
|
"command",
|
|
"command-light",
|
|
"command-medium-beta",
|
|
"command-xlarge-beta",
|
|
]
|
|
|
|
anthropic_models = ["claude-2", "claude-instant-1", "claude-instant-1.2"]
|
|
|
|
replicate_models = [
|
|
"replicate/"
|
|
] # placeholder, to make sure we accept any replicate model in our model_list
|
|
|
|
openrouter_models = [
|
|
"google/palm-2-codechat-bison",
|
|
"google/palm-2-chat-bison",
|
|
"openai/gpt-3.5-turbo",
|
|
"openai/gpt-3.5-turbo-16k",
|
|
"openai/gpt-4-32k",
|
|
"anthropic/claude-2",
|
|
"anthropic/claude-instant-v1",
|
|
"meta-llama/llama-2-13b-chat",
|
|
"meta-llama/llama-2-70b-chat",
|
|
]
|
|
|
|
vertex_chat_models = ["chat-bison", "chat-bison@001"]
|
|
|
|
|
|
vertex_text_models = ["text-bison", "text-bison@001"]
|
|
|
|
huggingface_models = [
|
|
"meta-llama/Llama-2-7b-hf",
|
|
"meta-llama/Llama-2-7b-chat-hf",
|
|
"meta-llama/Llama-2-13b-hf",
|
|
"meta-llama/Llama-2-13b-chat-hf",
|
|
"meta-llama/Llama-2-70b-hf",
|
|
"meta-llama/Llama-2-70b-chat-hf",
|
|
"meta-llama/Llama-2-7b",
|
|
"meta-llama/Llama-2-7b-chat",
|
|
"meta-llama/Llama-2-13b",
|
|
"meta-llama/Llama-2-13b-chat",
|
|
"meta-llama/Llama-2-70b",
|
|
"meta-llama/Llama-2-70b-chat",
|
|
] # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/completion/supported
|
|
|
|
ai21_models = ["j2-ultra", "j2-mid", "j2-light"]
|
|
|
|
model_list = (
|
|
open_ai_chat_completion_models
|
|
+ open_ai_text_completion_models
|
|
+ cohere_models
|
|
+ anthropic_models
|
|
+ replicate_models
|
|
+ openrouter_models
|
|
+ huggingface_models
|
|
+ vertex_chat_models
|
|
+ vertex_text_models
|
|
+ ai21_models
|
|
)
|
|
|
|
provider_list = [
|
|
"openai",
|
|
"cohere",
|
|
"anthropic",
|
|
"replicate",
|
|
"huggingface",
|
|
"together_ai",
|
|
"openrouter",
|
|
"vertex_ai",
|
|
"ai21",
|
|
]
|
|
####### EMBEDDING MODELS ###################
|
|
open_ai_embedding_models = ["text-embedding-ada-002"]
|
|
|
|
from .timeout import timeout
|
|
from .testing import *
|
|
from .utils import (
|
|
client,
|
|
exception_type,
|
|
get_optional_params,
|
|
modify_integration,
|
|
token_counter,
|
|
cost_per_token,
|
|
completion_cost,
|
|
get_litellm_params,
|
|
Logging
|
|
)
|
|
from .main import * # type: ignore
|
|
from .integrations import *
|
|
from openai.error import (
|
|
AuthenticationError,
|
|
InvalidRequestError,
|
|
RateLimitError,
|
|
ServiceUnavailableError,
|
|
OpenAIError,
|
|
)
|