phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-24 10:14:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge a4906a1afd into b82af5b826

Browse source
This commit is contained in:
Jorge Piedrahita Ortiz 2025-04-24 00:58:43 -07:00 • committed by GitHub
commit 22510c92c4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 341 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

1
docs/my-website/docs/completion/input.md
View file

@ -55,6 +55,7 @@ Use `litellm.get_supported_openai_params()` for an updated list of params for ea
|Bedrock| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | ✅ (model dependent) | |
|Sagemaker| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
|TogetherAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ | | | ✅ | | ✅ | ✅ | | | |
|Sambanova| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | ✅ | | ✅ | ✅ | | | |
|AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
|NLP Cloud| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
|Petals| ✅ | ✅ | | ✅ | ✅ | | | | | |

43
litellm/llms/sambanova/chat.py
View file

@ -4,7 +4,7 @@ Sambanova Chat Completions API
this is OpenAI compatible - no translation needed / occurs
"""
from typing import Optional
from typing import Optional, Union
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
@ -17,26 +17,28 @@ class SambanovaConfig(OpenAIGPTConfig):
"""
max_tokens: Optional[int] = None
response_format: Optional[dict] = None
seed: Optional[int] = None
temperature: Optional[float] = None
top_p: Optional[float] = None
top_k: Optional[int] = None
stop: Optional[Union[str, list]] = None
stream: Optional[bool] = None
top_p: Optional[int] = None
stream_options: Optional[dict] = None
tool_choice: Optional[str] = None
response_format: Optional[dict] = None
tools: Optional[list] = None
user: Optional[str] = None
def __init__(
self,
max_tokens: Optional[int] = None,
response_format: Optional[dict] = None,
seed: Optional[int] = None,
stop: Optional[str] = None,
stream: Optional[bool] = None,
stream_options: Optional[dict] = None,
temperature: Optional[float] = None,
top_p: Optional[int] = None,
top_p: Optional[float] = None,
top_k: Optional[int] = None,
tool_choice: Optional[str] = None,
tools: Optional[list] = None,
user: Optional[str] = None,
) -> None:
locals_ = locals().copy()
for key, value in locals_.items():
@ -56,12 +58,31 @@ class SambanovaConfig(OpenAIGPTConfig):
return [
"max_tokens",
"response_format",
"seed",
"stop",
"stream",
"stream_options",
"temperature",
"top_p",
"top_k",
"tool_choice",
"tools",
"user",
"parallel_tool_calls"
]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
"""
map max_completion_tokens param to max_tokens
"""
supported_openai_params = self.get_supported_openai_params(model=model)
for param, value in non_default_params.items():
if param == "max_completion_tokens":
optional_params["max_tokens"] = value
elif param in supported_openai_params:
optional_params[param] = value
return optional_params

177
litellm/model_prices_and_context_window_backup.json
View file

@ -11454,7 +11454,8 @@
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Meta-Llama-3.1-70B-Instruct": {
"max_tokens": 128000,
@ -11465,7 +11466,8 @@
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Meta-Llama-3.1-405B-Instruct": {
"max_tokens": 16000,
@ -11476,51 +11478,180 @@
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Meta-Llama-3.2-1B-Instruct": {
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 0.0000004,
"output_cost_per_token": 0.0000008,
"input_cost_per_token": 0.00000004,
"output_cost_per_token": 0.00000008,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"supports_function_calling": false,
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": false,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Meta-Llama-3.2-3B-Instruct": {
"max_tokens": 4000,
"max_input_tokens": 4000,
"max_output_tokens": 4000,
"input_cost_per_token": 0.0000008,
"output_cost_per_token": 0.0000016,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true
},
"sambanova/Qwen2.5-Coder-32B-Instruct": {
"max_tokens": 8000,
"max_input_tokens": 8000,
"max_output_tokens": 8000,
"input_cost_per_token": 0.00000008,
"output_cost_per_token": 0.00000016,
"litellm_provider": "sambanova",
"supports_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Llama-3.2-11B-Vision-Instruct": {
"max_tokens": 4000,
"max_input_tokens": 4000,
"max_output_tokens": 4000,
"input_cost_per_token": 0.00000015,
"output_cost_per_token": 0.00000030,
"litellm_provider": "sambanova",
"supports_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"supports_vision": true,
"source": "https://cloud.sambanova.ai/plans/pricing",
"metadata": {"notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount"}
},
"sambanova/Llama-3.2-90B-Vision-Instruct": {
"max_tokens": 4000,
"max_input_tokens": 4000,
"max_output_tokens": 4000,
"input_cost_per_token": 0.00000080,
"output_cost_per_token": 0.0000016,
"litellm_provider": "sambanova",
"supports_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"supports_vision": true,
"source": "https://cloud.sambanova.ai/plans/pricing",
"metadata": {"notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount"}
},
"sambanova/Meta-Llama-3.3-70B-Instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.0000006,
"output_cost_per_token": 0.0000012,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Llama-3.1-Swallow-8B-Instruct-v0.3": {
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 0.0000001,
"output_cost_per_token": 0.0000002,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Llama-3.1-Swallow-70B-Instruct-v0.3": {
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 0.0000006,
"output_cost_per_token": 0.0000012,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Llama-3.1-Tulu-3-405B": {
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 0.000005,
"output_cost_per_token": 0.000010,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Meta-Llama-Guard-3-8B": {
"max_tokens": 8000,
"max_input_tokens": 8000,
"max_output_tokens": 8000,
"input_cost_per_token": 0.0000003,
"output_cost_per_token": 0.0000003,
"litellm_provider": "sambanova",
"supports_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Qwen2.5-Coder-32B-Instruct": {
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000003,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Qwen2.5-72B-Instruct": {
"max_tokens": 8000,
"max_input_tokens": 8000,
"max_output_tokens": 8000,
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 0.000002,
"output_cost_per_token": 0.000004,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/QwQ-32B": {
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000003,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/DeepSeek-R1-Distill-Llama-70B": {
"max_tokens": 32000,
"max_input_tokens": 32000,
"max_output_tokens": 32000,
"input_cost_per_token": 0.0000007,
"output_cost_per_token": 0.0000014,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/DeepSeek-R1": {
"max_tokens": 8000,
"max_input_tokens": 8000,
"max_output_tokens": 8000,
"input_cost_per_token": 0.000005,
"output_cost_per_token": 0.000007,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"assemblyai/nano": {
"mode": "audio_transcription",

177
model_prices_and_context_window.json
View file

@ -11454,7 +11454,8 @@
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Meta-Llama-3.1-70B-Instruct": {
"max_tokens": 128000,
@ -11465,7 +11466,8 @@
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Meta-Llama-3.1-405B-Instruct": {
"max_tokens": 16000,
@ -11476,51 +11478,180 @@
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Meta-Llama-3.2-1B-Instruct": {
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 0.0000004,
"output_cost_per_token": 0.0000008,
"input_cost_per_token": 0.00000004,
"output_cost_per_token": 0.00000008,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"supports_function_calling": false,
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": false,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Meta-Llama-3.2-3B-Instruct": {
"max_tokens": 4000,
"max_input_tokens": 4000,
"max_output_tokens": 4000,
"input_cost_per_token": 0.0000008,
"output_cost_per_token": 0.0000016,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true
},
"sambanova/Qwen2.5-Coder-32B-Instruct": {
"max_tokens": 8000,
"max_input_tokens": 8000,
"max_output_tokens": 8000,
"input_cost_per_token": 0.00000008,
"output_cost_per_token": 0.00000016,
"litellm_provider": "sambanova",
"supports_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Llama-3.2-11B-Vision-Instruct": {
"max_tokens": 4000,
"max_input_tokens": 4000,
"max_output_tokens": 4000,
"input_cost_per_token": 0.00000015,
"output_cost_per_token": 0.00000030,
"litellm_provider": "sambanova",
"supports_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"supports_vision": true,
"source": "https://cloud.sambanova.ai/plans/pricing",
"metadata": {"notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount"}
},
"sambanova/Llama-3.2-90B-Vision-Instruct": {
"max_tokens": 4000,
"max_input_tokens": 4000,
"max_output_tokens": 4000,
"input_cost_per_token": 0.00000080,
"output_cost_per_token": 0.0000016,
"litellm_provider": "sambanova",
"supports_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"supports_vision": true,
"source": "https://cloud.sambanova.ai/plans/pricing",
"metadata": {"notes": "For vision models, images are converted to 6432 input tokens and are billed at that amount"}
},
"sambanova/Meta-Llama-3.3-70B-Instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.0000006,
"output_cost_per_token": 0.0000012,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Llama-3.1-Swallow-8B-Instruct-v0.3": {
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 0.0000001,
"output_cost_per_token": 0.0000002,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Llama-3.1-Swallow-70B-Instruct-v0.3": {
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 0.0000006,
"output_cost_per_token": 0.0000012,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Llama-3.1-Tulu-3-405B": {
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 0.000005,
"output_cost_per_token": 0.000010,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Meta-Llama-Guard-3-8B": {
"max_tokens": 8000,
"max_input_tokens": 8000,
"max_output_tokens": 8000,
"input_cost_per_token": 0.0000003,
"output_cost_per_token": 0.0000003,
"litellm_provider": "sambanova",
"supports_function_calling": false,
"mode": "chat",
"supports_tool_choice": false,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Qwen2.5-Coder-32B-Instruct": {
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000003,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/Qwen2.5-72B-Instruct": {
"max_tokens": 8000,
"max_input_tokens": 8000,
"max_output_tokens": 8000,
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 0.000002,
"output_cost_per_token": 0.000004,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/QwQ-32B": {
"max_tokens": 16000,
"max_input_tokens": 16000,
"max_output_tokens": 16000,
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000003,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/DeepSeek-R1-Distill-Llama-70B": {
"max_tokens": 32000,
"max_input_tokens": 32000,
"max_output_tokens": 32000,
"input_cost_per_token": 0.0000007,
"output_cost_per_token": 0.0000014,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"sambanova/DeepSeek-R1": {
"max_tokens": 8000,
"max_input_tokens": 8000,
"max_output_tokens": 8000,
"input_cost_per_token": 0.000005,
"output_cost_per_token": 0.000007,
"litellm_provider": "sambanova",
"supports_function_calling": true,
"mode": "chat",
"supports_tool_choice": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"assemblyai/nano": {
"mode": "audio_transcription",