diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 284ae16c56..9690a4d8b4 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -545,9 +545,12 @@ def completion_cost( # noqa: PLR0915 isinstance(completion_response, BaseModel) or isinstance(completion_response, dict) ): # tts returns a custom class - usage_obj: Optional[Union[dict, Usage]] = completion_response.get( # type: ignore - "usage", {} - ) + if isinstance(completion_response, dict): + usage_obj: Optional[Union[dict, Usage]] = completion_response.get( + "usage", {} + ) + else: + usage_obj = getattr(completion_response, "usage", {}) if isinstance(usage_obj, BaseModel) and not isinstance( usage_obj, litellm.Usage ): diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b3c9904c9f..f093437417 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1,6 +1,6 @@ { "sample_spec": { - "max_tokens": "set to max_output_tokens if provider specifies it. IF not set to max_tokens provider specifies", + "max_tokens": "LEGACY parameter. set to max_output_tokens if provider specifies it. IF not set to max_input_tokens, if provider specifies it.", "max_input_tokens": "max input tokens, if the provider specifies it. if not default to max_tokens", "max_output_tokens": "max output tokens, if the provider specifies it. if not default to max_tokens", "input_cost_per_token": 0.0000, @@ -16,75 +16,23 @@ "supports_response_schema": true, "supports_system_messages": true }, - "sambanova/Meta-Llama-3.1-8B-Instruct": { - "max_tokens": 16000, - "max_input_tokens": 16000, - "max_output_tokens": 16000, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.0000002, - "litellm_provider": "sambanova", - "supports_function_calling": true, - "mode": "chat" + "omni-moderation-latest": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 0, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openai", + "mode": "moderation" }, - "sambanova/Meta-Llama-3.1-70B-Instruct": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 0.0000006, - "output_cost_per_token": 0.0000012, - "litellm_provider": "sambanova", - "supports_function_calling": true, - "mode": "chat" - }, - "sambanova/Meta-Llama-3.1-405B-Instruct": { - "max_tokens": 16000, - "max_input_tokens": 16000, - "max_output_tokens": 16000, - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000010, - "litellm_provider": "sambanova", - "supports_function_calling": true, - "mode": "chat" - }, - "sambanova/Meta-Llama-3.2-1B-Instruct": { - "max_tokens": 16000, - "max_input_tokens": 16000, - "max_output_tokens": 16000, - "input_cost_per_token": 0.0000004, - "output_cost_per_token": 0.0000008, - "litellm_provider": "sambanova", - "supports_function_calling": true, - "mode": "chat" - }, - "sambanova/Meta-Llama-3.2-3B-Instruct": { - "max_tokens": 4000, - "max_input_tokens": 4000, - "max_output_tokens": 4000, - "input_cost_per_token": 0.0000008, - "output_cost_per_token": 0.0000016, - "litellm_provider": "sambanova", - "supports_function_calling": true, - "mode": "chat" - }, - "sambanova/Qwen2.5-Coder-32B-Instruct": { - "max_tokens": 8000, - "max_input_tokens": 8000, - "max_output_tokens": 8000, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000003, - "litellm_provider": "sambanova", - "supports_function_calling": true, - "mode": "chat" - }, - "sambanova/Qwen2.5-72B-Instruct": { - "max_tokens": 8000, - "max_input_tokens": 8000, - "max_output_tokens": 8000, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000004, - "litellm_provider": "sambanova", - "supports_function_calling": true, - "mode": "chat" + "omni-moderation-2024-09-26": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 0, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openai", + "mode": "moderation" }, "gpt-4": { "max_tokens": 4096, @@ -7939,5 +7887,75 @@ "mode": "embedding", "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "metadata": {"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."} + }, + "sambanova/Meta-Llama-3.1-8B-Instruct": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.0000002, + "litellm_provider": "sambanova", + "supports_function_calling": true, + "mode": "chat" + }, + "sambanova/Meta-Llama-3.1-70B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.0000006, + "output_cost_per_token": 0.0000012, + "litellm_provider": "sambanova", + "supports_function_calling": true, + "mode": "chat" + }, + "sambanova/Meta-Llama-3.1-405B-Instruct": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000010, + "litellm_provider": "sambanova", + "supports_function_calling": true, + "mode": "chat" + }, + "sambanova/Meta-Llama-3.2-1B-Instruct": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "input_cost_per_token": 0.0000004, + "output_cost_per_token": 0.0000008, + "litellm_provider": "sambanova", + "supports_function_calling": true, + "mode": "chat" + }, + "sambanova/Meta-Llama-3.2-3B-Instruct": { + "max_tokens": 4000, + "max_input_tokens": 4000, + "max_output_tokens": 4000, + "input_cost_per_token": 0.0000008, + "output_cost_per_token": 0.0000016, + "litellm_provider": "sambanova", + "supports_function_calling": true, + "mode": "chat" + }, + "sambanova/Qwen2.5-Coder-32B-Instruct": { + "max_tokens": 8000, + "max_input_tokens": 8000, + "max_output_tokens": 8000, + "input_cost_per_token": 0.0000015, + "output_cost_per_token": 0.000003, + "litellm_provider": "sambanova", + "supports_function_calling": true, + "mode": "chat" + }, + "sambanova/Qwen2.5-72B-Instruct": { + "max_tokens": 8000, + "max_input_tokens": 8000, + "max_output_tokens": 8000, + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000004, + "litellm_provider": "sambanova", + "supports_function_calling": true, + "mode": "chat" } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b3c9904c9f..f093437417 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1,6 +1,6 @@ { "sample_spec": { - "max_tokens": "set to max_output_tokens if provider specifies it. IF not set to max_tokens provider specifies", + "max_tokens": "LEGACY parameter. set to max_output_tokens if provider specifies it. IF not set to max_input_tokens, if provider specifies it.", "max_input_tokens": "max input tokens, if the provider specifies it. if not default to max_tokens", "max_output_tokens": "max output tokens, if the provider specifies it. if not default to max_tokens", "input_cost_per_token": 0.0000, @@ -16,75 +16,23 @@ "supports_response_schema": true, "supports_system_messages": true }, - "sambanova/Meta-Llama-3.1-8B-Instruct": { - "max_tokens": 16000, - "max_input_tokens": 16000, - "max_output_tokens": 16000, - "input_cost_per_token": 0.0000001, - "output_cost_per_token": 0.0000002, - "litellm_provider": "sambanova", - "supports_function_calling": true, - "mode": "chat" + "omni-moderation-latest": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 0, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openai", + "mode": "moderation" }, - "sambanova/Meta-Llama-3.1-70B-Instruct": { - "max_tokens": 128000, - "max_input_tokens": 128000, - "max_output_tokens": 128000, - "input_cost_per_token": 0.0000006, - "output_cost_per_token": 0.0000012, - "litellm_provider": "sambanova", - "supports_function_calling": true, - "mode": "chat" - }, - "sambanova/Meta-Llama-3.1-405B-Instruct": { - "max_tokens": 16000, - "max_input_tokens": 16000, - "max_output_tokens": 16000, - "input_cost_per_token": 0.000005, - "output_cost_per_token": 0.000010, - "litellm_provider": "sambanova", - "supports_function_calling": true, - "mode": "chat" - }, - "sambanova/Meta-Llama-3.2-1B-Instruct": { - "max_tokens": 16000, - "max_input_tokens": 16000, - "max_output_tokens": 16000, - "input_cost_per_token": 0.0000004, - "output_cost_per_token": 0.0000008, - "litellm_provider": "sambanova", - "supports_function_calling": true, - "mode": "chat" - }, - "sambanova/Meta-Llama-3.2-3B-Instruct": { - "max_tokens": 4000, - "max_input_tokens": 4000, - "max_output_tokens": 4000, - "input_cost_per_token": 0.0000008, - "output_cost_per_token": 0.0000016, - "litellm_provider": "sambanova", - "supports_function_calling": true, - "mode": "chat" - }, - "sambanova/Qwen2.5-Coder-32B-Instruct": { - "max_tokens": 8000, - "max_input_tokens": 8000, - "max_output_tokens": 8000, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000003, - "litellm_provider": "sambanova", - "supports_function_calling": true, - "mode": "chat" - }, - "sambanova/Qwen2.5-72B-Instruct": { - "max_tokens": 8000, - "max_input_tokens": 8000, - "max_output_tokens": 8000, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000004, - "litellm_provider": "sambanova", - "supports_function_calling": true, - "mode": "chat" + "omni-moderation-2024-09-26": { + "max_tokens": 32768, + "max_input_tokens": 32768, + "max_output_tokens": 0, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openai", + "mode": "moderation" }, "gpt-4": { "max_tokens": 4096, @@ -7939,5 +7887,75 @@ "mode": "embedding", "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "metadata": {"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."} + }, + "sambanova/Meta-Llama-3.1-8B-Instruct": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.0000002, + "litellm_provider": "sambanova", + "supports_function_calling": true, + "mode": "chat" + }, + "sambanova/Meta-Llama-3.1-70B-Instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.0000006, + "output_cost_per_token": 0.0000012, + "litellm_provider": "sambanova", + "supports_function_calling": true, + "mode": "chat" + }, + "sambanova/Meta-Llama-3.1-405B-Instruct": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000010, + "litellm_provider": "sambanova", + "supports_function_calling": true, + "mode": "chat" + }, + "sambanova/Meta-Llama-3.2-1B-Instruct": { + "max_tokens": 16000, + "max_input_tokens": 16000, + "max_output_tokens": 16000, + "input_cost_per_token": 0.0000004, + "output_cost_per_token": 0.0000008, + "litellm_provider": "sambanova", + "supports_function_calling": true, + "mode": "chat" + }, + "sambanova/Meta-Llama-3.2-3B-Instruct": { + "max_tokens": 4000, + "max_input_tokens": 4000, + "max_output_tokens": 4000, + "input_cost_per_token": 0.0000008, + "output_cost_per_token": 0.0000016, + "litellm_provider": "sambanova", + "supports_function_calling": true, + "mode": "chat" + }, + "sambanova/Qwen2.5-Coder-32B-Instruct": { + "max_tokens": 8000, + "max_input_tokens": 8000, + "max_output_tokens": 8000, + "input_cost_per_token": 0.0000015, + "output_cost_per_token": 0.000003, + "litellm_provider": "sambanova", + "supports_function_calling": true, + "mode": "chat" + }, + "sambanova/Qwen2.5-72B-Instruct": { + "max_tokens": 8000, + "max_input_tokens": 8000, + "max_output_tokens": 8000, + "input_cost_per_token": 0.000002, + "output_cost_per_token": 0.000004, + "litellm_provider": "sambanova", + "supports_function_calling": true, + "mode": "chat" } } diff --git a/tests/local_testing/test_completion_cost.py b/tests/local_testing/test_completion_cost.py index 585eac0c2a..54cee48e6b 100644 --- a/tests/local_testing/test_completion_cost.py +++ b/tests/local_testing/test_completion_cost.py @@ -2704,6 +2704,24 @@ def test_select_model_name_for_cost_calc(): assert return_model == "azure_ai/mistral-large" + +def test_moderations(): + from litellm import moderation + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.add_known_models() + + assert "omni-moderation-latest" in litellm.model_cost + print( + f"litellm.model_cost['omni-moderation-latest']: {litellm.model_cost['omni-moderation-latest']}" + ) + assert "omni-moderation-latest" in litellm.open_ai_chat_completion_models + + response = moderation("I am a bad person", model="omni-moderation-latest") + cost = completion_cost(response, model="omni-moderation-latest") + assert cost == 0 + def test_cost_calculator_azure_embedding(): from litellm.cost_calculator import response_cost_calculator from litellm.types.utils import EmbeddingResponse, Usage diff --git a/ui/litellm-dashboard/src/components/teams.tsx b/ui/litellm-dashboard/src/components/teams.tsx index 3a883d4cee..36a343637e 100644 --- a/ui/litellm-dashboard/src/components/teams.tsx +++ b/ui/litellm-dashboard/src/components/teams.tsx @@ -35,6 +35,9 @@ import { Col, Text, Grid, + Accordion, + AccordionHeader, + AccordionBody, } from "@tremor/react"; import { CogIcon } from "@heroicons/react/outline"; const isLocal = process.env.NODE_ENV === "development"; @@ -365,6 +368,13 @@ const Team: React.FC = ({ if (accessToken != null) { const newTeamAlias = formValues?.team_alias; const existingTeamAliases = teams?.map((t) => t.team_alias) ?? []; + let organizationId = formValues?.organization_id; + if (organizationId === "" || typeof organizationId !== 'string') { + formValues.organization_id = null; + } else { + formValues.organization_id = organizationId.trim(); + } + if (existingTeamAliases.includes(newTeamAlias)) { throw new Error( @@ -731,6 +741,25 @@ const Team: React.FC = ({ > + + + Additional Settings + + + + { + e.target.value = e.target.value.trim(); + }} + /> + + +
Create Team