mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-25 18:54:30 +00:00
LiteLLM Minor Fixes & Improvements (10/09/2024) (#6139)
* fix(utils.py): don't return 'none' response headers Fixes https://github.com/BerriAI/litellm/issues/6123 * fix(vertex_and_google_ai_studio_gemini.py): support parsing out additional properties and strict value for tool calls Fixes https://github.com/BerriAI/litellm/issues/6136 * fix(cost_calculator.py): set default character value to none Fixes https://github.com/BerriAI/litellm/issues/6133#issuecomment-2403290196 * fix(google.py): fix cost per token / cost per char conversion Fixes https://github.com/BerriAI/litellm/issues/6133#issuecomment-2403370287 * build(model_prices_and_context_window.json): update gemini pricing Fixes https://github.com/BerriAI/litellm/issues/6133 * build(model_prices_and_context_window.json): update gemini pricing * fix(litellm_logging.py): fix streaming caching logging when 'turn_off_message_logging' enabled Stores unredacted response in cache * build(model_prices_and_context_window.json): update gemini-1.5-flash pricing * fix(cost_calculator.py): fix default prompt_character count logic Fixes error in gemini cost calculation * fix(cost_calculator.py): fix cost calc for tts models
This commit is contained in:
Krish Dholakia
GitHub
parent
60baa65e0e
commit
6005450c8f
16 changed files with 788 additions and 534 deletions
165
litellm/utils.py
165
litellm/utils.py
|
|
@ -2771,6 +2771,11 @@ def get_optional_params_embeddings(
|
|||
|
||||
|
||||
def _remove_additional_properties(schema):
|
||||
"""
|
||||
clean out 'additionalProperties = False'. Causes vertexai/gemini OpenAI API Schema errors - https://github.com/langchain-ai/langchainjs/issues/5240
|
||||
|
||||
Relevant Issues: https://github.com/BerriAI/litellm/issues/6136, https://github.com/BerriAI/litellm/issues/6088
|
||||
"""
|
||||
if isinstance(schema, dict):
|
||||
# Remove the 'additionalProperties' key if it exists and is set to False
|
||||
if "additionalProperties" in schema and schema["additionalProperties"] is False:
|
||||
|
|
@ -2789,6 +2794,9 @@ def _remove_additional_properties(schema):
|
|||
|
||||
|
||||
def _remove_strict_from_schema(schema):
|
||||
"""
|
||||
Relevant Issues: https://github.com/BerriAI/litellm/issues/6136, https://github.com/BerriAI/litellm/issues/6088
|
||||
"""
|
||||
if isinstance(schema, dict):
|
||||
# Remove the 'additionalProperties' key if it exists and is set to False
|
||||
if "strict" in schema:
|
||||
|
|
@ -3000,37 +3008,6 @@ def get_optional_params(
|
|||
non_default_params["response_format"] = type_to_response_format_param(
|
||||
response_format=non_default_params["response_format"]
|
||||
)
|
||||
# # clean out 'additionalProperties = False'. Causes vertexai/gemini OpenAI API Schema errors - https://github.com/langchain-ai/langchainjs/issues/5240
|
||||
if (
|
||||
non_default_params["response_format"] is not None
|
||||
and non_default_params["response_format"]
|
||||
.get("json_schema", {})
|
||||
.get("schema")
|
||||
is not None
|
||||
and custom_llm_provider
|
||||
in [
|
||||
"gemini",
|
||||
"vertex_ai",
|
||||
"vertex_ai_beta",
|
||||
]
|
||||
):
|
||||
from litellm.llms.vertex_ai_and_google_ai_studio.common_utils import (
|
||||
_build_vertex_schema,
|
||||
)
|
||||
|
||||
old_schema = copy.deepcopy(
|
||||
non_default_params["response_format"]
|
||||
.get("json_schema", {})
|
||||
.get("schema")
|
||||
)
|
||||
new_schema = _remove_additional_properties(schema=old_schema)
|
||||
if isinstance(new_schema, list):
|
||||
for item in new_schema:
|
||||
if isinstance(item, dict):
|
||||
item = _build_vertex_schema(parameters=item)
|
||||
elif isinstance(new_schema, dict):
|
||||
new_schema = _build_vertex_schema(parameters=new_schema)
|
||||
non_default_params["response_format"]["json_schema"]["schema"] = new_schema
|
||||
if "tools" in non_default_params and isinstance(
|
||||
non_default_params, list
|
||||
): # fixes https://github.com/BerriAI/litellm/issues/4933
|
||||
|
|
@ -3197,7 +3174,7 @@ def get_optional_params(
|
|||
|
||||
if stream:
|
||||
optional_params["stream"] = stream
|
||||
#return optional_params
|
||||
# return optional_params
|
||||
if max_tokens is not None:
|
||||
if "vicuna" in model or "flan" in model:
|
||||
optional_params["max_length"] = max_tokens
|
||||
|
|
@ -4900,6 +4877,10 @@ def _strip_model_name(model: str) -> str:
|
|||
return strip_finetune
|
||||
|
||||
|
||||
def _get_model_info_from_model_cost(key: str) -> dict:
|
||||
return litellm.model_cost[key]
|
||||
|
||||
|
||||
def get_model_info(model: str, custom_llm_provider: Optional[str] = None) -> ModelInfo:
|
||||
"""
|
||||
Get a dict for the maximum tokens (context window), input_cost_per_token, output_cost_per_token for a given model.
|
||||
|
|
@ -5041,14 +5022,16 @@ def get_model_info(model: str, custom_llm_provider: Optional[str] = None) -> Mod
|
|||
"""
|
||||
Check if: (in order of specificity)
|
||||
1. 'custom_llm_provider/model' in litellm.model_cost. Checks "groq/llama3-8b-8192" if model="llama3-8b-8192" and custom_llm_provider="groq"
|
||||
2. 'combined_stripped_model_name' in litellm.model_cost. Checks if 'gemini/gemini-1.5-flash' in model map, if 'gemini/gemini-1.5-flash-001' given.
|
||||
3. 'stripped_model_name' in litellm.model_cost. Checks if 'ft:gpt-3.5-turbo' in model map, if 'ft:gpt-3.5-turbo:my-org:custom_suffix:id' given.
|
||||
4. 'model' in litellm.model_cost. Checks "groq/llama3-8b-8192" in litellm.model_cost if model="groq/llama3-8b-8192" and custom_llm_provider=None
|
||||
2. 'model' in litellm.model_cost. Checks "gemini-1.5-pro-002" in litellm.model_cost if model="gemini-1.5-pro-002" and custom_llm_provider=None
|
||||
3. 'combined_stripped_model_name' in litellm.model_cost. Checks if 'gemini/gemini-1.5-flash' in model map, if 'gemini/gemini-1.5-flash-001' given.
|
||||
4. 'stripped_model_name' in litellm.model_cost. Checks if 'ft:gpt-3.5-turbo' in model map, if 'ft:gpt-3.5-turbo:my-org:custom_suffix:id' given.
|
||||
5. 'split_model' in litellm.model_cost. Checks "llama3-8b-8192" in litellm.model_cost if model="groq/llama3-8b-8192"
|
||||
"""
|
||||
_model_info: Optional[Dict[str, Any]] = None
|
||||
key: Optional[str] = None
|
||||
if combined_model_name in litellm.model_cost:
|
||||
key = combined_model_name
|
||||
_model_info = litellm.model_cost[combined_model_name]
|
||||
_model_info = _get_model_info_from_model_cost(key=key)
|
||||
_model_info["supported_openai_params"] = supported_openai_params
|
||||
if (
|
||||
"litellm_provider" in _model_info
|
||||
|
|
@ -5059,58 +5042,10 @@ def get_model_info(model: str, custom_llm_provider: Optional[str] = None) -> Mod
|
|||
].startswith("vertex_ai"):
|
||||
pass
|
||||
else:
|
||||
raise Exception
|
||||
elif combined_stripped_model_name in litellm.model_cost:
|
||||
key = combined_stripped_model_name
|
||||
_model_info = litellm.model_cost[combined_stripped_model_name]
|
||||
_model_info["supported_openai_params"] = supported_openai_params
|
||||
if (
|
||||
"litellm_provider" in _model_info
|
||||
and _model_info["litellm_provider"] != custom_llm_provider
|
||||
):
|
||||
if custom_llm_provider == "vertex_ai" and _model_info[
|
||||
"litellm_provider"
|
||||
].startswith("vertex_ai"):
|
||||
pass
|
||||
elif custom_llm_provider == "fireworks_ai" and _model_info[
|
||||
"litellm_provider"
|
||||
].startswith("fireworks_ai"):
|
||||
pass
|
||||
else:
|
||||
raise Exception(
|
||||
"Got provider={}, Expected provider={}, for model={}".format(
|
||||
_model_info["litellm_provider"],
|
||||
custom_llm_provider,
|
||||
model,
|
||||
)
|
||||
)
|
||||
elif stripped_model_name in litellm.model_cost:
|
||||
key = stripped_model_name
|
||||
_model_info = litellm.model_cost[stripped_model_name]
|
||||
_model_info["supported_openai_params"] = supported_openai_params
|
||||
if (
|
||||
"litellm_provider" in _model_info
|
||||
and _model_info["litellm_provider"] != custom_llm_provider
|
||||
):
|
||||
if custom_llm_provider == "vertex_ai" and _model_info[
|
||||
"litellm_provider"
|
||||
].startswith("vertex_ai"):
|
||||
pass
|
||||
elif custom_llm_provider == "fireworks_ai" and _model_info[
|
||||
"litellm_provider"
|
||||
].startswith("fireworks_ai"):
|
||||
pass
|
||||
else:
|
||||
raise Exception(
|
||||
"Got provider={}, Expected provider={}, for model={}".format(
|
||||
_model_info["litellm_provider"],
|
||||
custom_llm_provider,
|
||||
model,
|
||||
)
|
||||
)
|
||||
elif model in litellm.model_cost:
|
||||
_model_info = None
|
||||
if _model_info is None and model in litellm.model_cost:
|
||||
key = model
|
||||
_model_info = litellm.model_cost[model]
|
||||
_model_info = _get_model_info_from_model_cost(key=key)
|
||||
_model_info["supported_openai_params"] = supported_openai_params
|
||||
if (
|
||||
"litellm_provider" in _model_info
|
||||
|
|
@ -5125,10 +5060,50 @@ def get_model_info(model: str, custom_llm_provider: Optional[str] = None) -> Mod
|
|||
].startswith("fireworks_ai"):
|
||||
pass
|
||||
else:
|
||||
raise Exception
|
||||
elif split_model in litellm.model_cost:
|
||||
_model_info = None
|
||||
if (
|
||||
_model_info is None
|
||||
and combined_stripped_model_name in litellm.model_cost
|
||||
):
|
||||
key = combined_stripped_model_name
|
||||
_model_info = _get_model_info_from_model_cost(key=key)
|
||||
_model_info["supported_openai_params"] = supported_openai_params
|
||||
if (
|
||||
"litellm_provider" in _model_info
|
||||
and _model_info["litellm_provider"] != custom_llm_provider
|
||||
):
|
||||
if custom_llm_provider == "vertex_ai" and _model_info[
|
||||
"litellm_provider"
|
||||
].startswith("vertex_ai"):
|
||||
pass
|
||||
elif custom_llm_provider == "fireworks_ai" and _model_info[
|
||||
"litellm_provider"
|
||||
].startswith("fireworks_ai"):
|
||||
pass
|
||||
else:
|
||||
_model_info = None
|
||||
if _model_info is None and stripped_model_name in litellm.model_cost:
|
||||
key = stripped_model_name
|
||||
_model_info = _get_model_info_from_model_cost(key=key)
|
||||
_model_info["supported_openai_params"] = supported_openai_params
|
||||
if (
|
||||
"litellm_provider" in _model_info
|
||||
and _model_info["litellm_provider"] != custom_llm_provider
|
||||
):
|
||||
if custom_llm_provider == "vertex_ai" and _model_info[
|
||||
"litellm_provider"
|
||||
].startswith("vertex_ai"):
|
||||
pass
|
||||
elif custom_llm_provider == "fireworks_ai" and _model_info[
|
||||
"litellm_provider"
|
||||
].startswith("fireworks_ai"):
|
||||
pass
|
||||
else:
|
||||
_model_info = None
|
||||
|
||||
if _model_info is None and split_model in litellm.model_cost:
|
||||
key = split_model
|
||||
_model_info = litellm.model_cost[split_model]
|
||||
_model_info = _get_model_info_from_model_cost(key=key)
|
||||
_model_info["supported_openai_params"] = supported_openai_params
|
||||
if (
|
||||
"litellm_provider" in _model_info
|
||||
|
|
@ -5143,8 +5118,8 @@ def get_model_info(model: str, custom_llm_provider: Optional[str] = None) -> Mod
|
|||
].startswith("fireworks_ai"):
|
||||
pass
|
||||
else:
|
||||
raise Exception
|
||||
else:
|
||||
_model_info = None
|
||||
if _model_info is None or key is None:
|
||||
raise ValueError(
|
||||
"This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
|
||||
)
|
||||
|
|
@ -5212,7 +5187,7 @@ def get_model_info(model: str, custom_llm_provider: Optional[str] = None) -> Mod
|
|||
litellm_provider=_model_info.get(
|
||||
"litellm_provider", custom_llm_provider
|
||||
),
|
||||
mode=_model_info.get("mode"),
|
||||
mode=_model_info.get("mode"), # type: ignore
|
||||
supported_openai_params=supported_openai_params,
|
||||
supports_system_messages=_model_info.get(
|
||||
"supports_system_messages", None
|
||||
|
|
@ -9260,10 +9235,6 @@ def process_response_headers(response_headers: Union[httpx.Headers, dict]) -> di
|
|||
processed_headers[k] = v
|
||||
else:
|
||||
additional_headers["{}-{}".format("llm_provider", k)] = v
|
||||
## GUARANTEE OPENAI HEADERS IN RESPONSE
|
||||
for item in OPENAI_RESPONSE_HEADERS:
|
||||
if item not in openai_headers:
|
||||
openai_headers[item] = None
|
||||
|
||||
additional_headers = {
|
||||
**openai_headers,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue