From c26b915ab1d896f346fc46fcb11da1e4e78ee9ee Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 19 Apr 2025 13:52:54 -0700 Subject: [PATCH] _update_kwargs_with_default_litellm_params --- ...odel_prices_and_context_window_backup.json | 29 ---------- litellm/router.py | 56 ++++++++++--------- 2 files changed, 30 insertions(+), 55 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b583de3b44..40d399aa68 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4979,35 +4979,6 @@ "supports_tool_choice": true }, "gemini-2.5-pro-exp-03-25": { - "max_tokens": 65536, - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_images_per_prompt": 3000, - "max_videos_per_prompt": 10, - "max_video_length": 1, - "max_audio_length_hours": 8.4, - "max_audio_per_prompt": 1, - "max_pdf_size_mb": 30, - "input_cost_per_token": 0, - "input_cost_per_token_above_200k_tokens": 0, - "output_cost_per_token": 0, - "output_cost_per_token_above_200k_tokens": 0, - "litellm_provider": "vertex_ai-language-models", - "mode": "chat", - "supports_system_messages": true, - "supports_function_calling": true, - "supports_vision": true, - "supports_audio_input": true, - "supports_video_input": true, - "supports_pdf_input": true, - "supports_response_schema": true, - "supports_tool_choice": true, - "supported_endpoints": ["/v1/chat/completions", "/v1/completions"], - "supported_modalities": ["text", "image", "audio", "video"], - "supported_output_modalities": ["text"], - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" - }, - "gemini-2.5-pro-preview-03-25": { "max_tokens": 65536, "max_input_tokens": 1048576, "max_output_tokens": 65536, diff --git a/litellm/router.py b/litellm/router.py index c934b1e9a8..92174adee3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -339,9 +339,9 @@ class Router: ) # names of models under litellm_params. ex. azure/chatgpt-v-2 self.deployment_latency_map = {} ### CACHING ### - cache_type: Literal[ - "local", "redis", "redis-semantic", "s3", "disk" - ] = "local" # default to an in-memory cache + cache_type: Literal["local", "redis", "redis-semantic", "s3", "disk"] = ( + "local" # default to an in-memory cache + ) redis_cache = None cache_config: Dict[str, Any] = {} @@ -562,9 +562,9 @@ class Router: ) ) - self.model_group_retry_policy: Optional[ - Dict[str, RetryPolicy] - ] = model_group_retry_policy + self.model_group_retry_policy: Optional[Dict[str, RetryPolicy]] = ( + model_group_retry_policy + ) self.allowed_fails_policy: Optional[AllowedFailsPolicy] = None if allowed_fails_policy is not None: @@ -1104,17 +1104,21 @@ class Router: ) -> None: """ Adds default litellm params to kwargs, if set. + + Handles inserting this as either "metadata" or "litellm_metadata" depending on the metadata_variable_name """ - self.default_litellm_params[ - metadata_variable_name - ] = self.default_litellm_params.pop("metadata", {}) - for k, v in self.default_litellm_params.items(): - if ( - k not in kwargs and v is not None - ): # prioritize model-specific params > default router params - kwargs[k] = v - elif k == metadata_variable_name: - kwargs[metadata_variable_name].update(v) + # 1) copy your defaults and pull out metadata + defaults = self.default_litellm_params.copy() + metadata_defaults = defaults.pop("metadata", {}) or {} + + # 2) add any non-metadata defaults that aren't already in kwargs + for key, value in defaults.items(): + if value is None: + continue + kwargs.setdefault(key, value) + + # 3) merge in metadata, this handles inserting this as either "metadata" or "litellm_metadata" + kwargs.setdefault(metadata_variable_name, {}).update(metadata_defaults) def _handle_clientside_credential( self, deployment: dict, kwargs: dict @@ -3243,11 +3247,11 @@ class Router: if isinstance(e, litellm.ContextWindowExceededError): if context_window_fallbacks is not None: - fallback_model_group: Optional[ - List[str] - ] = self._get_fallback_model_group_from_fallbacks( - fallbacks=context_window_fallbacks, - model_group=model_group, + fallback_model_group: Optional[List[str]] = ( + self._get_fallback_model_group_from_fallbacks( + fallbacks=context_window_fallbacks, + model_group=model_group, + ) ) if fallback_model_group is None: raise original_exception @@ -3279,11 +3283,11 @@ class Router: e.message += "\n{}".format(error_message) elif isinstance(e, litellm.ContentPolicyViolationError): if content_policy_fallbacks is not None: - fallback_model_group: Optional[ - List[str] - ] = self._get_fallback_model_group_from_fallbacks( - fallbacks=content_policy_fallbacks, - model_group=model_group, + fallback_model_group: Optional[List[str]] = ( + self._get_fallback_model_group_from_fallbacks( + fallbacks=content_policy_fallbacks, + model_group=model_group, + ) ) if fallback_model_group is None: raise original_exception