From 0415f1205e9eb38352d3eae8c65fec3002e54dae Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Fri, 11 Apr 2025 09:29:42 -0700 Subject: [PATCH] Litellm dev 04 10 2025 p3 (#9903) * feat(managed_files.py): encode file type in unified file id simplify calling gemini models * fix(common_utils.py): fix extracting file type from unified file id * fix(litellm_logging.py): create standard logging payload for create file call * fix: fix linting error --- litellm/litellm_core_utils/litellm_logging.py | 3 ++ .../prompt_templates/common_utils.py | 33 ++++++++++++++++++- litellm/proxy/_new_secret_config.yaml | 5 +-- litellm/proxy/_types.py | 4 --- litellm/proxy/hooks/managed_files.py | 21 +++++++++--- .../openai_files_endpoints/files_endpoints.py | 13 ++++++-- litellm/router.py | 3 -- litellm/types/llms/openai.py | 9 +++-- litellm/types/utils.py | 5 +++ ...ore_utils_prompt_templates_common_utils.py | 25 ++++++++++++++ 10 files changed, 99 insertions(+), 22 deletions(-) create mode 100644 tests/litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index ad6abd5575..af59901b33 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -56,6 +56,7 @@ from litellm.types.llms.openai import ( Batch, FineTuningJob, HttpxBinaryResponseContent, + OpenAIFileObject, ResponseCompletedEvent, ResponsesAPIResponse, ) @@ -902,6 +903,7 @@ class Logging(LiteLLMLoggingBaseClass): FineTuningJob, ResponsesAPIResponse, ResponseCompletedEvent, + OpenAIFileObject, LiteLLMRealtimeStreamLoggingObject, ], cache_hit: Optional[bool] = None, @@ -1095,6 +1097,7 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(logging_result, FineTuningJob) or isinstance(logging_result, LiteLLMBatch) or isinstance(logging_result, ResponsesAPIResponse) + or isinstance(logging_result, OpenAIFileObject) or isinstance(logging_result, LiteLLMRealtimeStreamLoggingObject) ): ## HIDDEN PARAMS ## diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 44b680d487..6d11cef325 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -4,6 +4,7 @@ Common utility functions used for translating messages across providers import io import mimetypes +import re from os import PathLike from typing import Dict, List, Literal, Mapping, Optional, Union, cast @@ -18,6 +19,7 @@ from litellm.types.utils import ( ExtractedFileData, FileTypes, ModelResponse, + SpecialEnums, StreamingChoices, ) @@ -325,6 +327,29 @@ def get_file_ids_from_messages(messages: List[AllMessageValues]) -> List[str]: return file_ids +def get_format_from_file_id(file_id: Optional[str]) -> Optional[str]: + """ + Gets format from file id + + unified_file_id = litellm_proxy:{};unified_id,{} + If not a unified file id, returns 'file' as default format + """ + if not file_id: + return None + try: + if file_id.startswith(SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value): + match = re.match( + f"{SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value}:(.*?);unified_id", + file_id, + ) + if match: + return match.group(1) + + return None + except Exception: + return None + + def update_messages_with_model_file_ids( messages: List[AllMessageValues], model_id: str, @@ -350,12 +375,18 @@ def update_messages_with_model_file_ids( file_object = cast(ChatCompletionFileObject, c) file_object_file_field = file_object["file"] file_id = file_object_file_field.get("file_id") + format = file_object_file_field.get( + "format", get_format_from_file_id(file_id) + ) + if file_id: provider_file_id = ( model_file_id_mapping.get(file_id, {}).get(model_id) or file_id ) file_object_file_field["file_id"] = provider_file_id + if format: + file_object_file_field["format"] = format return messages @@ -421,6 +452,7 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData: headers=file_headers, ) + def unpack_defs(schema, defs): properties = schema.get("properties", None) if properties is None: @@ -452,4 +484,3 @@ def unpack_defs(schema, defs): unpack_defs(ref, defs) value["items"] = ref continue - diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 3a3f3670cb..a6d90e4643 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -11,10 +11,6 @@ model_list: - model_name: "bedrock-nova" litellm_params: model: us.amazon.nova-pro-v1:0 - - model_name: "gemini-2.0-flash" - litellm_params: - model: gemini/gemini-2.0-flash - api_key: os.environ/GEMINI_API_KEY - model_name: openrouter_model litellm_params: model: openrouter/openrouter_model @@ -33,6 +29,7 @@ model_list: model_info: base_model: azure/gpt-4o-realtime-preview-2024-10-01 + litellm_settings: num_retries: 0 callbacks: ["prometheus"] diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 9a2a06fae9..456ef93496 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2692,10 +2692,6 @@ class PrismaCompatibleUpdateDBModel(TypedDict, total=False): updated_by: str -class SpecialEnums(enum.Enum): - LITELM_MANAGED_FILE_ID_PREFIX = "litellm_proxy/" - - class SpecialManagementEndpointEnums(enum.Enum): DEFAULT_ORGANIZATION = "default_organization" diff --git a/litellm/proxy/hooks/managed_files.py b/litellm/proxy/hooks/managed_files.py index 2d8d303931..2dd63171fd 100644 --- a/litellm/proxy/hooks/managed_files.py +++ b/litellm/proxy/hooks/managed_files.py @@ -9,10 +9,16 @@ from litellm import verbose_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.prompt_templates.common_utils import ( + extract_file_data, get_file_ids_from_messages, ) -from litellm.proxy._types import CallTypes, SpecialEnums, UserAPIKeyAuth -from litellm.types.llms.openai import OpenAIFileObject, OpenAIFilesPurpose +from litellm.proxy._types import CallTypes, UserAPIKeyAuth +from litellm.types.llms.openai import ( + CreateFileRequest, + OpenAIFileObject, + OpenAIFilesPurpose, +) +from litellm.types.utils import SpecialEnums if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -106,14 +112,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger): @staticmethod async def return_unified_file_id( file_objects: List[OpenAIFileObject], + create_file_request: CreateFileRequest, purpose: OpenAIFilesPurpose, internal_usage_cache: InternalUsageCache, litellm_parent_otel_span: Span, ) -> OpenAIFileObject: - unified_file_id = SpecialEnums.LITELM_MANAGED_FILE_ID_PREFIX.value + str( - uuid.uuid4() + ## GET THE FILE TYPE FROM THE CREATE FILE REQUEST + file_data = extract_file_data(create_file_request["file"]) + + file_type = file_data["content_type"] + + unified_file_id = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format( + file_type, str(uuid.uuid4()) ) + ## CREATE RESPONSE OBJECT ## CREATE RESPONSE OBJECT response = OpenAIFileObject( id=unified_file_id, diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index a26b04aebc..77458d9889 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -34,7 +34,11 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( from litellm.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles from litellm.proxy.utils import ProxyLogging from litellm.router import Router -from litellm.types.llms.openai import OpenAIFileObject, OpenAIFilesPurpose +from litellm.types.llms.openai import ( + CREATE_FILE_REQUESTS_PURPOSE, + OpenAIFileObject, + OpenAIFilesPurpose, +) router = APIRouter() @@ -147,6 +151,7 @@ async def create_file_for_each_model( responses.append(individual_response) response = await _PROXY_LiteLLMManagedFiles.return_unified_file_id( file_objects=responses, + create_file_request=_create_file_request, purpose=purpose, internal_usage_cache=proxy_logging_obj.internal_usage_cache, litellm_parent_otel_span=user_api_key_dict.parent_otel_span, @@ -232,7 +237,7 @@ async def create_file( # Cast purpose to OpenAIFilesPurpose type purpose = cast(OpenAIFilesPurpose, purpose) - data = {"purpose": purpose} + data = {} # Include original request and headers in the data data = await add_litellm_data_to_request( @@ -258,7 +263,9 @@ async def create_file( model=router_model, llm_router=llm_router ) - _create_file_request = CreateFileRequest(file=file_data, **data) + _create_file_request = CreateFileRequest( + file=file_data, purpose=cast(CREATE_FILE_REQUESTS_PURPOSE, purpose), **data + ) response: Optional[OpenAIFileObject] = None if ( diff --git a/litellm/router.py b/litellm/router.py index 9008a2bfe4..39b3ddcc16 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2760,9 +2760,6 @@ class Router: stripped_model, custom_llm_provider, _, _ = get_llm_provider( model=data["model"] ) - # kwargs["file"] = replace_model_in_jsonl( - # file_content=kwargs["file"], new_model_name=stripped_model - # ) response = litellm.acreate_file( **{ diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 483d7c4307..f36039cf44 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -288,7 +288,10 @@ class OpenAIFileObject(BaseModel): `error` field on `fine_tuning.job`. """ - _hidden_params: dict = {} + _hidden_params: dict = {"response_cost": 0.0} # no cost for writing a file + + +CREATE_FILE_REQUESTS_PURPOSE = Literal["assistants", "batch", "fine-tune"] # OpenAI Files Types @@ -307,8 +310,8 @@ class CreateFileRequest(TypedDict, total=False): timeout: Optional[float] = None """ - file: FileTypes - purpose: Literal["assistants", "batch", "fine-tune"] + file: Required[FileTypes] + purpose: Required[CREATE_FILE_REQUESTS_PURPOSE] extra_headers: Optional[Dict[str, str]] extra_body: Optional[Dict[str, str]] timeout: Optional[float] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d7d15bbccc..1bbec44b82 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2221,3 +2221,8 @@ class ExtractedFileData(TypedDict): content: bytes content_type: Optional[str] headers: Mapping[str, str] + + +class SpecialEnums(Enum): + LITELM_MANAGED_FILE_ID_PREFIX = "litellm_proxy" + LITELLM_MANAGED_FILE_COMPLETE_STR = "litellm_proxy:{};unified_id,{}" diff --git a/tests/litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py new file mode 100644 index 0000000000..2190c8cff0 --- /dev/null +++ b/tests/litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -0,0 +1,25 @@ +import json +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + get_format_from_file_id, + update_messages_with_model_file_ids, +) + + +def test_update_messages_with_model_file_ids(): + unified_file_id = ( + "litellm_proxy:application/pdf;unified_id,cbbe3534-8bf8-4386-af00-f5f6b7e370bf" + ) + + format = get_format_from_file_id(unified_file_id) + + assert format == "application/pdf"