diff --git a/tests/litellm/test_model_prices_and_context_window_schema.py b/tests/litellm/test_model_prices_and_context_window_schema.py deleted file mode 100644 index 085512da88..0000000000 --- a/tests/litellm/test_model_prices_and_context_window_schema.py +++ /dev/null @@ -1,108 +0,0 @@ -import json -from jsonschema import validate - -def test_model_prices_and_context_window_json_is_valid(): - ''' - Validates the `model_prices_and_context_window.json` file. - - If this test fails after you update the json, you need to update the schema or correct the change you made. - ''' - - INTENDED_SCHEMA = { - "type": "object", - "additionalProperties": { - "type": "object", - "properties": { - "cache_creation_input_audio_token_cost": {"type": "number"}, - "cache_creation_input_token_cost": {"type": "number"}, - "cache_read_input_token_cost": {"type": "number"}, - "deprecation_date": {"type": "string"}, - "input_cost_per_audio_per_second": {"type": "number"}, - "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, - "input_cost_per_audio_token": {"type": "number"}, - "input_cost_per_character": {"type": "number"}, - "input_cost_per_character_above_128k_tokens": {"type": "number"}, - "input_cost_per_image": {"type": "number"}, - "input_cost_per_image_above_128k_tokens": {"type": "number"}, - "input_cost_per_pixel": {"type": "number"}, - "input_cost_per_query": {"type": "number"}, - "input_cost_per_request": {"type": "number"}, - "input_cost_per_second": {"type": "number"}, - "input_cost_per_token": {"type": "number"}, - "input_cost_per_token_above_128k_tokens": {"type": "number"}, - "input_cost_per_token_batch_requests": {"type": "number"}, - "input_cost_per_token_batches": {"type": "number"}, - "input_cost_per_token_cache_hit": {"type": "number"}, - "input_cost_per_video_per_second": {"type": "number"}, - "input_cost_per_video_per_second_above_128k_tokens": {"type": "number"}, - "input_dbu_cost_per_token": {"type": "number"}, - "litellm_provider": {"type": "string"}, - "max_audio_length_hours": {"type": "number"}, - "max_audio_per_prompt": {"type": "number"}, - "max_document_chunks_per_query": {"type": "number"}, - "max_images_per_prompt": {"type": "number"}, - "max_input_tokens": {"type": "number"}, - "max_output_tokens": {"type": "number"}, - "max_pdf_size_mb": {"type": "number"}, - "max_query_tokens": {"type": "number"}, - "max_tokens": {"type": "number"}, - "max_tokens_per_document_chunk": {"type": "number"}, - "max_video_length": {"type": "number"}, - "max_videos_per_prompt": {"type": "number"}, - "metadata": {"type": "object"}, - "mode": { - "type": "string", - "enum": [ - "audio_speech", - "audio_transcription", - "chat", - "completion", - "embedding", - "image_generation", - "moderation", - "rerank" - ], - }, - "output_cost_per_audio_token": {"type": "number"}, - "output_cost_per_character": {"type": "number"}, - "output_cost_per_character_above_128k_tokens": {"type": "number"}, - "output_cost_per_image": {"type": "number"}, - "output_cost_per_pixel": {"type": "number"}, - "output_cost_per_second": {"type": "number"}, - "output_cost_per_token": {"type": "number"}, - "output_cost_per_token_above_128k_tokens": {"type": "number"}, - "output_cost_per_token_batches": {"type": "number"}, - "output_db_cost_per_token": {"type": "number"}, - "output_dbu_cost_per_token": {"type": "number"}, - "output_vector_size": {"type": "number"}, - "rpd": {"type": "number"}, - "rpm": {"type": "number"}, - "source": {"type": "string"}, - "supports_assistant_prefill": {"type": "boolean"}, - "supports_audio_input": {"type": "boolean"}, - "supports_audio_output": {"type": "boolean"}, - "supports_embedding_image_input": {"type": "boolean"}, - "supports_function_calling": {"type": "boolean"}, - "supports_image_input": {"type": "boolean"}, - "supports_parallel_function_calling": {"type": "boolean"}, - "supports_pdf_input": {"type": "boolean"}, - "supports_prompt_caching": {"type": "boolean"}, - "supports_response_schema": {"type": "boolean"}, - "supports_system_messages": {"type": "boolean"}, - "supports_tool_choice": {"type": "boolean"}, - "supports_video_input": {"type": "boolean"}, - "supports_vision": {"type": "boolean"}, - "tool_use_system_prompt_tokens": {"type": "number"}, - "tpm": {"type": "number"}, - }, - "additionalProperties": False, - }, - } - - with open("./model_prices_and_context_window.json", "r") as model_prices_file: - actual_json = json.load(model_prices_file) - assert isinstance(actual_json, dict) - actual_json.pop('sample_spec', None) # remove the sample, whose schema is inconsistent with the real data - - validate(actual_json, INTENDED_SCHEMA) - diff --git a/tests/local_testing/test_get_model_info.py b/tests/local_testing/test_get_model_info.py index 57da99135e..d71f3f7c24 100644 --- a/tests/local_testing/test_get_model_info.py +++ b/tests/local_testing/test_get_model_info.py @@ -3,6 +3,9 @@ import os import sys import traceback +import json + +from jsonschema import validate from typing import List, Dict, Any @@ -403,3 +406,137 @@ def test_get_model_info_cost_calculator_bedrock_region_cris_stripped(model, prov print("info", info) assert info["key"] == "us.anthropic.claude-3-haiku-20240307-v1:0" assert info["litellm_provider"] == "bedrock" + + +def test_aaamodel_prices_and_context_window_json_is_valid(): + """ + Validates the `model_prices_and_context_window.json` file. + + If this test fails after you update the json, you need to update the schema or correct the change you made. + """ + + INTENDED_SCHEMA = { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "cache_creation_input_audio_token_cost": {"type": "number"}, + "cache_creation_input_token_cost": {"type": "number"}, + "cache_read_input_token_cost": {"type": "number"}, + "deprecation_date": {"type": "string"}, + "input_cost_per_audio_per_second": {"type": "number"}, + "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, + "input_cost_per_audio_token": {"type": "number"}, + "input_cost_per_character": {"type": "number"}, + "input_cost_per_character_above_128k_tokens": {"type": "number"}, + "input_cost_per_image": {"type": "number"}, + "input_cost_per_image_above_128k_tokens": {"type": "number"}, + "input_cost_per_pixel": {"type": "number"}, + "input_cost_per_query": {"type": "number"}, + "input_cost_per_request": {"type": "number"}, + "input_cost_per_second": {"type": "number"}, + "input_cost_per_token": {"type": "number"}, + "input_cost_per_token_above_128k_tokens": {"type": "number"}, + "input_cost_per_token_batch_requests": {"type": "number"}, + "input_cost_per_token_batches": {"type": "number"}, + "input_cost_per_token_cache_hit": {"type": "number"}, + "input_cost_per_video_per_second": {"type": "number"}, + "input_cost_per_video_per_second_above_128k_tokens": {"type": "number"}, + "input_dbu_cost_per_token": {"type": "number"}, + "litellm_provider": {"type": "string"}, + "max_audio_length_hours": {"type": "number"}, + "max_audio_per_prompt": {"type": "number"}, + "max_document_chunks_per_query": {"type": "number"}, + "max_images_per_prompt": {"type": "number"}, + "max_input_tokens": {"type": "number"}, + "max_output_tokens": {"type": "number"}, + "max_pdf_size_mb": {"type": "number"}, + "max_query_tokens": {"type": "number"}, + "max_tokens": {"type": "number"}, + "max_tokens_per_document_chunk": {"type": "number"}, + "max_video_length": {"type": "number"}, + "max_videos_per_prompt": {"type": "number"}, + "metadata": {"type": "object"}, + "mode": { + "type": "string", + "enum": [ + "audio_speech", + "audio_transcription", + "chat", + "completion", + "embedding", + "image_generation", + "moderation", + "rerank", + "responses", + ], + }, + "output_cost_per_audio_token": {"type": "number"}, + "output_cost_per_character": {"type": "number"}, + "output_cost_per_character_above_128k_tokens": {"type": "number"}, + "output_cost_per_image": {"type": "number"}, + "output_cost_per_pixel": {"type": "number"}, + "output_cost_per_second": {"type": "number"}, + "output_cost_per_token": {"type": "number"}, + "output_cost_per_token_above_128k_tokens": {"type": "number"}, + "output_cost_per_token_batches": {"type": "number"}, + "output_db_cost_per_token": {"type": "number"}, + "output_dbu_cost_per_token": {"type": "number"}, + "output_vector_size": {"type": "number"}, + "rpd": {"type": "number"}, + "rpm": {"type": "number"}, + "source": {"type": "string"}, + "supports_assistant_prefill": {"type": "boolean"}, + "supports_audio_input": {"type": "boolean"}, + "supports_audio_output": {"type": "boolean"}, + "supports_embedding_image_input": {"type": "boolean"}, + "supports_function_calling": {"type": "boolean"}, + "supports_image_input": {"type": "boolean"}, + "supports_parallel_function_calling": {"type": "boolean"}, + "supports_pdf_input": {"type": "boolean"}, + "supports_prompt_caching": {"type": "boolean"}, + "supports_response_schema": {"type": "boolean"}, + "supports_system_messages": {"type": "boolean"}, + "supports_tool_choice": {"type": "boolean"}, + "supports_video_input": {"type": "boolean"}, + "supports_vision": {"type": "boolean"}, + "tool_use_system_prompt_tokens": {"type": "number"}, + "tpm": {"type": "number"}, + "supported_endpoints": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "/v1/responses", + "/v1/embeddings", + "/v1/chat/completions", + "/v1/completions", + "/v1/images/generations", + "/v1/images/variations", + "/v1/images/edits", + "/v1/batch", + ], + }, + }, + "supported_modalities": { + "type": "array", + "items": { + "type": "string", + "enum": ["text", "audio", "image", "video"], + }, + }, + "supports_native_streaming": {"type": "boolean"}, + }, + "additionalProperties": False, + }, + } + + prod_json = "./model_prices_and_context_window.json" + with open(prod_json, "r") as model_prices_file: + actual_json = json.load(model_prices_file) + assert isinstance(actual_json, dict) + actual_json.pop( + "sample_spec", None + ) # remove the sample, whose schema is inconsistent with the real data + + validate(actual_json, INTENDED_SCHEMA)