@@ -28,7 +28,7 @@ LiteLLM manages:
- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints
- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']`
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
-- Track spend & set budgets per project [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy)
+- Set Budgets & Rate limits per project [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy)
[**Jump to OpenAI Proxy Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs) diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 0c5fd6bb9..3f5596fc9 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -8,6 +8,7 @@ import TabItem from '@theme/TabItem'; Log Proxy Input, Output, Exceptions using Custom Callbacks, Langfuse, OpenTelemetry, LangFuse, DynamoDB, s3 Bucket - [Async Custom Callbacks](#custom-callback-class-async) +- [Async Custom Callback APIs](#custom-callback-apis-async) - [Logging to Langfuse](#logging-proxy-inputoutput---langfuse) - [Logging to s3 Buckets](#logging-proxy-inputoutput---s3-buckets) - [Logging to DynamoDB](#logging-proxy-inputoutput---dynamodb) @@ -297,6 +298,106 @@ ModelResponse( ``` +## Custom Callback APIs [Async] + +:::info + +This is an Enterprise only feature [Get Started with Enterprise here](https://github.com/BerriAI/litellm/tree/main/enterprise) + +::: + +Use this if you: +- Want to use custom callbacks written in a non Python programming language +- Want your callbacks to run on a different microservice + +#### Step 1. Create your generic logging API endpoint +Set up a generic API endpoint that can receive data in JSON format. The data will be included within a "data" field. + +Your server should support the following Request format: + +```shell +curl --location https://your-domain.com/log-event \ + --request POST \ + --header "Content-Type: application/json" \ + --data '{ + "data": { + "id": "chatcmpl-8sgE89cEQ4q9biRtxMvDfQU1O82PT", + "call_type": "acompletion", + "cache_hit": "None", + "startTime": "2024-02-15 16:18:44.336280", + "endTime": "2024-02-15 16:18:45.045539", + "model": "gpt-3.5-turbo", + "user": "ishaan-2", + "modelParameters": "{'temperature': 0.7, 'max_tokens': 10, 'user': 'ishaan-2', 'extra_body': {}}", + "messages": "[{'role': 'user', 'content': 'This is a test'}]", + "response": "ModelResponse(id='chatcmpl-8sgE89cEQ4q9biRtxMvDfQU1O82PT', choices=[Choices(finish_reason='length', index=0, message=Message(content='Great! How can I assist you with this test', role='assistant'))], created=1708042724, model='gpt-3.5-turbo-0613', object='chat.completion', system_fingerprint=None, usage=Usage(completion_tokens=10, prompt_tokens=11, total_tokens=21))", + "usage": "Usage(completion_tokens=10, prompt_tokens=11, total_tokens=21)", + "metadata": "{}", + "cost": "3.65e-05" + } + }' +``` + +Reference FastAPI Python Server + +Here's a reference FastAPI Server that is compatible with LiteLLM Proxy: + +```python +# this is an example endpoint to receive data from litellm +from fastapi import FastAPI, HTTPException, Request + +app = FastAPI() + + +@app.post("/log-event") +async def log_event(request: Request): + try: + print("Received /log-event request") + # Assuming the incoming request has JSON data + data = await request.json() + print("Received request data:") + print(data) + + # Your additional logic can go here + # For now, just printing the received data + + return {"message": "Request received successfully"} + except Exception as e: + print(f"Error processing request: {str(e)}") + import traceback + + traceback.print_exc() + raise HTTPException(status_code=500, detail="Internal Server Error") + + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="127.0.0.1", port=8000) + + +``` + + +#### Step 2. Set your `GENERIC_LOGGER_ENDPOINT` to the endpoint + route we should send callback logs to + +```shell +os.environ["GENERIC_LOGGER_ENDPOINT"] = "http://localhost:8000/log-event" +``` + +#### Step 3. Create a `config.yaml` file and set `litellm_settings`: `success_callback` = ["generic"] + +Example litellm proxy config.yaml +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo +litellm_settings: + success_callback: ["generic"] +``` + +Start the LiteLLM Proxy and make a test request to verify the logs reached your callback API + ## Logging Proxy Input/Output - Langfuse We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this will log all successfull LLM calls to langfuse diff --git a/enterprise/README.md b/enterprise/README.md index 0818e9a58..fd7e68fd2 100644 --- a/enterprise/README.md +++ b/enterprise/README.md @@ -1,11 +1,12 @@ ## LiteLLM Enterprise -Code in this folder is licensed under a commercial license. Please review the [LICENSE](/LICENSE.md) file within the /enterprise folder +Code in this folder is licensed under a commercial license. Please review the [LICENSE](./LICENSE.md) file within the /enterprise folder **These features are covered under the LiteLLM Enterprise contract** 👉 **Using in an Enterprise / Need specific features ?** Meet with us [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat?month=2024-02) ## Features: -- Custom API / microservice callbacks +- Custom API / microservice callbacks +- Google Text Moderation API diff --git a/enterprise/callbacks/example_logging_api.py b/enterprise/callbacks/example_logging_api.py new file mode 100644 index 000000000..57ea99a67 --- /dev/null +++ b/enterprise/callbacks/example_logging_api.py @@ -0,0 +1,31 @@ +# this is an example endpoint to receive data from litellm +from fastapi import FastAPI, HTTPException, Request + +app = FastAPI() + + +@app.post("/log-event") +async def log_event(request: Request): + try: + print("Received /log-event request") + # Assuming the incoming request has JSON data + data = await request.json() + print("Received request data:") + print(data) + + # Your additional logic can go here + # For now, just printing the received data + + return {"message": "Request received successfully"} + except Exception as e: + print(f"Error processing request: {str(e)}") + import traceback + + traceback.print_exc() + raise HTTPException(status_code=500, detail="Internal Server Error") + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="127.0.0.1", port=8000) diff --git a/enterprise/callbacks/generic_api_callback.py b/enterprise/callbacks/generic_api_callback.py new file mode 100644 index 000000000..076c13d5e --- /dev/null +++ b/enterprise/callbacks/generic_api_callback.py @@ -0,0 +1,128 @@ +# callback to make a request to an API endpoint + +#### What this does #### +# On success, logs events to Promptlayer +import dotenv, os +import requests + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.caching import DualCache + +from typing import Literal, Union + +dotenv.load_dotenv() # Loading env variables using dotenv +import traceback + + +#### What this does #### +# On success + failure, log events to Supabase + +import dotenv, os +import requests + +dotenv.load_dotenv() # Loading env variables using dotenv +import traceback +import datetime, subprocess, sys +import litellm, uuid +from litellm._logging import print_verbose, verbose_logger + + +class GenericAPILogger: + # Class variables or attributes + def __init__(self, endpoint=None, headers=None): + try: + if endpoint == None: + # check env for "GENERIC_LOGGER_ENDPOINT" + if os.getenv("GENERIC_LOGGER_ENDPOINT"): + # Do something with the endpoint + endpoint = os.getenv("GENERIC_LOGGER_ENDPOINT") + else: + # Handle the case when the endpoint is not found in the environment variables + raise ValueError( + f"endpoint not set for GenericAPILogger, GENERIC_LOGGER_ENDPOINT not found in environment variables" + ) + headers = headers or litellm.generic_logger_headers + self.endpoint = endpoint + self.headers = headers + + verbose_logger.debug( + f"in init GenericAPILogger, endpoint {self.endpoint}, headers {self.headers}" + ) + + pass + + except Exception as e: + print_verbose(f"Got exception on init GenericAPILogger client {str(e)}") + raise e + + # This is sync, because we run this in a separate thread. Running in a sepearate thread ensures it will never block an LLM API call + # Experience with s3, Langfuse shows that async logging events are complicated and can block LLM calls + def log_event( + self, kwargs, response_obj, start_time, end_time, user_id, print_verbose + ): + try: + verbose_logger.debug( + f"GenericAPILogger Logging - Enters logging function for model {kwargs}" + ) + + # construct payload to send custom logger + # follows the same params as langfuse.py + litellm_params = kwargs.get("litellm_params", {}) + metadata = ( + litellm_params.get("metadata", {}) or {} + ) # if litellm_params['metadata'] == None + messages = kwargs.get("messages") + cost = kwargs.get("response_cost", 0.0) + optional_params = kwargs.get("optional_params", {}) + call_type = kwargs.get("call_type", "litellm.completion") + cache_hit = kwargs.get("cache_hit", False) + usage = response_obj["usage"] + id = response_obj.get("id", str(uuid.uuid4())) + + # Build the initial payload + payload = { + "id": id, + "call_type": call_type, + "cache_hit": cache_hit, + "startTime": start_time, + "endTime": end_time, + "model": kwargs.get("model", ""), + "user": kwargs.get("user", ""), + "modelParameters": optional_params, + "messages": messages, + "response": response_obj, + "usage": usage, + "metadata": metadata, + "cost": cost, + } + + # Ensure everything in the payload is converted to str + for key, value in payload.items(): + try: + payload[key] = str(value) + except: + # non blocking if it can't cast to a str + pass + + import json + + data = { + "data": payload, + } + data = json.dumps(data) + print_verbose(f"\nGeneric Logger - Logging payload = {data}") + + # make request to endpoint with payload + response = requests.post(self.endpoint, json=data, headers=self.headers) + + response_status = response.status_code + response_text = response.text + + print_verbose( + f"Generic Logger - final response status = {response_status}, response text = {response_text}" + ) + return response + except Exception as e: + traceback.print_exc() + verbose_logger.debug(f"Generic - {str(e)}\n{traceback.format_exc()}") + pass diff --git a/enterprise/hooks/google_text_moderation.py b/enterprise/hooks/google_text_moderation.py new file mode 100644 index 000000000..b79a32805 --- /dev/null +++ b/enterprise/hooks/google_text_moderation.py @@ -0,0 +1,53 @@ +# +-----------------------------------------------+ +# +# Google Text Moderation +# https://cloud.google.com/natural-language/docs/moderating-text +# +# +-----------------------------------------------+ +# Thank you users! We ❤️ you! - Krrish & Ishaan + + +from typing import Optional, Literal, Union +import litellm, traceback, sys, uuid +from litellm.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth +from litellm.integrations.custom_logger import CustomLogger +from fastapi import HTTPException +from litellm._logging import verbose_proxy_logger +from litellm.utils import ( + ModelResponse, + EmbeddingResponse, + ImageResponse, + StreamingChoices, +) +from datetime import datetime +import aiohttp, asyncio + + +class _ENTERPRISE_GoogleTextModeration(CustomLogger): + user_api_key_cache = None + + # Class variables or attributes + def __init__(self, mock_testing: bool = False): + pass + + def print_verbose(self, print_statement): + try: + verbose_proxy_logger.debug(print_statement) + if litellm.set_verbose: + print(print_statement) # noqa + except: + pass + + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict, + call_type: str, + ): + """ + - Calls Google's Text Moderation API + - Rejects request if it fails safety check + """ + pass diff --git a/litellm/__init__.py b/litellm/__init__.py index 00d7488c9..a7f232f76 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -146,6 +146,7 @@ model_cost_map_url: str = "https://raw.githubusercontent.com/BerriAI/litellm/mai suppress_debug_info = False dynamodb_table_name: Optional[str] = None s3_callback_params: Optional[Dict] = None +generic_logger_headers: Optional[Dict] = None default_key_generate_params: Optional[Dict] = None upperbound_key_generate_params: Optional[Dict] = None default_team_settings: Optional[List] = None diff --git a/litellm/_redis.py b/litellm/_redis.py index 4484926d4..69ff6f3f2 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -98,6 +98,9 @@ def _get_redis_client_logic(**env_overrides): def get_redis_client(**env_overrides): redis_kwargs = _get_redis_client_logic(**env_overrides) if "url" in redis_kwargs and redis_kwargs["url"] is not None: + redis_kwargs.pop( + "connection_pool", None + ) # redis.from_url doesn't support setting your own connection pool return redis.Redis.from_url(**redis_kwargs) return redis.Redis(**redis_kwargs) @@ -105,6 +108,9 @@ def get_redis_client(**env_overrides): def get_redis_async_client(**env_overrides): redis_kwargs = _get_redis_client_logic(**env_overrides) if "url" in redis_kwargs and redis_kwargs["url"] is not None: + redis_kwargs.pop( + "connection_pool", None + ) # redis.from_url doesn't support setting your own connection pool return async_redis.Redis.from_url(**redis_kwargs) return async_redis.Redis( socket_timeout=5, diff --git a/litellm/caching.py b/litellm/caching.py index 564972068..567b9aadb 100644 --- a/litellm/caching.py +++ b/litellm/caching.py @@ -124,7 +124,7 @@ class RedisCache(BaseCache): self.redis_client.set(name=key, value=str(value), ex=ttl) except Exception as e: # NON blocking - notify users Redis is throwing an exception - logging.debug("LiteLLM Caching: set() - Got exception from REDIS : ", e) + print_verbose("LiteLLM Caching: set() - Got exception from REDIS : ", e) async def async_set_cache(self, key, value, **kwargs): _redis_client = self.init_async_client() @@ -134,10 +134,12 @@ class RedisCache(BaseCache): f"Set ASYNC Redis Cache: key: {key}\nValue {value}\nttl={ttl}" ) try: - await redis_client.set(name=key, value=json.dumps(value), ex=ttl) + await redis_client.set( + name=key, value=json.dumps(value), ex=ttl, get=True + ) except Exception as e: # NON blocking - notify users Redis is throwing an exception - logging.debug("LiteLLM Caching: set() - Got exception from REDIS : ", e) + print_verbose("LiteLLM Caching: set() - Got exception from REDIS : ", e) async def async_set_cache_pipeline(self, cache_list, ttl=None): """ diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index b548f22bc..7ce843339 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -259,6 +259,7 @@ class LangFuseLogger: if key in [ "user_api_key", "user_api_key_user_id", + "user_api_key_team_id", "semantic-similarity", ]: tags.append(f"{key}:{value}") diff --git a/litellm/llms/vertex_ai.py b/litellm/llms/vertex_ai.py index 816ded394..2e5fb3dd3 100644 --- a/litellm/llms/vertex_ai.py +++ b/litellm/llms/vertex_ai.py @@ -343,24 +343,31 @@ def completion( llm_model = CodeChatModel.from_pretrained(model) mode = "chat" request_str += f"llm_model = CodeChatModel.from_pretrained({model})\n" - else: # assume vertex model garden - client = aiplatform.gapic.PredictionServiceClient( - client_options=client_options + elif model == "private": + mode = "private" + model = optional_params.pop("model_id", None) + # private endpoint requires a dict instead of JSON + instances = [optional_params.copy()] + instances[0]["prompt"] = prompt + llm_model = aiplatform.PrivateEndpoint( + endpoint_name=model, + project=vertex_project, + location=vertex_location, ) + request_str += f"llm_model = aiplatform.PrivateEndpoint(endpoint_name={model}, project={vertex_project}, location={vertex_location})\n" + else: # assume vertex model garden on public endpoint + mode = "custom" - instances = [optional_params] + instances = [optional_params.copy()] instances[0]["prompt"] = prompt instances = [ json_format.ParseDict(instance_dict, Value()) for instance_dict in instances ] - llm_model = client.endpoint_path( - project=vertex_project, location=vertex_location, endpoint=model - ) - - mode = "custom" - request_str += f"llm_model = client.endpoint_path(project={vertex_project}, location={vertex_location}, endpoint={model})\n" + # Will determine the API used based on async parameter + llm_model = None + # NOTE: async prediction and streaming under "private" mode isn't supported by aiplatform right now if acompletion == True: data = { "llm_model": llm_model, @@ -532,9 +539,6 @@ def completion( """ Vertex AI Model Garden """ - request_str += ( - f"client.predict(endpoint={llm_model}, instances={instances})\n" - ) ## LOGGING logging_obj.pre_call( input=prompt, @@ -544,11 +548,21 @@ def completion( "request_str": request_str, }, ) - - response = client.predict( - endpoint=llm_model, - instances=instances, + llm_model = aiplatform.gapic.PredictionServiceClient( + client_options=client_options + ) + request_str += f"llm_model = aiplatform.gapic.PredictionServiceClient(client_options={client_options})\n" + endpoint_path = llm_model.endpoint_path( + project=vertex_project, location=vertex_location, endpoint=model + ) + request_str += ( + f"llm_model.predict(endpoint={endpoint_path}, instances={instances})\n" + ) + response = llm_model.predict( + endpoint=endpoint_path, + instances=instances ).predictions + completion_response = response[0] if ( isinstance(completion_response, str) @@ -558,6 +572,36 @@ def completion( if "stream" in optional_params and optional_params["stream"] == True: response = TextStreamer(completion_response) return response + elif mode == "private": + """ + Vertex AI Model Garden deployed on private endpoint + """ + ## LOGGING + logging_obj.pre_call( + input=prompt, + api_key=None, + additional_args={ + "complete_input_dict": optional_params, + "request_str": request_str, + }, + ) + request_str += ( + f"llm_model.predict(instances={instances})\n" + ) + response = llm_model.predict( + instances=instances + ).predictions + + completion_response = response[0] + if ( + isinstance(completion_response, str) + and "\nOutput:\n" in completion_response + ): + completion_response = completion_response.split("\nOutput:\n", 1)[1] + if "stream" in optional_params and optional_params["stream"] == True: + response = TextStreamer(completion_response) + return response + ## LOGGING logging_obj.post_call( input=prompt, api_key=None, original_response=completion_response @@ -722,17 +766,6 @@ async def async_completion( Vertex AI Model Garden """ from google.cloud import aiplatform - - async_client = aiplatform.gapic.PredictionServiceAsyncClient( - client_options=client_options - ) - llm_model = async_client.endpoint_path( - project=vertex_project, location=vertex_location, endpoint=model - ) - - request_str += ( - f"client.predict(endpoint={llm_model}, instances={instances})\n" - ) ## LOGGING logging_obj.pre_call( input=prompt, @@ -743,8 +776,18 @@ async def async_completion( }, ) - response_obj = await async_client.predict( - endpoint=llm_model, + llm_model = aiplatform.gapic.PredictionServiceAsyncClient( + client_options=client_options + ) + request_str += f"llm_model = aiplatform.gapic.PredictionServiceAsyncClient(client_options={client_options})\n" + endpoint_path = llm_model.endpoint_path( + project=vertex_project, location=vertex_location, endpoint=model + ) + request_str += ( + f"llm_model.predict(endpoint={endpoint_path}, instances={instances})\n" + ) + response_obj = await llm_model.predict( + endpoint=endpoint_path, instances=instances, ) response = response_obj.predictions @@ -754,6 +797,23 @@ async def async_completion( and "\nOutput:\n" in completion_response ): completion_response = completion_response.split("\nOutput:\n", 1)[1] + + elif mode == "private": + request_str += ( + f"llm_model.predict_async(instances={instances})\n" + ) + response_obj = await llm_model.predict_async( + instances=instances, + ) + + response = response_obj.predictions + completion_response = response[0] + if ( + isinstance(completion_response, str) + and "\nOutput:\n" in completion_response + ): + completion_response = completion_response.split("\nOutput:\n", 1)[1] + ## LOGGING logging_obj.post_call( input=prompt, api_key=None, original_response=completion_response @@ -894,15 +954,8 @@ async def async_streaming( response = llm_model.predict_streaming_async(prompt, **optional_params) elif mode == "custom": from google.cloud import aiplatform + stream = optional_params.pop("stream", None) - async_client = aiplatform.gapic.PredictionServiceAsyncClient( - client_options=client_options - ) - llm_model = async_client.endpoint_path( - project=vertex_project, location=vertex_location, endpoint=model - ) - - request_str += f"client.predict(endpoint={llm_model}, instances={instances})\n" ## LOGGING logging_obj.pre_call( input=prompt, @@ -912,9 +965,34 @@ async def async_streaming( "request_str": request_str, }, ) + llm_model = aiplatform.gapic.PredictionServiceAsyncClient( + client_options=client_options + ) + request_str += f"llm_model = aiplatform.gapic.PredictionServiceAsyncClient(client_options={client_options})\n" + endpoint_path = llm_model.endpoint_path( + project=vertex_project, location=vertex_location, endpoint=model + ) + request_str += f"client.predict(endpoint={endpoint_path}, instances={instances})\n" + response_obj = await llm_model.predict( + endpoint=endpoint_path, + instances=instances, + ) - response_obj = await async_client.predict( - endpoint=llm_model, + response = response_obj.predictions + completion_response = response[0] + if ( + isinstance(completion_response, str) + and "\nOutput:\n" in completion_response + ): + completion_response = completion_response.split("\nOutput:\n", 1)[1] + if stream: + response = TextStreamer(completion_response) + + elif mode == "private": + stream = optional_params.pop("stream", None) + _ = instances[0].pop("stream", None) + request_str += f"llm_model.predict_async(instances={instances})\n" + response_obj = await llm_model.predict_async( instances=instances, ) response = response_obj.predictions @@ -924,8 +1002,9 @@ async def async_streaming( and "\nOutput:\n" in completion_response ): completion_response = completion_response.split("\nOutput:\n", 1)[1] - if "stream" in optional_params and optional_params["stream"] == True: + if stream: response = TextStreamer(completion_response) + streamwrapper = CustomStreamWrapper( completion_stream=response, model=model, diff --git a/litellm/main.py b/litellm/main.py index 53567ee7e..93ea3c644 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -10,6 +10,7 @@ import os, openai, sys, json, inspect, uuid, datetime, threading from typing import Any, Literal, Union from functools import partial + import dotenv, traceback, random, asyncio, time, contextvars from copy import deepcopy import httpx diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7a7f154e3..fa50b06f1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -642,21 +642,40 @@ "mode": "chat" }, "gemini-pro": { - "max_tokens": 30720, + "max_tokens": 32760, "max_output_tokens": 2048, "input_cost_per_token": 0.00000025, "output_cost_per_token": 0.0000005, "litellm_provider": "vertex_ai-language-models", "mode": "chat" }, + "gemini-1.0-pro": { + "max_tokens": 32760, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00000025, + "output_cost_per_token": 0.0000005, + "litellm_provider": "vertex_ai-language-models", + "mode": "chat" + }, "gemini-pro-vision": { - "max_tokens": 30720, + "max_tokens": 16384, "max_output_tokens": 2048, "input_cost_per_token": 0.00000025, "output_cost_per_token": 0.0000005, "litellm_provider": "vertex_ai-vision-models", "mode": "chat" }, + "gemini-1.0-pro-vision": { + "max_tokens": 16384, + "max_output_tokens": 2048, + "max_images_per_prompt": 16, + "max_videos_per_prompt": 1, + "max_video_length": 2, + "input_cost_per_token": 0.00000025, + "output_cost_per_token": 0.0000005, + "litellm_provider": "vertex_ai-vision-models", + "mode": "chat" + }, "textembedding-gecko": { "max_tokens": 3072, "max_input_tokens": 3072, diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html index 68e7eafd9..03563194a 100644 --- a/litellm/proxy/_experimental/out/404.html +++ b/litellm/proxy/_experimental/out/404.html @@ -1 +1 @@ -