diff --git a/docs/my-website/docs/observability/langsmith_integration.md b/docs/my-website/docs/observability/langsmith_integration.md index 78c7e3119..b115866d5 100644 --- a/docs/my-website/docs/observability/langsmith_integration.md +++ b/docs/my-website/docs/observability/langsmith_integration.md @@ -71,6 +71,23 @@ response = litellm.completion( ) print(response) ``` + +### Make LiteLLM Proxy use Custom `LANGSMITH_BASE_URL` + +If you're using a custom LangSmith instance, you can set the +`LANGSMITH_BASE_URL` environment variable to point to your instance. +For example, you can make LiteLLM Proxy log to a local LangSmith instance with +this config: + +```yaml +litellm_settings: + success_callback: ["langsmith"] + +environment_variables: + LANGSMITH_BASE_URL: "http://localhost:1984" + LANGSMITH_PROJECT: "litellm-proxy" +``` + ## Support & Talk to Founders - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index bf8f72bea..38be0c433 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -232,7 +232,6 @@ response = completion( model="anthropic/claude-3-opus-20240229", messages=messages, tools=tools, - extra_headers={"anthropic-beta": "tools-2024-05-16"}, ) ``` @@ -247,7 +246,6 @@ response = completion( messages=messages, tools=tools, tool_choice={"type": "tool", "name": "get_weather"}, - extra_headers={"anthropic-beta": "tools-2024-05-16"}, ) ``` diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index 1831164be..8904faa45 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -1,7 +1,8 @@ +import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# ✨ Enterprise Features - Content Mod, SSO +# ✨ Enterprise Features - Content Mod, SSO, Custom Swagger Features here are behind a commercial license in our `/enterprise` folder. [**See Code**](https://github.com/BerriAI/litellm/tree/main/enterprise) @@ -20,6 +21,7 @@ Features: - ✅ Reject calls (incoming / outgoing) with Banned Keywords (e.g. competitors) - ✅ Don't log/store specific requests to Langfuse, Sentry, etc. (eg confidential LLM requests) - ✅ Tracking Spend for Custom Tags +- ✅ Custom Branding + Routes on Swagger Docs @@ -526,4 +528,39 @@ curl -X GET "http://0.0.0.0:4000/spend/tags" \ \ No newline at end of file +## Tracking Spend per User --> + +## Swagger Docs - Custom Routes + Branding + +:::info + +Requires a LiteLLM Enterprise key to use. Request one [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) + +::: + +Set LiteLLM Key in your environment + +```bash +LITELLM_LICENSE="" +``` + +### Customize Title + Description + +In your environment, set: + +```bash +DOCS_TITLE="TotalGPT" +DOCS_DESCRIPTION="Sample Company Description" +``` + +### Customize Routes + +Hide admin routes from users. + +In your environment, set: + +```bash +DOCS_FILTERED="True" # only shows openai routes to user +``` + + \ No newline at end of file diff --git a/docs/my-website/img/custom_swagger.png b/docs/my-website/img/custom_swagger.png new file mode 100644 index 000000000..e17c0882b Binary files /dev/null and b/docs/my-website/img/custom_swagger.png differ diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 92e440215..3e25b4ee7 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -44,6 +44,8 @@ class LangsmithLogger: print_verbose( f"Langsmith Logging - project_name: {project_name}, run_name {run_name}" ) + langsmith_base_url = os.getenv("LANGSMITH_BASE_URL", "https://api.smith.langchain.com") + try: print_verbose( f"Langsmith Logging - Enters logging function for model {kwargs}" @@ -86,8 +88,12 @@ class LangsmithLogger: "end_time": end_time, } + url = f"{langsmith_base_url}/runs" + print_verbose( + f"Langsmith Logging - About to send data to {url} ..." + ) response = requests.post( - "https://api.smith.langchain.com/runs", + url=url, json=data, headers={"x-api-key": self.langsmith_api_key}, ) diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index 2e02ffe59..f14dabc03 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -507,7 +507,7 @@ class AnthropicChatCompletion(BaseLLM): _is_function_call = True if "anthropic-beta" not in headers: # default to v1 of "anthropic-beta" - headers["anthropic-beta"] = "tools-2024-04-04" + headers["anthropic-beta"] = "tools-2024-05-16" anthropic_tools = [] for tool in optional_params["tools"]: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 61e6de53d..b900b623b 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -87,10 +87,10 @@ class LiteLLMRoutes(enum.Enum): # models "/models", "/v1/models", + # token counter + "/utils/token_counter", ] - llm_utils_routes: List = ["utils/token_counter"] - info_routes: List = [ "/key/info", "/team/info", diff --git a/litellm/proxy/auth/litellm_license.py b/litellm/proxy/auth/litellm_license.py new file mode 100644 index 000000000..c6c61ad4e --- /dev/null +++ b/litellm/proxy/auth/litellm_license.py @@ -0,0 +1,42 @@ +# What is this? +## If litellm license in env, checks if it's valid +import os +from litellm.llms.custom_httpx.http_handler import HTTPHandler + + +class LicenseCheck: + """ + - Check if license in env + - Returns if license is valid + """ + + base_url = "https://license.litellm.ai" + + def __init__(self) -> None: + self.license_str = os.getenv("LITELLM_LICENSE", None) + self.http_handler = HTTPHandler() + + def _verify(self, license_str: str) -> bool: + url = "{}/verify_license/{}".format(self.base_url, license_str) + + try: # don't impact user, if call fails + response = self.http_handler.get(url=url) + + response.raise_for_status() + + response_json = response.json() + + premium = response_json["verify"] + + assert isinstance(premium, bool) + + return premium + except Exception as e: + return False + + def is_premium(self) -> bool: + if self.license_str is None: + return False + elif self._verify(license_str=self.license_str): + return True + return False diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 06d5958f2..972af4012 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -110,6 +110,7 @@ from litellm.router import LiteLLM_Params, Deployment, updateDeployment from litellm.router import ModelInfo as RouterModelInfo from litellm._logging import verbose_router_logger, verbose_proxy_logger from litellm.proxy.auth.handle_jwt import JWTHandler +from litellm.proxy.auth.litellm_license import LicenseCheck from litellm.proxy.hooks.prompt_injection_detection import ( _OPTIONAL_PromptInjectionDetection, ) @@ -150,6 +151,7 @@ from fastapi.responses import ( ORJSONResponse, JSONResponse, ) +from fastapi.openapi.utils import get_openapi from fastapi.responses import RedirectResponse from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles @@ -169,17 +171,30 @@ except Exception as e: except Exception as e: pass +_license_check = LicenseCheck() +premium_user: bool = _license_check.is_premium() + ui_link = f"/ui/" ui_message = ( f"👉 [```LiteLLM Admin Panel on /ui```]({ui_link}). Create, Edit Keys with SSO" ) +### CUSTOM BRANDING [ENTERPRISE FEATURE] ### _docs_url = None if os.getenv("NO_DOCS", "False") == "True" else "/" +_title = os.getenv("DOCS_TITLE", "LiteLLM API") if premium_user else "LiteLLM API" +_description = ( + os.getenv( + "DOCS_DESCRIPTION", + f"Proxy Server to call 100+ LLMs in the OpenAI format\n\n{ui_message}", + ) + if premium_user + else f"Proxy Server to call 100+ LLMs in the OpenAI format\n\n{ui_message}" +) app = FastAPI( docs_url=_docs_url, - title="LiteLLM API", - description=f"Proxy Server to call 100+ LLMs in the OpenAI format\n\n{ui_message}", + title=_title, + description=_description, version=version, root_path=os.environ.get( "SERVER_ROOT_PATH", "" @@ -187,6 +202,31 @@ app = FastAPI( ) +### CUSTOM API DOCS [ENTERPRISE FEATURE] ### +# Custom OpenAPI schema generator to include only selected routes +def custom_openapi(): + if app.openapi_schema: + return app.openapi_schema + openapi_schema = get_openapi( + title=app.title, + version=app.version, + description=app.description, + routes=app.routes, + ) + # Filter routes to include only specific ones + openai_routes = LiteLLMRoutes.openai_routes.value + paths_to_include: dict = {} + for route in openai_routes: + paths_to_include[route] = openapi_schema["paths"][route] + openapi_schema["paths"] = paths_to_include + app.openapi_schema = openapi_schema + return app.openapi_schema + + +if os.getenv("DOCS_FILTERED", "False") == "True" and premium_user: + app.openapi = custom_openapi # type: ignore + + class ProxyException(Exception): # NOTE: DO NOT MODIFY THIS # This is used to map exactly to OPENAI Exceptions diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 57fb6d33e..2a6a1fbf6 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -3032,7 +3032,6 @@ def test_mistral_anyscale_stream(): print(chunk["choices"][0]["delta"].get("content", ""), end="") -# test_mistral_anyscale_stream() # test_completion_anyscale_2() # def test_completion_with_fallbacks_multiple_keys(): # print(f"backup key 1: {os.getenv('BACKUP_OPENAI_API_KEY_1')}") diff --git a/litellm/utils.py b/litellm/utils.py index 800a9cdab..6442ecf6e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5122,7 +5122,7 @@ def get_optional_params( or "tools" in non_default_params ): if ( - custom_llm_provider != "openai" + custom_llm_provider == "ollama" and custom_llm_provider != "text-completion-openai" and custom_llm_provider != "azure" and custom_llm_provider != "vertex_ai" @@ -5136,6 +5136,8 @@ def get_optional_params( and custom_llm_provider != "cohere" and custom_llm_provider != "bedrock" and custom_llm_provider != "ollama_chat" + and custom_llm_provider != "openrouter" + and custom_llm_provider not in litellm.openai_compatible_providers ): if custom_llm_provider == "ollama": # ollama actually supports json output diff --git a/pyproject.toml b/pyproject.toml index 015380b89..b3cc49079 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.37.13" +version = "1.37.15" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -79,7 +79,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.37.13" +version = "1.37.15" version_files = [ "pyproject.toml:^version" ]