From 986a5267907b4263db6af8c86b462aff2808cdd8 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 8 Mar 2024 16:25:54 -0800 Subject: [PATCH 1/6] (feat) disable logging per request --- litellm/main.py | 2 ++ litellm/utils.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 63649844a..76742eb06 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -563,6 +563,7 @@ def completion( "caching_groups", "ttl", "cache", + "no-log", ] default_params = openai_params + litellm_params non_default_params = { @@ -2417,6 +2418,7 @@ def embedding( "caching_groups", "ttl", "cache", + "no-log", ] default_params = openai_params + litellm_params non_default_params = { diff --git a/litellm/utils.py b/litellm/utils.py index 38836a4bc..01fab699a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2985,12 +2985,15 @@ def client(original_function): print_verbose( f"Async Wrapper: Completed Call, calling async_success_handler: {logging_obj.async_success_handler}" ) - asyncio.create_task( - logging_obj.async_success_handler(result, start_time, end_time) - ) - threading.Thread( - target=logging_obj.success_handler, args=(result, start_time, end_time) - ).start() + # check if user does not want this to be logged + if kwargs.get("no-log", False) == False: + asyncio.create_task( + logging_obj.async_success_handler(result, start_time, end_time) + ) + threading.Thread( + target=logging_obj.success_handler, + args=(result, start_time, end_time), + ).start() # RETURN RESULT if hasattr(result, "_hidden_params"): From 4ff68c85623d4c3a52cde986a69d9f29ee0c4ab5 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 8 Mar 2024 16:26:25 -0800 Subject: [PATCH 2/6] (docs) no log requests --- docs/my-website/docs/proxy/enterprise.md | 51 +++++++++++++++++++----- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/docs/my-website/docs/proxy/enterprise.md b/docs/my-website/docs/proxy/enterprise.md index a4f3ea7b1..e0c5374f0 100644 --- a/docs/my-website/docs/proxy/enterprise.md +++ b/docs/my-website/docs/proxy/enterprise.md @@ -12,14 +12,16 @@ Features here are behind a commercial license in our `/enterprise` folder. [**Se ::: Features: -- [ ] Content Moderation with LlamaGuard -- [ ] Content Moderation with Google Text Moderations -- [ ] Content Moderation with LLM Guard -- [ ] Reject calls from Blocked User list -- [ ] Reject calls (incoming / outgoing) with Banned Keywords (e.g. competitors) -- [ ] Tracking Spend for Custom Tags +- ✅ Content Moderation with LlamaGuard +- ✅ Content Moderation with Google Text Moderations +- ✅ Content Moderation with LLM Guard +- ✅ Reject calls from Blocked User list +- ✅ Reject calls (incoming / outgoing) with Banned Keywords (e.g. competitors) +- ✅ Don't log/store specific requests (eg confidential LLM requests) +- ✅ Tracking Spend for Custom Tags -## Content Moderation with LlamaGuard +## Content Moderation +### Content Moderation with LlamaGuard Currently works with Sagemaker's LlamaGuard endpoint. @@ -39,7 +41,7 @@ os.environ["AWS_SECRET_ACCESS_KEY"] = "" os.environ["AWS_REGION_NAME"] = "" ``` -### Customize LlamaGuard prompt +#### Customize LlamaGuard prompt To modify the unsafe categories llama guard evaluates against, just create your own version of [this category list](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/llamaguard_prompt.txt) @@ -51,7 +53,7 @@ callbacks: ["llamaguard_moderations"] llamaguard_unsafe_content_categories: /path/to/llamaguard_prompt.txt ``` -## Content Moderation with LLM Guard +### Content Moderation with LLM Guard Set the LLM Guard API Base in your environment @@ -78,7 +80,7 @@ Expected results: LLM Guard: Received response - {"sanitized_prompt": "hello world", "is_valid": true, "scanners": { "Regex": 0.0 }} ``` -## Content Moderation with Google Text Moderation +### Content Moderation with Google Text Moderation Requires your GOOGLE_APPLICATION_CREDENTIALS to be set in your .env (same as VertexAI). @@ -89,7 +91,7 @@ litellm_settings: callbacks: ["google_text_moderation"] ``` -### Set custom confidence thresholds +#### Set custom confidence thresholds Google Moderations checks the test against several categories. [Source](https://cloud.google.com/natural-language/docs/moderating-text#safety_attribute_confidence_scores) @@ -133,6 +135,33 @@ Here are the category specific values: | "legal" | legal_threshold: 0.1 | +## Incognito Requests - Don't log anything + +When `no-log=True`, the request will **not be logged on any callbacks** and there will be **no server logs on litellm** + +```python +import openai +client = openai.OpenAI( + api_key="anything", # proxy api-key + base_url="http://0.0.0.0:8000" # litellm proxy +) + +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ + "no-log": True + } +) + +print(response) +``` + ## Enable Blocked User Lists If any call is made to proxy with this user id, it'll be rejected - use this if you want to let users opt-out of ai features From d6dc28f0ed2d11329d23370071fef7f903e5f599 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 8 Mar 2024 16:27:53 -0800 Subject: [PATCH 3/6] (fix) proxy setting success callbacks --- litellm/proxy/proxy_config.yaml | 9 +++------ litellm/proxy/proxy_server.py | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 654a50b2f..76c9ed04c 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -5,12 +5,9 @@ model_list: api_base: os.environ/AZURE_API_BASE api_key: os.environ/AZURE_API_KEY api_version: "2023-07-01-preview" - - model_name: azure-gpt-3.5 - litellm_params: - model: gpt-3.5-turbo - api_key: os.environ/OPENAI_API_KEY - model_info: - access_groups: ["public"] +litellm_settings: + set_verbose: True + success_callback: ["langfuse"] router_settings: set_verbose: True debug_level: "DEBUG" \ No newline at end of file diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 19ac5c961..810fdc0d1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1675,9 +1675,9 @@ class ProxyConfig: # these are litellm callbacks - "langfuse", "sentry", "wandb" else: litellm.success_callback.append(callback) - verbose_proxy_logger.debug( + print( # noqa f"{blue_color_code} Initialized Success Callbacks - {litellm.success_callback} {reset_color_code}" - ) + ) # noqa elif key == "failure_callback": litellm.failure_callback = [] From ddd231a8c23557131a099f5ea26f7b1ba5f7a4a4 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 8 Mar 2024 16:46:38 -0800 Subject: [PATCH 4/6] (feat) use no-log as a litellm param --- litellm/main.py | 3 +++ litellm/utils.py | 24 ++++++++++++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 76742eb06..c8a8dbc1c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -487,6 +487,8 @@ def completion( ### ASYNC CALLS ### acompletion = kwargs.get("acompletion", False) client = kwargs.get("client", None) + ### Admin Controls ### + no_log = kwargs.get("no-log", False) ######## end of unpacking kwargs ########### openai_params = [ "functions", @@ -727,6 +729,7 @@ def completion( model_info=model_info, proxy_server_request=proxy_server_request, preset_cache_key=preset_cache_key, + no_log=no_log, ) logging.update_environment_variables( model=model, diff --git a/litellm/utils.py b/litellm/utils.py index 01fab699a..163223ab2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1279,6 +1279,10 @@ class Logging: for callback in callbacks: try: + litellm_params = self.model_call_details.get("litellm_params", {}) + if litellm_params.get("no-log", False) == True: + print_verbose("no-log request, skipping logging") + continue if callback == "lite_debugger": print_verbose("reaches lite_debugger for logging!") print_verbose(f"liteDebuggerClient: {liteDebuggerClient}") @@ -1708,6 +1712,9 @@ class Logging: verbose_logger.debug(f"Async success callbacks: {callbacks}") for callback in callbacks: try: + if kwargs.get("no-log", False) == True: + print_verbose("no-log request, skipping logging") + continue if callback == "cache" and litellm.cache is not None: # set_cache once complete streaming response is built print_verbose("async success_callback: reaches cache for logging!") @@ -2986,14 +2993,13 @@ def client(original_function): f"Async Wrapper: Completed Call, calling async_success_handler: {logging_obj.async_success_handler}" ) # check if user does not want this to be logged - if kwargs.get("no-log", False) == False: - asyncio.create_task( - logging_obj.async_success_handler(result, start_time, end_time) - ) - threading.Thread( - target=logging_obj.success_handler, - args=(result, start_time, end_time), - ).start() + asyncio.create_task( + logging_obj.async_success_handler(result, start_time, end_time) + ) + threading.Thread( + target=logging_obj.success_handler, + args=(result, start_time, end_time), + ).start() # RETURN RESULT if hasattr(result, "_hidden_params"): @@ -3895,6 +3901,7 @@ def get_litellm_params( proxy_server_request=None, acompletion=None, preset_cache_key=None, + no_log=None, ): litellm_params = { "acompletion": acompletion, @@ -3911,6 +3918,7 @@ def get_litellm_params( "model_info": model_info, "proxy_server_request": proxy_server_request, "preset_cache_key": preset_cache_key, + "no-log": no_log, "stream_response": {}, # litellm_call_id: ModelResponse Dict } From 0a538fe679a066608bbf997d5d96d646f52adb90 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 8 Mar 2024 16:56:20 -0800 Subject: [PATCH 5/6] (feat) use no-log to disable per request logging --- litellm/utils.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 163223ab2..32f0f765b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1281,8 +1281,13 @@ class Logging: try: litellm_params = self.model_call_details.get("litellm_params", {}) if litellm_params.get("no-log", False) == True: - print_verbose("no-log request, skipping logging") - continue + # proxy cost tracking cal backs should run + if not ( + isinstance(callback, CustomLogger) + and "_PROXY_" in callback.__class__.__name__ + ): + print_verbose("no-log request, skipping logging") + continue if callback == "lite_debugger": print_verbose("reaches lite_debugger for logging!") print_verbose(f"liteDebuggerClient: {liteDebuggerClient}") @@ -1711,6 +1716,16 @@ class Logging: callbacks = litellm._async_success_callback verbose_logger.debug(f"Async success callbacks: {callbacks}") for callback in callbacks: + # check if callback can run for this request + litellm_params = self.model_call_details.get("litellm_params", {}) + if litellm_params.get("no-log", False) == True: + # proxy cost tracking cal backs should run + if not ( + isinstance(callback, CustomLogger) + and "_PROXY_" in callback.__class__.__name__ + ): + print_verbose("no-log request, skipping logging") + continue try: if kwargs.get("no-log", False) == True: print_verbose("no-log request, skipping logging") From 5850ff470f589959cc4d23fe7b3275b649c81088 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 8 Mar 2024 20:42:12 -0800 Subject: [PATCH 6/6] (feat) disable/enable logging --- litellm/_logging.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/litellm/_logging.py b/litellm/_logging.py index 438fa9743..26693c15e 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -31,6 +31,18 @@ def _turn_on_debug(): verbose_proxy_logger.setLevel(level=logging.DEBUG) # set proxy logs to debug +def _disable_debugging(): + verbose_logger.disabled = True + verbose_router_logger.disabled = True + verbose_proxy_logger.disabled = True + + +def _enable_debugging(): + verbose_logger.disabled = False + verbose_router_logger.disabled = False + verbose_proxy_logger.disabled = False + + def print_verbose(print_statement): try: if set_verbose: