From 6c30f18f8cb6f9e6e2bfa16fd68b90c20bd4aeec Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 4 Sep 2024 13:04:19 -0700 Subject: [PATCH] docs new presidio language controls --- .../docs/proxy/guardrails/pii_masking_v2.md | 129 ++++++++++++++++++ docs/my-website/docs/proxy/pii_masking.md | 10 +- docs/my-website/sidebars.js | 12 +- .../guardrails/guardrail_hooks/presidio.py | 5 +- litellm/proxy/proxy_config.yaml | 13 +- 5 files changed, 154 insertions(+), 15 deletions(-) create mode 100644 docs/my-website/docs/proxy/guardrails/pii_masking_v2.md diff --git a/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md b/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md new file mode 100644 index 000000000..592c0e6bf --- /dev/null +++ b/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md @@ -0,0 +1,129 @@ +import Image from '@theme/IdealImage'; + +# PII Masking - Presidio + +## Quick Start + +LiteLLM supports [Microsoft Presidio](https://github.com/microsoft/presidio/) for PII masking. + +### 1. Define Guardrails on your LiteLLM config.yaml + +Define your guardrails under the `guardrails` section +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: openai/gpt-3.5-turbo + api_key: os.environ/OPENAI_API_KEY + +guardrails: + - guardrail_name: "presidio-pre-guard" + litellm_params: + guardrail: presidio # supported values: "aporia", "bedrock", "lakera", "presidio" + mode: "pre_call" +``` + +Set the following env vars + +```bash +export PRESIDIO_ANALYZER_API_BASE="http://localhost:5002" +export PRESIDIO_ANONYMIZER_API_BASE="http://localhost:5001" +``` + +#### Supported values for `mode` + +- `pre_call` Run **before** LLM call, on **input** +- `post_call` Run **after** LLM call, on **input & output** + + +### 2. Start LiteLLM Gateway + + +```shell +litellm --config config.yaml --detailed_debug +``` + +### 3. Test request + +**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)** + + + + +Expect this to mask `Jane Doe` since it's PII + +```shell +curl http://localhost:4000/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "Hello my name is Jane Doe"} + ], + "guardrails": ["presidio-pre-guard"], + }' +``` + +Expected response on failure + +```shell +{ + "id": "chatcmpl-A3qSC39K7imjGbZ8xCDacGJZBoTJQ", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Hello, ! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + } + } + ], + "created": 1725479980, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_5bd87c427a", + "usage": { + "completion_tokens": 13, + "prompt_tokens": 14, + "total_tokens": 27 + }, + "service_tier": null +} +``` + + + + + +```shell +curl http://localhost:4000/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "Hello good morning"} + ], + "guardrails": ["presidio-pre-guard"], + }' +``` + + + + + + + +## Set `language` per request + +## Output parsing + +## Ad Hoc Recognizers + + + + diff --git a/docs/my-website/docs/proxy/pii_masking.md b/docs/my-website/docs/proxy/pii_masking.md index 8106765f4..83e4965a4 100644 --- a/docs/my-website/docs/proxy/pii_masking.md +++ b/docs/my-website/docs/proxy/pii_masking.md @@ -1,6 +1,14 @@ import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; -# PII Masking +# PII Masking - LiteLLM Gateway (Deprecated Version) + +:::warning + +This is deprecated, please use [our new Presidio pii masking integration](./guardrails/pii_masking_v2) + +::: LiteLLM supports [Microsoft Presidio](https://github.com/microsoft/presidio/) for PII masking. diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 0abb5144f..f3780b84e 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -67,7 +67,15 @@ const sidebars = { { type: "category", label: "🛡️ [Beta] Guardrails", - items: ["proxy/guardrails/quick_start", "proxy/guardrails/aporia_api", "proxy/guardrails/lakera_ai", "proxy/guardrails/bedrock", "proxy/guardrails/custom_guardrail", "prompt_injection"], + items: [ + "proxy/guardrails/quick_start", + "proxy/guardrails/aporia_api", + "proxy/guardrails/lakera_ai", + "proxy/guardrails/bedrock", + "proxy/guardrails/pii_masking_v2", + "proxy/guardrails/custom_guardrail", + "prompt_injection" + ], }, { type: "category", @@ -101,7 +109,6 @@ const sidebars = { "proxy/model_management", "proxy/health", "proxy/debugging", - "proxy/pii_masking", "proxy/call_hooks", "proxy/rules", "proxy/cli", @@ -291,6 +298,7 @@ const sidebars = { "data_security", "migration_policy", "contributing", + "proxy/pii_masking", "rules", "proxy_server", { diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index a44bdaa9f..eb894ecbb 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -396,8 +396,9 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): if "metadata" in data: _metadata = data["metadata"] _guardrail_config = _metadata.get("guardrail_config") - _presidio_config = PresidioPerRequestConfig(**_guardrail_config) - return _presidio_config + if _guardrail_config: + _presidio_config = PresidioPerRequestConfig(**_guardrail_config) + return _presidio_config return None diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 913c2e946..0aaadc13f 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,22 +1,15 @@ model_list: - - model_name: fake-openai-endpoint + - model_name: openai/* litellm_params: - model: openai/fake - api_base: https://exampleopenaiendpoint-production.up.railway.app/ + model: gpt-3.5-turbo api_key: os.environ/OPENAI_API_KEY - - model_name: gpt-3.5-turbo-end-user-test - litellm_params: - model: azure/chatgpt-v-2 - api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ - api_version: "2023-05-15" - api_key: os.environ/AZURE_API_KEY litellm_settings: success_callback: ["prometheus"] failure_callback: ["prometheus"] guardrails: - - guardrail_name: "presidio" + - guardrail_name: "presidio-pre-guard" litellm_params: guardrail: presidio # supported values: "aporia", "lakera", "presidio" mode: "pre_call" # pre_call, during_call, post_call