forked from phoenix/litellm-mirror
fix(llm_guard.py): add streaming hook for moderation calls
This commit is contained in:
parent
0a5b8f0e4e
commit
49847347d0
4 changed files with 36 additions and 25 deletions
|
@ -101,19 +101,16 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
|
|||
- Use the sanitized prompt returned
|
||||
- LLM Guard can handle things like PII Masking, etc.
|
||||
"""
|
||||
if "messages" in data:
|
||||
safety_check_messages = data["messages"][
|
||||
-1
|
||||
] # get the last response - llama guard has a 4k token limit
|
||||
if (
|
||||
isinstance(safety_check_messages, dict)
|
||||
and "content" in safety_check_messages
|
||||
and isinstance(safety_check_messages["content"], str)
|
||||
):
|
||||
await self.moderation_check(safety_check_messages["content"])
|
||||
|
||||
return data
|
||||
|
||||
async def async_post_call_streaming_hook(
|
||||
self, user_api_key_dict: UserAPIKeyAuth, response: str
|
||||
):
|
||||
if response is not None:
|
||||
await self.moderation_check(text=response)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
# llm_guard = _ENTERPRISE_LLMGuard()
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue