phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-23 15:53:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

safety

Browse source
This commit is contained in:
Sai Soundararaj 2025-07-01 15:41:45 -07:00
parent 8fc72e4669
commit 74b7bfd75d
4 changed files with 45 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

16
llama_stack/apis/safety/safety.py
View file

@ -17,6 +17,12 @@ from llama_stack.schema_utils import json_schema_type, webmethod
@json_schema_type
class ViolationLevel(Enum):
"""Severity level of a safety violation.
:cvar INFO: Informational level violation that does not require action
:cvar WARN: Warning level violation that suggests caution but allows continuation
:cvar ERROR: Error level violation that requires blocking or intervention
"""
INFO = "info"
WARN = "warn"
ERROR = "error"
@ -24,6 +30,12 @@ class ViolationLevel(Enum):
@json_schema_type
class SafetyViolation(BaseModel):
"""Details of a safety violation detected by content moderation.
:param violation_level: Severity level of the violation
:param user_message: (Optional) Message to convey to the user about the violation
:param metadata: Additional metadata including specific violation codes for debugging and telemetry
"""
violation_level: ViolationLevel
# what message should you convey to the user
@ -36,6 +48,10 @@ class SafetyViolation(BaseModel):
@json_schema_type
class RunShieldResponse(BaseModel):
"""Response from running a safety shield.
:param violation: (Optional) Safety violation detected by the shield, if any
"""
violation: SafetyViolation | None = None