phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-29 15:23:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

safety

Browse source
This commit is contained in:
Sai Soundararaj 2025-07-01 15:41:45 -07:00
parent 8fc72e4669
commit 74b7bfd75d
4 changed files with 45 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

6
api_update_plan.md
View file

@ -219,9 +219,9 @@ Before finalizing documentation, verify:
- Minimal code changes - primarily docstring additions only
### File Processing Order (Complete Absolute Paths)
1. `/Users/saip/Documents/GitHub/llama-stack/llama_stack/apis/agents/agents.py` - Core agent system (start here, most complete)
2. `/Users/saip/Documents/GitHub/llama-stack/llama_stack/apis/inference/inference.py` - Core LLM functionality
3. `/Users/saip/Documents/GitHub/llama-stack/llama_stack/apis/safety/safety.py` - Safety and moderation
[x] 1. `/Users/saip/Documents/GitHub/llama-stack/llama_stack/apis/agents/agents.py` - Core agent system (start here, most complete)
[x] 2. `/Users/saip/Documents/GitHub/llama-stack/llama_stack/apis/inference/inference.py` - Core LLM functionality
[x] 3. `/Users/saip/Documents/GitHub/llama-stack/llama_stack/apis/safety/safety.py` - Safety and moderation
4. `/Users/saip/Documents/GitHub/llama-stack/llama_stack/apis/models/models.py` - Model metadata and management
5. `/Users/saip/Documents/GitHub/llama-stack/llama_stack/apis/tools/tools.py` - Tool system APIs
6. `/Users/saip/Documents/GitHub/llama-stack/llama_stack/apis/tools/rag_tool.py` - RAG tool runtime

21
docs/_static/llama-stack-spec.html vendored
View file

@ -6801,10 +6801,12 @@
"type": "object",
"properties": {
"violation_level": {
"$ref": "#/components/schemas/ViolationLevel"
"$ref": "#/components/schemas/ViolationLevel",
"description": "Severity level of the violation"
},
"user_message": {
"type": "string"
"type": "string",
"description": "(Optional) Message to convey to the user about the violation"
},
"metadata": {
"type": "object",
@ -6829,7 +6831,8 @@
"type": "object"
}
]
}
},
"description": "Additional metadata including specific violation codes for debugging and telemetry"
}
},
"additionalProperties": false,
@ -6837,7 +6840,8 @@
"violation_level",
"metadata"
],
"title": "SafetyViolation"
"title": "SafetyViolation",
"description": "Details of a safety violation detected by content moderation."
},
"ShieldCallStep": {
"type": "object",
@ -7140,7 +7144,8 @@
"warn",
"error"
],
"title": "ViolationLevel"
"title": "ViolationLevel",
"description": "Severity level of a safety violation."
},
"AgentTurnResponseEvent": {
"type": "object",
@ -15758,11 +15763,13 @@
"type": "object",
"properties": {
"violation": {
"$ref": "#/components/schemas/SafetyViolation"
"$ref": "#/components/schemas/SafetyViolation",
"description": "(Optional) Safety violation detected by the shield, if any"
}
},
"additionalProperties": false,
"title": "RunShieldResponse"
"title": "RunShieldResponse",
"description": "Response from running a safety shield."
},
"SaveSpansToDatasetRequest": {
"type": "object",

12
docs/_static/llama-stack-spec.yaml vendored
View file

@ -4888,8 +4888,11 @@ components:
properties:
violation_level:
$ref: '#/components/schemas/ViolationLevel'
description: Severity level of the violation
user_message:
type: string
description: >-
(Optional) Message to convey to the user about the violation
metadata:
type: object
additionalProperties:
@ -4900,11 +4903,16 @@ components:
- type: string
- type: array
- type: object
description: >-
Additional metadata including specific violation codes for debugging and
telemetry
additionalProperties: false
required:
- violation_level
- metadata
title: SafetyViolation
description: >-
Details of a safety violation detected by content moderation.
ShieldCallStep:
type: object
properties:
@ -5122,6 +5130,7 @@ components:
- warn
- error
title: ViolationLevel
description: Severity level of a safety violation.
AgentTurnResponseEvent:
type: object
properties:
@ -11128,8 +11137,11 @@ components:
properties:
violation:
$ref: '#/components/schemas/SafetyViolation'
description: >-
(Optional) Safety violation detected by the shield, if any
additionalProperties: false
title: RunShieldResponse
description: Response from running a safety shield.
SaveSpansToDatasetRequest:
type: object
properties:

16
llama_stack/apis/safety/safety.py
View file

@ -17,6 +17,12 @@ from llama_stack.schema_utils import json_schema_type, webmethod
@json_schema_type
class ViolationLevel(Enum):
"""Severity level of a safety violation.
:cvar INFO: Informational level violation that does not require action
:cvar WARN: Warning level violation that suggests caution but allows continuation
:cvar ERROR: Error level violation that requires blocking or intervention
"""
INFO = "info"
WARN = "warn"
ERROR = "error"
@ -24,6 +30,12 @@ class ViolationLevel(Enum):
@json_schema_type
class SafetyViolation(BaseModel):
"""Details of a safety violation detected by content moderation.
:param violation_level: Severity level of the violation
:param user_message: (Optional) Message to convey to the user about the violation
:param metadata: Additional metadata including specific violation codes for debugging and telemetry
"""
violation_level: ViolationLevel
# what message should you convey to the user
@ -36,6 +48,10 @@ class SafetyViolation(BaseModel):
@json_schema_type
class RunShieldResponse(BaseModel):
"""Response from running a safety shield.
:param violation: (Optional) Safety violation detected by the shield, if any
"""
violation: SafetyViolation | None = None