From 0c442cdd8fd61d146e99ca2f1df6743c2ebf0df7 Mon Sep 17 00:00:00 2001 From: Emilio Garcia Date: Mon, 17 Nov 2025 12:20:36 -0500 Subject: [PATCH] fix(rebase): clean up errors after rebasing and patch the dashboard --- scripts/telemetry/llama-stack-dashboard.json | 360 ++++++++++++++----- src/llama_stack/core/routers/safety.py | 5 +- 2 files changed, 274 insertions(+), 91 deletions(-) diff --git a/scripts/telemetry/llama-stack-dashboard.json b/scripts/telemetry/llama-stack-dashboard.json index a9f8ac7a2..a8db9713c 100644 --- a/scripts/telemetry/llama-stack-dashboard.json +++ b/scripts/telemetry/llama-stack-dashboard.json @@ -1,11 +1,24 @@ { "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": null, + "id": 1, "links": [], "liveNow": false, "panels": [ @@ -16,11 +29,40 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "auto", - "fillOpacity": 10 + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -32,7 +74,8 @@ } ] } - } + }, + "overrides": [] }, "gridPos": { "h": 8, @@ -40,15 +83,16 @@ "x": 0, "y": 0 }, - "id": 1, + "id": 2, "options": { "legend": { "calcs": [], - "displayMode": "table", + "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "none" } @@ -59,9 +103,112 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "llama_stack_completion_tokens_total", - "legendFormat": "{{model_id}} ({{provider_id}})", - "refId": "A" + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(gen_ai_request_model) (llama_stack_gen_ai_client_token_usage_sum{gen_ai_token_type=\"input\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Prompt Tokens", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(gen_ai_request_model) (llama_stack_gen_ai_client_token_usage_sum{gen_ai_token_type=\"output\"})", + "fullMetaSearch": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false } ], "title": "Completion Tokens", @@ -74,78 +221,40 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "auto", - "fillOpacity": 10 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "llama_stack_prompt_tokens_total", - "legendFormat": "Prompt - {{model_id}}", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "llama_stack_tokens_total", - "legendFormat": "Total - {{model_id}}", - "refId": "B" - } - ], - "title": "Prompt & Total Tokens", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "linear", - "showPoints": "auto", - "fillOpacity": 10 + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -158,7 +267,8 @@ ] }, "unit": "ms" - } + }, + "overrides": [] }, "gridPos": { "h": 8, @@ -175,6 +285,7 @@ "showLegend": true }, "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "none" } @@ -219,7 +330,8 @@ } ] } - } + }, + "overrides": [] }, "gridPos": { "h": 8, @@ -240,8 +352,11 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { @@ -272,7 +387,8 @@ } ] } - } + }, + "overrides": [] }, "gridPos": { "h": 8, @@ -293,8 +409,11 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "11.0.0", "targets": [ { "datasource": { @@ -315,11 +434,40 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "auto", - "fillOpacity": 10 + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -332,7 +480,8 @@ ] }, "unit": "reqps" - } + }, + "overrides": [] }, "gridPos": { "h": 8, @@ -349,6 +498,7 @@ "showLegend": true }, "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "none" } @@ -374,11 +524,40 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "auto", - "fillOpacity": 10 + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -391,7 +570,8 @@ ] }, "unit": "Bps" - } + }, + "overrides": [] }, "gridPos": { "h": 8, @@ -408,6 +588,7 @@ "showLegend": true }, "tooltip": { + "maxHeight": 600, "mode": "multi", "sort": "none" } @@ -437,7 +618,7 @@ } ], "refresh": "5s", - "schemaVersion": 38, + "schemaVersion": 39, "tags": [ "llama-stack" ], @@ -445,13 +626,14 @@ "list": [] }, "time": { - "from": "now-15m", + "from": "now-3h", "to": "now" }, + "timeRangeUpdatedDuringEditOrView": false, "timepicker": {}, "timezone": "browser", "title": "Llama Stack Metrics", "uid": "llama-stack-metrics", - "version": 0, + "version": 17, "weekStart": "" } diff --git a/src/llama_stack/core/routers/safety.py b/src/llama_stack/core/routers/safety.py index 7f15bc4b9..10c21ea88 100644 --- a/src/llama_stack/core/routers/safety.py +++ b/src/llama_stack/core/routers/safety.py @@ -6,12 +6,13 @@ from typing import Any +from opentelemetry import trace + from llama_stack.core.datatypes import SafetyConfig from llama_stack.log import get_logger +from llama_stack.telemetry.helpers import safety_request_span_attributes, safety_span_name from llama_stack_api import ModerationObject, OpenAIMessageParam, RoutingTable, RunShieldResponse, Safety, Shield -from opentelemetry import trace - logger = get_logger(name=__name__, category="core::routers") tracer = trace.get_tracer(__name__)