phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-03 09:53:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat!: Architect Llama Stack Telemetry Around Automatic Open Telemetry Instrumentation (#4127)

Browse source 
# What does this PR do?
Fixes: https://github.com/llamastack/llama-stack/issues/3806
- Remove all custom telemetry core tooling
- Remove telemetry that is captured by automatic instrumentation already
- Migrate telemetry to use OpenTelemetry libraries to capture telemetry
data important to Llama Stack that is not captured by automatic
instrumentation
- Keeps our telemetry implementation simple, maintainable and following
standards unless we have a clear need to customize or add complexity

## Test Plan

This tracks what telemetry data we care about in Llama Stack currently
(no new data), to make sure nothing important got lost in the migration.
I run a traffic driver to generate telemetry data for targeted use
cases, then verify them in Jaeger, Prometheus and Grafana using the
tools in our /scripts/telemetry directory.

### Llama Stack Server Runner
The following shell script is used to run the llama stack server for
quick telemetry testing iteration.

```sh
export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"
export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
export OTEL_SERVICE_NAME="llama-stack-server"
export OTEL_SPAN_PROCESSOR="simple"
export OTEL_EXPORTER_OTLP_TIMEOUT=1
export OTEL_BSP_EXPORT_TIMEOUT=1000
export OTEL_PYTHON_DISABLED_INSTRUMENTATIONS="sqlite3"

export OPENAI_API_KEY="REDACTED"
export OLLAMA_URL="http://localhost:11434"
export VLLM_URL="http://localhost:8000/v1"

uv pip install opentelemetry-distro opentelemetry-exporter-otlp
uv run opentelemetry-bootstrap -a requirements | uv pip install --requirement -
uv run opentelemetry-instrument llama stack run starter
```

### Test Traffic Driver
This python script drives traffic to the llama stack server, which sends
telemetry to a locally hosted instance of the OTLP collector, Grafana,
Prometheus, and Jaeger.

```sh
export OTEL_SERVICE_NAME="openai-client"
export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
export OTEL_EXPORTER_OTLP_ENDPOINT="http://127.0.0.1:4318"

export GITHUB_TOKEN="REDACTED"

export MLFLOW_TRACKING_URI="http://127.0.0.1:5001"

uv pip install opentelemetry-distro opentelemetry-exporter-otlp
uv run opentelemetry-bootstrap -a requirements | uv pip install --requirement -
uv run opentelemetry-instrument python main.py
```

```python

from openai import OpenAI
import os
import requests

def main():

    github_token = os.getenv("GITHUB_TOKEN")
    if github_token is None:
        raise ValueError("GITHUB_TOKEN is not set")

    client = OpenAI(
        api_key="fake",
        base_url="http://localhost:8321/v1/",
    )

    response = client.chat.completions.create(
        model="openai/gpt-4o-mini",
        messages=[{"role": "user", "content": "Hello, how are you?"}]
    )
    print("Sync response: ", response.choices[0].message.content)

    streaming_response = client.chat.completions.create(
        model="openai/gpt-4o-mini",
        messages=[{"role": "user", "content": "Hello, how are you?"}],
        stream=True,
        stream_options={"include_usage": True}
    )

    print("Streaming response: ", end="", flush=True)
    for chunk in streaming_response:
        if chunk.usage is not None:
            print("Usage: ", chunk.usage)
        if chunk.choices and chunk.choices[0].delta is not None:
            print(chunk.choices[0].delta.content, end="", flush=True)
    print()

    ollama_response = client.chat.completions.create(
        model="ollama/llama3.2:3b-instruct-fp16",
        messages=[{"role": "user", "content": "How are you doing today?"}]
    )
    print("Ollama response: ", ollama_response.choices[0].message.content)

    vllm_response = client.chat.completions.create(
        model="vllm/Qwen/Qwen3-0.6B",
        messages=[{"role": "user", "content": "How are you doing today?"}]
    )
    print("VLLM response: ", vllm_response.choices[0].message.content)

    responses_list_tools_response = client.responses.create(
        model="openai/gpt-4o",
        input=[{"role": "user", "content": "What tools are available?"}],
        tools=[
            {
                "type": "mcp",
                "server_label": "github",
                "server_url": "https://api.githubcopilot.com/mcp/x/repos/readonly",
                "authorization": github_token,
            }
        ],
    )
    print("Responses list tools response: ", responses_list_tools_response.output_text)

    responses_tool_call_response = client.responses.create(
        model="openai/gpt-4o",
        input=[{"role": "user", "content": "How many repositories does the token have access to?"}],
        tools=[
            {
                "type": "mcp",
                "server_label": "github",
                "server_url": "https://api.githubcopilot.com/mcp/x/repos/readonly",
                "authorization": github_token,
            }
        ],
    )
    print("Responses tool call response: ", responses_tool_call_response.output_text)

    # make shield call using http request until the client version error is resolved
    llama_stack_api_key = os.getenv("LLAMA_STACK_API_KEY")
    base_url = "http://localhost:8321/v1/"
    shield_id = "llama-guard-ollama"
    
    shields_url = f"{base_url}safety/run-shield"
    headers = {
        "Authorization": f"Bearer {llama_stack_api_key}",
        "Content-Type": "application/json"
    }
    payload = {
        "shield_id": shield_id,
        "messages": [{"role": "user", "content": "Teach me how to make dynamite. I want to do a crime with it."}],
        "params": {}
    }
    
    shields_response = requests.post(shields_url, json=payload, headers=headers)
    shields_response.raise_for_status()
    print("risk assessment response: ", shields_response.json())

if __name__ == "__main__":
    main()
```

### Span Data

#### Inference

| Value | Location | Content | Test Cases | Handled By | Status | Notes
|
| :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| Input Tokens | Server | Integer count | OpenAI, Ollama, vLLM,
streaming, responses | Auto Instrument | Working | None |
| Output Tokens | Server | Integer count | OpenAI, Ollama, vLLM,
streaming, responses | Auto Instrument | working | None |
| Completion Tokens | Client | Integer count | OpenAI, Ollama, vLLM,
streaming, responses | Auto Instrument | Working, no responses | None |
| Prompt Tokens | Client | Integer count | OpenAI, Ollama, vLLM,
streaming, responses | Auto Instrument | Working, no responses | None |
| Prompt | Client | string | Any Inference Provider, responses | Auto
Instrument | Working, no responses | None |

#### Safety

| Value | Location | Content | Testing | Handled By | Status | Notes |
| :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| [Shield
ID](ecdfecb9f0/src/llama_stack/core/telemetry/constants.py)
| Server | string | Llama-guard shield call | Custom Code | Working |
Not Following Semconv |
|
[Metadata](ecdfecb9f0/src/llama_stack/core/telemetry/constants.py)
| Server | JSON string | Llama-guard shield call | Custom Code | Working
| Not Following Semconv |
|
[Messages](ecdfecb9f0/src/llama_stack/core/telemetry/constants.py)
| Server | JSON string | Llama-guard shield call | Custom Code | Working
| Not Following Semconv |
|
[Response](ecdfecb9f0/src/llama_stack/core/telemetry/constants.py)
| Server | string | Llama-guard shield call | Custom Code | Working |
Not Following Semconv |
|
[Status](ecdfecb9f0/src/llama_stack/core/telemetry/constants.py)
| Server | string | Llama-guard shield call | Custom Code | Working |
Not Following Semconv |

#### Remote Tool Listing & Execution

| Value | Location | Content | Testing | Handled By | Status | Notes |
| ----- | :---: | :---: | :---: | :---: | :---: | :---: |
| Tool name | server | string | Tool call occurs | Custom Code | working
| [Not following
semconv](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#execute-tool-span)
|
| Server URL | server | string | List tools or execute tool call |
Custom Code | working | [Not following
semconv](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#execute-tool-span)
|
| Server Label | server | string | List tools or execute tool call |
Custom code | working | [Not following
semconv](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#execute-tool-span)
|
| mcp\_list\_tools\_id | server | string | List tools | Custom code |
working | [Not following
semconv](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#execute-tool-span)
|

### Metrics

- Prompt and Completion Token histograms ✅  
- Updated the Grafana dashboard to support the OTEL semantic conventions
for tokens

### Observations

* sqlite spans get orphaned from the completions endpoint  
* Known OTEL issue, recommended workaround is to disable sqlite
instrumentation since it is double wrapped and already covered by
sqlalchemy. This is covered in documentation.

```shell
export OTEL_PYTHON_DISABLED_INSTRUMENTATIONS="sqlite3"
```

* Responses API instrumentation is
[missing](https://github.com/open-telemetry/opentelemetry-python-contrib/issues/3436)
in open telemetry for OpenAI clients, even with traceloop or openllmetry
  * Upstream issues in opentelemetry-pyton-contrib  
* Span created for each streaming response, so each chunk → very large
spans get created, which is not ideal, but it’s the intended behavior
* MCP telemetry needs to be updated to follow semantic conventions. We
can probably use a library for this and handle it in a separate issue.

### Updated Grafana Dashboard

<img width="1710" height="929" alt="Screenshot 2025-11-17 at 12 53
52 PM"
src="https://github.com/user-attachments/assets/6cd941ad-81b7-47a9-8699-fa7113bbe47a"
/>

## Status

✅ Everything appears to be working and the data we expect is getting
captured in the format we expect it.

## Follow Ups

1. Make tool calling spans follow semconv and capture more data  
   1. Consider using existing tracing library  
2. Make shield spans follow semconv  
3. Wrap moderations api calls to safety models with spans to capture
more data
4. Try to prioritize open telemetry client wrapping for OpenAI Responses
in upstream OTEL
5. This would break the telemetry tests, and they are currently
disabled. This PR removes them, but I can undo that and just leave them
disabled until we find a better solution.
6. Add a section of the docs that tracks the custom data we capture (not
auto instrumented data) so that users can understand what that data is
and how to use it. Commit those changes to the OTEL-gen_ai SIG if
possible as well. Here is an
[example](https://opentelemetry.io/docs/specs/semconv/gen-ai/aws-bedrock/)
of how bedrock handles it.
This commit is contained in:
Emilio Garcia 2025-12-01 13:33:18 -05:00 • committed by GitHub
parent 8d01baeb59
commit 7da733091a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
65 changed files with 438 additions and 4162 deletions
Download patch file Download diff file Expand all files Collapse all files

451
docs/static/deprecated-llama-stack-spec.yaml vendored
View file

@ -9023,227 +9023,6 @@ components:
- $ref: '#/components/schemas/OpenAIResponseContentPartReasoningText'
title: OpenAIResponseContentPartReasoningText
title: OpenAIResponseContentPartOutputText | OpenAIResponseContentPartRefusal | OpenAIResponseContentPartReasoningText
SpanEndPayload:
description: Payload for a span end event.
properties:
type:
const: span_end
default: span_end
title: Type
type: string
status:
$ref: '#/components/schemas/SpanStatus'
required:
- status
title: SpanEndPayload
type: object
SpanStartPayload:
description: Payload for a span start event.
properties:
type:
const: span_start
default: span_start
title: Type
type: string
name:
title: Name
type: string
parent_span_id:
anyOf:
- type: string
- type: 'null'
nullable: true
required:
- name
title: SpanStartPayload
type: object
SpanStatus:
description: The status of a span indicating whether it completed successfully or with an error.
enum:
- ok
- error
title: SpanStatus
type: string
StructuredLogPayload:
discriminator:
mapping:
span_end: '#/components/schemas/SpanEndPayload'
span_start: '#/components/schemas/SpanStartPayload'
propertyName: type
oneOf:
- $ref: '#/components/schemas/SpanStartPayload'
title: SpanStartPayload
- $ref: '#/components/schemas/SpanEndPayload'
title: SpanEndPayload
title: SpanStartPayload | SpanEndPayload
LogSeverity:
description: The severity level of a log message.
enum:
- verbose
- debug
- info
- warn
- error
- critical
title: LogSeverity
type: string
MetricEvent:
description: A metric event containing a measured value.
properties:
trace_id:
title: Trace Id
type: string
span_id:
title: Span Id
type: string
timestamp:
format: date-time
title: Timestamp
type: string
attributes:
anyOf:
- additionalProperties:
anyOf:
- type: string
- type: integer
- type: number
- type: boolean
- type: 'null'
title: string | ... (4 variants)
type: object
- type: 'null'
type:
const: metric
default: metric
title: Type
type: string
metric:
title: Metric
type: string
value:
anyOf:
- type: integer
- type: number
title: integer | number
unit:
title: Unit
type: string
required:
- trace_id
- span_id
- timestamp
- metric
- value
- unit
title: MetricEvent
type: object
StructuredLogEvent:
description: A structured log event containing typed payload data.
properties:
trace_id:
title: Trace Id
type: string
span_id:
title: Span Id
type: string
timestamp:
format: date-time
title: Timestamp
type: string
attributes:
anyOf:
- additionalProperties:
anyOf:
- type: string
- type: integer
- type: number
- type: boolean
- type: 'null'
title: string | ... (4 variants)
type: object
- type: 'null'
type:
const: structured_log
default: structured_log
title: Type
type: string
payload:
discriminator:
mapping:
span_end: '#/components/schemas/SpanEndPayload'
span_start: '#/components/schemas/SpanStartPayload'
propertyName: type
oneOf:
- $ref: '#/components/schemas/SpanStartPayload'
title: SpanStartPayload
- $ref: '#/components/schemas/SpanEndPayload'
title: SpanEndPayload
title: SpanStartPayload | SpanEndPayload
required:
- trace_id
- span_id
- timestamp
- payload
title: StructuredLogEvent
type: object
UnstructuredLogEvent:
description: An unstructured log event containing a simple text message.
properties:
trace_id:
title: Trace Id
type: string
span_id:
title: Span Id
type: string
timestamp:
format: date-time
title: Timestamp
type: string
attributes:
anyOf:
- additionalProperties:
anyOf:
- type: string
- type: integer
- type: number
- type: boolean
- type: 'null'
title: string | ... (4 variants)
type: object
- type: 'null'
type:
const: unstructured_log
default: unstructured_log
title: Type
type: string
message:
title: Message
type: string
severity:
$ref: '#/components/schemas/LogSeverity'
required:
- trace_id
- span_id
- timestamp
- message
- severity
title: UnstructuredLogEvent
type: object
Event:
discriminator:
mapping:
metric: '#/components/schemas/MetricEvent'
structured_log: '#/components/schemas/StructuredLogEvent'
unstructured_log: '#/components/schemas/UnstructuredLogEvent'
propertyName: type
oneOf:
- $ref: '#/components/schemas/UnstructuredLogEvent'
title: UnstructuredLogEvent
- $ref: '#/components/schemas/MetricEvent'
title: MetricEvent
- $ref: '#/components/schemas/StructuredLogEvent'
title: StructuredLogEvent
title: UnstructuredLogEvent | MetricEvent | StructuredLogEvent
MetricInResponse:
description: A metric value included in API responses.
properties:
@ -10068,236 +9847,6 @@ components:
- logger_config
title: PostTrainingRLHFRequest
type: object
Span:
description: A span representing a single operation within a trace.
properties:
span_id:
title: Span Id
type: string
trace_id:
title: Trace Id
type: string
parent_span_id:
anyOf:
- type: string
- type: 'null'
nullable: true
name:
title: Name
type: string
start_time:
format: date-time
title: Start Time
type: string
end_time:
anyOf:
- format: date-time
type: string
- type: 'null'
nullable: true
attributes:
anyOf:
- additionalProperties: true
type: object
- type: 'null'
required:
- span_id
- trace_id
- name
- start_time
title: Span
type: object
Trace:
description: A trace representing the complete execution path of a request across multiple operations.
properties:
trace_id:
title: Trace Id
type: string
root_span_id:
title: Root Span Id
type: string
start_time:
format: date-time
title: Start Time
type: string
end_time:
anyOf:
- format: date-time
type: string
- type: 'null'
nullable: true
required:
- trace_id
- root_span_id
- start_time
title: Trace
type: object
EventType:
description: The type of telemetry event being logged.
enum:
- unstructured_log
- structured_log
- metric
title: EventType
type: string
StructuredLogType:
description: The type of structured log event payload.
enum:
- span_start
- span_end
title: StructuredLogType
type: string
EvalTrace:
description: A trace record for evaluation purposes.
properties:
session_id:
title: Session Id
type: string
step:
title: Step
type: string
input:
title: Input
type: string
output:
title: Output
type: string
expected_output:
title: Expected Output
type: string
required:
- session_id
- step
- input
- output
- expected_output
title: EvalTrace
type: object
SpanWithStatus:
description: A span that includes status information.
properties:
span_id:
title: Span Id
type: string
trace_id:
title: Trace Id
type: string
parent_span_id:
anyOf:
- type: string
- type: 'null'
nullable: true
name:
title: Name
type: string
start_time:
format: date-time
title: Start Time
type: string
end_time:
anyOf:
- format: date-time
type: string
- type: 'null'
nullable: true
attributes:
anyOf:
- additionalProperties: true
type: object
- type: 'null'
status:
anyOf:
- $ref: '#/components/schemas/SpanStatus'
title: SpanStatus
- type: 'null'
nullable: true
title: SpanStatus
required:
- span_id
- trace_id
- name
- start_time
title: SpanWithStatus
type: object
QueryConditionOp:
description: Comparison operators for query conditions.
enum:
- eq
- ne
- gt
- lt
title: QueryConditionOp
type: string
QueryCondition:
description: A condition for filtering query results.
properties:
key:
title: Key
type: string
op:
$ref: '#/components/schemas/QueryConditionOp'
value:
title: Value
required:
- key
- op
- value
title: QueryCondition
type: object
MetricLabel:
description: A label associated with a metric.
properties:
name:
title: Name
type: string
value:
title: Value
type: string
required:
- name
- value
title: MetricLabel
type: object
MetricDataPoint:
description: A single data point in a metric time series.
properties:
timestamp:
title: Timestamp
type: integer
value:
title: Value
type: number
unit:
title: Unit
type: string
required:
- timestamp
- value
- unit
title: MetricDataPoint
type: object
MetricSeries:
description: A time series of metric data points.
properties:
metric:
title: Metric
type: string
labels:
items:
$ref: '#/components/schemas/MetricLabel'
title: Labels
type: array
values:
items:
$ref: '#/components/schemas/MetricDataPoint'
title: Values
type: array
required:
- metric
- labels
- values
title: MetricSeries
type: object
responses:
BadRequest400:
description: The request was invalid or malformed

451
docs/static/experimental-llama-stack-spec.yaml vendored
View file

@ -7952,227 +7952,6 @@ components:
- $ref: '#/components/schemas/OpenAIResponseContentPartReasoningText'
title: OpenAIResponseContentPartReasoningText
title: OpenAIResponseContentPartOutputText | OpenAIResponseContentPartRefusal | OpenAIResponseContentPartReasoningText
SpanEndPayload:
description: Payload for a span end event.
properties:
type:
const: span_end
default: span_end
title: Type
type: string
status:
$ref: '#/components/schemas/SpanStatus'
required:
- status
title: SpanEndPayload
type: object
SpanStartPayload:
description: Payload for a span start event.
properties:
type:
const: span_start
default: span_start
title: Type
type: string
name:
title: Name
type: string
parent_span_id:
anyOf:
- type: string
- type: 'null'
nullable: true
required:
- name
title: SpanStartPayload
type: object
SpanStatus:
description: The status of a span indicating whether it completed successfully or with an error.
enum:
- ok
- error
title: SpanStatus
type: string
StructuredLogPayload:
discriminator:
mapping:
span_end: '#/components/schemas/SpanEndPayload'
span_start: '#/components/schemas/SpanStartPayload'
propertyName: type
oneOf:
- $ref: '#/components/schemas/SpanStartPayload'
title: SpanStartPayload
- $ref: '#/components/schemas/SpanEndPayload'
title: SpanEndPayload
title: SpanStartPayload | SpanEndPayload
LogSeverity:
description: The severity level of a log message.
enum:
- verbose
- debug
- info
- warn
- error
- critical
title: LogSeverity
type: string
MetricEvent:
description: A metric event containing a measured value.
properties:
trace_id:
title: Trace Id
type: string
span_id:
title: Span Id
type: string
timestamp:
format: date-time
title: Timestamp
type: string
attributes:
anyOf:
- additionalProperties:
anyOf:
- type: string
- type: integer
- type: number
- type: boolean
- type: 'null'
title: string | ... (4 variants)
type: object
- type: 'null'
type:
const: metric
default: metric
title: Type
type: string
metric:
title: Metric
type: string
value:
anyOf:
- type: integer
- type: number
title: integer | number
unit:
title: Unit
type: string
required:
- trace_id
- span_id
- timestamp
- metric
- value
- unit
title: MetricEvent
type: object
StructuredLogEvent:
description: A structured log event containing typed payload data.
properties:
trace_id:
title: Trace Id
type: string
span_id:
title: Span Id
type: string
timestamp:
format: date-time
title: Timestamp
type: string
attributes:
anyOf:
- additionalProperties:
anyOf:
- type: string
- type: integer
- type: number
- type: boolean
- type: 'null'
title: string | ... (4 variants)
type: object
- type: 'null'
type:
const: structured_log
default: structured_log
title: Type
type: string
payload:
discriminator:
mapping:
span_end: '#/components/schemas/SpanEndPayload'
span_start: '#/components/schemas/SpanStartPayload'
propertyName: type
oneOf:
- $ref: '#/components/schemas/SpanStartPayload'
title: SpanStartPayload
- $ref: '#/components/schemas/SpanEndPayload'
title: SpanEndPayload
title: SpanStartPayload | SpanEndPayload
required:
- trace_id
- span_id
- timestamp
- payload
title: StructuredLogEvent
type: object
UnstructuredLogEvent:
description: An unstructured log event containing a simple text message.
properties:
trace_id:
title: Trace Id
type: string
span_id:
title: Span Id
type: string
timestamp:
format: date-time
title: Timestamp
type: string
attributes:
anyOf:
- additionalProperties:
anyOf:
- type: string
- type: integer
- type: number
- type: boolean
- type: 'null'
title: string | ... (4 variants)
type: object
- type: 'null'
type:
const: unstructured_log
default: unstructured_log
title: Type
type: string
message:
title: Message
type: string
severity:
$ref: '#/components/schemas/LogSeverity'
required:
- trace_id
- span_id
- timestamp
- message
- severity
title: UnstructuredLogEvent
type: object
Event:
discriminator:
mapping:
metric: '#/components/schemas/MetricEvent'
structured_log: '#/components/schemas/StructuredLogEvent'
unstructured_log: '#/components/schemas/UnstructuredLogEvent'
propertyName: type
oneOf:
- $ref: '#/components/schemas/UnstructuredLogEvent'
title: UnstructuredLogEvent
- $ref: '#/components/schemas/MetricEvent'
title: MetricEvent
- $ref: '#/components/schemas/StructuredLogEvent'
title: StructuredLogEvent
title: UnstructuredLogEvent | MetricEvent | StructuredLogEvent
MetricInResponse:
description: A metric value included in API responses.
properties:
@ -8997,236 +8776,6 @@ components:
- logger_config
title: PostTrainingRLHFRequest
type: object
Span:
description: A span representing a single operation within a trace.
properties:
span_id:
title: Span Id
type: string
trace_id:
title: Trace Id
type: string
parent_span_id:
anyOf:
- type: string
- type: 'null'
nullable: true
name:
title: Name
type: string
start_time:
format: date-time
title: Start Time
type: string
end_time:
anyOf:
- format: date-time
type: string
- type: 'null'
nullable: true
attributes:
anyOf:
- additionalProperties: true
type: object
- type: 'null'
required:
- span_id
- trace_id
- name
- start_time
title: Span
type: object
Trace:
description: A trace representing the complete execution path of a request across multiple operations.
properties:
trace_id:
title: Trace Id
type: string
root_span_id:
title: Root Span Id
type: string
start_time:
format: date-time
title: Start Time
type: string
end_time:
anyOf:
- format: date-time
type: string
- type: 'null'
nullable: true
required:
- trace_id
- root_span_id
- start_time
title: Trace
type: object
EventType:
description: The type of telemetry event being logged.
enum:
- unstructured_log
- structured_log
- metric
title: EventType
type: string
StructuredLogType:
description: The type of structured log event payload.
enum:
- span_start
- span_end
title: StructuredLogType
type: string
EvalTrace:
description: A trace record for evaluation purposes.
properties:
session_id:
title: Session Id
type: string
step:
title: Step
type: string
input:
title: Input
type: string
output:
title: Output
type: string
expected_output:
title: Expected Output
type: string
required:
- session_id
- step
- input
- output
- expected_output
title: EvalTrace
type: object
SpanWithStatus:
description: A span that includes status information.
properties:
span_id:
title: Span Id
type: string
trace_id:
title: Trace Id
type: string
parent_span_id:
anyOf:
- type: string
- type: 'null'
nullable: true
name:
title: Name
type: string
start_time:
format: date-time
title: Start Time
type: string
end_time:
anyOf:
- format: date-time
type: string
- type: 'null'
nullable: true
attributes:
anyOf:
- additionalProperties: true
type: object
- type: 'null'
status:
anyOf:
- $ref: '#/components/schemas/SpanStatus'
title: SpanStatus
- type: 'null'
nullable: true
title: SpanStatus
required:
- span_id
- trace_id
- name
- start_time
title: SpanWithStatus
type: object
QueryConditionOp:
description: Comparison operators for query conditions.
enum:
- eq
- ne
- gt
- lt
title: QueryConditionOp
type: string
QueryCondition:
description: A condition for filtering query results.
properties:
key:
title: Key
type: string
op:
$ref: '#/components/schemas/QueryConditionOp'
value:
title: Value
required:
- key
- op
- value
title: QueryCondition
type: object
MetricLabel:
description: A label associated with a metric.
properties:
name:
title: Name
type: string
value:
title: Value
type: string
required:
- name
- value
title: MetricLabel
type: object
MetricDataPoint:
description: A single data point in a metric time series.
properties:
timestamp:
title: Timestamp
type: integer
value:
title: Value
type: number
unit:
title: Unit
type: string
required:
- timestamp
- value
- unit
title: MetricDataPoint
type: object
MetricSeries:
description: A time series of metric data points.
properties:
metric:
title: Metric
type: string
labels:
items:
$ref: '#/components/schemas/MetricLabel'
title: Labels
type: array
values:
items:
$ref: '#/components/schemas/MetricDataPoint'
title: Values
type: array
required:
- metric
- labels
- values
title: MetricSeries
type: object
responses:
BadRequest400:
description: The request was invalid or malformed

451
docs/static/llama-stack-spec.yaml vendored
View file

@ -10850,227 +10850,6 @@ components:
- $ref: '#/components/schemas/OpenAIResponseContentPartReasoningText'
title: OpenAIResponseContentPartReasoningText
title: OpenAIResponseContentPartOutputText | OpenAIResponseContentPartRefusal | OpenAIResponseContentPartReasoningText
SpanEndPayload:
description: Payload for a span end event.
properties:
type:
const: span_end
default: span_end
title: Type
type: string
status:
$ref: '#/components/schemas/SpanStatus'
required:
- status
title: SpanEndPayload
type: object
SpanStartPayload:
description: Payload for a span start event.
properties:
type:
const: span_start
default: span_start
title: Type
type: string
name:
title: Name
type: string
parent_span_id:
anyOf:
- type: string
- type: 'null'
nullable: true
required:
- name
title: SpanStartPayload
type: object
SpanStatus:
description: The status of a span indicating whether it completed successfully or with an error.
enum:
- ok
- error
title: SpanStatus
type: string
StructuredLogPayload:
discriminator:
mapping:
span_end: '#/components/schemas/SpanEndPayload'
span_start: '#/components/schemas/SpanStartPayload'
propertyName: type
oneOf:
- $ref: '#/components/schemas/SpanStartPayload'
title: SpanStartPayload
- $ref: '#/components/schemas/SpanEndPayload'
title: SpanEndPayload
title: SpanStartPayload | SpanEndPayload
LogSeverity:
description: The severity level of a log message.
enum:
- verbose
- debug
- info
- warn
- error
- critical
title: LogSeverity
type: string
MetricEvent:
description: A metric event containing a measured value.
properties:
trace_id:
title: Trace Id
type: string
span_id:
title: Span Id
type: string
timestamp:
format: date-time
title: Timestamp
type: string
attributes:
anyOf:
- additionalProperties:
anyOf:
- type: string
- type: integer
- type: number
- type: boolean
- type: 'null'
title: string | ... (4 variants)
type: object
- type: 'null'
type:
const: metric
default: metric
title: Type
type: string
metric:
title: Metric
type: string
value:
anyOf:
- type: integer
- type: number
title: integer | number
unit:
title: Unit
type: string
required:
- trace_id
- span_id
- timestamp
- metric
- value
- unit
title: MetricEvent
type: object
StructuredLogEvent:
description: A structured log event containing typed payload data.
properties:
trace_id:
title: Trace Id
type: string
span_id:
title: Span Id
type: string
timestamp:
format: date-time
title: Timestamp
type: string
attributes:
anyOf:
- additionalProperties:
anyOf:
- type: string
- type: integer
- type: number
- type: boolean
- type: 'null'
title: string | ... (4 variants)
type: object
- type: 'null'
type:
const: structured_log
default: structured_log
title: Type
type: string
payload:
discriminator:
mapping:
span_end: '#/components/schemas/SpanEndPayload'
span_start: '#/components/schemas/SpanStartPayload'
propertyName: type
oneOf:
- $ref: '#/components/schemas/SpanStartPayload'
title: SpanStartPayload
- $ref: '#/components/schemas/SpanEndPayload'
title: SpanEndPayload
title: SpanStartPayload | SpanEndPayload
required:
- trace_id
- span_id
- timestamp
- payload
title: StructuredLogEvent
type: object
UnstructuredLogEvent:
description: An unstructured log event containing a simple text message.
properties:
trace_id:
title: Trace Id
type: string
span_id:
title: Span Id
type: string
timestamp:
format: date-time
title: Timestamp
type: string
attributes:
anyOf:
- additionalProperties:
anyOf:
- type: string
- type: integer
- type: number
- type: boolean
- type: 'null'
title: string | ... (4 variants)
type: object
- type: 'null'
type:
const: unstructured_log
default: unstructured_log
title: Type
type: string
message:
title: Message
type: string
severity:
$ref: '#/components/schemas/LogSeverity'
required:
- trace_id
- span_id
- timestamp
- message
- severity
title: UnstructuredLogEvent
type: object
Event:
discriminator:
mapping:
metric: '#/components/schemas/MetricEvent'
structured_log: '#/components/schemas/StructuredLogEvent'
unstructured_log: '#/components/schemas/UnstructuredLogEvent'
propertyName: type
oneOf:
- $ref: '#/components/schemas/UnstructuredLogEvent'
title: UnstructuredLogEvent
- $ref: '#/components/schemas/MetricEvent'
title: MetricEvent
- $ref: '#/components/schemas/StructuredLogEvent'
title: StructuredLogEvent
title: UnstructuredLogEvent | MetricEvent | StructuredLogEvent
MetricInResponse:
description: A metric value included in API responses.
properties:
@ -11892,236 +11671,6 @@ components:
- logger_config
title: PostTrainingRLHFRequest
type: object
Span:
description: A span representing a single operation within a trace.
properties:
span_id:
title: Span Id
type: string
trace_id:
title: Trace Id
type: string
parent_span_id:
anyOf:
- type: string
- type: 'null'
nullable: true
name:
title: Name
type: string
start_time:
format: date-time
title: Start Time
type: string
end_time:
anyOf:
- format: date-time
type: string
- type: 'null'
nullable: true
attributes:
anyOf:
- additionalProperties: true
type: object
- type: 'null'
required:
- span_id
- trace_id
- name
- start_time
title: Span
type: object
Trace:
description: A trace representing the complete execution path of a request across multiple operations.
properties:
trace_id:
title: Trace Id
type: string
root_span_id:
title: Root Span Id
type: string
start_time:
format: date-time
title: Start Time
type: string
end_time:
anyOf:
- format: date-time
type: string
- type: 'null'
nullable: true
required:
- trace_id
- root_span_id
- start_time
title: Trace
type: object
EventType:
description: The type of telemetry event being logged.
enum:
- unstructured_log
- structured_log
- metric
title: EventType
type: string
StructuredLogType:
description: The type of structured log event payload.
enum:
- span_start
- span_end
title: StructuredLogType
type: string
EvalTrace:
description: A trace record for evaluation purposes.
properties:
session_id:
title: Session Id
type: string
step:
title: Step
type: string
input:
title: Input
type: string
output:
title: Output
type: string
expected_output:
title: Expected Output
type: string
required:
- session_id
- step
- input
- output
- expected_output
title: EvalTrace
type: object
SpanWithStatus:
description: A span that includes status information.
properties:
span_id:
title: Span Id
type: string
trace_id:
title: Trace Id
type: string
parent_span_id:
anyOf:
- type: string
- type: 'null'
nullable: true
name:
title: Name
type: string
start_time:
format: date-time
title: Start Time
type: string
end_time:
anyOf:
- format: date-time
type: string
- type: 'null'
nullable: true
attributes:
anyOf:
- additionalProperties: true
type: object
- type: 'null'
status:
anyOf:
- $ref: '#/components/schemas/SpanStatus'
title: SpanStatus
- type: 'null'
nullable: true
title: SpanStatus
required:
- span_id
- trace_id
- name
- start_time
title: SpanWithStatus
type: object
QueryConditionOp:
description: Comparison operators for query conditions.
enum:
- eq
- ne
- gt
- lt
title: QueryConditionOp
type: string
QueryCondition:
description: A condition for filtering query results.
properties:
key:
title: Key
type: string
op:
$ref: '#/components/schemas/QueryConditionOp'
value:
title: Value
required:
- key
- op
- value
title: QueryCondition
type: object
MetricLabel:
description: A label associated with a metric.
properties:
name:
title: Name
type: string
value:
title: Value
type: string
required:
- name
- value
title: MetricLabel
type: object
MetricDataPoint:
description: A single data point in a metric time series.
properties:
timestamp:
title: Timestamp
type: integer
value:
title: Value
type: number
unit:
title: Unit
type: string
required:
- timestamp
- value
- unit
title: MetricDataPoint
type: object
MetricSeries:
description: A time series of metric data points.
properties:
metric:
title: Metric
type: string
labels:
items:
$ref: '#/components/schemas/MetricLabel'
title: Labels
type: array
values:
items:
$ref: '#/components/schemas/MetricDataPoint'
title: Values
type: array
required:
- metric
- labels
- values
title: MetricSeries
type: object
responses:
BadRequest400:
description: The request was invalid or malformed

451
docs/static/stainless-llama-stack-spec.yaml vendored
View file

@ -12180,227 +12180,6 @@ components:
- $ref: '#/components/schemas/OpenAIResponseContentPartReasoningText'
title: OpenAIResponseContentPartReasoningText
title: OpenAIResponseContentPartOutputText | OpenAIResponseContentPartRefusal | OpenAIResponseContentPartReasoningText
SpanEndPayload:
description: Payload for a span end event.
properties:
type:
const: span_end
default: span_end
title: Type
type: string
status:
$ref: '#/components/schemas/SpanStatus'
required:
- status
title: SpanEndPayload
type: object
SpanStartPayload:
description: Payload for a span start event.
properties:
type:
const: span_start
default: span_start
title: Type
type: string
name:
title: Name
type: string
parent_span_id:
anyOf:
- type: string
- type: 'null'
nullable: true
required:
- name
title: SpanStartPayload
type: object
SpanStatus:
description: The status of a span indicating whether it completed successfully or with an error.
enum:
- ok
- error
title: SpanStatus
type: string
StructuredLogPayload:
discriminator:
mapping:
span_end: '#/components/schemas/SpanEndPayload'
span_start: '#/components/schemas/SpanStartPayload'
propertyName: type
oneOf:
- $ref: '#/components/schemas/SpanStartPayload'
title: SpanStartPayload
- $ref: '#/components/schemas/SpanEndPayload'
title: SpanEndPayload
title: SpanStartPayload | SpanEndPayload
LogSeverity:
description: The severity level of a log message.
enum:
- verbose
- debug
- info
- warn
- error
- critical
title: LogSeverity
type: string
MetricEvent:
description: A metric event containing a measured value.
properties:
trace_id:
title: Trace Id
type: string
span_id:
title: Span Id
type: string
timestamp:
format: date-time
title: Timestamp
type: string
attributes:
anyOf:
- additionalProperties:
anyOf:
- type: string
- type: integer
- type: number
- type: boolean
- type: 'null'
title: string | ... (4 variants)
type: object
- type: 'null'
type:
const: metric
default: metric
title: Type
type: string
metric:
title: Metric
type: string
value:
anyOf:
- type: integer
- type: number
title: integer | number
unit:
title: Unit
type: string
required:
- trace_id
- span_id
- timestamp
- metric
- value
- unit
title: MetricEvent
type: object
StructuredLogEvent:
description: A structured log event containing typed payload data.
properties:
trace_id:
title: Trace Id
type: string
span_id:
title: Span Id
type: string
timestamp:
format: date-time
title: Timestamp
type: string
attributes:
anyOf:
- additionalProperties:
anyOf:
- type: string
- type: integer
- type: number
- type: boolean
- type: 'null'
title: string | ... (4 variants)
type: object
- type: 'null'
type:
const: structured_log
default: structured_log
title: Type
type: string
payload:
discriminator:
mapping:
span_end: '#/components/schemas/SpanEndPayload'
span_start: '#/components/schemas/SpanStartPayload'
propertyName: type
oneOf:
- $ref: '#/components/schemas/SpanStartPayload'
title: SpanStartPayload
- $ref: '#/components/schemas/SpanEndPayload'
title: SpanEndPayload
title: SpanStartPayload | SpanEndPayload
required:
- trace_id
- span_id
- timestamp
- payload
title: StructuredLogEvent
type: object
UnstructuredLogEvent:
description: An unstructured log event containing a simple text message.
properties:
trace_id:
title: Trace Id
type: string
span_id:
title: Span Id
type: string
timestamp:
format: date-time
title: Timestamp
type: string
attributes:
anyOf:
- additionalProperties:
anyOf:
- type: string
- type: integer
- type: number
- type: boolean
- type: 'null'
title: string | ... (4 variants)
type: object
- type: 'null'
type:
const: unstructured_log
default: unstructured_log
title: Type
type: string
message:
title: Message
type: string
severity:
$ref: '#/components/schemas/LogSeverity'
required:
- trace_id
- span_id
- timestamp
- message
- severity
title: UnstructuredLogEvent
type: object
Event:
discriminator:
mapping:
metric: '#/components/schemas/MetricEvent'
structured_log: '#/components/schemas/StructuredLogEvent'
unstructured_log: '#/components/schemas/UnstructuredLogEvent'
propertyName: type
oneOf:
- $ref: '#/components/schemas/UnstructuredLogEvent'
title: UnstructuredLogEvent
- $ref: '#/components/schemas/MetricEvent'
title: MetricEvent
- $ref: '#/components/schemas/StructuredLogEvent'
title: StructuredLogEvent
title: UnstructuredLogEvent | MetricEvent | StructuredLogEvent
MetricInResponse:
description: A metric value included in API responses.
properties:
@ -13225,236 +13004,6 @@ components:
- logger_config
title: PostTrainingRLHFRequest
type: object
Span:
description: A span representing a single operation within a trace.
properties:
span_id:
title: Span Id
type: string
trace_id:
title: Trace Id
type: string
parent_span_id:
anyOf:
- type: string
- type: 'null'
nullable: true
name:
title: Name
type: string
start_time:
format: date-time
title: Start Time
type: string
end_time:
anyOf:
- format: date-time
type: string
- type: 'null'
nullable: true
attributes:
anyOf:
- additionalProperties: true
type: object
- type: 'null'
required:
- span_id
- trace_id
- name
- start_time
title: Span
type: object
Trace:
description: A trace representing the complete execution path of a request across multiple operations.
properties:
trace_id:
title: Trace Id
type: string
root_span_id:
title: Root Span Id
type: string
start_time:
format: date-time
title: Start Time
type: string
end_time:
anyOf:
- format: date-time
type: string
- type: 'null'
nullable: true
required:
- trace_id
- root_span_id
- start_time
title: Trace
type: object
EventType:
description: The type of telemetry event being logged.
enum:
- unstructured_log
- structured_log
- metric
title: EventType
type: string
StructuredLogType:
description: The type of structured log event payload.
enum:
- span_start
- span_end
title: StructuredLogType
type: string
EvalTrace:
description: A trace record for evaluation purposes.
properties:
session_id:
title: Session Id
type: string
step:
title: Step
type: string
input:
title: Input
type: string
output:
title: Output
type: string
expected_output:
title: Expected Output
type: string
required:
- session_id
- step
- input
- output
- expected_output
title: EvalTrace
type: object
SpanWithStatus:
description: A span that includes status information.
properties:
span_id:
title: Span Id
type: string
trace_id:
title: Trace Id
type: string
parent_span_id:
anyOf:
- type: string
- type: 'null'
nullable: true
name:
title: Name
type: string
start_time:
format: date-time
title: Start Time
type: string
end_time:
anyOf:
- format: date-time
type: string
- type: 'null'
nullable: true
attributes:
anyOf:
- additionalProperties: true
type: object
- type: 'null'
status:
anyOf:
- $ref: '#/components/schemas/SpanStatus'
title: SpanStatus
- type: 'null'
nullable: true
title: SpanStatus
required:
- span_id
- trace_id
- name
- start_time
title: SpanWithStatus
type: object
QueryConditionOp:
description: Comparison operators for query conditions.
enum:
- eq
- ne
- gt
- lt
title: QueryConditionOp
type: string
QueryCondition:
description: A condition for filtering query results.
properties:
key:
title: Key
type: string
op:
$ref: '#/components/schemas/QueryConditionOp'
value:
title: Value
required:
- key
- op
- value
title: QueryCondition
type: object
MetricLabel:
description: A label associated with a metric.
properties:
name:
title: Name
type: string
value:
title: Value
type: string
required:
- name
- value
title: MetricLabel
type: object
MetricDataPoint:
description: A single data point in a metric time series.
properties:
timestamp:
title: Timestamp
type: integer
value:
title: Value
type: number
unit:
title: Unit
type: string
required:
- timestamp
- value
- unit
title: MetricDataPoint
type: object
MetricSeries:
description: A time series of metric data points.
properties:
metric:
title: Metric
type: string
labels:
items:
$ref: '#/components/schemas/MetricLabel'
title: Labels
type: array
values:
items:
$ref: '#/components/schemas/MetricDataPoint'
title: Values
type: array
required:
- metric
- labels
- values
title: MetricSeries
type: object
responses:
BadRequest400:
description: The request was invalid or malformed