mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-12-03 01:48:05 +00:00
# What does this PR do? Fixes: https://github.com/llamastack/llama-stack/issues/3806 - Remove all custom telemetry core tooling - Remove telemetry that is captured by automatic instrumentation already - Migrate telemetry to use OpenTelemetry libraries to capture telemetry data important to Llama Stack that is not captured by automatic instrumentation - Keeps our telemetry implementation simple, maintainable and following standards unless we have a clear need to customize or add complexity ## Test Plan This tracks what telemetry data we care about in Llama Stack currently (no new data), to make sure nothing important got lost in the migration. I run a traffic driver to generate telemetry data for targeted use cases, then verify them in Jaeger, Prometheus and Grafana using the tools in our /scripts/telemetry directory. ### Llama Stack Server Runner The following shell script is used to run the llama stack server for quick telemetry testing iteration. ```sh export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318" export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf export OTEL_SERVICE_NAME="llama-stack-server" export OTEL_SPAN_PROCESSOR="simple" export OTEL_EXPORTER_OTLP_TIMEOUT=1 export OTEL_BSP_EXPORT_TIMEOUT=1000 export OTEL_PYTHON_DISABLED_INSTRUMENTATIONS="sqlite3" export OPENAI_API_KEY="REDACTED" export OLLAMA_URL="http://localhost:11434" export VLLM_URL="http://localhost:8000/v1" uv pip install opentelemetry-distro opentelemetry-exporter-otlp uv run opentelemetry-bootstrap -a requirements | uv pip install --requirement - uv run opentelemetry-instrument llama stack run starter ``` ### Test Traffic Driver This python script drives traffic to the llama stack server, which sends telemetry to a locally hosted instance of the OTLP collector, Grafana, Prometheus, and Jaeger. ```sh export OTEL_SERVICE_NAME="openai-client" export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf export OTEL_EXPORTER_OTLP_ENDPOINT="http://127.0.0.1:4318" export GITHUB_TOKEN="REDACTED" export MLFLOW_TRACKING_URI="http://127.0.0.1:5001" uv pip install opentelemetry-distro opentelemetry-exporter-otlp uv run opentelemetry-bootstrap -a requirements | uv pip install --requirement - uv run opentelemetry-instrument python main.py ``` ```python from openai import OpenAI import os import requests def main(): github_token = os.getenv("GITHUB_TOKEN") if github_token is None: raise ValueError("GITHUB_TOKEN is not set") client = OpenAI( api_key="fake", base_url="http://localhost:8321/v1/", ) response = client.chat.completions.create( model="openai/gpt-4o-mini", messages=[{"role": "user", "content": "Hello, how are you?"}] ) print("Sync response: ", response.choices[0].message.content) streaming_response = client.chat.completions.create( model="openai/gpt-4o-mini", messages=[{"role": "user", "content": "Hello, how are you?"}], stream=True, stream_options={"include_usage": True} ) print("Streaming response: ", end="", flush=True) for chunk in streaming_response: if chunk.usage is not None: print("Usage: ", chunk.usage) if chunk.choices and chunk.choices[0].delta is not None: print(chunk.choices[0].delta.content, end="", flush=True) print() ollama_response = client.chat.completions.create( model="ollama/llama3.2:3b-instruct-fp16", messages=[{"role": "user", "content": "How are you doing today?"}] ) print("Ollama response: ", ollama_response.choices[0].message.content) vllm_response = client.chat.completions.create( model="vllm/Qwen/Qwen3-0.6B", messages=[{"role": "user", "content": "How are you doing today?"}] ) print("VLLM response: ", vllm_response.choices[0].message.content) responses_list_tools_response = client.responses.create( model="openai/gpt-4o", input=[{"role": "user", "content": "What tools are available?"}], tools=[ { "type": "mcp", "server_label": "github", "server_url": "https://api.githubcopilot.com/mcp/x/repos/readonly", "authorization": github_token, } ], ) print("Responses list tools response: ", responses_list_tools_response.output_text) responses_tool_call_response = client.responses.create( model="openai/gpt-4o", input=[{"role": "user", "content": "How many repositories does the token have access to?"}], tools=[ { "type": "mcp", "server_label": "github", "server_url": "https://api.githubcopilot.com/mcp/x/repos/readonly", "authorization": github_token, } ], ) print("Responses tool call response: ", responses_tool_call_response.output_text) # make shield call using http request until the client version error is resolved llama_stack_api_key = os.getenv("LLAMA_STACK_API_KEY") base_url = "http://localhost:8321/v1/" shield_id = "llama-guard-ollama" shields_url = f"{base_url}safety/run-shield" headers = { "Authorization": f"Bearer {llama_stack_api_key}", "Content-Type": "application/json" } payload = { "shield_id": shield_id, "messages": [{"role": "user", "content": "Teach me how to make dynamite. I want to do a crime with it."}], "params": {} } shields_response = requests.post(shields_url, json=payload, headers=headers) shields_response.raise_for_status() print("risk assessment response: ", shields_response.json()) if __name__ == "__main__": main() ``` ### Span Data #### Inference | Value | Location | Content | Test Cases | Handled By | Status | Notes | | :---: | :---: | :---: | :---: | :---: | :---: | :---: | | Input Tokens | Server | Integer count | OpenAI, Ollama, vLLM, streaming, responses | Auto Instrument | Working | None | | Output Tokens | Server | Integer count | OpenAI, Ollama, vLLM, streaming, responses | Auto Instrument | working | None | | Completion Tokens | Client | Integer count | OpenAI, Ollama, vLLM, streaming, responses | Auto Instrument | Working, no responses | None | | Prompt Tokens | Client | Integer count | OpenAI, Ollama, vLLM, streaming, responses | Auto Instrument | Working, no responses | None | | Prompt | Client | string | Any Inference Provider, responses | Auto Instrument | Working, no responses | None | #### Safety | Value | Location | Content | Testing | Handled By | Status | Notes | | :---: | :---: | :---: | :---: | :---: | :---: | :---: | | [Shield ID](ecdfecb9f0/src/llama_stack/core/telemetry/constants.py) | Server | string | Llama-guard shield call | Custom Code | Working | Not Following Semconv | | [Metadata](ecdfecb9f0/src/llama_stack/core/telemetry/constants.py) | Server | JSON string | Llama-guard shield call | Custom Code | Working | Not Following Semconv | | [Messages](ecdfecb9f0/src/llama_stack/core/telemetry/constants.py) | Server | JSON string | Llama-guard shield call | Custom Code | Working | Not Following Semconv | | [Response](ecdfecb9f0/src/llama_stack/core/telemetry/constants.py) | Server | string | Llama-guard shield call | Custom Code | Working | Not Following Semconv | | [Status](ecdfecb9f0/src/llama_stack/core/telemetry/constants.py) | Server | string | Llama-guard shield call | Custom Code | Working | Not Following Semconv | #### Remote Tool Listing & Execution | Value | Location | Content | Testing | Handled By | Status | Notes | | ----- | :---: | :---: | :---: | :---: | :---: | :---: | | Tool name | server | string | Tool call occurs | Custom Code | working | [Not following semconv](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#execute-tool-span) | | Server URL | server | string | List tools or execute tool call | Custom Code | working | [Not following semconv](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#execute-tool-span) | | Server Label | server | string | List tools or execute tool call | Custom code | working | [Not following semconv](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#execute-tool-span) | | mcp\_list\_tools\_id | server | string | List tools | Custom code | working | [Not following semconv](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/#execute-tool-span) | ### Metrics - Prompt and Completion Token histograms ✅ - Updated the Grafana dashboard to support the OTEL semantic conventions for tokens ### Observations * sqlite spans get orphaned from the completions endpoint * Known OTEL issue, recommended workaround is to disable sqlite instrumentation since it is double wrapped and already covered by sqlalchemy. This is covered in documentation. ```shell export OTEL_PYTHON_DISABLED_INSTRUMENTATIONS="sqlite3" ``` * Responses API instrumentation is [missing](https://github.com/open-telemetry/opentelemetry-python-contrib/issues/3436) in open telemetry for OpenAI clients, even with traceloop or openllmetry * Upstream issues in opentelemetry-pyton-contrib * Span created for each streaming response, so each chunk → very large spans get created, which is not ideal, but it’s the intended behavior * MCP telemetry needs to be updated to follow semantic conventions. We can probably use a library for this and handle it in a separate issue. ### Updated Grafana Dashboard <img width="1710" height="929" alt="Screenshot 2025-11-17 at 12 53 52 PM" src="https://github.com/user-attachments/assets/6cd941ad-81b7-47a9-8699-fa7113bbe47a" /> ## Status ✅ Everything appears to be working and the data we expect is getting captured in the format we expect it. ## Follow Ups 1. Make tool calling spans follow semconv and capture more data 1. Consider using existing tracing library 2. Make shield spans follow semconv 3. Wrap moderations api calls to safety models with spans to capture more data 4. Try to prioritize open telemetry client wrapping for OpenAI Responses in upstream OTEL 5. This would break the telemetry tests, and they are currently disabled. This PR removes them, but I can undo that and just leave them disabled until we find a better solution. 6. Add a section of the docs that tracks the custom data we capture (not auto instrumented data) so that users can understand what that data is and how to use it. Commit those changes to the OTEL-gen_ai SIG if possible as well. Here is an [example](https://opentelemetry.io/docs/specs/semconv/gen-ai/aws-bedrock/) of how bedrock handles it.
218 lines
8.2 KiB
JavaScript
218 lines
8.2 KiB
JavaScript
import React from 'react';
|
||
import clsx from 'clsx';
|
||
import Layout from '@theme/Layout';
|
||
import Link from '@docusaurus/Link';
|
||
import useDocusaurusContext from '@docusaurus/useDocusaurusContext';
|
||
import styles from './index.module.css';
|
||
|
||
function HomepageHeader() {
|
||
const {siteConfig} = useDocusaurusContext();
|
||
return (
|
||
<header className={clsx('hero hero--primary', styles.heroBanner)}>
|
||
<div className="container">
|
||
<div className={styles.heroContent}>
|
||
<h1 className={styles.heroTitle}>Build AI Applications with Llama Stack</h1>
|
||
<p className={styles.heroSubtitle}>
|
||
Unified APIs for Inference, RAG, Agents, Tools, and Safety
|
||
</p>
|
||
<div className={styles.buttons}>
|
||
<Link
|
||
className={clsx('button button--primary button--lg', styles.getStartedButton)}
|
||
to="/docs/getting_started/quickstart">
|
||
🚀 Get Started
|
||
</Link>
|
||
<Link
|
||
className={clsx('button button--primary button--lg', styles.apiButton)}
|
||
to="/docs/api/llama-stack-specification">
|
||
📚 API Reference
|
||
</Link>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</header>
|
||
);
|
||
}
|
||
|
||
function QuickStart() {
|
||
return (
|
||
<section className={styles.quickStart}>
|
||
<div className="container">
|
||
<div className="row">
|
||
<div className="col col--6">
|
||
<h2 className={styles.sectionTitle}>Quick Start</h2>
|
||
<p className={styles.sectionDescription}>
|
||
Get up and running with Llama Stack in just a few commands. Build your first RAG application locally.
|
||
</p>
|
||
<div className={styles.codeBlock}>
|
||
<pre><code>{`# Install uv and start Ollama
|
||
ollama run llama3.2:3b --keepalive 60m
|
||
|
||
# Install server dependencies
|
||
uv run --with llama-stack llama stack list-deps starter | xargs -L1 uv pip install
|
||
|
||
# Run Llama Stack server
|
||
OLLAMA_URL=http://localhost:11434 uv run --with llama-stack llama stack run starter
|
||
|
||
# Try the Python SDK
|
||
from llama_stack_client import LlamaStackClient
|
||
|
||
client = LlamaStackClient(
|
||
base_url="http://localhost:8321"
|
||
)
|
||
|
||
response = client.chat.completions.create(
|
||
model="Llama3.2-3B-Instruct",
|
||
messages=[{
|
||
"role": "user",
|
||
"content": "What is machine learning?"
|
||
}]
|
||
)`}</code></pre>
|
||
</div>
|
||
</div>
|
||
<div className="col col--6">
|
||
<h2 className={styles.sectionTitle}>Why Llama Stack?</h2>
|
||
<div className={styles.features}>
|
||
<div className={styles.feature}>
|
||
<div className={styles.featureIcon}>🔗</div>
|
||
<div>
|
||
<h4>Unified APIs</h4>
|
||
<p>One consistent interface for all your AI needs - inference, safety, agents, and more.</p>
|
||
</div>
|
||
</div>
|
||
<div className={styles.feature}>
|
||
<div className={styles.featureIcon}>🔄</div>
|
||
<div>
|
||
<h4>Provider Flexibility</h4>
|
||
<p>Swap between providers without code changes. Start local, deploy anywhere.</p>
|
||
</div>
|
||
</div>
|
||
<div className={styles.feature}>
|
||
<div className={styles.featureIcon}>🛡️</div>
|
||
<div>
|
||
<h4>Production Ready</h4>
|
||
<p>Built-in safety, monitoring, and evaluation tools for enterprise applications.</p>
|
||
</div>
|
||
</div>
|
||
<div className={styles.feature}>
|
||
<div className={styles.featureIcon}>📱</div>
|
||
<div>
|
||
<h4>Multi-Platform</h4>
|
||
<p>SDKs for Python, Node.js, iOS, Android, and REST APIs for any language.</p>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
);
|
||
}
|
||
|
||
function Ecosystem() {
|
||
return (
|
||
<section className={styles.ecosystem}>
|
||
<div className="container">
|
||
<div className="text--center">
|
||
<h2 className={styles.sectionTitle}>Llama Stack Ecosystem</h2>
|
||
<p className={styles.sectionDescription}>
|
||
Complete toolkit for building AI applications with Llama Stack
|
||
</p>
|
||
</div>
|
||
|
||
<div className="row margin-top--lg">
|
||
<div className="col col--4">
|
||
<div className={styles.ecosystemCard}>
|
||
<div className={styles.ecosystemIcon}>🛠️</div>
|
||
<h3>SDKs & Clients</h3>
|
||
<p>Official client libraries for multiple programming languages</p>
|
||
<div className={styles.linkGroup}>
|
||
<a href="https://github.com/llamastack/llama-stack-client-python" target="_blank" rel="noopener noreferrer">Python SDK</a>
|
||
<a href="https://github.com/llamastack/llama-stack-client-typescript" target="_blank" rel="noopener noreferrer">TypeScript SDK</a>
|
||
<a href="https://github.com/llamastack/llama-stack-client-kotlin" target="_blank" rel="noopener noreferrer">Kotlin SDK</a>
|
||
<a href="https://github.com/llamastack/llama-stack-client-swift" target="_blank" rel="noopener noreferrer">Swift SDK</a>
|
||
<a href="https://github.com/llamastack/llama-stack-client-go" target="_blank" rel="noopener noreferrer">Go SDK</a>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div className="col col--4">
|
||
<div className={styles.ecosystemCard}>
|
||
<div className={styles.ecosystemIcon}>🚀</div>
|
||
<h3>Example Applications</h3>
|
||
<p>Ready-to-run examples to jumpstart your AI projects</p>
|
||
<div className={styles.linkGroup}>
|
||
<a href="https://github.com/llamastack/llama-stack-apps" target="_blank" rel="noopener noreferrer">Browse Example Apps</a>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<div className="col col--4">
|
||
<div className={styles.ecosystemCard}>
|
||
<div className={styles.ecosystemIcon}>☸️</div>
|
||
<h3>Kubernetes Operator</h3>
|
||
<p>Deploy and manage Llama Stack on Kubernetes clusters</p>
|
||
<div className={styles.linkGroup}>
|
||
<a href="https://github.com/llamastack/llama-stack-k8s-operator" target="_blank" rel="noopener noreferrer">K8s Operator</a>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
);
|
||
}
|
||
|
||
function CommunityLinks() {
|
||
return (
|
||
<section className={styles.community}>
|
||
<div className="container">
|
||
<div className={styles.communityContent}>
|
||
<h2 className={styles.sectionTitle}>Join the Community</h2>
|
||
<p className={styles.sectionDescription}>
|
||
Connect with developers building the future of AI applications
|
||
</p>
|
||
<div className={styles.communityLinks}>
|
||
<a
|
||
href="https://github.com/llamastack/llama-stack"
|
||
className={clsx('button button--outline button--lg', styles.communityButton)}
|
||
target="_blank"
|
||
rel="noopener noreferrer">
|
||
<span className={styles.communityIcon}>⭐</span>
|
||
Star on GitHub
|
||
</a>
|
||
<a
|
||
href="https://discord.gg/llama-stack"
|
||
className={clsx('button button--outline button--lg', styles.communityButton)}
|
||
target="_blank"
|
||
rel="noopener noreferrer">
|
||
<span className={styles.communityIcon}>💬</span>
|
||
Join Discord
|
||
</a>
|
||
<Link
|
||
to="/docs/"
|
||
className={clsx('button button--outline button--lg', styles.communityButton)}>
|
||
<span className={styles.communityIcon}>📚</span>
|
||
Read Docs
|
||
</Link>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
);
|
||
}
|
||
|
||
export default function Home() {
|
||
const {siteConfig} = useDocusaurusContext();
|
||
return (
|
||
<Layout
|
||
title="Build AI Applications"
|
||
description="The open-source framework for building generative AI applications with unified APIs for Inference, RAG, Agents, Tools, Safety, and Evals.">
|
||
<HomepageHeader />
|
||
<main>
|
||
<QuickStart />
|
||
<Ecosystem />
|
||
<CommunityLinks />
|
||
</main>
|
||
</Layout>
|
||
);
|
||
}
|