feat(telemetry:major): End to End Testing, Metric Capture, SQL Alchemy Injection

tests/integration/telemetry/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+

tests/integration/telemetry/mocking/README.md

@@ -0,0 +1,148 @@
+# Mock Server Infrastructure
+
+This directory contains mock servers for E2E telemetry testing.
+
+## Structure
+
+```
+mocking/
+├── README.md          ← You are here
+├── __init__.py        ← Module exports
+├── mock_base.py       ← Pydantic base class for all mocks
+├── servers.py         ← Mock server implementations
+└── harness.py         ← Async startup harness
+```
+
+## Files
+
+### `mock_base.py` - Base Class
+Pydantic base model that all mock servers must inherit from.
+
+**Contract:**
+```python
+class MockServerBase(BaseModel):
+    async def await_start(self):
+        # Start server and wait until ready
+        ...
+    
+    def stop(self):
+        # Stop server and cleanup
+        ...
+```
+
+### `servers.py` - Mock Implementations
+Contains:
+- **MockOTLPCollector** - Receives OTLP telemetry (port 4318)
+- **MockVLLMServer** - Simulates vLLM inference API (port 8000)
+
+### `harness.py` - Startup Orchestration
+Provides:
+- **MockServerConfig** - Pydantic config for server registration
+- **start_mock_servers_async()** - Starts servers in parallel
+- **stop_mock_servers()** - Stops all servers
+
+## Creating a New Mock Server
+
+### Step 1: Implement the Server
+
+Add to `servers.py`:
+```python
+class MockRedisServer(MockServerBase):
+    """Mock Redis server."""
+    
+    port: int = Field(default=6379)
+    
+    # Non-Pydantic fields
+    server: Any = Field(default=None, exclude=True)
+    
+    def model_post_init(self, __context):
+        self.server = None
+    
+    async def await_start(self):
+        """Start Redis mock and wait until ready."""
+        # Start your server
+        self.server = create_redis_server(self.port)
+        self.server.start()
+        
+        # Wait for port to be listening
+        for _ in range(10):
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            if sock.connect_ex(('localhost', self.port)) == 0:
+                sock.close()
+                return  # Ready!
+            await asyncio.sleep(0.1)
+    
+    def stop(self):
+        if self.server:
+            self.server.stop()
+```
+
+### Step 2: Register in Test
+
+In `test_otel_e2e.py`, add to MOCK_SERVERS list:
+```python
+MOCK_SERVERS = [
+    # ... existing servers ...
+    MockServerConfig(
+        name="Mock Redis",
+        server_class=MockRedisServer,
+        init_kwargs={"port": 6379},
+    ),
+]
+```
+
+### Step 3: Done!
+
+The harness automatically:
+- Creates the server instance
+- Calls `await_start()` in parallel with other servers
+- Returns when all are ready
+- Stops all servers on teardown
+
+## Benefits
+
+✅ **Parallel Startup** - All servers start simultaneously  
+✅ **Type-Safe** - Pydantic validation  
+✅ **Simple** - Just implement 2 methods  
+✅ **Fast** - No HTTP polling, direct port checking  
+✅ **Clean** - Async/await pattern  
+
+## Usage in Tests
+
+```python
+@pytest.fixture(scope="module")
+def mock_servers():
+    servers = asyncio.run(start_mock_servers_async(MOCK_SERVERS))
+    yield servers
+    stop_mock_servers(servers)
+
+# Access specific servers
+@pytest.fixture(scope="module")
+def mock_redis(mock_servers):
+    return mock_servers["Mock Redis"]
+```
+
+## Key Design Decisions
+
+### Why Pydantic?
+- Type safety for server configuration
+- Built-in validation
+- Clear interface contract
+
+### Why `await_start()` instead of HTTP `/ready`?
+- Faster (no HTTP round-trip)
+- Simpler (direct port checking)
+- More reliable (internal state, not external endpoint)
+
+### Why separate harness?
+- Reusable across different test files
+- Easy to add new servers
+- Centralized error handling
+
+## Examples
+
+See `test_otel_e2e.py` for real-world usage:
+- Line ~200: MOCK_SERVERS configuration
+- Line ~230: Convenience fixtures
+- Line ~240: Using servers in tests
+

tests/integration/telemetry/mocking/__init__.py

@@ -0,0 +1,29 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Mock server infrastructure for telemetry E2E testing.
+
+This module provides:
+- MockServerBase: Pydantic base class for all mock servers
+- MockOTLPCollector: Mock OTLP telemetry collector
+- MockVLLMServer: Mock vLLM inference server
+- Mock server harness for parallel async startup
+"""
+
+from .mock_base import MockServerBase
+from .servers import MockOTLPCollector, MockVLLMServer
+from .harness import MockServerConfig, start_mock_servers_async, stop_mock_servers
+
+__all__ = [
+    "MockServerBase",
+    "MockOTLPCollector",
+    "MockVLLMServer",
+    "MockServerConfig",
+    "start_mock_servers_async",
+    "stop_mock_servers",
+]
+

tests/integration/telemetry/mocking/harness.py

@@ -0,0 +1,107 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Mock server startup harness for parallel initialization.
+
+HOW TO ADD A NEW MOCK SERVER:
+1. Import your mock server class
+2. Add it to MOCK_SERVERS list with configuration
+3. Done! It will start in parallel with others.
+"""
+
+import asyncio
+from typing import Any, Dict, List
+
+from pydantic import BaseModel, Field
+
+from .mock_base import MockServerBase
+
+
+class MockServerConfig(BaseModel):
+    """
+    Configuration for a mock server to start.
+    
+    **TO ADD A NEW MOCK SERVER:**
+    Just create a MockServerConfig instance with your server class.
+    
+    Example:
+        MockServerConfig(
+            name="Mock MyService",
+            server_class=MockMyService,
+            init_kwargs={"port": 9000, "config_param": "value"},
+        )
+    """
+    
+    model_config = {"arbitrary_types_allowed": True}
+    
+    name: str = Field(description="Display name for logging")
+    server_class: type = Field(description="Mock server class (must inherit from MockServerBase)")
+    init_kwargs: Dict[str, Any] = Field(default_factory=dict, description="Kwargs to pass to server constructor")
+
+
+async def start_mock_servers_async(mock_servers_config: List[MockServerConfig]) -> Dict[str, MockServerBase]:
+    """
+    Start all mock servers in parallel and wait for them to be ready.
+    
+    **HOW IT WORKS:**
+    1. Creates all server instances
+    2. Calls await_start() on all servers in parallel
+    3. Returns when all are ready
+    
+    **SIMPLE TO USE:**
+        servers = await start_mock_servers_async([config1, config2, ...])
+    
+    Args:
+        mock_servers_config: List of mock server configurations
+    
+    Returns:
+        Dict mapping server name to server instance
+    """
+    servers = {}
+    start_tasks = []
+    
+    # Create all servers and prepare start tasks
+    for config in mock_servers_config:
+        server = config.server_class(**config.init_kwargs)
+        servers[config.name] = server
+        start_tasks.append(server.await_start())
+    
+    # Start all servers in parallel
+    try:
+        await asyncio.gather(*start_tasks)
+        
+        # Print readiness confirmation
+        for name in servers.keys():
+            print(f"[INFO] {name} ready")
+        
+    except Exception as e:
+        # If any server fails, stop all servers
+        for server in servers.values():
+            try:
+                server.stop()
+            except:
+                pass
+        raise RuntimeError(f"Failed to start mock servers: {e}")
+    
+    return servers
+
+
+def stop_mock_servers(servers: Dict[str, Any]):
+    """
+    Stop all mock servers.
+    
+    Args:
+        servers: Dict of server instances from start_mock_servers_async()
+    """
+    for name, server in servers.items():
+        try:
+            if hasattr(server, 'get_request_count'):
+                print(f"\n[INFO] {name} received {server.get_request_count()} requests")
+            server.stop()
+        except Exception as e:
+            print(f"[WARN] Error stopping {name}: {e}")
+

tests/integration/telemetry/mocking/mock_base.py

@@ -0,0 +1,69 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Base class for mock servers with async startup support.
+
+All mock servers should inherit from MockServerBase and implement await_start().
+"""
+
+import asyncio
+from abc import abstractmethod
+from pydantic import BaseModel, Field
+
+
+class MockServerBase(BaseModel):
+    """
+    Pydantic base model for mock servers.
+    
+    **TO CREATE A NEW MOCK SERVER:**
+    1. Inherit from this class
+    2. Implement async def await_start(self)
+    3. Implement def stop(self)
+    4. Done!
+    
+    Example:
+        class MyMockServer(MockServerBase):
+            port: int = 8080
+            
+            async def await_start(self):
+                # Start your server
+                self.server = create_server()
+                self.server.start()
+                # Wait until ready (can check internal state, no HTTP needed)
+                while not self.server.is_listening():
+                    await asyncio.sleep(0.1)
+            
+            def stop(self):
+                if self.server:
+                    self.server.stop()
+    """
+    
+    model_config = {"arbitrary_types_allowed": True}
+    
+    @abstractmethod
+    async def await_start(self):
+        """
+        Start the server and wait until it's ready.
+        
+        This method should:
+        1. Start the server (synchronous or async)
+        2. Wait until the server is fully ready to accept requests
+        3. Return when ready
+        
+        Subclasses can check internal state directly - no HTTP polling needed!
+        """
+        ...
+    
+    @abstractmethod
+    def stop(self):
+        """
+        Stop the server and clean up resources.
+        
+        This method should gracefully shut down the server.
+        """
+        ...
+

tests/integration/telemetry/mocking/servers.py

@@ -0,0 +1,387 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Mock servers for OpenTelemetry E2E testing.
+
+This module provides mock servers for testing telemetry:
+- MockOTLPCollector: Receives and stores OTLP telemetry exports
+- MockVLLMServer: Simulates vLLM inference API with valid OpenAI responses
+
+These mocks allow E2E testing without external dependencies.
+"""
+
+import asyncio
+import http.server
+import json
+import socket
+import threading
+import time
+from typing import Any, Dict, List
+
+from pydantic import Field
+
+from .mock_base import MockServerBase
+
+
+class MockOTLPCollector(MockServerBase):
+    """
+    Mock OTLP collector HTTP server.
+    
+    Receives real OTLP exports from Llama Stack and stores them for verification.
+    Runs on localhost:4318 (standard OTLP HTTP port).
+    
+    Usage:
+        collector = MockOTLPCollector()
+        await collector.await_start()
+        # ... run tests ...
+        print(f"Received {collector.get_trace_count()} traces")
+        collector.stop()
+    """
+    
+    port: int = Field(default=4318, description="Port to run collector on")
+    
+    # Non-Pydantic fields (set after initialization)
+    traces: List[Dict] = Field(default_factory=list, exclude=True)
+    metrics: List[Dict] = Field(default_factory=list, exclude=True)
+    server: Any = Field(default=None, exclude=True)
+    server_thread: Any = Field(default=None, exclude=True)
+    
+    def model_post_init(self, __context):
+        """Initialize after Pydantic validation."""
+        self.traces = []
+        self.metrics = []
+        self.server = None
+        self.server_thread = None
+    
+    def _create_handler_class(self):
+        """Create the HTTP handler class for this collector instance."""
+        collector_self = self
+        
+        class OTLPHandler(http.server.BaseHTTPRequestHandler):
+            """HTTP request handler for OTLP requests."""
+            
+            def log_message(self, format, *args):
+                """Suppress HTTP server logs."""
+                pass
+            
+            def do_GET(self):
+                """Handle GET requests."""
+                # No readiness endpoint needed - using await_start() instead
+                self.send_response(404)
+                self.end_headers()
+            
+            def do_POST(self):
+                """Handle OTLP POST requests."""
+                content_length = int(self.headers.get('Content-Length', 0))
+                body = self.rfile.read(content_length) if content_length > 0 else b''
+                
+                # Store the export request
+                if '/v1/traces' in self.path:
+                    collector_self.traces.append({
+                        'body': body,
+                        'timestamp': time.time(),
+                    })
+                elif '/v1/metrics' in self.path:
+                    collector_self.metrics.append({
+                        'body': body,
+                        'timestamp': time.time(),
+                    })
+                
+                # Always return success (200 OK)
+                self.send_response(200)
+                self.send_header('Content-Type', 'application/json')
+                self.end_headers()
+                self.wfile.write(b'{}')
+        
+        return OTLPHandler
+    
+    async def await_start(self):
+        """
+        Start the OTLP collector and wait until ready.
+        
+        This method is async and can be awaited to ensure the server is ready.
+        """
+        # Create handler and start the HTTP server
+        handler_class = self._create_handler_class()
+        self.server = http.server.HTTPServer(('localhost', self.port), handler_class)
+        self.server_thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+        self.server_thread.start()
+        
+        # Wait for server to be listening on the port
+        for _ in range(10):
+            try:
+                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                result = sock.connect_ex(('localhost', self.port))
+                sock.close()
+                if result == 0:
+                    # Port is listening
+                    return
+            except:
+                pass
+            await asyncio.sleep(0.1)
+        
+        raise RuntimeError(f"OTLP collector failed to start on port {self.port}")
+    
+    def stop(self):
+        """Stop the OTLP collector server."""
+        if self.server:
+            self.server.shutdown()
+            self.server.server_close()
+    
+    def clear(self):
+        """Clear all captured telemetry data."""
+        self.traces = []
+        self.metrics = []
+    
+    def get_trace_count(self) -> int:
+        """Get number of trace export requests received."""
+        return len(self.traces)
+    
+    def get_metric_count(self) -> int:
+        """Get number of metric export requests received."""
+        return len(self.metrics)
+    
+    def get_all_traces(self) -> List[Dict]:
+        """Get all captured trace exports."""
+        return self.traces
+    
+    def get_all_metrics(self) -> List[Dict]:
+        """Get all captured metric exports."""
+        return self.metrics
+
+
+class MockVLLMServer(MockServerBase):
+    """
+    Mock vLLM inference server with OpenAI-compatible API.
+    
+    Returns valid OpenAI Python client response objects for:
+    - Chat completions (/v1/chat/completions)
+    - Text completions (/v1/completions)
+    - Model listing (/v1/models)
+    
+    Runs on localhost:8000 (standard vLLM port).
+    
+    Usage:
+        server = MockVLLMServer(models=["my-model"])
+        await server.await_start()
+        # ... make inference calls ...
+        print(f"Handled {server.get_request_count()} requests")
+        server.stop()
+    """
+    
+    port: int = Field(default=8000, description="Port to run server on")
+    models: List[str] = Field(
+        default_factory=lambda: ["meta-llama/Llama-3.2-1B-Instruct"],
+        description="List of model IDs to serve"
+    )
+    
+    # Non-Pydantic fields
+    requests_received: List[Dict] = Field(default_factory=list, exclude=True)
+    server: Any = Field(default=None, exclude=True)
+    server_thread: Any = Field(default=None, exclude=True)
+    
+    def model_post_init(self, __context):
+        """Initialize after Pydantic validation."""
+        self.requests_received = []
+        self.server = None
+        self.server_thread = None
+    
+    def _create_handler_class(self):
+        """Create the HTTP handler class for this vLLM instance."""
+        server_self = self
+        
+        class VLLMHandler(http.server.BaseHTTPRequestHandler):
+            """HTTP request handler for vLLM API."""
+            
+            def log_message(self, format, *args):
+                """Suppress HTTP server logs."""
+                pass
+            
+            def log_request(self, code='-', size='-'):
+                """Log incoming requests for debugging."""
+                print(f"[DEBUG] Mock vLLM received: {self.command} {self.path} -> {code}")
+            
+            def do_GET(self):
+                """Handle GET requests (models list, health check)."""
+                # Log GET requests too
+                server_self.requests_received.append({
+                    'path': self.path,
+                    'method': 'GET',
+                    'timestamp': time.time(),
+                })
+                
+                if self.path == '/v1/models':
+                    response = self._create_models_list_response()
+                    self._send_json_response(200, response)
+                
+                elif self.path == '/health' or self.path == '/v1/health':
+                    self._send_json_response(200, {"status": "healthy"})
+                
+                else:
+                    self.send_response(404)
+                    self.end_headers()
+            
+            def do_POST(self):
+                """Handle POST requests (chat/text completions)."""
+                content_length = int(self.headers.get('Content-Length', 0))
+                body = self.rfile.read(content_length) if content_length > 0 else b'{}'
+                
+                try:
+                    request_data = json.loads(body)
+                except:
+                    request_data = {}
+                
+                # Log the request
+                server_self.requests_received.append({
+                    'path': self.path,
+                    'body': request_data,
+                    'timestamp': time.time(),
+                })
+                
+                # Route to appropriate handler
+                if '/chat/completions' in self.path:
+                    response = self._create_chat_completion_response(request_data)
+                    self._send_json_response(200, response)
+                
+                elif '/completions' in self.path:
+                    response = self._create_text_completion_response(request_data)
+                    self._send_json_response(200, response)
+                
+                else:
+                    self._send_json_response(200, {"status": "ok"})
+            
+            # ----------------------------------------------------------------
+            # Response Generators
+            # **TO MODIFY RESPONSES:** Edit these methods
+            # ----------------------------------------------------------------
+            
+            def _create_models_list_response(self) -> Dict:
+                """Create OpenAI models list response with configured models."""
+                return {
+                    "object": "list",
+                    "data": [
+                        {
+                            "id": model_id,
+                            "object": "model",
+                            "created": int(time.time()),
+                            "owned_by": "meta",
+                        }
+                        for model_id in server_self.models
+                    ]
+                }
+            
+            def _create_chat_completion_response(self, request_data: Dict) -> Dict:
+                """
+                Create OpenAI ChatCompletion response.
+                
+                Returns a valid response matching openai.types.ChatCompletion
+                """
+                return {
+                    "id": "chatcmpl-test123",
+                    "object": "chat.completion",
+                    "created": int(time.time()),
+                    "model": request_data.get("model", "meta-llama/Llama-3.2-1B-Instruct"),
+                    "choices": [{
+                        "index": 0,
+                        "message": {
+                            "role": "assistant",
+                            "content": "This is a test response from mock vLLM server.",
+                            "tool_calls": None,
+                        },
+                        "logprobs": None,
+                        "finish_reason": "stop",
+                    }],
+                    "usage": {
+                        "prompt_tokens": 25,
+                        "completion_tokens": 15,
+                        "total_tokens": 40,
+                        "completion_tokens_details": None,
+                    },
+                    "system_fingerprint": None,
+                    "service_tier": None,
+                }
+            
+            def _create_text_completion_response(self, request_data: Dict) -> Dict:
+                """
+                Create OpenAI Completion response.
+                
+                Returns a valid response matching openai.types.Completion
+                """
+                return {
+                    "id": "cmpl-test123",
+                    "object": "text_completion",
+                    "created": int(time.time()),
+                    "model": request_data.get("model", "meta-llama/Llama-3.2-1B-Instruct"),
+                    "choices": [{
+                        "text": "This is a test completion.",
+                        "index": 0,
+                        "logprobs": None,
+                        "finish_reason": "stop",
+                    }],
+                    "usage": {
+                        "prompt_tokens": 10,
+                        "completion_tokens": 8,
+                        "total_tokens": 18,
+                        "completion_tokens_details": None,
+                    },
+                    "system_fingerprint": None,
+                }
+            
+            def _send_json_response(self, status_code: int, data: Dict):
+                """Helper to send JSON response."""
+                self.send_response(status_code)
+                self.send_header('Content-Type', 'application/json')
+                self.end_headers()
+                self.wfile.write(json.dumps(data).encode())
+        
+        return VLLMHandler
+    
+    async def await_start(self):
+        """
+        Start the vLLM server and wait until ready.
+        
+        This method is async and can be awaited to ensure the server is ready.
+        """
+        # Create handler and start the HTTP server
+        handler_class = self._create_handler_class()
+        self.server = http.server.HTTPServer(('localhost', self.port), handler_class)
+        self.server_thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+        self.server_thread.start()
+        
+        # Wait for server to be listening on the port
+        for _ in range(10):
+            try:
+                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+                result = sock.connect_ex(('localhost', self.port))
+                sock.close()
+                if result == 0:
+                    # Port is listening
+                    return
+            except:
+                pass
+            await asyncio.sleep(0.1)
+        
+        raise RuntimeError(f"vLLM server failed to start on port {self.port}")
+    
+    def stop(self):
+        """Stop the vLLM server."""
+        if self.server:
+            self.server.shutdown()
+            self.server.server_close()
+    
+    def clear(self):
+        """Clear request history."""
+        self.requests_received = []
+    
+    def get_request_count(self) -> int:
+        """Get number of requests received."""
+        return len(self.requests_received)
+    
+    def get_all_requests(self) -> List[Dict]:
+        """Get all received requests with their bodies."""
+        return self.requests_received
+

tests/integration/telemetry/test_otel_e2e.py

@@ -0,0 +1,455 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+End-to-end integration tests for OpenTelemetry with automatic instrumentation.
+
+HOW THIS WORKS:
+1. Starts a mock OTLP collector (HTTP server) to receive telemetry
+2. Starts a mock vLLM server to handle inference requests
+3. Starts REAL Llama Stack with: opentelemetry-instrument llama stack run
+4. Makes REAL API calls to the stack
+5. Verifies telemetry was exported to the mock collector
+
+WHERE TO MAKE CHANGES:
+- Add test cases → See TEST_CASES list below (line ~70)
+- Add mock servers → See MOCK_SERVERS list in mock_servers fixture (line ~200)
+- Modify mock behavior → See mocking/servers.py
+- Change stack config → See llama_stack_server fixture (line ~250)
+- Add assertions → See TestOTelE2EWithRealServer class (line ~370)
+
+RUNNING THE TESTS:
+- Quick (mock servers only): pytest test_otel_e2e.py::TestMockServers -v
+- Full E2E (slow): pytest test_otel_e2e.py::TestOTelE2EWithRealServer -v -m slow
+"""
+
+# ============================================================================
+# IMPORTS
+# ============================================================================
+
+import os
+import socket
+import subprocess
+import time
+from typing import Any, Dict, List
+
+import pytest
+import requests
+import yaml
+from pydantic import BaseModel, Field
+
+# Mock servers are in the mocking/ subdirectory
+from .mocking import (
+    MockOTLPCollector,
+    MockVLLMServer,
+    MockServerConfig,
+    start_mock_servers_async,
+    stop_mock_servers,
+)
+
+
+# ============================================================================
+# DATA MODELS
+# ============================================================================
+
+class TelemetryTestCase(BaseModel):
+    """
+    Pydantic model defining expected telemetry for an API call.
+    
+    **TO ADD A NEW TEST CASE:** Add to TEST_CASES list below.
+    """
+    
+    name: str = Field(description="Unique test case identifier")
+    http_method: str = Field(description="HTTP method (GET, POST, etc.)")
+    api_path: str = Field(description="API path (e.g., '/v1/models')")
+    request_body: Dict[str, Any] | None = Field(default=None)
+    expected_http_status: int = Field(default=200)
+    expected_trace_exports: int = Field(default=1, description="Minimum number of trace exports expected")
+    expected_metric_exports: int = Field(default=0, description="Minimum number of metric exports expected")
+    should_have_error_span: bool = Field(default=False)
+
+
+# ============================================================================
+# TEST CONFIGURATION
+# **TO ADD NEW TESTS:** Add TelemetryTestCase instances here
+# ============================================================================
+
+TEST_CASES = [
+    TelemetryTestCase(
+        name="models_list",
+        http_method="GET",
+        api_path="/v1/models",
+        expected_trace_exports=1,
+        expected_metric_exports=1,  # HTTP metrics from OTel provider middleware
+    ),
+    TelemetryTestCase(
+        name="chat_completion",
+        http_method="POST",
+        api_path="/v1/inference/chat_completion",
+        request_body={
+            "model": "meta-llama/Llama-3.2-1B-Instruct",
+            "messages": [{"role": "user", "content": "Hello!"}],
+        },
+        expected_trace_exports=2,  # Stack request + vLLM backend call
+        expected_metric_exports=1,  # HTTP metrics (duration, count, active_requests)
+    ),
+]
+
+
+# ============================================================================
+# TEST INFRASTRUCTURE
+# ============================================================================
+
+class TelemetryTestRunner:
+    """
+    Executes TelemetryTestCase instances against real Llama Stack.
+    
+    **HOW IT WORKS:**
+    1. Makes real HTTP request to the stack
+    2. Waits for telemetry export
+    3. Verifies exports were sent to mock collector
+    """
+    
+    def __init__(self, base_url: str, collector: MockOTLPCollector):
+        self.base_url = base_url
+        self.collector = collector
+    
+    def run_test_case(self, test_case: TelemetryTestCase, verbose: bool = False) -> bool:
+        """Execute a single test case and verify telemetry."""
+        initial_traces = self.collector.get_trace_count()
+        initial_metrics = self.collector.get_metric_count()
+        
+        if verbose:
+            print(f"\n--- {test_case.name} ---")
+            print(f"  {test_case.http_method} {test_case.api_path}")
+        
+        # Make real HTTP request to Llama Stack
+        try:
+            url = f"{self.base_url}{test_case.api_path}"
+            
+            if test_case.http_method == "GET":
+                response = requests.get(url, timeout=5)
+            elif test_case.http_method == "POST":
+                response = requests.post(url, json=test_case.request_body or {}, timeout=5)
+            else:
+                response = requests.request(test_case.http_method, url, timeout=5)
+            
+            if verbose:
+                print(f"  HTTP Response: {response.status_code}")
+            
+            status_match = response.status_code == test_case.expected_http_status
+            
+        except requests.exceptions.RequestException as e:
+            if verbose:
+                print(f"  Request failed: {e}")
+            status_match = False
+        
+        # Wait for automatic instrumentation to export telemetry
+        # Traces export immediately, metrics export every 1 second (configured via env var)
+        time.sleep(2.0)  # Wait for both traces and metrics to export
+        
+        # Verify traces were exported to mock collector
+        new_traces = self.collector.get_trace_count() - initial_traces
+        traces_exported = new_traces >= test_case.expected_trace_exports
+        
+        # Verify metrics were exported (if expected)
+        new_metrics = self.collector.get_metric_count() - initial_metrics
+        metrics_exported = new_metrics >= test_case.expected_metric_exports
+        
+        if verbose:
+            print(f"  Expected: >={test_case.expected_trace_exports} trace exports, >={test_case.expected_metric_exports} metric exports")
+            print(f"  Actual: {new_traces} trace exports, {new_metrics} metric exports")
+            result = status_match and traces_exported and metrics_exported
+            print(f"  Result: {'PASS' if result else 'FAIL'}")
+        
+        return status_match and traces_exported and metrics_exported
+    
+    def run_all_test_cases(self, test_cases: List[TelemetryTestCase], verbose: bool = True) -> Dict[str, bool]:
+        """Run all test cases and return results."""
+        results = {}
+        for test_case in test_cases:
+            results[test_case.name] = self.run_test_case(test_case, verbose=verbose)
+        return results
+
+
+# ============================================================================
+# HELPER FUNCTIONS
+# ============================================================================
+
+def is_port_available(port: int) -> bool:
+    """Check if a TCP port is available for binding."""
+    try:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+            sock.bind(('localhost', port))
+            return True
+    except OSError:
+        return False
+
+
+# ============================================================================
+# PYTEST FIXTURES
+# ============================================================================
+
+@pytest.fixture(scope="module")
+def mock_servers():
+    """
+    Fixture: Start all mock servers in parallel using async harness.
+    
+    **TO ADD A NEW MOCK SERVER:**
+    Just add a MockServerConfig to the MOCK_SERVERS list below.
+    """
+    import asyncio
+    
+    # ========================================================================
+    # MOCK SERVER CONFIGURATION
+    # **TO ADD A NEW MOCK:** Just add a MockServerConfig instance below
+    # 
+    # Example:
+    #   MockServerConfig(
+    #       name="Mock MyService",
+    #       server_class=MockMyService,  # Must inherit from MockServerBase
+    #       init_kwargs={"port": 9000, "param": "value"},
+    #   ),
+    # ========================================================================
+    MOCK_SERVERS = [
+        MockServerConfig(
+            name="Mock OTLP Collector",
+            server_class=MockOTLPCollector,
+            init_kwargs={"port": 4318},
+        ),
+        MockServerConfig(
+            name="Mock vLLM Server",
+            server_class=MockVLLMServer,
+            init_kwargs={
+                "port": 8000,
+                "models": ["meta-llama/Llama-3.2-1B-Instruct"],
+            },
+        ),
+        # Add more mock servers here - they will start in parallel automatically!
+    ]
+    
+    # Start all servers in parallel
+    servers = asyncio.run(start_mock_servers_async(MOCK_SERVERS))
+    
+    # Verify vLLM models
+    models_response = requests.get("http://localhost:8000/v1/models", timeout=1)
+    models_data = models_response.json()
+    print(f"[INFO] Mock vLLM serving {len(models_data['data'])} models: {[m['id'] for m in models_data['data']]}")
+    
+    yield servers
+    
+    # Stop all servers
+    stop_mock_servers(servers)
+
+
+@pytest.fixture(scope="module")
+def mock_otlp_collector(mock_servers):
+    """Convenience fixture to get OTLP collector from mock_servers."""
+    return mock_servers["Mock OTLP Collector"]
+
+
+@pytest.fixture(scope="module")
+def mock_vllm_server(mock_servers):
+    """Convenience fixture to get vLLM server from mock_servers."""
+    return mock_servers["Mock vLLM Server"]
+
+
+@pytest.fixture(scope="module")
+def llama_stack_server(tmp_path_factory, mock_otlp_collector, mock_vllm_server):
+    """
+    Fixture: Start real Llama Stack server with automatic OTel instrumentation.
+    
+    **THIS IS THE MAIN FIXTURE** - it runs:
+        opentelemetry-instrument llama stack run --config run.yaml
+    
+    **TO MODIFY STACK CONFIG:** Edit run_config dict below
+    """
+    config_dir = tmp_path_factory.mktemp("otel-stack-config")
+    
+    # Ensure mock vLLM is ready and accessible before starting Llama Stack
+    print(f"\n[INFO] Verifying mock vLLM is accessible at http://localhost:8000...")
+    try:
+        vllm_models = requests.get("http://localhost:8000/v1/models", timeout=2)
+        print(f"[INFO] Mock vLLM models endpoint response: {vllm_models.status_code}")
+    except Exception as e:
+        pytest.fail(f"Mock vLLM not accessible before starting Llama Stack: {e}")
+    
+    # Create run.yaml with inference provider
+    # **TO ADD MORE PROVIDERS:** Add to providers dict
+    run_config = {
+        "image_name": "test-otel-e2e",
+        "apis": ["inference"],
+        "providers": {
+            "inference": [
+                {
+                    "provider_id": "vllm",
+                    "provider_type": "remote::vllm",
+                    "config": {
+                        "url": "http://localhost:8000/v1",
+                    },
+                },
+            ],
+        },
+        "models": [
+            {
+                "model_id": "meta-llama/Llama-3.2-1B-Instruct",
+                "provider_id": "vllm",
+            }
+        ],
+    }
+    
+    config_file = config_dir / "run.yaml"
+    with open(config_file, "w") as f:
+        yaml.dump(run_config, f)
+    
+    # Find available port for Llama Stack
+    port = 5555
+    while not is_port_available(port) and port < 5600:
+        port += 1
+    
+    if port >= 5600:
+        pytest.skip("No available ports for test server")
+    
+    # Set environment variables for OTel instrumentation
+    # NOTE: These only affect the subprocess, not other tests
+    env = os.environ.copy()
+    env["OTEL_EXPORTER_OTLP_ENDPOINT"] = "http://localhost:4318"
+    env["OTEL_EXPORTER_OTLP_PROTOCOL"] = "http/protobuf"  # Ensure correct protocol
+    env["OTEL_SERVICE_NAME"] = "llama-stack-e2e-test"
+    env["LLAMA_STACK_PORT"] = str(port)
+    env["OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED"] = "true"
+    
+    # Configure fast metric export for testing (default is 60 seconds)
+    # This makes metrics export every 500ms instead of every 60 seconds
+    env["OTEL_METRIC_EXPORT_INTERVAL"] = "500"  # milliseconds
+    env["OTEL_METRIC_EXPORT_TIMEOUT"] = "1000"  # milliseconds
+    
+    # Disable inference recording to ensure real requests to our mock vLLM
+    # This is critical - without this, Llama Stack replays cached responses
+    # Safe to remove here as it only affects the subprocess environment
+    if "LLAMA_STACK_TEST_INFERENCE_MODE" in env:
+        del env["LLAMA_STACK_TEST_INFERENCE_MODE"]
+    
+    # Start server with automatic instrumentation
+    cmd = [
+        "opentelemetry-instrument",  # ← Automatic instrumentation wrapper
+        "llama", "stack", "run",
+        str(config_file),
+        "--port", str(port),
+    ]
+    
+    print(f"\n[INFO] Starting Llama Stack with OTel instrumentation on port {port}")
+    print(f"[INFO] Command: {' '.join(cmd)}")
+    
+    process = subprocess.Popen(
+        cmd,
+        env=env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    
+    # Wait for server to start
+    max_wait = 30
+    base_url = f"http://localhost:{port}"
+    
+    for i in range(max_wait):
+        try:
+            response = requests.get(f"{base_url}/v1/health", timeout=1)
+            if response.status_code == 200:
+                print(f"[INFO] Server ready at {base_url}")
+                break
+        except requests.exceptions.RequestException:
+            if i == max_wait - 1:
+                process.terminate()
+                stdout, stderr = process.communicate(timeout=5)
+                pytest.fail(f"Server failed to start.\nStdout: {stdout}\nStderr: {stderr}")
+            time.sleep(1)
+    
+    yield {
+        'base_url': base_url,
+        'port': port,
+        'collector': mock_otlp_collector,
+        'vllm_server': mock_vllm_server,
+    }
+    
+    # Cleanup
+    print(f"\n[INFO] Stopping Llama Stack server")
+    process.terminate()
+    try:
+        process.wait(timeout=5)
+    except subprocess.TimeoutExpired:
+        process.kill()
+
+
+# ============================================================================
+# TESTS: End-to-End with Real Stack
+# **THESE RUN SLOW** - marked with @pytest.mark.slow
+# **TO ADD NEW E2E TESTS:** Add methods to this class
+# ============================================================================
+
+@pytest.mark.slow
+class TestOTelE2E:
+    """
+    End-to-end tests with real Llama Stack server.
+    
+    These tests verify the complete flow:
+    - Real Llama Stack with opentelemetry-instrument
+    - Real API calls
+    - Real automatic instrumentation
+    - Mock OTLP collector captures exports
+    """
+    
+    def test_server_starts_with_auto_instrumentation(self, llama_stack_server):
+        """Verify server starts successfully with opentelemetry-instrument."""
+        base_url = llama_stack_server['base_url']
+        
+        # Try different health check endpoints
+        health_endpoints = ["/health", "/v1/health", "/"]
+        server_responding = False
+        
+        for endpoint in health_endpoints:
+            try:
+                response = requests.get(f"{base_url}{endpoint}", timeout=5)
+                print(f"\n[DEBUG] {endpoint} -> {response.status_code}")
+                if response.status_code == 200:
+                    server_responding = True
+                    break
+            except Exception as e:
+                print(f"[DEBUG] {endpoint} failed: {e}")
+        
+        assert server_responding, f"Server not responding on any endpoint at {base_url}"
+        
+        print(f"\n[PASS] Llama Stack running with OTel at {base_url}")
+    
+    def test_all_test_cases_via_runner(self, llama_stack_server):
+        """
+        **MAIN TEST:** Run all TelemetryTestCase instances.
+        
+        This executes all test cases defined in TEST_CASES list.
+        **TO ADD MORE TESTS:** Add to TEST_CASES at top of file
+        """
+        base_url = llama_stack_server['base_url']
+        collector = llama_stack_server['collector']
+        
+        # Create test runner
+        runner = TelemetryTestRunner(base_url, collector)
+        
+        # Execute all test cases
+        results = runner.run_all_test_cases(TEST_CASES, verbose=True)
+        
+        # Print summary
+        print(f"\n{'='*50}")
+        print(f"TEST CASE SUMMARY")
+        print(f"{'='*50}")
+        passed = sum(1 for p in results.values() if p)
+        total = len(results)
+        print(f"Passed: {passed}/{total}\n")
+        
+        for name, result in results.items():
+            status = "[PASS]" if result else "[FAIL]"
+            print(f"  {status} {name}")
+        print(f"{'='*50}\n")

tests/integration/telemetry/test_otel_provider.py

@@ -1,532 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-"""
-Integration tests for OpenTelemetry provider.
-
-These tests verify that the OTel provider correctly:
-- Initializes within the Llama Stack
-- Captures expected metrics (counters, histograms, up/down counters)
-- Captures expected spans/traces
-- Exports telemetry data to an OTLP collector (in-memory for testing)
-
-Tests use in-memory exporters to avoid external dependencies and can run in GitHub Actions.
-"""
-
-import os
-import time
-from collections import defaultdict
-from unittest.mock import patch
-
-import pytest
-from opentelemetry.sdk.metrics.export import InMemoryMetricReader
-from opentelemetry.sdk.trace.export import SimpleSpanProcessor
-from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
-
-from llama_stack.providers.inline.telemetry.otel.config import OTelTelemetryConfig
-from llama_stack.providers.inline.telemetry.otel.otel import OTelTelemetryProvider
-
-
-@pytest.fixture(scope="module")
-def in_memory_span_exporter():
-    """Create an in-memory span exporter to capture traces."""
-    return InMemorySpanExporter()
-
-
-@pytest.fixture(scope="module")
-def in_memory_metric_reader():
-    """Create an in-memory metric reader to capture metrics."""
-    return InMemoryMetricReader()
-
-
-@pytest.fixture(scope="module")
-def otel_provider_with_memory_exporters(in_memory_span_exporter, in_memory_metric_reader):
-    """
-    Create an OTelTelemetryProvider configured with in-memory exporters.
-    
-    This allows us to capture and verify telemetry data without external services.
-    Returns a dict with 'provider', 'span_exporter', and 'metric_reader'.
-    """
-    # Set mock environment to avoid warnings
-    os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = "http://localhost:4318"
-    
-    config = OTelTelemetryConfig(
-        service_name="test-llama-stack-otel",
-        service_version="1.0.0-test",
-        deployment_environment="ci-test",
-        span_processor="simple",
-    )
-    
-    # Patch the provider to use in-memory exporters
-    with patch.object(
-        OTelTelemetryProvider,
-        'model_post_init',
-        lambda self, _: _init_with_memory_exporters(
-            self, config, in_memory_span_exporter, in_memory_metric_reader
-        )
-    ):
-        provider = OTelTelemetryProvider(config=config)
-        yield {
-            'provider': provider,
-            'span_exporter': in_memory_span_exporter,
-            'metric_reader': in_memory_metric_reader
-        }
-
-
-def _init_with_memory_exporters(provider, config, span_exporter, metric_reader):
-    """Helper to initialize provider with in-memory exporters."""
-    import threading
-    from opentelemetry import metrics, trace
-    from opentelemetry.sdk.metrics import MeterProvider
-    from opentelemetry.sdk.resources import Attributes, Resource
-    from opentelemetry.sdk.trace import TracerProvider
-    
-    # Initialize pydantic private attributes
-    if provider.__pydantic_private__ is None:
-        provider.__pydantic_private__ = {}
-    
-    provider._lock = threading.Lock()
-    provider._counters = {}
-    provider._up_down_counters = {}
-    provider._histograms = {}
-    provider._gauges = {}
-    
-    # Create resource attributes
-    attributes: Attributes = {
-        key: value
-        for key, value in {
-            "service.name": config.service_name,
-            "service.version": config.service_version,
-            "deployment.environment": config.deployment_environment,
-        }.items()
-        if value is not None
-    }
-    
-    resource = Resource.create(attributes)
-    
-    # Configure tracer provider with in-memory exporter
-    tracer_provider = TracerProvider(resource=resource)
-    tracer_provider.add_span_processor(SimpleSpanProcessor(span_exporter))
-    trace.set_tracer_provider(tracer_provider)
-    
-    # Configure meter provider with in-memory reader
-    meter_provider = MeterProvider(
-        resource=resource,
-        metric_readers=[metric_reader]
-    )
-    metrics.set_meter_provider(meter_provider)
-
-
-class TestOTelProviderInitialization:
-    """Test OTel provider initialization within Llama Stack."""
-    
-    def test_provider_initializes_successfully(self, otel_provider_with_memory_exporters):
-        """Test that the OTel provider initializes without errors."""
-        provider = otel_provider_with_memory_exporters['provider']
-        span_exporter = otel_provider_with_memory_exporters['span_exporter']
-        
-        assert provider is not None
-        assert provider.config.service_name == "test-llama-stack-otel"
-        assert provider.config.service_version == "1.0.0-test"
-        assert provider.config.deployment_environment == "ci-test"
-    
-    def test_provider_has_thread_safety_mechanisms(self, otel_provider_with_memory_exporters):
-        """Test that the provider has thread-safety mechanisms in place."""
-        provider = otel_provider_with_memory_exporters['provider']
-        
-        assert hasattr(provider, "_lock")
-        assert provider._lock is not None
-        assert hasattr(provider, "_counters")
-        assert hasattr(provider, "_histograms")
-        assert hasattr(provider, "_up_down_counters")
-
-
-class TestOTelMetricsCapture:
-    """Test that OTel provider captures expected metrics."""
-    
-    def test_counter_metric_is_captured(self, otel_provider_with_memory_exporters):
-        """Test that counter metrics are captured."""
-        provider = otel_provider_with_memory_exporters['provider']
-        metric_reader = otel_provider_with_memory_exporters['metric_reader']
-        
-        # Record counter metrics
-        provider.record_count("llama.requests.total", 1.0, attributes={"endpoint": "/chat"})
-        provider.record_count("llama.requests.total", 1.0, attributes={"endpoint": "/chat"})
-        provider.record_count("llama.requests.total", 1.0, attributes={"endpoint": "/embeddings"})
-        
-        # Force metric collection - collect() triggers the reader to gather metrics
-        metric_reader.collect()
-        metric_reader.collect()
-        metrics_data = metric_reader.get_metrics_data()
-        
-        # Verify metrics were captured
-        assert metrics_data is not None
-        assert len(metrics_data.resource_metrics) > 0
-        
-        # Find our counter metric
-        found_counter = False
-        for resource_metric in metrics_data.resource_metrics:
-            for scope_metric in resource_metric.scope_metrics:
-                for metric in scope_metric.metrics:
-                    if metric.name == "llama.requests.total":
-                        found_counter = True
-                        # Verify it's a counter with data points
-                        assert hasattr(metric.data, "data_points")
-                        assert len(metric.data.data_points) > 0
-        
-        assert found_counter, "Counter metric 'llama.requests.total' was not captured"
-    
-    def test_histogram_metric_is_captured(self, otel_provider_with_memory_exporters):
-        """Test that histogram metrics are captured."""
-        provider = otel_provider_with_memory_exporters['provider']
-        metric_reader = otel_provider_with_memory_exporters['metric_reader']
-        
-        # Record histogram metrics with various values
-        latencies = [10.5, 25.3, 50.1, 100.7, 250.2]
-        for latency in latencies:
-            provider.record_histogram(
-                "llama.inference.latency",
-                latency,
-                attributes={"model": "llama-3.2"}
-            )
-        
-        # Force metric collection
-        metric_reader.collect()
-        metrics_data = metric_reader.get_metrics_data()
-        
-        # Find our histogram metric
-        found_histogram = False
-        for resource_metric in metrics_data.resource_metrics:
-            for scope_metric in resource_metric.scope_metrics:
-                for metric in scope_metric.metrics:
-                    if metric.name == "llama.inference.latency":
-                        found_histogram = True
-                        # Verify it's a histogram
-                        assert hasattr(metric.data, "data_points")
-                        data_point = metric.data.data_points[0]
-                        # Histograms should have count and sum
-                        assert hasattr(data_point, "count")
-                        assert data_point.count == len(latencies)
-        
-        assert found_histogram, "Histogram metric 'llama.inference.latency' was not captured"
-    
-    def test_up_down_counter_metric_is_captured(self, otel_provider_with_memory_exporters):
-        """Test that up/down counter metrics are captured."""
-        provider = otel_provider_with_memory_exporters['provider']
-        metric_reader = otel_provider_with_memory_exporters['metric_reader']
-        
-        # Record up/down counter metrics
-        provider.record_up_down_counter("llama.active.sessions", 5)
-        provider.record_up_down_counter("llama.active.sessions", 3)
-        provider.record_up_down_counter("llama.active.sessions", -2)
-        
-        # Force metric collection
-        metric_reader.collect()
-        metrics_data = metric_reader.get_metrics_data()
-        
-        # Find our up/down counter metric
-        found_updown = False
-        for resource_metric in metrics_data.resource_metrics:
-            for scope_metric in resource_metric.scope_metrics:
-                for metric in scope_metric.metrics:
-                    if metric.name == "llama.active.sessions":
-                        found_updown = True
-                        assert hasattr(metric.data, "data_points")
-                        assert len(metric.data.data_points) > 0
-        
-        assert found_updown, "Up/Down counter metric 'llama.active.sessions' was not captured"
-    
-    def test_metrics_with_attributes_are_captured(self, otel_provider_with_memory_exporters):
-        """Test that metric attributes/labels are preserved."""
-        provider = otel_provider_with_memory_exporters['provider']
-        metric_reader = otel_provider_with_memory_exporters['metric_reader']
-        
-        # Record metrics with different attributes
-        provider.record_count("llama.tokens.generated", 150.0, attributes={
-            "model": "llama-3.2-1b",
-            "user": "test-user"
-        })
-        
-        # Force metric collection
-        metric_reader.collect()
-        metrics_data = metric_reader.get_metrics_data()
-        
-        # Verify attributes are preserved
-        found_with_attributes = False
-        for resource_metric in metrics_data.resource_metrics:
-            for scope_metric in resource_metric.scope_metrics:
-                for metric in scope_metric.metrics:
-                    if metric.name == "llama.tokens.generated":
-                        data_point = metric.data.data_points[0]
-                        # Check attributes - they're already a dict in the SDK
-                        attrs = data_point.attributes if isinstance(data_point.attributes, dict) else {}
-                        if "model" in attrs and "user" in attrs:
-                            found_with_attributes = True
-                            assert attrs["model"] == "llama-3.2-1b"
-                            assert attrs["user"] == "test-user"
-        
-        assert found_with_attributes, "Metrics with attributes were not properly captured"
-    
-    def test_multiple_metric_types_coexist(self, otel_provider_with_memory_exporters):
-        """Test that different metric types can coexist."""
-        provider = otel_provider_with_memory_exporters['provider']
-        metric_reader = otel_provider_with_memory_exporters['metric_reader']
-        
-        # Record various metric types
-        provider.record_count("test.counter", 1.0)
-        provider.record_histogram("test.histogram", 42.0)
-        provider.record_up_down_counter("test.gauge", 10)
-        
-        # Force metric collection
-        metric_reader.collect()
-        metrics_data = metric_reader.get_metrics_data()
-        
-        # Count unique metrics
-        metric_names = set()
-        for resource_metric in metrics_data.resource_metrics:
-            for scope_metric in resource_metric.scope_metrics:
-                for metric in scope_metric.metrics:
-                    metric_names.add(metric.name)
-        
-        # Should have all three metrics
-        assert "test.counter" in metric_names
-        assert "test.histogram" in metric_names
-        assert "test.gauge" in metric_names
-
-
-class TestOTelSpansCapture:
-    """Test that OTel provider captures expected spans/traces."""
-    
-    def test_basic_span_is_captured(self, otel_provider_with_memory_exporters):
-        """Test that basic spans are captured."""
-        provider = otel_provider_with_memory_exporters['provider']
-        metric_reader = otel_provider_with_memory_exporters['metric_reader']
-        span_exporter = otel_provider_with_memory_exporters['span_exporter']
-        
-        # Create a span
-        span = provider.custom_trace("llama.inference.request")
-        span.end()
-        
-        # Get captured spans
-        spans = span_exporter.get_finished_spans()
-        
-        assert len(spans) > 0
-        assert any(span.name == "llama.inference.request" for span in spans)
-    
-    def test_span_with_attributes_is_captured(self, otel_provider_with_memory_exporters):
-        """Test that span attributes are preserved."""
-        provider = otel_provider_with_memory_exporters['provider']
-        span_exporter = otel_provider_with_memory_exporters['span_exporter']
-        
-        # Create a span with attributes
-        span = provider.custom_trace(
-            "llama.chat.completion",
-            attributes={
-                "model.id": "llama-3.2-1b",
-                "user.id": "test-user-123",
-                "request.id": "req-abc-123"
-            }
-        )
-        span.end()
-        
-        # Get captured spans
-        spans = span_exporter.get_finished_spans()
-        
-        # Find our span
-        our_span = None
-        for s in spans:
-            if s.name == "llama.chat.completion":
-                our_span = s
-                break
-        
-        assert our_span is not None, "Span 'llama.chat.completion' was not captured"
-        
-        # Verify attributes
-        attrs = dict(our_span.attributes)
-        assert attrs.get("model.id") == "llama-3.2-1b"
-        assert attrs.get("user.id") == "test-user-123"
-        assert attrs.get("request.id") == "req-abc-123"
-    
-    def test_multiple_spans_are_captured(self, otel_provider_with_memory_exporters):
-        """Test that multiple spans are captured."""
-        provider = otel_provider_with_memory_exporters['provider']
-        span_exporter = otel_provider_with_memory_exporters['span_exporter']
-        
-        # Create multiple spans
-        span_names = [
-            "llama.request.validate",
-            "llama.model.load",
-            "llama.inference.execute",
-            "llama.response.format"
-        ]
-        
-        for name in span_names:
-            span = provider.custom_trace(name)
-            time.sleep(0.01)  # Small delay to ensure ordering
-            span.end()
-        
-        # Get captured spans
-        spans = span_exporter.get_finished_spans()
-        captured_names = {span.name for span in spans}
-        
-        # Verify all spans were captured
-        for expected_name in span_names:
-            assert expected_name in captured_names, f"Span '{expected_name}' was not captured"
-    
-    def test_span_has_service_metadata(self, otel_provider_with_memory_exporters):
-        """Test that spans include service metadata."""
-        provider = otel_provider_with_memory_exporters['provider']
-        span_exporter = otel_provider_with_memory_exporters['span_exporter']
-        
-        # Create a span
-        span = provider.custom_trace("test.span")
-        span.end()
-        
-        # Get captured spans
-        spans = span_exporter.get_finished_spans()
-        
-        assert len(spans) > 0
-        
-        # Check resource attributes
-        span = spans[0]
-        resource_attrs = dict(span.resource.attributes)
-        
-        assert resource_attrs.get("service.name") == "test-llama-stack-otel"
-        assert resource_attrs.get("service.version") == "1.0.0-test"
-        assert resource_attrs.get("deployment.environment") == "ci-test"
-
-
-class TestOTelDataExport:
-    """Test that telemetry data can be exported to OTLP collector."""
-    
-    def test_metrics_are_exportable(self, otel_provider_with_memory_exporters):
-        """Test that metrics can be exported."""
-        provider = otel_provider_with_memory_exporters['provider']
-        metric_reader = otel_provider_with_memory_exporters['metric_reader']
-        
-        # Record metrics
-        provider.record_count("export.test.counter", 5.0)
-        provider.record_histogram("export.test.histogram", 123.45)
-        
-        # Force export
-        metric_reader.collect()
-        metrics_data = metric_reader.get_metrics_data()
-        
-        # Verify data structure is exportable
-        assert metrics_data is not None
-        assert hasattr(metrics_data, "resource_metrics")
-        assert len(metrics_data.resource_metrics) > 0
-        
-        # Verify resource attributes are present (needed for OTLP export)
-        resource = metrics_data.resource_metrics[0].resource
-        assert resource is not None
-        assert len(resource.attributes) > 0
-    
-    def test_spans_are_exportable(self, otel_provider_with_memory_exporters):
-        """Test that spans can be exported."""
-        provider = otel_provider_with_memory_exporters['provider']
-        span_exporter = otel_provider_with_memory_exporters['span_exporter']
-        
-        # Create spans
-        span1 = provider.custom_trace("export.test.span1")
-        span1.end()
-        
-        span2 = provider.custom_trace("export.test.span2")
-        span2.end()
-        
-        # Get exported spans
-        spans = span_exporter.get_finished_spans()
-        
-        # Verify spans have required OTLP fields
-        assert len(spans) >= 2
-        for span in spans:
-            assert span.name is not None
-            assert span.context is not None
-            assert span.context.trace_id is not None
-            assert span.context.span_id is not None
-            assert span.resource is not None
-    
-    def test_concurrent_export_is_safe(self, otel_provider_with_memory_exporters):
-        """Test that concurrent metric/span recording doesn't break export."""
-        import concurrent.futures
-        provider = otel_provider_with_memory_exporters['provider']
-        metric_reader = otel_provider_with_memory_exporters['metric_reader']
-        span_exporter = otel_provider_with_memory_exporters['span_exporter']
-        
-        def record_data(thread_id):
-            for i in range(10):
-                provider.record_count(f"concurrent.counter.{thread_id}", 1.0)
-                span = provider.custom_trace(f"concurrent.span.{thread_id}.{i}")
-                span.end()
-        
-        # Record from multiple threads
-        with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
-            futures = [executor.submit(record_data, i) for i in range(5)]
-            concurrent.futures.wait(futures)
-        
-        # Verify export still works
-        metric_reader.collect()
-        metrics_data = metric_reader.get_metrics_data()
-        spans = span_exporter.get_finished_spans()
-        
-        assert metrics_data is not None
-        assert len(spans) >= 50  # 5 threads * 10 spans each
-
-
-@pytest.mark.integration
-class TestOTelProviderIntegration:
-    """End-to-end integration tests simulating real usage."""
-    
-    def test_complete_inference_workflow_telemetry(self, otel_provider_with_memory_exporters):
-        """Simulate a complete inference workflow with telemetry."""
-        provider = otel_provider_with_memory_exporters['provider']
-        metric_reader = otel_provider_with_memory_exporters['metric_reader']
-        span_exporter = otel_provider_with_memory_exporters['span_exporter']
-        
-        # Simulate inference workflow
-        request_span = provider.custom_trace(
-            "llama.inference.request",
-            attributes={"model": "llama-3.2-1b", "user": "test"}
-        )
-        
-        # Track metrics during inference
-        provider.record_count("llama.requests.received", 1.0)
-        provider.record_up_down_counter("llama.requests.in_flight", 1)
-        
-        # Simulate processing time
-        time.sleep(0.01)
-        provider.record_histogram("llama.request.duration_ms", 10.5)
-        
-        # Track tokens
-        provider.record_count("llama.tokens.input", 25.0)
-        provider.record_count("llama.tokens.output", 150.0)
-        
-        # End request
-        provider.record_up_down_counter("llama.requests.in_flight", -1)
-        provider.record_count("llama.requests.completed", 1.0)
-        request_span.end()
-        
-        # Verify all telemetry was captured
-        metric_reader.collect()
-        metrics_data = metric_reader.get_metrics_data()
-        spans = span_exporter.get_finished_spans()
-        
-        # Check metrics exist
-        metric_names = set()
-        for rm in metrics_data.resource_metrics:
-            for sm in rm.scope_metrics:
-                for m in sm.metrics:
-                    metric_names.add(m.name)
-        
-        assert "llama.requests.received" in metric_names
-        assert "llama.requests.in_flight" in metric_names
-        assert "llama.request.duration_ms" in metric_names
-        assert "llama.tokens.input" in metric_names
-        assert "llama.tokens.output" in metric_names
-        
-        # Check span exists
-        assert any(s.name == "llama.inference.request" for s in spans)
-