llama-stack-mirror/tests/unit/providers/telemetry/test_agent_metrics_histogram.py

# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.

from unittest.mock import Mock

import pytest

from llama_stack.apis.telemetry import MetricEvent, MetricType
from llama_stack.providers.inline.telemetry.meta_reference.config import TelemetryConfig
from llama_stack.providers.inline.telemetry.meta_reference.telemetry import TelemetryAdapter


class TestAgentMetricsHistogram:
    """Unit tests for histogram support in telemetry adapter for agent metrics"""

    @pytest.fixture
    def telemetry_config(self):
        """Basic telemetry config for testing"""
        return TelemetryConfig(
            service_name="test-service",
            sinks=[],
        )

    @pytest.fixture
    def telemetry_adapter(self, telemetry_config):
        """TelemetryAdapter with mocked meter"""
        adapter = TelemetryAdapter(telemetry_config, {})
        # Mock the meter to avoid OpenTelemetry setup
        adapter.meter = Mock()
        return adapter

    def test_get_or_create_histogram_new(self, telemetry_adapter):
        """Test creating a new histogram"""
        mock_histogram = Mock()
        telemetry_adapter.meter.create_histogram.return_value = mock_histogram

        # Clear global storage to ensure clean state
        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["histograms"] = {}

        result = telemetry_adapter._get_or_create_histogram("test_histogram", "s", [0.1, 0.5, 1.0, 5.0, 10.0])

        assert result == mock_histogram
        telemetry_adapter.meter.create_histogram.assert_called_once_with(
            name="test_histogram",
            unit="s",
            description="Histogram for test_histogram",
        )
        assert _GLOBAL_STORAGE["histograms"]["test_histogram"] == mock_histogram

    def test_get_or_create_histogram_existing(self, telemetry_adapter):
        """Test retrieving an existing histogram"""
        mock_histogram = Mock()

        # Pre-populate global storage
        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["histograms"] = {"existing_histogram": mock_histogram}

        result = telemetry_adapter._get_or_create_histogram("existing_histogram", "ms")

        assert result == mock_histogram
        # Should not create a new histogram
        telemetry_adapter.meter.create_histogram.assert_not_called()

    def test_log_metric_duration_histogram(self, telemetry_adapter):
        """Test logging duration metrics creates histogram"""
        mock_histogram = Mock()
        telemetry_adapter.meter.create_histogram.return_value = mock_histogram

        # Clear global storage
        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["histograms"] = {}

        metric_event = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="llama_stack_agent_workflow_duration_seconds",
            value=15.7,
            timestamp=1234567890.0,
            unit="s",
            attributes={"agent_id": "test-agent"},
            metric_type=MetricType.HISTOGRAM,
        )

        telemetry_adapter._log_metric(metric_event)

        # Verify histogram was created and recorded
        telemetry_adapter.meter.create_histogram.assert_called_once_with(
            name="llama_stack_agent_workflow_duration_seconds",
            unit="s",
            description="Histogram for llama_stack_agent_workflow_duration_seconds",
        )
        mock_histogram.record.assert_called_once_with(15.7, attributes={"agent_id": "test-agent"})

    def test_log_metric_duration_histogram_default_buckets(self, telemetry_adapter):
        """Test that duration metrics use default buckets"""
        mock_histogram = Mock()
        telemetry_adapter.meter.create_histogram.return_value = mock_histogram

        # Clear global storage
        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["histograms"] = {}

        metric_event = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="custom_duration_seconds",
            value=5.2,
            timestamp=1234567890.0,
            unit="s",
            attributes={},
            metric_type=MetricType.HISTOGRAM,
        )

        telemetry_adapter._log_metric(metric_event)

        # Verify histogram was created (buckets are not passed to create_histogram in OpenTelemetry)
        mock_histogram.record.assert_called_once_with(5.2, attributes={})

    def test_log_metric_non_duration_counter(self, telemetry_adapter):
        """Test that non-duration metrics still use counters"""
        mock_counter = Mock()
        telemetry_adapter.meter.create_counter.return_value = mock_counter

        # Clear global storage
        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["counters"] = {}

        metric_event = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="llama_stack_agent_workflows_total",
            value=1,
            timestamp=1234567890.0,
            unit="1",
            attributes={"agent_id": "test-agent", "status": "completed"},
        )

        telemetry_adapter._log_metric(metric_event)

        # Verify counter was used, not histogram
        telemetry_adapter.meter.create_counter.assert_called_once()
        telemetry_adapter.meter.create_histogram.assert_not_called()
        mock_counter.add.assert_called_once_with(1, attributes={"agent_id": "test-agent", "status": "completed"})

    def test_log_metric_no_meter(self, telemetry_adapter):
        """Test metric logging when meter is None"""
        telemetry_adapter.meter = None

        metric_event = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="test_duration_seconds",
            value=1.0,
            timestamp=1234567890.0,
            unit="s",
            attributes={},
        )

        # Should not raise exception
        telemetry_adapter._log_metric(metric_event)

    def test_histogram_name_detection_patterns(self, telemetry_adapter):
        """Test various duration metric name patterns"""
        mock_histogram = Mock()
        telemetry_adapter.meter.create_histogram.return_value = mock_histogram

        # Clear global storage
        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["histograms"] = {}

        duration_metrics = [
            "workflow_duration_seconds",
            "request_duration_seconds",
            "processing_duration_seconds",
            "llama_stack_agent_workflow_duration_seconds",
        ]

        for metric_name in duration_metrics:
            _GLOBAL_STORAGE["histograms"] = {}  # Reset for each test

            metric_event = MetricEvent(
                trace_id="123",
                span_id="456",
                metric=metric_name,
                value=1.0,
                timestamp=1234567890.0,
                unit="s",
                attributes={},
                metric_type=MetricType.HISTOGRAM,
            )

            telemetry_adapter._log_metric(metric_event)
            mock_histogram.record.assert_called()

        # Reset call count for negative test
        mock_histogram.record.reset_mock()
        telemetry_adapter.meter.create_histogram.reset_mock()

        # Test non-duration metric
        non_duration_metric = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="workflow_total",  # No "_duration_seconds" suffix
            value=1,
            timestamp=1234567890.0,
            unit="1",
            attributes={},
        )

        telemetry_adapter._log_metric(non_duration_metric)

        # Should not create histogram for non-duration metric
        telemetry_adapter.meter.create_histogram.assert_not_called()
        mock_histogram.record.assert_not_called()

    def test_histogram_global_storage_isolation(self, telemetry_adapter):
        """Test that histogram storage doesn't interfere with counters"""
        mock_histogram = Mock()
        mock_counter = Mock()

        telemetry_adapter.meter.create_histogram.return_value = mock_histogram
        telemetry_adapter.meter.create_counter.return_value = mock_counter

        # Clear global storage
        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["histograms"] = {}
        _GLOBAL_STORAGE["counters"] = {}

        # Create histogram
        duration_metric = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="test_duration_seconds",
            value=1.0,
            timestamp=1234567890.0,
            unit="s",
            attributes={},
            metric_type=MetricType.HISTOGRAM,
        )
        telemetry_adapter._log_metric(duration_metric)

        # Create counter
        counter_metric = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="test_counter",
            value=1,
            timestamp=1234567890.0,
            unit="1",
            attributes={},
        )
        telemetry_adapter._log_metric(counter_metric)

        # Verify both were created and stored separately
        assert "test_duration_seconds" in _GLOBAL_STORAGE["histograms"]
        assert "test_counter" in _GLOBAL_STORAGE["counters"]
        assert "test_duration_seconds" not in _GLOBAL_STORAGE["counters"]
        assert "test_counter" not in _GLOBAL_STORAGE["histograms"]

    def test_histogram_buckets_parameter_ignored(self, telemetry_adapter):
        """Test that buckets parameter doesn't affect histogram creation (OpenTelemetry handles buckets internally)"""
        mock_histogram = Mock()
        telemetry_adapter.meter.create_histogram.return_value = mock_histogram

        # Clear global storage
        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["histograms"] = {}

        # Call with buckets parameter
        result = telemetry_adapter._get_or_create_histogram(
            "test_histogram", "s", buckets=[0.1, 0.5, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0]
        )

        # Buckets are not passed to OpenTelemetry create_histogram
        telemetry_adapter.meter.create_histogram.assert_called_once_with(
            name="test_histogram",
            unit="s",
            description="Histogram for test_histogram",
        )
        assert result == mock_histogram