llama-stack-mirror/tests/unit/providers/telemetry/test_agent_metrics_histogram.py

# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.

from unittest.mock import Mock

import pytest

from llama_stack.apis.telemetry import MetricEvent, MetricType
from llama_stack.providers.inline.telemetry.meta_reference.config import TelemetryConfig
from llama_stack.providers.inline.telemetry.meta_reference.telemetry import TelemetryAdapter


class TestAgentMetricsHistogram:
    """Tests for agent histogram metrics"""

    @pytest.fixture
    def config(self):
        return TelemetryConfig(service_name="test-service", sinks=[])

    @pytest.fixture
    def adapter(self, config):
        adapter = TelemetryAdapter(config, {})
        adapter.meter = Mock()  # skip otel setup
        return adapter

    def test_histogram_creation(self, adapter):
        mock_hist = Mock()
        adapter.meter.create_histogram.return_value = mock_hist

        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["histograms"] = {}

        result = adapter._get_or_create_histogram("test_histogram", "s")

        assert result == mock_hist
        adapter.meter.create_histogram.assert_called_once_with(
            name="test_histogram",
            unit="s",
            description="test histogram",
        )
        assert _GLOBAL_STORAGE["histograms"]["test_histogram"] == mock_hist

    def test_histogram_reuse(self, adapter):
        mock_hist = Mock()
        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["histograms"] = {"existing_histogram": mock_hist}

        result = adapter._get_or_create_histogram("existing_histogram", "ms")

        assert result == mock_hist
        adapter.meter.create_histogram.assert_not_called()

    def test_workflow_duration_histogram(self, adapter):
        mock_hist = Mock()
        adapter.meter.create_histogram.return_value = mock_hist

        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["histograms"] = {}

        event = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="llama_stack_agent_workflow_duration_seconds",
            value=15.7,
            timestamp=1234567890.0,
            unit="s",
            attributes={"agent_id": "test-agent"},
            metric_type=MetricType.HISTOGRAM,
        )

        adapter._log_metric(event)

        adapter.meter.create_histogram.assert_called_once_with(
            name="llama_stack_agent_workflow_duration_seconds",
            unit="s",
            description="llama stack agent workflow duration seconds",
        )
        mock_hist.record.assert_called_once_with(15.7, attributes={"agent_id": "test-agent"})

    def test_duration_buckets_configured_via_views(self, adapter):
        mock_hist = Mock()
        adapter.meter.create_histogram.return_value = mock_hist

        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["histograms"] = {}

        event = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="custom_duration_seconds",
            value=5.2,
            timestamp=1234567890.0,
            unit="s",
            attributes={},
            metric_type=MetricType.HISTOGRAM,
        )

        adapter._log_metric(event)

        # buckets configured via otel views, not passed to create_histogram
        mock_hist.record.assert_called_once_with(5.2, attributes={})

    def test_non_duration_uses_counter(self, adapter):
        mock_counter = Mock()
        adapter.meter.create_counter.return_value = mock_counter

        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["counters"] = {}

        event = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="llama_stack_agent_workflows_total",
            value=1,
            timestamp=1234567890.0,
            unit="1",
            attributes={"agent_id": "test-agent", "status": "completed"},
        )

        adapter._log_metric(event)

        adapter.meter.create_counter.assert_called_once()
        adapter.meter.create_histogram.assert_not_called()
        mock_counter.add.assert_called_once_with(1, attributes={"agent_id": "test-agent", "status": "completed"})

    def test_no_meter_doesnt_crash(self, adapter):
        adapter.meter = None

        event = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="test_duration_seconds",
            value=1.0,
            timestamp=1234567890.0,
            unit="s",
            attributes={},
        )

        adapter._log_metric(event)  # shouldn't crash

    def test_histogram_vs_counter_by_type(self, adapter):
        mock_hist = Mock()
        mock_counter = Mock()
        adapter.meter.create_histogram.return_value = mock_hist
        adapter.meter.create_counter.return_value = mock_counter

        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["histograms"] = {}
        _GLOBAL_STORAGE["counters"] = {}

        # histogram metric
        hist_event = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="workflow_duration_seconds",
            value=1.0,
            timestamp=1234567890.0,
            unit="s",
            attributes={},
            metric_type=MetricType.HISTOGRAM,
        )
        adapter._log_metric(hist_event)
        mock_hist.record.assert_called()

        # counter metric (default type)
        counter_event = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="workflow_total",
            value=1,
            timestamp=1234567890.0,
            unit="1",
            attributes={},
        )
        adapter._log_metric(counter_event)
        mock_counter.add.assert_called()

    def test_storage_separation(self, adapter):
        mock_hist = Mock()
        mock_counter = Mock()
        adapter.meter.create_histogram.return_value = mock_hist
        adapter.meter.create_counter.return_value = mock_counter

        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["histograms"] = {}
        _GLOBAL_STORAGE["counters"] = {}

        # create both types
        hist_event = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="test_duration_seconds",
            value=1.0,
            timestamp=1234567890.0,
            unit="s",
            attributes={},
            metric_type=MetricType.HISTOGRAM,
        )
        counter_event = MetricEvent(
            trace_id="123",
            span_id="456",
            metric="test_counter",
            value=1,
            timestamp=1234567890.0,
            unit="1",
            attributes={},
        )

        adapter._log_metric(hist_event)
        adapter._log_metric(counter_event)

        # check they're stored separately
        assert "test_duration_seconds" in _GLOBAL_STORAGE["histograms"]
        assert "test_counter" in _GLOBAL_STORAGE["counters"]
        assert "test_duration_seconds" not in _GLOBAL_STORAGE["counters"]
        assert "test_counter" not in _GLOBAL_STORAGE["histograms"]

    def test_histogram_uses_views_for_buckets(self, adapter):
        mock_hist = Mock()
        adapter.meter.create_histogram.return_value = mock_hist

        from llama_stack.providers.inline.telemetry.meta_reference.telemetry import _GLOBAL_STORAGE

        _GLOBAL_STORAGE["histograms"] = {}

        result = adapter._get_or_create_histogram("test_histogram", "s")

        # buckets come from otel views, not create_histogram params
        adapter.meter.create_histogram.assert_called_once_with(
            name="test_histogram",
            unit="s",
            description="test histogram",
        )
        assert result == mock_hist