From 3c1a2c3d6606a907bf9edb644c28c07336afc2db Mon Sep 17 00:00:00 2001 From: Dinesh Yeduguru Date: Mon, 27 Jan 2025 11:20:28 -0800 Subject: [PATCH] Fix telemetry init (#885) # What does this PR do? When you re-initialize the library client in a notebook, we were seeing this error: ``` Getting traces for session_id=5c8d1969-0957-49d2-b852-32cbb8ef8caf --------------------------------------------------------------------------- AttributeError Traceback (most recent call last) [](https://localhost:8080/#) in () 7 agent_logs = [] 8 ----> 9 for span in client.telemetry.query_spans( 10 attribute_filters=[ 11 {"key": "session_id", "op": "eq", "value": session_id}, 10 frames [/usr/local/lib/python3.11/dist-packages/llama_stack/providers/inline/telemetry/meta_reference/telemetry.py](https://localhost:8080/#) in query_traces(self, attribute_filters, limit, offset, order_by) 246 ) -> QueryTracesResponse: 247 return QueryTracesResponse( --> 248 data=await self.trace_store.query_traces( 249 attribute_filters=attribute_filters, 250 limit=limit, AttributeError: 'TelemetryAdapter' object has no attribute 'trace_store' ``` This is happening because the we were skipping some required steps for the object state as part of the global _TRACE_PROVIDER check. This PR moves the initialization of the object state out of the TRACE_PROVIDER init. --- .../inline/telemetry/meta_reference/telemetry.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/llama_stack/providers/inline/telemetry/meta_reference/telemetry.py b/llama_stack/providers/inline/telemetry/meta_reference/telemetry.py index aeeed1ac0..569d02f50 100644 --- a/llama_stack/providers/inline/telemetry/meta_reference/telemetry.py +++ b/llama_stack/providers/inline/telemetry/meta_reference/telemetry.py @@ -81,6 +81,11 @@ class TelemetryAdapter(TelemetryDatasetMixin, Telemetry): ) global _TRACER_PROVIDER + # Initialize the correct span processor based on the provider state. + # This is needed since once the span processor is set, it cannot be unset. + # Recreating the telemetry adapter multiple times will result in duplicate span processors. + # Since the library client can be recreated multiple times in a notebook, + # the kernel will hold on to the span processor and cause duplicate spans to be written. if _TRACER_PROVIDER is None: provider = TracerProvider(resource=resource) trace.set_tracer_provider(provider) @@ -100,14 +105,18 @@ class TelemetryAdapter(TelemetryDatasetMixin, Telemetry): resource=resource, metric_readers=[metric_reader] ) metrics.set_meter_provider(metric_provider) - self.meter = metrics.get_meter(__name__) if TelemetrySink.SQLITE in self.config.sinks: trace.get_tracer_provider().add_span_processor( SQLiteSpanProcessor(self.config.sqlite_db_path) ) - self.trace_store = SQLiteTraceStore(self.config.sqlite_db_path) if TelemetrySink.CONSOLE in self.config.sinks: trace.get_tracer_provider().add_span_processor(ConsoleSpanProcessor()) + + if TelemetrySink.OTEL in self.config.sinks: + self.meter = metrics.get_meter(__name__) + if TelemetrySink.SQLITE in self.config.sinks: + self.trace_store = SQLiteTraceStore(self.config.sqlite_db_path) + self._lock = _global_lock async def initialize(self) -> None: