From 9da1b2779338dfcc443b347d005e81113bd8eb4e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 11:35:17 -0700 Subject: [PATCH 01/11] feat - low raw request on OTEL --- litellm/integrations/opentelemetry.py | 152 ++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index c8b34c477d..46c0c1a86e 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -22,6 +22,7 @@ LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm") LITELLM_RESOURCE = { "service.name": os.getenv("OTEL_SERVICE_NAME", "litellm"), } +RAW_REQUEST_SPAN_NAME = "RAW_GENAI_REQUEST" @dataclass @@ -200,6 +201,7 @@ class OpenTelemetry(CustomLogger): ) _parent_context, parent_otel_span = self._get_span_context(kwargs) + # Span 1: Requst sent to litellm SDK span = self.tracer.start_span( name=self._get_span_name(kwargs), start_time=self._to_ns(start_time), @@ -208,6 +210,18 @@ class OpenTelemetry(CustomLogger): span.set_status(Status(StatusCode.OK)) self.set_attributes(span, kwargs, response_obj) span.end(end_time=self._to_ns(end_time)) + + # Span 2: Raw Request / Response to LLM + raw_request_span = self.tracer.start_span( + name=RAW_REQUEST_SPAN_NAME, + start_time=self._to_ns(start_time), + context=_parent_context, + ) + + raw_request_span.set_status(Status(StatusCode.OK)) + self.set_raw_request_attributes(raw_request_span, kwargs, response_obj) + raw_request_span.end(end_time=self._to_ns(end_time)) + if parent_otel_span is not None: parent_otel_span.end(end_time=self._to_ns(datetime.now())) @@ -324,6 +338,144 @@ class OpenTelemetry(CustomLogger): usage.get("prompt_tokens"), ) + def set_anthropic_raw_request_attributes(self, span: Span, kwargs, response_obj): + from opentelemetry.semconv.ai import SpanAttributes + + optional_params = kwargs.get("optional_params", {}) + litellm_params = kwargs.get("litellm_params", {}) or {} + custom_llm_provider = "anthropic" + + _raw_response = kwargs.get("original_response") + _additional_args = kwargs.get("additional_args", {}) or {} + complete_input_dict = _additional_args.get("complete_input_dict") + ############################################# + ########## LLM Request Attributes ########### + ############################################# + + # OTEL Attributes for the RAW Request to https://docs.anthropic.com/en/api/messages + if complete_input_dict: + if complete_input_dict.get("model"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_MODEL, complete_input_dict.get("model") + ) + + if complete_input_dict.get("messages"): + for idx, prompt in enumerate(complete_input_dict.get("messages")): + span.set_attribute( + f"{SpanAttributes.LLM_PROMPTS}.{idx}.role", + prompt.get("role"), + ) + span.set_attribute( + f"{SpanAttributes.LLM_PROMPTS}.{idx}.content", + prompt.get("content"), + ) + if complete_input_dict.get("max_tokens"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_MAX_TOKENS, + complete_input_dict.get("max_tokens"), + ) + + if complete_input_dict.get("temperature"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_TEMPERATURE, + complete_input_dict.get("temperature"), + ) + + if complete_input_dict.get("top_p"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_TOP_P, complete_input_dict.get("top_p") + ) + + if complete_input_dict.get("stream"): + span.set_attribute( + SpanAttributes.LLM_IS_STREAMING, complete_input_dict.get("stream") + ) + + if complete_input_dict.get("tools"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_FUNCTIONS, + complete_input_dict.get("tools"), + ) + + if complete_input_dict.get("user"): + span.set_attribute( + SpanAttributes.LLM_USER, complete_input_dict.get("user") + ) + + ############################################# + ########## LLM Response Attributes ########## + ############################################# + if _raw_response: + # cast sr -> dict + import json + + _raw_response = json.loads(_raw_response) + + # The unique identifier for the completion. + if _raw_response.get("id"): + span.set_attribute("gen_ai.response.id", _raw_response.get("id")) + + # The model used to generate the response. + if _raw_response.get("model"): + span.set_attribute( + SpanAttributes.LLM_RESPONSE_MODEL, _raw_response.get("model") + ) + + if _raw_response.get("content"): + for idx, choice in enumerate(_raw_response.get("content")): + if choice.get("type"): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.type", + choice.get("type"), + ) + + if choice.get("text"): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content", + choice.get("text"), + ) + + if choice.get("id"): + # https://docs.anthropic.com/en/docs/tool-use + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.id", + choice.get("id"), + ) + + if choice.get("name"): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.name", + choice.get("name"), + ) + + if choice.get("input"): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.input", + choice.get("input"), + ) + + pass + + def set_openai_raw_request_attributes(self, span: Span, kwargs, response_obj): + from opentelemetry.semconv.ai import SpanAttributes + + pass + + def set_default_raw_request_attributes(self, span: Span, kwargs, response_obj): + from opentelemetry.semconv.ai import SpanAttributes + + pass + + def set_raw_request_attributes(self, span: Span, kwargs, response_obj): + litellm_params = kwargs.get("litellm_params", {}) or {} + custom_llm_provider = litellm_params.get("custom_llm_provider", "Unknown") + if custom_llm_provider == "anthropic": + self.set_anthropic_raw_request_attributes(span, kwargs, response_obj) + elif custom_llm_provider == "openai": + self.set_openai_raw_request_attributes(span, kwargs, response_obj) + else: + self.set_default_raw_request_attributes(span, kwargs, response_obj) + def _to_ns(self, dt): return int(dt.timestamp() * 1e9) From 7f86dc859e32422e7ac11412136bf8e78861d0c5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 14:01:59 -0700 Subject: [PATCH 02/11] feat - set span attributes OTEL with raw request / response --- litellm/integrations/opentelemetry.py | 236 +++++++++----------------- 1 file changed, 83 insertions(+), 153 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 46c0c1a86e..449f524eec 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -1,6 +1,7 @@ import os from dataclasses import dataclass from datetime import datetime +import litellm from litellm.integrations.custom_logger import CustomLogger from litellm._logging import verbose_logger @@ -193,6 +194,7 @@ class OpenTelemetry(CustomLogger): def _handle_sucess(self, kwargs, response_obj, start_time, end_time): from opentelemetry.trace import Status, StatusCode + from opentelemetry import trace verbose_logger.debug( "OpenTelemetry Logger: Logging kwargs: %s, OTEL config settings=%s", @@ -209,18 +211,18 @@ class OpenTelemetry(CustomLogger): ) span.set_status(Status(StatusCode.OK)) self.set_attributes(span, kwargs, response_obj) - span.end(end_time=self._to_ns(end_time)) # Span 2: Raw Request / Response to LLM raw_request_span = self.tracer.start_span( name=RAW_REQUEST_SPAN_NAME, start_time=self._to_ns(start_time), - context=_parent_context, + context=trace.set_span_in_context(span), ) raw_request_span.set_status(Status(StatusCode.OK)) self.set_raw_request_attributes(raw_request_span, kwargs, response_obj) raw_request_span.end(end_time=self._to_ns(end_time)) + span.end(end_time=self._to_ns(end_time)) if parent_otel_span is not None: parent_otel_span.end(end_time=self._to_ns(datetime.now())) @@ -251,7 +253,8 @@ class OpenTelemetry(CustomLogger): ############################################# # The name of the LLM a request is being made to - span.set_attribute(SpanAttributes.LLM_REQUEST_MODEL, kwargs.get("model")) + if kwargs.get("model"): + span.set_attribute(SpanAttributes.LLM_REQUEST_MODEL, kwargs.get("model")) # The Generative AI Provider: Azure, OpenAI, etc. span.set_attribute( @@ -260,64 +263,87 @@ class OpenTelemetry(CustomLogger): ) # The maximum number of tokens the LLM generates for a request. - span.set_attribute( - SpanAttributes.LLM_REQUEST_MAX_TOKENS, optional_params.get("max_tokens") - ) + if optional_params.get("max_tokens"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_MAX_TOKENS, optional_params.get("max_tokens") + ) # The temperature setting for the LLM request. - span.set_attribute( - SpanAttributes.LLM_REQUEST_TEMPERATURE, optional_params.get("temperature") - ) + if optional_params.get("temperature"): + span.set_attribute( + SpanAttributes.LLM_REQUEST_TEMPERATURE, + optional_params.get("temperature"), + ) # The top_p sampling setting for the LLM request. - span.set_attribute( - SpanAttributes.LLM_REQUEST_TOP_P, optional_params.get("top_p") - ) - - span.set_attribute( - SpanAttributes.LLM_IS_STREAMING, optional_params.get("stream") - ) - - span.set_attribute( - SpanAttributes.LLM_REQUEST_FUNCTIONS, - optional_params.get("tools"), - ) - - span.set_attribute(SpanAttributes.LLM_USER, optional_params.get("user")) - - for idx, prompt in enumerate(kwargs.get("messages")): + if optional_params.get("top_p"): span.set_attribute( - f"{SpanAttributes.LLM_PROMPTS}.{idx}.role", - prompt.get("role"), - ) - span.set_attribute( - f"{SpanAttributes.LLM_PROMPTS}.{idx}.content", - prompt.get("content"), + SpanAttributes.LLM_REQUEST_TOP_P, optional_params.get("top_p") ) + span.set_attribute( + SpanAttributes.LLM_IS_STREAMING, optional_params.get("stream", False) + ) + + if optional_params.get("tools"): + # cast to str - since OTEL only accepts string values + _tools = str(optional_params.get("tools")) + span.set_attribute(SpanAttributes.LLM_REQUEST_FUNCTIONS, _tools) + + if optional_params.get("user"): + span.set_attribute(SpanAttributes.LLM_USER, optional_params.get("user")) + + if kwargs.get("messages"): + for idx, prompt in enumerate(kwargs.get("messages")): + if prompt.get("role"): + span.set_attribute( + f"{SpanAttributes.LLM_PROMPTS}.{idx}.role", + prompt.get("role"), + ) + + if prompt.get("content"): + span.set_attribute( + f"{SpanAttributes.LLM_PROMPTS}.{idx}.content", + prompt.get("content"), + ) ############################################# ########## LLM Response Attributes ########## ############################################# + if response_obj.get("choices"): + for idx, choice in enumerate(response_obj.get("choices")): + if choice.get("finish_reason"): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.finish_reason", + choice.get("finish_reason"), + ) + if choice.get("message"): + if choice.get("message").get("role"): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role", + choice.get("message").get("role"), + ) + if choice.get("message").get("content"): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content", + choice.get("message").get("content"), + ) - for idx, choice in enumerate(response_obj.get("choices")): - span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.finish_reason", - choice.get("finish_reason"), - ) - span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role", - choice.get("message").get("role"), - ) - span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content", - choice.get("message").get("content"), - ) + if choice.get("message").get("tool_calls"): + _tool_calls = choice.get("message").get("tool_calls") + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.tool_calls", + _tool_calls, + ) # The unique identifier for the completion. - span.set_attribute("gen_ai.response.id", response_obj.get("id")) + if response_obj.get("id"): + span.set_attribute("gen_ai.response.id", response_obj.get("id")) # The model used to generate the response. - span.set_attribute(SpanAttributes.LLM_RESPONSE_MODEL, response_obj.get("model")) + if response_obj.get("model"): + span.set_attribute( + SpanAttributes.LLM_RESPONSE_MODEL, response_obj.get("model") + ) usage = response_obj.get("usage") if usage: @@ -338,7 +364,7 @@ class OpenTelemetry(CustomLogger): usage.get("prompt_tokens"), ) - def set_anthropic_raw_request_attributes(self, span: Span, kwargs, response_obj): + def set_raw_request_attributes(self, span: Span, kwargs, response_obj): from opentelemetry.semconv.ai import SpanAttributes optional_params = kwargs.get("optional_params", {}) @@ -354,52 +380,12 @@ class OpenTelemetry(CustomLogger): # OTEL Attributes for the RAW Request to https://docs.anthropic.com/en/api/messages if complete_input_dict: - if complete_input_dict.get("model"): + for param, val in complete_input_dict.items(): + if not isinstance(val, str): + val = str(val) span.set_attribute( - SpanAttributes.LLM_REQUEST_MODEL, complete_input_dict.get("model") - ) - - if complete_input_dict.get("messages"): - for idx, prompt in enumerate(complete_input_dict.get("messages")): - span.set_attribute( - f"{SpanAttributes.LLM_PROMPTS}.{idx}.role", - prompt.get("role"), - ) - span.set_attribute( - f"{SpanAttributes.LLM_PROMPTS}.{idx}.content", - prompt.get("content"), - ) - if complete_input_dict.get("max_tokens"): - span.set_attribute( - SpanAttributes.LLM_REQUEST_MAX_TOKENS, - complete_input_dict.get("max_tokens"), - ) - - if complete_input_dict.get("temperature"): - span.set_attribute( - SpanAttributes.LLM_REQUEST_TEMPERATURE, - complete_input_dict.get("temperature"), - ) - - if complete_input_dict.get("top_p"): - span.set_attribute( - SpanAttributes.LLM_REQUEST_TOP_P, complete_input_dict.get("top_p") - ) - - if complete_input_dict.get("stream"): - span.set_attribute( - SpanAttributes.LLM_IS_STREAMING, complete_input_dict.get("stream") - ) - - if complete_input_dict.get("tools"): - span.set_attribute( - SpanAttributes.LLM_REQUEST_FUNCTIONS, - complete_input_dict.get("tools"), - ) - - if complete_input_dict.get("user"): - span.set_attribute( - SpanAttributes.LLM_USER, complete_input_dict.get("user") + f"gen_ai.request.{param}", + val, ) ############################################# @@ -410,72 +396,16 @@ class OpenTelemetry(CustomLogger): import json _raw_response = json.loads(_raw_response) - - # The unique identifier for the completion. - if _raw_response.get("id"): - span.set_attribute("gen_ai.response.id", _raw_response.get("id")) - - # The model used to generate the response. - if _raw_response.get("model"): + for param, val in _raw_response.items(): + if not isinstance(val, str): + val = str(val) span.set_attribute( - SpanAttributes.LLM_RESPONSE_MODEL, _raw_response.get("model") + f"gen_ai.response.{param}", + val, ) - if _raw_response.get("content"): - for idx, choice in enumerate(_raw_response.get("content")): - if choice.get("type"): - span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.type", - choice.get("type"), - ) - - if choice.get("text"): - span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content", - choice.get("text"), - ) - - if choice.get("id"): - # https://docs.anthropic.com/en/docs/tool-use - span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.id", - choice.get("id"), - ) - - if choice.get("name"): - span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.name", - choice.get("name"), - ) - - if choice.get("input"): - span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.input", - choice.get("input"), - ) - pass - def set_openai_raw_request_attributes(self, span: Span, kwargs, response_obj): - from opentelemetry.semconv.ai import SpanAttributes - - pass - - def set_default_raw_request_attributes(self, span: Span, kwargs, response_obj): - from opentelemetry.semconv.ai import SpanAttributes - - pass - - def set_raw_request_attributes(self, span: Span, kwargs, response_obj): - litellm_params = kwargs.get("litellm_params", {}) or {} - custom_llm_provider = litellm_params.get("custom_llm_provider", "Unknown") - if custom_llm_provider == "anthropic": - self.set_anthropic_raw_request_attributes(span, kwargs, response_obj) - elif custom_llm_provider == "openai": - self.set_openai_raw_request_attributes(span, kwargs, response_obj) - else: - self.set_default_raw_request_attributes(span, kwargs, response_obj) - def _to_ns(self, dt): return int(dt.timestamp() * 1e9) From 4d7ff7c79b48bd6278138aa53ddc0e32867a1c6b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 14:12:00 -0700 Subject: [PATCH 03/11] fix otel - handle vision images content --- litellm/integrations/opentelemetry.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 449f524eec..5b82ac3b3c 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -302,6 +302,8 @@ class OpenTelemetry(CustomLogger): ) if prompt.get("content"): + if not isinstance(prompt.get("content"), str): + prompt["content"] = str(prompt.get("content")) span.set_attribute( f"{SpanAttributes.LLM_PROMPTS}.{idx}.content", prompt.get("content"), From 29fed0e1981a8bc969b8493f6af25d79f881409d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 14:16:51 -0700 Subject: [PATCH 04/11] otel test - callbacks --- litellm/tests/test_opentelemetry.py | 68 +++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/litellm/tests/test_opentelemetry.py b/litellm/tests/test_opentelemetry.py index 4626cab838..ebeecf254a 100644 --- a/litellm/tests/test_opentelemetry.py +++ b/litellm/tests/test_opentelemetry.py @@ -11,25 +11,77 @@ import pytest verbose_logger.setLevel(logging.DEBUG) -@pytest.mark.skip(reason="new test") +# @pytest.mark.skip(reason="new test") def test_otel_callback(): exporter = InMemorySpanExporter() - + litellm.set_verbose = True litellm.callbacks = [OpenTelemetry(OpenTelemetryConfig(exporter=exporter))] - litellm.completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "hi"}], - ) - asyncio.run( litellm.acompletion( model="gpt-3.5-turbo", messages=[{"role": "user", "content": "hi"}], + temperature=0.1, + user="OTEL_USER", ) ) time.sleep(4) spans = exporter.get_finished_spans() - assert len(spans) == 1 + 1 + print("spans", spans) + assert len(spans) == 2 + + +@pytest.mark.parametrize( + "model", + ["anthropic/claude-3-opus-20240229"], +) +@pytest.mark.skip(reason="Local only test. WIP.") +def test_completion_claude_3_function_call_with_otel(model): + litellm.set_verbose = True + + litellm.callbacks = [OpenTelemetry(OpenTelemetryConfig())] + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } + ] + messages = [ + { + "role": "user", + "content": "What's the weather like in Boston today in Fahrenheit?", + } + ] + try: + # test without max tokens + response = litellm.completion( + model=model, + messages=messages, + tools=tools, + tool_choice={ + "type": "function", + "function": {"name": "get_current_weather"}, + }, + drop_params=True, + ) + + print("response from LiteLLM", response) + + except Exception as e: + pytest.fail(f"Error occurred: {e}") From 22fa537d980bee749a910ed6d0e914f3a8a838d4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 14:22:44 -0700 Subject: [PATCH 05/11] otel - use correct raw request name --- litellm/integrations/opentelemetry.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 5b82ac3b3c..b0caee61b8 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -23,7 +23,8 @@ LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm") LITELLM_RESOURCE = { "service.name": os.getenv("OTEL_SERVICE_NAME", "litellm"), } -RAW_REQUEST_SPAN_NAME = "RAW_GENAI_REQUEST" +RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request" +LITELLM_REQUEST_SPAN_NAME = "litellm_request" @dataclass @@ -412,7 +413,7 @@ class OpenTelemetry(CustomLogger): return int(dt.timestamp() * 1e9) def _get_span_name(self, kwargs): - return f"litellm-{kwargs.get('call_type', 'completion')}" + return LITELLM_REQUEST_SPAN_NAME def _get_span_context(self, kwargs): from opentelemetry.trace.propagation.tracecontext import ( From e7f6c5d6510667dfa3d90a08dc7ae89c193e0e79 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 14:37:06 -0700 Subject: [PATCH 06/11] fix otel don't log raw request / response when turn_off_message_logging --- litellm/integrations/opentelemetry.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index b0caee61b8..92ae9e280d 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -213,16 +213,20 @@ class OpenTelemetry(CustomLogger): span.set_status(Status(StatusCode.OK)) self.set_attributes(span, kwargs, response_obj) - # Span 2: Raw Request / Response to LLM - raw_request_span = self.tracer.start_span( - name=RAW_REQUEST_SPAN_NAME, - start_time=self._to_ns(start_time), - context=trace.set_span_in_context(span), - ) + if litellm.turn_off_message_logging is True: + pass + else: + # Span 2: Raw Request / Response to LLM + raw_request_span = self.tracer.start_span( + name=RAW_REQUEST_SPAN_NAME, + start_time=self._to_ns(start_time), + context=trace.set_span_in_context(span), + ) + + raw_request_span.set_status(Status(StatusCode.OK)) + self.set_raw_request_attributes(raw_request_span, kwargs, response_obj) + raw_request_span.end(end_time=self._to_ns(end_time)) - raw_request_span.set_status(Status(StatusCode.OK)) - self.set_raw_request_attributes(raw_request_span, kwargs, response_obj) - raw_request_span.end(end_time=self._to_ns(end_time)) span.end(end_time=self._to_ns(end_time)) if parent_otel_span is not None: @@ -372,7 +376,6 @@ class OpenTelemetry(CustomLogger): optional_params = kwargs.get("optional_params", {}) litellm_params = kwargs.get("litellm_params", {}) or {} - custom_llm_provider = "anthropic" _raw_response = kwargs.get("original_response") _additional_args = kwargs.get("additional_args", {}) or {} From 62d31411035d44334b9aaca0c44a80e4561f5c53 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 15:24:14 -0700 Subject: [PATCH 07/11] test_otel_callback --- litellm/tests/test_opentelemetry.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/litellm/tests/test_opentelemetry.py b/litellm/tests/test_opentelemetry.py index ebeecf254a..92d7b85153 100644 --- a/litellm/tests/test_opentelemetry.py +++ b/litellm/tests/test_opentelemetry.py @@ -17,13 +17,11 @@ def test_otel_callback(): litellm.set_verbose = True litellm.callbacks = [OpenTelemetry(OpenTelemetryConfig(exporter=exporter))] - asyncio.run( - litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "hi"}], - temperature=0.1, - user="OTEL_USER", - ) + litellm.completion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + temperature=0.1, + user="OTEL_USER", ) time.sleep(4) From bcfe62b80194e61c75a3fc9276caba65707d08ef Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 18:24:00 -0700 Subject: [PATCH 08/11] fix test otel logger --- .../{test_opentelemetry.py => test_async_opentelemetry.py} | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) rename litellm/tests/{test_opentelemetry.py => test_async_opentelemetry.py} (95%) diff --git a/litellm/tests/test_opentelemetry.py b/litellm/tests/test_async_opentelemetry.py similarity index 95% rename from litellm/tests/test_opentelemetry.py rename to litellm/tests/test_async_opentelemetry.py index 92d7b85153..26af43a1d6 100644 --- a/litellm/tests/test_opentelemetry.py +++ b/litellm/tests/test_async_opentelemetry.py @@ -12,19 +12,20 @@ verbose_logger.setLevel(logging.DEBUG) # @pytest.mark.skip(reason="new test") -def test_otel_callback(): +@pytest.mark.asyncio +async def test_otel_callback(): exporter = InMemorySpanExporter() litellm.set_verbose = True litellm.callbacks = [OpenTelemetry(OpenTelemetryConfig(exporter=exporter))] - litellm.completion( + await litellm.acompletion( model="gpt-3.5-turbo", messages=[{"role": "user", "content": "hi"}], temperature=0.1, user="OTEL_USER", ) - time.sleep(4) + await asyncio.sleep(4) spans = exporter.get_finished_spans() print("spans", spans) From 638c4acbb20e347c9b60655c6a8152817d2ebabb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 19:13:10 -0700 Subject: [PATCH 09/11] fix - show custom llm provider on OTEL span --- litellm/integrations/opentelemetry.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 92ae9e280d..30fd79be06 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -293,7 +293,9 @@ class OpenTelemetry(CustomLogger): if optional_params.get("tools"): # cast to str - since OTEL only accepts string values _tools = str(optional_params.get("tools")) - span.set_attribute(SpanAttributes.LLM_REQUEST_FUNCTIONS, _tools) + span.set_attribute( + SpanAttributes.LLM_REQUEST_FUNCTIONS, optional_params.get("tools") + ) if optional_params.get("user"): span.set_attribute(SpanAttributes.LLM_USER, optional_params.get("user")) @@ -330,6 +332,10 @@ class OpenTelemetry(CustomLogger): choice.get("message").get("role"), ) if choice.get("message").get("content"): + if not isinstance(choice.get("message").get("content"), str): + choice["message"]["content"] = str( + choice.get("message").get("content") + ) span.set_attribute( f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content", choice.get("message").get("content"), @@ -337,6 +343,8 @@ class OpenTelemetry(CustomLogger): if choice.get("message").get("tool_calls"): _tool_calls = choice.get("message").get("tool_calls") + if not isinstance(_tool_calls, str): + _tool_calls = str(_tool_calls) span.set_attribute( f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.tool_calls", _tool_calls, @@ -376,6 +384,7 @@ class OpenTelemetry(CustomLogger): optional_params = kwargs.get("optional_params", {}) litellm_params = kwargs.get("litellm_params", {}) or {} + custom_llm_provider = litellm_params.get("custom_llm_provider", "Unknown") _raw_response = kwargs.get("original_response") _additional_args = kwargs.get("additional_args", {}) or {} @@ -390,7 +399,7 @@ class OpenTelemetry(CustomLogger): if not isinstance(val, str): val = str(val) span.set_attribute( - f"gen_ai.request.{param}", + f"llm.{custom_llm_provider}.{param}", val, ) @@ -406,7 +415,7 @@ class OpenTelemetry(CustomLogger): if not isinstance(val, str): val = str(val) span.set_attribute( - f"gen_ai.response.{param}", + f"llm.{custom_llm_provider}.{param}", val, ) From 21ac0efaae21cff69784bc5bc6b3d63eec4f896e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 19:40:29 -0700 Subject: [PATCH 10/11] fix - otel set tools attribute --- litellm/integrations/opentelemetry.py | 49 +++++++++++++++++++++------ 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 30fd79be06..01d9661fc6 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -244,6 +244,31 @@ class OpenTelemetry(CustomLogger): self.set_attributes(span, kwargs, response_obj) span.end(end_time=self._to_ns(end_time)) + def set_tools_attributes(self, span: Span, tools): + from opentelemetry.semconv.ai import SpanAttributes + import json + + if not tools: + return + + try: + for i, tool in enumerate(tools): + function = tool.get("function") + if not function: + continue + + prefix = f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.{i}" + span.set_attribute(f"{prefix}.name", function.get("name")) + span.set_attribute(f"{prefix}.description", function.get("description")) + span.set_attribute( + f"{prefix}.parameters", json.dumps(function.get("parameters")) + ) + except Exception as e: + verbose_logger.error( + "OpenTelemetry: Error setting tools attributes: %s", str(e) + ) + pass + def set_attributes(self, span: Span, kwargs, response_obj): from opentelemetry.semconv.ai import SpanAttributes @@ -291,11 +316,8 @@ class OpenTelemetry(CustomLogger): ) if optional_params.get("tools"): - # cast to str - since OTEL only accepts string values - _tools = str(optional_params.get("tools")) - span.set_attribute( - SpanAttributes.LLM_REQUEST_FUNCTIONS, optional_params.get("tools") - ) + tools = optional_params["tools"] + self.set_tools_attributes(span, tools) if optional_params.get("user"): span.set_attribute(SpanAttributes.LLM_USER, optional_params.get("user")) @@ -341,13 +363,18 @@ class OpenTelemetry(CustomLogger): choice.get("message").get("content"), ) - if choice.get("message").get("tool_calls"): - _tool_calls = choice.get("message").get("tool_calls") - if not isinstance(_tool_calls, str): - _tool_calls = str(_tool_calls) + message = choice.get("message") + if not isinstance(message, dict): + message = message.dict() + tool_calls = message.get("tool_calls") + if tool_calls: span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.tool_calls", - _tool_calls, + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.function_call.name", + tool_calls[0].get("function").get("name"), + ) + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.function_call.arguments", + tool_calls[0].get("function").get("arguments"), ) # The unique identifier for the completion. From 6f40fc6a8149147ae87c107b039004c6b05c71cb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 8 Jun 2024 19:43:03 -0700 Subject: [PATCH 11/11] fix OTEL test --- litellm/tests/test_async_opentelemetry.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/tests/test_async_opentelemetry.py b/litellm/tests/test_async_opentelemetry.py index 26af43a1d6..4d174c0505 100644 --- a/litellm/tests/test_async_opentelemetry.py +++ b/litellm/tests/test_async_opentelemetry.py @@ -11,7 +11,9 @@ import pytest verbose_logger.setLevel(logging.DEBUG) -# @pytest.mark.skip(reason="new test") +@pytest.mark.skip( + reason="new test. WIP. works locally but not on CI. Still figuring this out" +) @pytest.mark.asyncio async def test_otel_callback(): exporter = InMemorySpanExporter()