From 7ec414a3cf53fa7f7c88eba9a226d5c90c7b808f Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Fri, 11 Oct 2024 16:49:53 +0300 Subject: [PATCH] Feat: Add Langtrace integration (#5341) * Feat: Add Langtrace integration * add langtrace service name * fix timestamps for traces * add tests * Discard Callback + use existing otel logger * cleanup * remove print statments * remove callback * add docs * docs * add logging docs * format logging * remove emoji and add litellm proxy example * format logging * format `logging.md` * add langtrace docs to logging.md * sync conflict --- .../observability/langtrace_integration.md | 63 ++++++++++ docs/my-website/docs/proxy/logging.md | 41 +++++++ litellm/__init__.py | 1 + litellm/integrations/langtrace.py | 108 ++++++++++++++++++ litellm/integrations/opentelemetry.py | 7 ++ litellm/litellm_core_utils/litellm_logging.py | 38 ++++++ litellm/tests/test_langtrace.py | 33 ++++++ 7 files changed, 291 insertions(+) create mode 100644 docs/my-website/docs/observability/langtrace_integration.md create mode 100644 litellm/integrations/langtrace.py create mode 100644 litellm/tests/test_langtrace.py diff --git a/docs/my-website/docs/observability/langtrace_integration.md b/docs/my-website/docs/observability/langtrace_integration.md new file mode 100644 index 000000000..1188b06fd --- /dev/null +++ b/docs/my-website/docs/observability/langtrace_integration.md @@ -0,0 +1,63 @@ +import Image from '@theme/IdealImage'; + +# Langtrace AI + +Monitor, evaluate & improve your LLM apps + +## Pre-Requisites + +Make an account on [Langtrace AI](https://langtrace.ai/login) + +## Quick Start + +Use just 2 lines of code, to instantly log your responses **across all providers** with langtrace + +```python +litellm.callbacks = ["langtrace"] +langtrace.init() +``` + +```python +import litellm +import os +from langtrace_python_sdk import langtrace + +# Langtrace API Keys +os.environ["LANGTRACE_API_KEY"] = "" + +# LLM API Keys +os.environ['OPENAI_API_KEY']="" + +# set langtrace as a callback, litellm will send the data to langtrace +litellm.callbacks = ["langtrace"] + +# init langtrace +langtrace.init() + +# openai call +response = completion( + model="gpt-4o", + messages=[ + {"content": "respond only in Yoda speak.", "role": "system"}, + {"content": "Hello, how are you?", "role": "user"}, + ], +) +print(response) +``` + +### Using with LiteLLM Proxy + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + +litellm_settings: + callbacks: ["langtrace"] + +environment_variables: + LANGTRACE_API_KEY: "141a****" +``` diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 89ed0bda5..b03843438 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -1307,6 +1307,47 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ Expect to see your log on Langfuse + +## Logging LLM IO to Langtrace + +1. Set `success_callback: ["langtrace"]` on litellm config.yaml + +```yaml +model_list: + - model_name: gpt-4 + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + +litellm_settings: + callbacks: ["langtrace"] + +environment_variables: + LANGTRACE_API_KEY: "141a****" +``` + +2. Start Proxy + +``` +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data ' { + "model": "fake-openai-endpoint", + "messages": [ + { + "role": "user", + "content": "Hello, Claude gm!" + } + ], + } +' ## Logging LLM IO to Galileo [BETA] diff --git a/litellm/__init__.py b/litellm/__init__.py index 55276570b..9379caac1 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -51,6 +51,7 @@ _custom_logger_compatible_callbacks_literal = Literal[ "galileo", "braintrust", "arize", + "langtrace", "gcs_bucket", "opik", ] diff --git a/litellm/integrations/langtrace.py b/litellm/integrations/langtrace.py new file mode 100644 index 000000000..f5dcfacdf --- /dev/null +++ b/litellm/integrations/langtrace.py @@ -0,0 +1,108 @@ +import traceback +import json +from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy._types import SpanAttributes + +from typing import TYPE_CHECKING, Any, Optional, Union + +if TYPE_CHECKING: + from opentelemetry.trace import Span as _Span + + Span = _Span +else: + Span = Any + + +class LangtraceAttributes: + """ + This class is used to save trace attributes to Langtrace's spans + """ + + def set_langtrace_attributes(self, span: Span, kwargs, response_obj): + """ + This function is used to log the event to Langtrace + """ + + vendor = kwargs.get("litellm_params").get("custom_llm_provider") + optional_params = kwargs.get("optional_params", {}) + options = {**kwargs, **optional_params} + self.set_request_attributes(span, options, vendor) + self.set_response_attributes(span, response_obj) + self.set_usage_attributes(span, response_obj) + + def set_request_attributes(self, span: Span, kwargs, vendor): + """ + This function is used to get span attributes for the LLM request + """ + span_attributes = { + "gen_ai.operation.name": "chat", + "langtrace.service.name": vendor, + SpanAttributes.LLM_REQUEST_MODEL.value: kwargs.get("model"), + SpanAttributes.LLM_IS_STREAMING.value: kwargs.get("stream"), + SpanAttributes.LLM_REQUEST_TEMPERATURE.value: kwargs.get("temperature"), + SpanAttributes.LLM_TOP_K.value: kwargs.get("top_k"), + SpanAttributes.LLM_REQUEST_TOP_P.value: kwargs.get("top_p"), + SpanAttributes.LLM_USER.value: kwargs.get("user"), + SpanAttributes.LLM_REQUEST_MAX_TOKENS.value: kwargs.get("max_tokens"), + SpanAttributes.LLM_RESPONSE_STOP_REASON.value: kwargs.get("stop"), + SpanAttributes.LLM_FREQUENCY_PENALTY.value: kwargs.get("frequency_penalty"), + SpanAttributes.LLM_PRESENCE_PENALTY.value: kwargs.get("presence_penalty"), + } + + prompts = kwargs.get("messages") + + if prompts: + span.add_event( + name="gen_ai.content.prompt", + attributes={SpanAttributes.LLM_PROMPTS.value: json.dumps(prompts)}, + ) + + self.set_span_attributes(span, span_attributes) + + def set_response_attributes(self, span: Span, response_obj): + """ + This function is used to get span attributes for the LLM response + """ + response_attributes = { + "gen_ai.response_id": response_obj.get("id"), + "gen_ai.system_fingerprint": response_obj.get("system_fingerprint"), + SpanAttributes.LLM_RESPONSE_MODEL.value: response_obj.get("model"), + } + completions = [] + for choice in response_obj.get("choices", []): + role = choice.get("message").get("role") + content = choice.get("message").get("content") + completions.append({"role": role, "content": content}) + + span.add_event( + name="gen_ai.content.completion", + attributes={SpanAttributes.LLM_COMPLETIONS: json.dumps(completions)}, + ) + + self.set_span_attributes(span, response_attributes) + + def set_usage_attributes(self, span: Span, response_obj): + """ + This function is used to get span attributes for the LLM usage + """ + usage = response_obj.get("usage") + if usage: + usage_attributes = { + SpanAttributes.LLM_USAGE_PROMPT_TOKENS.value: usage.get( + "prompt_tokens" + ), + SpanAttributes.LLM_USAGE_COMPLETION_TOKENS.value: usage.get( + "completion_tokens" + ), + SpanAttributes.LLM_USAGE_TOTAL_TOKENS.value: usage.get("total_tokens"), + } + self.set_span_attributes(span, usage_attributes) + + def set_span_attributes(self, span: Span, attributes): + """ + This function is used to set span attributes + """ + for key, value in attributes.items(): + if not value: + continue + span.set_attribute(key, value) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index a26a45ebe..fada59ab2 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -352,6 +352,13 @@ class OpenTelemetry(CustomLogger): set_arize_ai_attributes(span, kwargs, response_obj) return + elif self.callback_name == "langtrace": + from litellm.integrations.langtrace import LangtraceAttributes + + LangtraceAttributes().set_langtrace_attributes( + span, kwargs, response_obj + ) + return from litellm.proxy._types import SpanAttributes optional_params = kwargs.get("optional_params", {}) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 2d70975c9..ce97f1c6f 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2531,6 +2531,31 @@ def _init_custom_logger_compatible_class( dynamic_rate_limiter_obj.update_variables(llm_router=llm_router) _in_memory_loggers.append(dynamic_rate_limiter_obj) return dynamic_rate_limiter_obj # type: ignore + elif logging_integration == "langtrace": + if "LANGTRACE_API_KEY" not in os.environ: + raise ValueError("LANGTRACE_API_KEY not found in environment variables") + + from litellm.integrations.opentelemetry import ( + OpenTelemetry, + OpenTelemetryConfig, + ) + + otel_config = OpenTelemetryConfig( + exporter="otlp_http", + endpoint="https://langtrace.ai/api/trace", + ) + os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = ( + f"api_key={os.getenv('LANGTRACE_API_KEY')}" + ) + for callback in _in_memory_loggers: + if ( + isinstance(callback, OpenTelemetry) + and callback.callback_name == "langtrace" + ): + return callback # type: ignore + _otel_logger = OpenTelemetry(config=otel_config, callback_name="langtrace") + _in_memory_loggers.append(_otel_logger) + return _otel_logger # type: ignore def get_custom_logger_compatible_class( @@ -2612,6 +2637,19 @@ def get_custom_logger_compatible_class( for callback in _in_memory_loggers: if isinstance(callback, _PROXY_DynamicRateLimitHandler): return callback # type: ignore + + elif logging_integration == "langtrace": + from litellm.integrations.opentelemetry import OpenTelemetry + + if "LANGTRACE_API_KEY" not in os.environ: + raise ValueError("LANGTRACE_API_KEY not found in environment variables") + + for callback in _in_memory_loggers: + if ( + isinstance(callback, OpenTelemetry) + and callback.callback_name == "langtrace" + ): + return callback return None diff --git a/litellm/tests/test_langtrace.py b/litellm/tests/test_langtrace.py new file mode 100644 index 000000000..803bae521 --- /dev/null +++ b/litellm/tests/test_langtrace.py @@ -0,0 +1,33 @@ +import os +import sys +import time + +import pytest +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from langtrace_python_sdk import langtrace + +import litellm + +sys.path.insert(0, os.path.abspath("../..")) + + +@pytest.fixture() +def exporter(): + exporter = InMemorySpanExporter() + langtrace.init(batch=False, custom_remote_exporter=exporter) + litellm.success_callback = ["langtrace"] + litellm.set_verbose = True + + return exporter + + +@pytest.mark.parametrize("model", ["claude-2.1", "gpt-3.5-turbo"]) +def test_langtrace_logging(exporter, model): + litellm.completion( + model=model, + messages=[{"role": "user", "content": "This is a test"}], + max_tokens=1000, + temperature=0.7, + timeout=5, + mock_response="hi", + )