Feat: Add Langtrace integration (#5341)

* Feat: Add Langtrace integration

* add langtrace service name

* fix timestamps for traces

* add tests

* Discard Callback + use existing otel logger

* cleanup

* remove print statments

* remove callback

* add docs

* docs

* add logging docs

* format logging

* remove emoji and add litellm proxy example

* format logging

* format `logging.md`

* add langtrace docs to logging.md

* sync conflict

docs/my-website/docs/observability/langtrace_integration.md

@@ -0,0 +1,63 @@
+import Image from '@theme/IdealImage';
+
+# Langtrace AI
+
+Monitor, evaluate & improve your LLM apps
+
+## Pre-Requisites
+
+Make an account on [Langtrace AI](https://langtrace.ai/login)
+
+## Quick Start
+
+Use just 2 lines of code, to instantly log your responses **across all providers** with langtrace
+
+```python
+litellm.callbacks = ["langtrace"]
+langtrace.init()
+```
+
+```python
+import litellm
+import os
+from langtrace_python_sdk import langtrace
+
+# Langtrace API Keys
+os.environ["LANGTRACE_API_KEY"] = "<your-api-key>"
+
+# LLM API Keys
+os.environ['OPENAI_API_KEY']="<openai-api-key>"
+
+# set langtrace as a callback, litellm will send the data to langtrace
+litellm.callbacks = ["langtrace"]
+
+#  init langtrace
+langtrace.init()
+
+# openai call
+response = completion(
+    model="gpt-4o",
+    messages=[
+        {"content": "respond only in Yoda speak.", "role": "system"},
+        {"content": "Hello, how are you?", "role": "user"},
+    ],
+)
+print(response)
+```
+
+### Using with LiteLLM Proxy
+
+```yaml
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: openai/fake
+      api_key: fake-key
+      api_base: https://exampleopenaiendpoint-production.up.railway.app/
+
+litellm_settings:
+  callbacks: ["langtrace"]
+
+environment_variables:
+  LANGTRACE_API_KEY: "141a****"
+```

docs/my-website/docs/proxy/logging.md

@@ -1307,6 +1307,47 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
 Expect to see your log on Langfuse
 <Image img={require('../../img/langsmith_new.png')} />
 
+
+## Logging LLM IO to Langtrace
+
+1. Set `success_callback: ["langtrace"]` on litellm config.yaml
+
+```yaml
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: openai/fake
+      api_key: fake-key
+      api_base: https://exampleopenaiendpoint-production.up.railway.app/
+
+litellm_settings:
+  callbacks: ["langtrace"]
+
+environment_variables:
+    LANGTRACE_API_KEY: "141a****"
+```
+
+2. Start Proxy
+
+```
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--data ' {
+      "model": "fake-openai-endpoint",
+      "messages": [
+        {
+          "role": "user",
+          "content": "Hello, Claude gm!"
+        }
+      ],
+    }
+'
 ## Logging LLM IO to Galileo
 
 [BETA]

litellm/__init__.py

@@ -51,6 +51,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
     "galileo",
     "braintrust",
     "arize",
+    "langtrace",
     "gcs_bucket",
     "opik",
 ]

litellm/integrations/langtrace.py

@@ -0,0 +1,108 @@
+import traceback
+import json
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.proxy._types import SpanAttributes
+
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+if TYPE_CHECKING:
+    from opentelemetry.trace import Span as _Span
+
+    Span = _Span
+else:
+    Span = Any
+
+
+class LangtraceAttributes:
+    """
+    This class is used to save trace attributes to Langtrace's spans
+    """
+
+    def set_langtrace_attributes(self, span: Span, kwargs, response_obj):
+        """
+        This function is used to log the event to Langtrace
+        """
+
+        vendor = kwargs.get("litellm_params").get("custom_llm_provider")
+        optional_params = kwargs.get("optional_params", {})
+        options = {**kwargs, **optional_params}
+        self.set_request_attributes(span, options, vendor)
+        self.set_response_attributes(span, response_obj)
+        self.set_usage_attributes(span, response_obj)
+
+    def set_request_attributes(self, span: Span, kwargs, vendor):
+        """
+        This function is used to get span attributes for the LLM request
+        """
+        span_attributes = {
+            "gen_ai.operation.name": "chat",
+            "langtrace.service.name": vendor,
+            SpanAttributes.LLM_REQUEST_MODEL.value: kwargs.get("model"),
+            SpanAttributes.LLM_IS_STREAMING.value: kwargs.get("stream"),
+            SpanAttributes.LLM_REQUEST_TEMPERATURE.value: kwargs.get("temperature"),
+            SpanAttributes.LLM_TOP_K.value: kwargs.get("top_k"),
+            SpanAttributes.LLM_REQUEST_TOP_P.value: kwargs.get("top_p"),
+            SpanAttributes.LLM_USER.value: kwargs.get("user"),
+            SpanAttributes.LLM_REQUEST_MAX_TOKENS.value: kwargs.get("max_tokens"),
+            SpanAttributes.LLM_RESPONSE_STOP_REASON.value: kwargs.get("stop"),
+            SpanAttributes.LLM_FREQUENCY_PENALTY.value: kwargs.get("frequency_penalty"),
+            SpanAttributes.LLM_PRESENCE_PENALTY.value: kwargs.get("presence_penalty"),
+        }
+
+        prompts = kwargs.get("messages")
+
+        if prompts:
+            span.add_event(
+                name="gen_ai.content.prompt",
+                attributes={SpanAttributes.LLM_PROMPTS.value: json.dumps(prompts)},
+            )
+
+        self.set_span_attributes(span, span_attributes)
+
+    def set_response_attributes(self, span: Span, response_obj):
+        """
+        This function is used to get span attributes for the LLM response
+        """
+        response_attributes = {
+            "gen_ai.response_id": response_obj.get("id"),
+            "gen_ai.system_fingerprint": response_obj.get("system_fingerprint"),
+            SpanAttributes.LLM_RESPONSE_MODEL.value: response_obj.get("model"),
+        }
+        completions = []
+        for choice in response_obj.get("choices", []):
+            role = choice.get("message").get("role")
+            content = choice.get("message").get("content")
+            completions.append({"role": role, "content": content})
+
+        span.add_event(
+            name="gen_ai.content.completion",
+            attributes={SpanAttributes.LLM_COMPLETIONS: json.dumps(completions)},
+        )
+
+        self.set_span_attributes(span, response_attributes)
+
+    def set_usage_attributes(self, span: Span, response_obj):
+        """
+        This function is used to get span attributes for the LLM usage
+        """
+        usage = response_obj.get("usage")
+        if usage:
+            usage_attributes = {
+                SpanAttributes.LLM_USAGE_PROMPT_TOKENS.value: usage.get(
+                    "prompt_tokens"
+                ),
+                SpanAttributes.LLM_USAGE_COMPLETION_TOKENS.value: usage.get(
+                    "completion_tokens"
+                ),
+                SpanAttributes.LLM_USAGE_TOTAL_TOKENS.value: usage.get("total_tokens"),
+            }
+            self.set_span_attributes(span, usage_attributes)
+
+    def set_span_attributes(self, span: Span, attributes):
+        """
+        This function is used to set span attributes
+        """
+        for key, value in attributes.items():
+            if not value:
+                continue
+            span.set_attribute(key, value)

litellm/integrations/opentelemetry.py

@@ -352,6 +352,13 @@ class OpenTelemetry(CustomLogger):
 
                 set_arize_ai_attributes(span, kwargs, response_obj)
                 return
+            elif self.callback_name == "langtrace":
+                from litellm.integrations.langtrace import LangtraceAttributes
+
+                LangtraceAttributes().set_langtrace_attributes(
+                    span, kwargs, response_obj
+                )
+                return
             from litellm.proxy._types import SpanAttributes
 
             optional_params = kwargs.get("optional_params", {})

litellm/litellm_core_utils/litellm_logging.py

@@ -2531,6 +2531,31 @@ def _init_custom_logger_compatible_class(
             dynamic_rate_limiter_obj.update_variables(llm_router=llm_router)
         _in_memory_loggers.append(dynamic_rate_limiter_obj)
         return dynamic_rate_limiter_obj  # type: ignore
+    elif logging_integration == "langtrace":
+        if "LANGTRACE_API_KEY" not in os.environ:
+            raise ValueError("LANGTRACE_API_KEY not found in environment variables")
+
+        from litellm.integrations.opentelemetry import (
+            OpenTelemetry,
+            OpenTelemetryConfig,
+        )
+
+        otel_config = OpenTelemetryConfig(
+            exporter="otlp_http",
+            endpoint="https://langtrace.ai/api/trace",
+        )
+        os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
+            f"api_key={os.getenv('LANGTRACE_API_KEY')}"
+        )
+        for callback in _in_memory_loggers:
+            if (
+                isinstance(callback, OpenTelemetry)
+                and callback.callback_name == "langtrace"
+            ):
+                return callback  # type: ignore
+        _otel_logger = OpenTelemetry(config=otel_config, callback_name="langtrace")
+        _in_memory_loggers.append(_otel_logger)
+        return _otel_logger  # type: ignore
 
 
 def get_custom_logger_compatible_class(
@@ -2612,6 +2637,19 @@ def get_custom_logger_compatible_class(
         for callback in _in_memory_loggers:
             if isinstance(callback, _PROXY_DynamicRateLimitHandler):
                 return callback  # type: ignore
+
+    elif logging_integration == "langtrace":
+        from litellm.integrations.opentelemetry import OpenTelemetry
+
+        if "LANGTRACE_API_KEY" not in os.environ:
+            raise ValueError("LANGTRACE_API_KEY not found in environment variables")
+
+        for callback in _in_memory_loggers:
+            if (
+                isinstance(callback, OpenTelemetry)
+                and callback.callback_name == "langtrace"
+            ):
+                return callback
     return None
 
 

litellm/tests/test_langtrace.py

@@ -0,0 +1,33 @@
+import os
+import sys
+import time
+
+import pytest
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from langtrace_python_sdk import langtrace
+
+import litellm
+
+sys.path.insert(0, os.path.abspath("../.."))
+
+
+@pytest.fixture()
+def exporter():
+    exporter = InMemorySpanExporter()
+    langtrace.init(batch=False, custom_remote_exporter=exporter)
+    litellm.success_callback = ["langtrace"]
+    litellm.set_verbose = True
+
+    return exporter
+
+
+@pytest.mark.parametrize("model", ["claude-2.1", "gpt-3.5-turbo"])
+def test_langtrace_logging(exporter, model):
+    litellm.completion(
+        model=model,
+        messages=[{"role": "user", "content": "This is a test"}],
+        max_tokens=1000,
+        temperature=0.7,
+        timeout=5,
+        mock_response="hi",
+    )