forked from phoenix/litellm-mirror
Feat: Add Langtrace integration (#5341)
* Feat: Add Langtrace integration * add langtrace service name * fix timestamps for traces * add tests * Discard Callback + use existing otel logger * cleanup * remove print statments * remove callback * add docs * docs * add logging docs * format logging * remove emoji and add litellm proxy example * format logging * format `logging.md` * add langtrace docs to logging.md * sync conflict
This commit is contained in:
parent
42174fde4e
commit
7ec414a3cf
7 changed files with 291 additions and 0 deletions
63
docs/my-website/docs/observability/langtrace_integration.md
Normal file
63
docs/my-website/docs/observability/langtrace_integration.md
Normal file
|
@ -0,0 +1,63 @@
|
||||||
|
import Image from '@theme/IdealImage';
|
||||||
|
|
||||||
|
# Langtrace AI
|
||||||
|
|
||||||
|
Monitor, evaluate & improve your LLM apps
|
||||||
|
|
||||||
|
## Pre-Requisites
|
||||||
|
|
||||||
|
Make an account on [Langtrace AI](https://langtrace.ai/login)
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
Use just 2 lines of code, to instantly log your responses **across all providers** with langtrace
|
||||||
|
|
||||||
|
```python
|
||||||
|
litellm.callbacks = ["langtrace"]
|
||||||
|
langtrace.init()
|
||||||
|
```
|
||||||
|
|
||||||
|
```python
|
||||||
|
import litellm
|
||||||
|
import os
|
||||||
|
from langtrace_python_sdk import langtrace
|
||||||
|
|
||||||
|
# Langtrace API Keys
|
||||||
|
os.environ["LANGTRACE_API_KEY"] = "<your-api-key>"
|
||||||
|
|
||||||
|
# LLM API Keys
|
||||||
|
os.environ['OPENAI_API_KEY']="<openai-api-key>"
|
||||||
|
|
||||||
|
# set langtrace as a callback, litellm will send the data to langtrace
|
||||||
|
litellm.callbacks = ["langtrace"]
|
||||||
|
|
||||||
|
# init langtrace
|
||||||
|
langtrace.init()
|
||||||
|
|
||||||
|
# openai call
|
||||||
|
response = completion(
|
||||||
|
model="gpt-4o",
|
||||||
|
messages=[
|
||||||
|
{"content": "respond only in Yoda speak.", "role": "system"},
|
||||||
|
{"content": "Hello, how are you?", "role": "user"},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
print(response)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Using with LiteLLM Proxy
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
model_list:
|
||||||
|
- model_name: gpt-4
|
||||||
|
litellm_params:
|
||||||
|
model: openai/fake
|
||||||
|
api_key: fake-key
|
||||||
|
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||||
|
|
||||||
|
litellm_settings:
|
||||||
|
callbacks: ["langtrace"]
|
||||||
|
|
||||||
|
environment_variables:
|
||||||
|
LANGTRACE_API_KEY: "141a****"
|
||||||
|
```
|
|
@ -1307,6 +1307,47 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||||
Expect to see your log on Langfuse
|
Expect to see your log on Langfuse
|
||||||
<Image img={require('../../img/langsmith_new.png')} />
|
<Image img={require('../../img/langsmith_new.png')} />
|
||||||
|
|
||||||
|
|
||||||
|
## Logging LLM IO to Langtrace
|
||||||
|
|
||||||
|
1. Set `success_callback: ["langtrace"]` on litellm config.yaml
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
model_list:
|
||||||
|
- model_name: gpt-4
|
||||||
|
litellm_params:
|
||||||
|
model: openai/fake
|
||||||
|
api_key: fake-key
|
||||||
|
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||||
|
|
||||||
|
litellm_settings:
|
||||||
|
callbacks: ["langtrace"]
|
||||||
|
|
||||||
|
environment_variables:
|
||||||
|
LANGTRACE_API_KEY: "141a****"
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Start Proxy
|
||||||
|
|
||||||
|
```
|
||||||
|
litellm --config /path/to/config.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
3. Test it!
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl --location 'http://0.0.0.0:4000/chat/completions' \
|
||||||
|
--header 'Content-Type: application/json' \
|
||||||
|
--data ' {
|
||||||
|
"model": "fake-openai-endpoint",
|
||||||
|
"messages": [
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": "Hello, Claude gm!"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
'
|
||||||
## Logging LLM IO to Galileo
|
## Logging LLM IO to Galileo
|
||||||
|
|
||||||
[BETA]
|
[BETA]
|
||||||
|
|
|
@ -51,6 +51,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
|
||||||
"galileo",
|
"galileo",
|
||||||
"braintrust",
|
"braintrust",
|
||||||
"arize",
|
"arize",
|
||||||
|
"langtrace",
|
||||||
"gcs_bucket",
|
"gcs_bucket",
|
||||||
"opik",
|
"opik",
|
||||||
]
|
]
|
||||||
|
|
108
litellm/integrations/langtrace.py
Normal file
108
litellm/integrations/langtrace.py
Normal file
|
@ -0,0 +1,108 @@
|
||||||
|
import traceback
|
||||||
|
import json
|
||||||
|
from litellm.integrations.custom_logger import CustomLogger
|
||||||
|
from litellm.proxy._types import SpanAttributes
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from opentelemetry.trace import Span as _Span
|
||||||
|
|
||||||
|
Span = _Span
|
||||||
|
else:
|
||||||
|
Span = Any
|
||||||
|
|
||||||
|
|
||||||
|
class LangtraceAttributes:
|
||||||
|
"""
|
||||||
|
This class is used to save trace attributes to Langtrace's spans
|
||||||
|
"""
|
||||||
|
|
||||||
|
def set_langtrace_attributes(self, span: Span, kwargs, response_obj):
|
||||||
|
"""
|
||||||
|
This function is used to log the event to Langtrace
|
||||||
|
"""
|
||||||
|
|
||||||
|
vendor = kwargs.get("litellm_params").get("custom_llm_provider")
|
||||||
|
optional_params = kwargs.get("optional_params", {})
|
||||||
|
options = {**kwargs, **optional_params}
|
||||||
|
self.set_request_attributes(span, options, vendor)
|
||||||
|
self.set_response_attributes(span, response_obj)
|
||||||
|
self.set_usage_attributes(span, response_obj)
|
||||||
|
|
||||||
|
def set_request_attributes(self, span: Span, kwargs, vendor):
|
||||||
|
"""
|
||||||
|
This function is used to get span attributes for the LLM request
|
||||||
|
"""
|
||||||
|
span_attributes = {
|
||||||
|
"gen_ai.operation.name": "chat",
|
||||||
|
"langtrace.service.name": vendor,
|
||||||
|
SpanAttributes.LLM_REQUEST_MODEL.value: kwargs.get("model"),
|
||||||
|
SpanAttributes.LLM_IS_STREAMING.value: kwargs.get("stream"),
|
||||||
|
SpanAttributes.LLM_REQUEST_TEMPERATURE.value: kwargs.get("temperature"),
|
||||||
|
SpanAttributes.LLM_TOP_K.value: kwargs.get("top_k"),
|
||||||
|
SpanAttributes.LLM_REQUEST_TOP_P.value: kwargs.get("top_p"),
|
||||||
|
SpanAttributes.LLM_USER.value: kwargs.get("user"),
|
||||||
|
SpanAttributes.LLM_REQUEST_MAX_TOKENS.value: kwargs.get("max_tokens"),
|
||||||
|
SpanAttributes.LLM_RESPONSE_STOP_REASON.value: kwargs.get("stop"),
|
||||||
|
SpanAttributes.LLM_FREQUENCY_PENALTY.value: kwargs.get("frequency_penalty"),
|
||||||
|
SpanAttributes.LLM_PRESENCE_PENALTY.value: kwargs.get("presence_penalty"),
|
||||||
|
}
|
||||||
|
|
||||||
|
prompts = kwargs.get("messages")
|
||||||
|
|
||||||
|
if prompts:
|
||||||
|
span.add_event(
|
||||||
|
name="gen_ai.content.prompt",
|
||||||
|
attributes={SpanAttributes.LLM_PROMPTS.value: json.dumps(prompts)},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.set_span_attributes(span, span_attributes)
|
||||||
|
|
||||||
|
def set_response_attributes(self, span: Span, response_obj):
|
||||||
|
"""
|
||||||
|
This function is used to get span attributes for the LLM response
|
||||||
|
"""
|
||||||
|
response_attributes = {
|
||||||
|
"gen_ai.response_id": response_obj.get("id"),
|
||||||
|
"gen_ai.system_fingerprint": response_obj.get("system_fingerprint"),
|
||||||
|
SpanAttributes.LLM_RESPONSE_MODEL.value: response_obj.get("model"),
|
||||||
|
}
|
||||||
|
completions = []
|
||||||
|
for choice in response_obj.get("choices", []):
|
||||||
|
role = choice.get("message").get("role")
|
||||||
|
content = choice.get("message").get("content")
|
||||||
|
completions.append({"role": role, "content": content})
|
||||||
|
|
||||||
|
span.add_event(
|
||||||
|
name="gen_ai.content.completion",
|
||||||
|
attributes={SpanAttributes.LLM_COMPLETIONS: json.dumps(completions)},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.set_span_attributes(span, response_attributes)
|
||||||
|
|
||||||
|
def set_usage_attributes(self, span: Span, response_obj):
|
||||||
|
"""
|
||||||
|
This function is used to get span attributes for the LLM usage
|
||||||
|
"""
|
||||||
|
usage = response_obj.get("usage")
|
||||||
|
if usage:
|
||||||
|
usage_attributes = {
|
||||||
|
SpanAttributes.LLM_USAGE_PROMPT_TOKENS.value: usage.get(
|
||||||
|
"prompt_tokens"
|
||||||
|
),
|
||||||
|
SpanAttributes.LLM_USAGE_COMPLETION_TOKENS.value: usage.get(
|
||||||
|
"completion_tokens"
|
||||||
|
),
|
||||||
|
SpanAttributes.LLM_USAGE_TOTAL_TOKENS.value: usage.get("total_tokens"),
|
||||||
|
}
|
||||||
|
self.set_span_attributes(span, usage_attributes)
|
||||||
|
|
||||||
|
def set_span_attributes(self, span: Span, attributes):
|
||||||
|
"""
|
||||||
|
This function is used to set span attributes
|
||||||
|
"""
|
||||||
|
for key, value in attributes.items():
|
||||||
|
if not value:
|
||||||
|
continue
|
||||||
|
span.set_attribute(key, value)
|
|
@ -352,6 +352,13 @@ class OpenTelemetry(CustomLogger):
|
||||||
|
|
||||||
set_arize_ai_attributes(span, kwargs, response_obj)
|
set_arize_ai_attributes(span, kwargs, response_obj)
|
||||||
return
|
return
|
||||||
|
elif self.callback_name == "langtrace":
|
||||||
|
from litellm.integrations.langtrace import LangtraceAttributes
|
||||||
|
|
||||||
|
LangtraceAttributes().set_langtrace_attributes(
|
||||||
|
span, kwargs, response_obj
|
||||||
|
)
|
||||||
|
return
|
||||||
from litellm.proxy._types import SpanAttributes
|
from litellm.proxy._types import SpanAttributes
|
||||||
|
|
||||||
optional_params = kwargs.get("optional_params", {})
|
optional_params = kwargs.get("optional_params", {})
|
||||||
|
|
|
@ -2531,6 +2531,31 @@ def _init_custom_logger_compatible_class(
|
||||||
dynamic_rate_limiter_obj.update_variables(llm_router=llm_router)
|
dynamic_rate_limiter_obj.update_variables(llm_router=llm_router)
|
||||||
_in_memory_loggers.append(dynamic_rate_limiter_obj)
|
_in_memory_loggers.append(dynamic_rate_limiter_obj)
|
||||||
return dynamic_rate_limiter_obj # type: ignore
|
return dynamic_rate_limiter_obj # type: ignore
|
||||||
|
elif logging_integration == "langtrace":
|
||||||
|
if "LANGTRACE_API_KEY" not in os.environ:
|
||||||
|
raise ValueError("LANGTRACE_API_KEY not found in environment variables")
|
||||||
|
|
||||||
|
from litellm.integrations.opentelemetry import (
|
||||||
|
OpenTelemetry,
|
||||||
|
OpenTelemetryConfig,
|
||||||
|
)
|
||||||
|
|
||||||
|
otel_config = OpenTelemetryConfig(
|
||||||
|
exporter="otlp_http",
|
||||||
|
endpoint="https://langtrace.ai/api/trace",
|
||||||
|
)
|
||||||
|
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
|
||||||
|
f"api_key={os.getenv('LANGTRACE_API_KEY')}"
|
||||||
|
)
|
||||||
|
for callback in _in_memory_loggers:
|
||||||
|
if (
|
||||||
|
isinstance(callback, OpenTelemetry)
|
||||||
|
and callback.callback_name == "langtrace"
|
||||||
|
):
|
||||||
|
return callback # type: ignore
|
||||||
|
_otel_logger = OpenTelemetry(config=otel_config, callback_name="langtrace")
|
||||||
|
_in_memory_loggers.append(_otel_logger)
|
||||||
|
return _otel_logger # type: ignore
|
||||||
|
|
||||||
|
|
||||||
def get_custom_logger_compatible_class(
|
def get_custom_logger_compatible_class(
|
||||||
|
@ -2612,6 +2637,19 @@ def get_custom_logger_compatible_class(
|
||||||
for callback in _in_memory_loggers:
|
for callback in _in_memory_loggers:
|
||||||
if isinstance(callback, _PROXY_DynamicRateLimitHandler):
|
if isinstance(callback, _PROXY_DynamicRateLimitHandler):
|
||||||
return callback # type: ignore
|
return callback # type: ignore
|
||||||
|
|
||||||
|
elif logging_integration == "langtrace":
|
||||||
|
from litellm.integrations.opentelemetry import OpenTelemetry
|
||||||
|
|
||||||
|
if "LANGTRACE_API_KEY" not in os.environ:
|
||||||
|
raise ValueError("LANGTRACE_API_KEY not found in environment variables")
|
||||||
|
|
||||||
|
for callback in _in_memory_loggers:
|
||||||
|
if (
|
||||||
|
isinstance(callback, OpenTelemetry)
|
||||||
|
and callback.callback_name == "langtrace"
|
||||||
|
):
|
||||||
|
return callback
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
|
33
litellm/tests/test_langtrace.py
Normal file
33
litellm/tests/test_langtrace.py
Normal file
|
@ -0,0 +1,33 @@
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||||
|
from langtrace_python_sdk import langtrace
|
||||||
|
|
||||||
|
import litellm
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.abspath("../.."))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def exporter():
|
||||||
|
exporter = InMemorySpanExporter()
|
||||||
|
langtrace.init(batch=False, custom_remote_exporter=exporter)
|
||||||
|
litellm.success_callback = ["langtrace"]
|
||||||
|
litellm.set_verbose = True
|
||||||
|
|
||||||
|
return exporter
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("model", ["claude-2.1", "gpt-3.5-turbo"])
|
||||||
|
def test_langtrace_logging(exporter, model):
|
||||||
|
litellm.completion(
|
||||||
|
model=model,
|
||||||
|
messages=[{"role": "user", "content": "This is a test"}],
|
||||||
|
max_tokens=1000,
|
||||||
|
temperature=0.7,
|
||||||
|
timeout=5,
|
||||||
|
mock_response="hi",
|
||||||
|
)
|
Loading…
Add table
Add a link
Reference in a new issue