phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

Revert "Revert "fix: Log errors in Traceloop Integration (reverts previous revert)""

Browse source
This commit is contained in:
Nir Gazit 2024-05-30 04:02:20 +03:00
parent b7fcec8835
commit b8d97c688c
4 changed files with 177 additions and 140 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.circleci/config.yml
View file

@ -43,7 +43,7 @@ jobs:
pip install "langfuse==2.27.1"
pip install "logfire==0.29.0"
pip install numpydoc
pip install traceloop-sdk==0.18.2
pip install traceloop-sdk==0.21.1
pip install openai
pip install prisma
pip install "httpx==0.24.1"

229
litellm/integrations/traceloop.py
View file

@ -1,114 +1,153 @@
import traceback
from litellm._logging import verbose_logger
import litellm
class TraceloopLogger:
def __init__(self):
from traceloop.sdk.tracing.tracing import TracerWrapper
from traceloop.sdk import Traceloop
try:
from traceloop.sdk.tracing.tracing import TracerWrapper
from traceloop.sdk import Traceloop
from traceloop.sdk.instruments import Instruments
except ModuleNotFoundError as e:
verbose_logger.error(
f"Traceloop not installed, try running 'pip install traceloop-sdk' to fix this error: {e}\n{traceback.format_exc()}"
)
Traceloop.init(app_name="Litellm-Server", disable_batch=True)
Traceloop.init(
app_name="Litellm-Server",
disable_batch=True,
instruments=[
Instruments.CHROMA,
Instruments.PINECONE,
Instruments.WEAVIATE,
Instruments.LLAMA_INDEX,
Instruments.LANGCHAIN,
],
)
self.tracer_wrapper = TracerWrapper()
def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
from opentelemetry.trace import SpanKind
def log_event(
self,
kwargs,
response_obj,
start_time,
end_time,
user_id,
print_verbose,
level="DEFAULT",
status_message=None,
):
from opentelemetry import trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from opentelemetry.semconv.ai import SpanAttributes
try:
print_verbose(
f"Traceloop Logging - Enters logging function for model {kwargs}"
)
tracer = self.tracer_wrapper.get_tracer()
model = kwargs.get("model")
# LiteLLM uses the standard OpenAI library, so it's already handled by Traceloop SDK
if kwargs.get("litellm_params").get("custom_llm_provider") == "openai":
return
optional_params = kwargs.get("optional_params", {})
with tracer.start_as_current_span(
"litellm.completion",
kind=SpanKind.CLIENT,
) as span:
if span.is_recording():
span = tracer.start_span(
"litellm.completion", kind=SpanKind.CLIENT, start_time=start_time
)
if span.is_recording():
span.set_attribute(
SpanAttributes.LLM_REQUEST_MODEL, kwargs.get("model")
)
if "stop" in optional_params:
span.set_attribute(
SpanAttributes.LLM_REQUEST_MODEL, kwargs.get("model")
SpanAttributes.LLM_CHAT_STOP_SEQUENCES,
optional_params.get("stop"),
)
if "stop" in optional_params:
span.set_attribute(
SpanAttributes.LLM_CHAT_STOP_SEQUENCES,
optional_params.get("stop"),
)
if "frequency_penalty" in optional_params:
span.set_attribute(
SpanAttributes.LLM_FREQUENCY_PENALTY,
optional_params.get("frequency_penalty"),
)
if "presence_penalty" in optional_params:
span.set_attribute(
SpanAttributes.LLM_PRESENCE_PENALTY,
optional_params.get("presence_penalty"),
)
if "top_p" in optional_params:
span.set_attribute(
SpanAttributes.LLM_TOP_P, optional_params.get("top_p")
)
if "tools" in optional_params or "functions" in optional_params:
span.set_attribute(
SpanAttributes.LLM_REQUEST_FUNCTIONS,
optional_params.get(
"tools", optional_params.get("functions")
),
)
if "user" in optional_params:
span.set_attribute(
SpanAttributes.LLM_USER, optional_params.get("user")
)
if "max_tokens" in optional_params:
span.set_attribute(
SpanAttributes.LLM_REQUEST_MAX_TOKENS,
kwargs.get("max_tokens"),
)
if "temperature" in optional_params:
span.set_attribute(
SpanAttributes.LLM_TEMPERATURE, kwargs.get("temperature")
)
for idx, prompt in enumerate(kwargs.get("messages")):
span.set_attribute(
f"{SpanAttributes.LLM_PROMPTS}.{idx}.role",
prompt.get("role"),
)
span.set_attribute(
f"{SpanAttributes.LLM_PROMPTS}.{idx}.content",
prompt.get("content"),
)
if "frequency_penalty" in optional_params:
span.set_attribute(
SpanAttributes.LLM_RESPONSE_MODEL, response_obj.get("model")
SpanAttributes.LLM_FREQUENCY_PENALTY,
optional_params.get("frequency_penalty"),
)
if "presence_penalty" in optional_params:
span.set_attribute(
SpanAttributes.LLM_PRESENCE_PENALTY,
optional_params.get("presence_penalty"),
)
if "top_p" in optional_params:
span.set_attribute(
SpanAttributes.LLM_TOP_P, optional_params.get("top_p")
)
if "tools" in optional_params or "functions" in optional_params:
span.set_attribute(
SpanAttributes.LLM_REQUEST_FUNCTIONS,
optional_params.get("tools", optional_params.get("functions")),
)
if "user" in optional_params:
span.set_attribute(
SpanAttributes.LLM_USER, optional_params.get("user")
)
if "max_tokens" in optional_params:
span.set_attribute(
SpanAttributes.LLM_REQUEST_MAX_TOKENS,
kwargs.get("max_tokens"),
)
if "temperature" in optional_params:
span.set_attribute(
SpanAttributes.LLM_REQUEST_TEMPERATURE,
kwargs.get("temperature"),
)
usage = response_obj.get("usage")
if usage:
span.set_attribute(
SpanAttributes.LLM_USAGE_TOTAL_TOKENS,
usage.get("total_tokens"),
)
span.set_attribute(
SpanAttributes.LLM_USAGE_COMPLETION_TOKENS,
usage.get("completion_tokens"),
)
span.set_attribute(
SpanAttributes.LLM_USAGE_PROMPT_TOKENS,
usage.get("prompt_tokens"),
)
for idx, choice in enumerate(response_obj.get("choices")):
span.set_attribute(
f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.finish_reason",
choice.get("finish_reason"),
)
span.set_attribute(
f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role",
choice.get("message").get("role"),
)
span.set_attribute(
f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content",
choice.get("message").get("content"),
)
for idx, prompt in enumerate(kwargs.get("messages")):
span.set_attribute(
f"{SpanAttributes.LLM_PROMPTS}.{idx}.role",
prompt.get("role"),
)
span.set_attribute(
f"{SpanAttributes.LLM_PROMPTS}.{idx}.content",
prompt.get("content"),
)
span.set_attribute(
SpanAttributes.LLM_RESPONSE_MODEL, response_obj.get("model")
)
usage = response_obj.get("usage")
if usage:
span.set_attribute(
SpanAttributes.LLM_USAGE_TOTAL_TOKENS,
usage.get("total_tokens"),
)
span.set_attribute(
SpanAttributes.LLM_USAGE_COMPLETION_TOKENS,
usage.get("completion_tokens"),
)
span.set_attribute(
SpanAttributes.LLM_USAGE_PROMPT_TOKENS,
usage.get("prompt_tokens"),
)
for idx, choice in enumerate(response_obj.get("choices")):
span.set_attribute(
f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.finish_reason",
choice.get("finish_reason"),
)
span.set_attribute(
f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role",
choice.get("message").get("role"),
)
span.set_attribute(
f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content",
choice.get("message").get("content"),
)
if (
level == "ERROR"
and status_message is not None
and isinstance(status_message, str)
):
span.record_exception(Exception(status_message))
span.set_status(Status(StatusCode.ERROR, status_message))
span.end(end_time)
except Exception as e:
print_verbose(f"Traceloop Layer Error - {e}")

74
litellm/tests/test_traceloop.py
View file

@ -1,49 +1,35 @@
# Commented out for now - since traceloop break ci/cd
# import sys
# import os
# import io, asyncio
import sys
import os
import time
import pytest
import litellm
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from traceloop.sdk import Traceloop
# sys.path.insert(0, os.path.abspath('../..'))
# from litellm import completion
# import litellm
# litellm.num_retries = 3
# litellm.success_callback = [""]
# import time
# import pytest
# from traceloop.sdk import Traceloop
# Traceloop.init(app_name="test-litellm", disable_batch=True)
sys.path.insert(0, os.path.abspath("../.."))
# def test_traceloop_logging():
# try:
# litellm.set_verbose = True
# response = litellm.completion(
# model="gpt-3.5-turbo",
# messages=[{"role": "user", "content":"This is a test"}],
# max_tokens=1000,
# temperature=0.7,
# timeout=5,
# )
# print(f"response: {response}")
# except Exception as e:
# pytest.fail(f"An exception occurred - {e}")
# # test_traceloop_logging()
@pytest.fixture()
def exporter():
exporter = InMemorySpanExporter()
Traceloop.init(
app_name="test_litellm",
disable_batch=True,
exporter=exporter,
)
litellm.success_callback = ["traceloop"]
litellm.set_verbose = True
return exporter
# # def test_traceloop_logging_async():
# # try:
# # litellm.set_verbose = True
# # async def test_acompletion():
# # return await litellm.acompletion(
# # model="gpt-3.5-turbo",
# # messages=[{"role": "user", "content":"This is a test"}],
# # max_tokens=1000,
# # temperature=0.7,
# # timeout=5,
# # )
# # response = asyncio.run(test_acompletion())
# # print(f"response: {response}")
# # except Exception as e:
# # pytest.fail(f"An exception occurred - {e}")
# # test_traceloop_logging_async()
@pytest.mark.parametrize("model", ["claude-instant-1.2", "gpt-3.5-turbo"])
def test_traceloop_logging(exporter, model):
litellm.completion(
model=model,
messages=[{"role": "user", "content": "This is a test"}],
max_tokens=1000,
temperature=0.7,
timeout=5,
)

12
litellm/utils.py
View file

@ -2027,6 +2027,7 @@ class Logging:
response_obj=result,
start_time=start_time,
end_time=end_time,
user_id=kwargs.get("user", None),
print_verbose=print_verbose,
)
if callback == "s3":
@ -2598,6 +2599,17 @@ class Logging:
level="ERROR",
kwargs=self.model_call_details,
)
if callback == "traceloop":
traceloopLogger.log_event(
start_time=start_time,
end_time=end_time,
response_obj=None,
user_id=kwargs.get("user", None),
print_verbose=print_verbose,
status_message=str(exception),
level="ERROR",
kwargs=self.model_call_details,
)
if callback == "prometheus":
global prometheusLogger
verbose_logger.debug("reaches prometheus for success logging!")