From b8d97c688cc695183072562bb994063e9fafdd93 Mon Sep 17 00:00:00 2001 From: Nir Gazit Date: Thu, 30 May 2024 04:02:20 +0300 Subject: [PATCH] Revert "Revert "fix: Log errors in Traceloop Integration (reverts previous revert)"" --- .circleci/config.yml | 2 +- litellm/integrations/traceloop.py | 229 +++++++++++++++++------------- litellm/tests/test_traceloop.py | 74 ++++------ litellm/utils.py | 12 ++ 4 files changed, 177 insertions(+), 140 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 27f79ed51..e6d988bae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,7 @@ jobs: pip install "langfuse==2.27.1" pip install "logfire==0.29.0" pip install numpydoc - pip install traceloop-sdk==0.18.2 + pip install traceloop-sdk==0.21.1 pip install openai pip install prisma pip install "httpx==0.24.1" diff --git a/litellm/integrations/traceloop.py b/litellm/integrations/traceloop.py index bbdb9a1b0..39d62028e 100644 --- a/litellm/integrations/traceloop.py +++ b/litellm/integrations/traceloop.py @@ -1,114 +1,153 @@ +import traceback +from litellm._logging import verbose_logger +import litellm + + class TraceloopLogger: def __init__(self): - from traceloop.sdk.tracing.tracing import TracerWrapper - from traceloop.sdk import Traceloop + try: + from traceloop.sdk.tracing.tracing import TracerWrapper + from traceloop.sdk import Traceloop + from traceloop.sdk.instruments import Instruments + except ModuleNotFoundError as e: + verbose_logger.error( + f"Traceloop not installed, try running 'pip install traceloop-sdk' to fix this error: {e}\n{traceback.format_exc()}" + ) - Traceloop.init(app_name="Litellm-Server", disable_batch=True) + Traceloop.init( + app_name="Litellm-Server", + disable_batch=True, + instruments=[ + Instruments.CHROMA, + Instruments.PINECONE, + Instruments.WEAVIATE, + Instruments.LLAMA_INDEX, + Instruments.LANGCHAIN, + ], + ) self.tracer_wrapper = TracerWrapper() - def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose): - from opentelemetry.trace import SpanKind + def log_event( + self, + kwargs, + response_obj, + start_time, + end_time, + user_id, + print_verbose, + level="DEFAULT", + status_message=None, + ): + from opentelemetry import trace + from opentelemetry.trace import SpanKind, Status, StatusCode from opentelemetry.semconv.ai import SpanAttributes try: + print_verbose( + f"Traceloop Logging - Enters logging function for model {kwargs}" + ) + tracer = self.tracer_wrapper.get_tracer() - model = kwargs.get("model") - - # LiteLLM uses the standard OpenAI library, so it's already handled by Traceloop SDK - if kwargs.get("litellm_params").get("custom_llm_provider") == "openai": - return - optional_params = kwargs.get("optional_params", {}) - with tracer.start_as_current_span( - "litellm.completion", - kind=SpanKind.CLIENT, - ) as span: - if span.is_recording(): + span = tracer.start_span( + "litellm.completion", kind=SpanKind.CLIENT, start_time=start_time + ) + + if span.is_recording(): + span.set_attribute( + SpanAttributes.LLM_REQUEST_MODEL, kwargs.get("model") + ) + if "stop" in optional_params: span.set_attribute( - SpanAttributes.LLM_REQUEST_MODEL, kwargs.get("model") + SpanAttributes.LLM_CHAT_STOP_SEQUENCES, + optional_params.get("stop"), ) - if "stop" in optional_params: - span.set_attribute( - SpanAttributes.LLM_CHAT_STOP_SEQUENCES, - optional_params.get("stop"), - ) - if "frequency_penalty" in optional_params: - span.set_attribute( - SpanAttributes.LLM_FREQUENCY_PENALTY, - optional_params.get("frequency_penalty"), - ) - if "presence_penalty" in optional_params: - span.set_attribute( - SpanAttributes.LLM_PRESENCE_PENALTY, - optional_params.get("presence_penalty"), - ) - if "top_p" in optional_params: - span.set_attribute( - SpanAttributes.LLM_TOP_P, optional_params.get("top_p") - ) - if "tools" in optional_params or "functions" in optional_params: - span.set_attribute( - SpanAttributes.LLM_REQUEST_FUNCTIONS, - optional_params.get( - "tools", optional_params.get("functions") - ), - ) - if "user" in optional_params: - span.set_attribute( - SpanAttributes.LLM_USER, optional_params.get("user") - ) - if "max_tokens" in optional_params: - span.set_attribute( - SpanAttributes.LLM_REQUEST_MAX_TOKENS, - kwargs.get("max_tokens"), - ) - if "temperature" in optional_params: - span.set_attribute( - SpanAttributes.LLM_TEMPERATURE, kwargs.get("temperature") - ) - - for idx, prompt in enumerate(kwargs.get("messages")): - span.set_attribute( - f"{SpanAttributes.LLM_PROMPTS}.{idx}.role", - prompt.get("role"), - ) - span.set_attribute( - f"{SpanAttributes.LLM_PROMPTS}.{idx}.content", - prompt.get("content"), - ) - + if "frequency_penalty" in optional_params: span.set_attribute( - SpanAttributes.LLM_RESPONSE_MODEL, response_obj.get("model") + SpanAttributes.LLM_FREQUENCY_PENALTY, + optional_params.get("frequency_penalty"), + ) + if "presence_penalty" in optional_params: + span.set_attribute( + SpanAttributes.LLM_PRESENCE_PENALTY, + optional_params.get("presence_penalty"), + ) + if "top_p" in optional_params: + span.set_attribute( + SpanAttributes.LLM_TOP_P, optional_params.get("top_p") + ) + if "tools" in optional_params or "functions" in optional_params: + span.set_attribute( + SpanAttributes.LLM_REQUEST_FUNCTIONS, + optional_params.get("tools", optional_params.get("functions")), + ) + if "user" in optional_params: + span.set_attribute( + SpanAttributes.LLM_USER, optional_params.get("user") + ) + if "max_tokens" in optional_params: + span.set_attribute( + SpanAttributes.LLM_REQUEST_MAX_TOKENS, + kwargs.get("max_tokens"), + ) + if "temperature" in optional_params: + span.set_attribute( + SpanAttributes.LLM_REQUEST_TEMPERATURE, + kwargs.get("temperature"), ) - usage = response_obj.get("usage") - if usage: - span.set_attribute( - SpanAttributes.LLM_USAGE_TOTAL_TOKENS, - usage.get("total_tokens"), - ) - span.set_attribute( - SpanAttributes.LLM_USAGE_COMPLETION_TOKENS, - usage.get("completion_tokens"), - ) - span.set_attribute( - SpanAttributes.LLM_USAGE_PROMPT_TOKENS, - usage.get("prompt_tokens"), - ) - for idx, choice in enumerate(response_obj.get("choices")): - span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.finish_reason", - choice.get("finish_reason"), - ) - span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role", - choice.get("message").get("role"), - ) - span.set_attribute( - f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content", - choice.get("message").get("content"), - ) + for idx, prompt in enumerate(kwargs.get("messages")): + span.set_attribute( + f"{SpanAttributes.LLM_PROMPTS}.{idx}.role", + prompt.get("role"), + ) + span.set_attribute( + f"{SpanAttributes.LLM_PROMPTS}.{idx}.content", + prompt.get("content"), + ) + + span.set_attribute( + SpanAttributes.LLM_RESPONSE_MODEL, response_obj.get("model") + ) + usage = response_obj.get("usage") + if usage: + span.set_attribute( + SpanAttributes.LLM_USAGE_TOTAL_TOKENS, + usage.get("total_tokens"), + ) + span.set_attribute( + SpanAttributes.LLM_USAGE_COMPLETION_TOKENS, + usage.get("completion_tokens"), + ) + span.set_attribute( + SpanAttributes.LLM_USAGE_PROMPT_TOKENS, + usage.get("prompt_tokens"), + ) + + for idx, choice in enumerate(response_obj.get("choices")): + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.finish_reason", + choice.get("finish_reason"), + ) + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role", + choice.get("message").get("role"), + ) + span.set_attribute( + f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.content", + choice.get("message").get("content"), + ) + + if ( + level == "ERROR" + and status_message is not None + and isinstance(status_message, str) + ): + span.record_exception(Exception(status_message)) + span.set_status(Status(StatusCode.ERROR, status_message)) + + span.end(end_time) except Exception as e: print_verbose(f"Traceloop Layer Error - {e}") diff --git a/litellm/tests/test_traceloop.py b/litellm/tests/test_traceloop.py index 405a8a357..f96973628 100644 --- a/litellm/tests/test_traceloop.py +++ b/litellm/tests/test_traceloop.py @@ -1,49 +1,35 @@ -# Commented out for now - since traceloop break ci/cd -# import sys -# import os -# import io, asyncio +import sys +import os +import time +import pytest +import litellm +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceloop.sdk import Traceloop -# sys.path.insert(0, os.path.abspath('../..')) - -# from litellm import completion -# import litellm -# litellm.num_retries = 3 -# litellm.success_callback = [""] -# import time -# import pytest -# from traceloop.sdk import Traceloop -# Traceloop.init(app_name="test-litellm", disable_batch=True) +sys.path.insert(0, os.path.abspath("../..")) -# def test_traceloop_logging(): -# try: -# litellm.set_verbose = True -# response = litellm.completion( -# model="gpt-3.5-turbo", -# messages=[{"role": "user", "content":"This is a test"}], -# max_tokens=1000, -# temperature=0.7, -# timeout=5, -# ) -# print(f"response: {response}") -# except Exception as e: -# pytest.fail(f"An exception occurred - {e}") -# # test_traceloop_logging() +@pytest.fixture() +def exporter(): + exporter = InMemorySpanExporter() + Traceloop.init( + app_name="test_litellm", + disable_batch=True, + exporter=exporter, + ) + litellm.success_callback = ["traceloop"] + litellm.set_verbose = True + + return exporter -# # def test_traceloop_logging_async(): -# # try: -# # litellm.set_verbose = True -# # async def test_acompletion(): -# # return await litellm.acompletion( -# # model="gpt-3.5-turbo", -# # messages=[{"role": "user", "content":"This is a test"}], -# # max_tokens=1000, -# # temperature=0.7, -# # timeout=5, -# # ) -# # response = asyncio.run(test_acompletion()) -# # print(f"response: {response}") -# # except Exception as e: -# # pytest.fail(f"An exception occurred - {e}") -# # test_traceloop_logging_async() +@pytest.mark.parametrize("model", ["claude-instant-1.2", "gpt-3.5-turbo"]) +def test_traceloop_logging(exporter, model): + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "This is a test"}], + max_tokens=1000, + temperature=0.7, + timeout=5, + ) diff --git a/litellm/utils.py b/litellm/utils.py index ea0f46c14..95d9160ef 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2027,6 +2027,7 @@ class Logging: response_obj=result, start_time=start_time, end_time=end_time, + user_id=kwargs.get("user", None), print_verbose=print_verbose, ) if callback == "s3": @@ -2598,6 +2599,17 @@ class Logging: level="ERROR", kwargs=self.model_call_details, ) + if callback == "traceloop": + traceloopLogger.log_event( + start_time=start_time, + end_time=end_time, + response_obj=None, + user_id=kwargs.get("user", None), + print_verbose=print_verbose, + status_message=str(exception), + level="ERROR", + kwargs=self.model_call_details, + ) if callback == "prometheus": global prometheusLogger verbose_logger.debug("reaches prometheus for success logging!")