diff --git a/.circleci/config.yml b/.circleci/config.yml index 4b762ca82..1326c1368 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,6 +41,7 @@ jobs: pip install langchain pip install lunary==0.2.5 pip install "langfuse==2.27.1" + pip install "logfire==0.29.0" pip install numpydoc pip install traceloop-sdk==0.0.69 pip install openai @@ -86,7 +87,6 @@ jobs: exit 1 fi cd .. - # Run pytest and generate JUnit XML report - run: @@ -94,7 +94,7 @@ jobs: command: | pwd ls - python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5 + python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 120m # Store test results @@ -170,6 +170,7 @@ jobs: pip install "aioboto3==12.3.0" pip install langchain pip install "langfuse>=2.0.0" + pip install "logfire==0.29.0" pip install numpydoc pip install prisma pip install fastapi @@ -223,7 +224,7 @@ jobs: name: Start outputting logs command: docker logs -f my-app background: true - - run: + - run: name: Wait for app to be ready command: dockerize -wait http://localhost:4000 -timeout 1m - run: @@ -231,7 +232,7 @@ jobs: command: | pwd ls - python -m pytest -vv tests/ -x --junitxml=test-results/junit.xml --durations=5 + python -m pytest -vv tests/ -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 120m # Store test results @@ -253,7 +254,7 @@ jobs: name: Copy model_prices_and_context_window File to model_prices_and_context_window_backup command: | cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json - + - run: name: Check if litellm dir was updated or if pyproject.toml was modified command: | @@ -338,4 +339,4 @@ workflows: filters: branches: only: - - main \ No newline at end of file + - main diff --git a/litellm/integrations/logfire_logger.py b/litellm/integrations/logfire_logger.py new file mode 100644 index 000000000..edc83a35e --- /dev/null +++ b/litellm/integrations/logfire_logger.py @@ -0,0 +1,145 @@ +#### What this does #### +# On success + failure, log events to Logfire + +import dotenv, os + +dotenv.load_dotenv() # Loading env variables using dotenv +import traceback +import uuid +from litellm._logging import print_verbose, verbose_logger + +from typing import Any, Dict, NamedTuple +from typing_extensions import LiteralString + + +class SpanConfig(NamedTuple): + message_template: LiteralString + span_data: Dict[str, Any] + + +class LogfireLogger: + # Class variables or attributes + def __init__(self): + try: + verbose_logger.debug(f"in init logfire logger") + import logfire + + # only setting up logfire if we are sending to logfire + # in testing, we don't want to send to logfire + if logfire.DEFAULT_LOGFIRE_INSTANCE.config.send_to_logfire: + logfire.configure(token=os.getenv("LOGFIRE_TOKEN")) + except Exception as e: + print_verbose(f"Got exception on init logfire client {str(e)}") + raise e + + def _get_span_config(self, payload) -> SpanConfig: + if ( + payload["call_type"] == "completion" + or payload["call_type"] == "acompletion" + ): + return SpanConfig( + message_template="Chat Completion with {request_data[model]!r}", + span_data={"request_data": payload}, + ) + elif ( + payload["call_type"] == "embedding" or payload["call_type"] == "aembedding" + ): + return SpanConfig( + message_template="Embedding Creation with {request_data[model]!r}", + span_data={"request_data": payload}, + ) + elif ( + payload["call_type"] == "image_generation" + or payload["call_type"] == "aimage_generation" + ): + return SpanConfig( + message_template="Image Generation with {request_data[model]!r}", + span_data={"request_data": payload}, + ) + else: + return SpanConfig( + message_template="Litellm Call with {request_data[model]!r}", + span_data={"request_data": payload}, + ) + + async def _async_log_event( + self, kwargs, response_obj, start_time, end_time, print_verbose, user_id + ): + self.log_event(kwargs, response_obj, start_time, end_time, print_verbose) + + def log_event( + self, kwargs, response_obj, start_time, end_time, user_id, print_verbose + ): + try: + import logfire + + verbose_logger.debug( + f"logfire Logging - Enters logging function for model {kwargs}" + ) + litellm_params = kwargs.get("litellm_params", {}) + metadata = ( + litellm_params.get("metadata", {}) or {} + ) # if litellm_params['metadata'] == None + messages = kwargs.get("messages") + optional_params = kwargs.get("optional_params", {}) + call_type = kwargs.get("call_type", "completion") + cache_hit = kwargs.get("cache_hit", False) + usage = response_obj.get("usage", {}) + id = response_obj.get("id", str(uuid.uuid4())) + try: + response_time = (end_time - start_time).total_seconds() + except: + response_time = None + + # Clean Metadata before logging - never log raw metadata + # the raw metadata can contain circular references which leads to infinite recursion + # we clean out all extra litellm metadata params before logging + clean_metadata = {} + if isinstance(metadata, dict): + for key, value in metadata.items(): + # clean litellm metadata before logging + if key in [ + "endpoint", + "caching_groups", + "previous_models", + ]: + continue + else: + clean_metadata[key] = value + + # Build the initial payload + payload = { + "id": id, + "call_type": call_type, + "cache_hit": cache_hit, + "startTime": start_time, + "endTime": end_time, + "responseTime (seconds)": response_time, + "model": kwargs.get("model", ""), + "user": kwargs.get("user", ""), + "modelParameters": optional_params, + "spend": kwargs.get("response_cost", 0), + "messages": messages, + "response": response_obj, + "usage": usage, + "metadata": clean_metadata, + } + logfire_openai = logfire.with_settings(custom_scope_suffix="openai") + + message_template, span_data = self._get_span_config(payload) + with logfire_openai.span( + message_template, + **span_data, + ): + pass + print_verbose(f"\ndd Logger - Logging payload = {payload}") + + print_verbose( + f"Logfire Layer Logging - final response object: {response_obj}" + ) + except Exception as e: + traceback.print_exc() + verbose_logger.debug( + f"Logfire Layer Error - {str(e)}\n{traceback.format_exc()}" + ) + pass diff --git a/litellm/utils.py b/litellm/utils.py index ec296e9dc..1a0310928 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6,7 +6,6 @@ # +-----------------------------------------------+ # # Thank you users! We ❤️ you! - Krrish & Ishaan - import sys, re, binascii, struct import litellm import dotenv, json, traceback, threading, base64, ast @@ -67,6 +66,7 @@ from .integrations.supabase import Supabase from .integrations.lunary import LunaryLogger from .integrations.prompt_layer import PromptLayerLogger from .integrations.langsmith import LangsmithLogger +from .integrations.logfire_logger import LogfireLogger from .integrations.weights_biases import WeightsBiasesLogger from .integrations.custom_logger import CustomLogger from .integrations.langfuse import LangFuseLogger @@ -128,6 +128,7 @@ heliconeLogger = None athinaLogger = None promptLayerLogger = None langsmithLogger = None +logfireLogger = None weightsBiasesLogger = None customLogger = None langFuseLogger = None @@ -1059,7 +1060,7 @@ class CallTypes(Enum): # Logging function -> log the exact model details + what's being sent | Non-BlockingP class Logging: - global supabaseClient, liteDebuggerClient, promptLayerLogger, weightsBiasesLogger, langsmithLogger, capture_exception, add_breadcrumb, lunaryLogger + global supabaseClient, liteDebuggerClient, promptLayerLogger, weightsBiasesLogger, langsmithLogger, logfireLogger, capture_exception, add_breadcrumb, lunaryLogger def __init__( self, @@ -1628,6 +1629,33 @@ class Logging: end_time=end_time, print_verbose=print_verbose, ) + if callback == "logfire": + global logfireLogger + verbose_logger.debug("reaches logfire for success logging!") + kwargs = {} + for k, v in self.model_call_details.items(): + if ( + k != "original_response" + ): # copy.deepcopy raises errors as this could be a coroutine + kwargs[k] = v + + # this only logs streaming once, complete_streaming_response exists i.e when stream ends + if self.stream: + if "complete_streaming_response" not in kwargs: + break + else: + print_verbose("reaches logfire for streaming logging!") + result = kwargs["complete_streaming_response"] + + logfireLogger.log_event( + kwargs=self.model_call_details, + response_obj=result, + start_time=start_time, + end_time=end_time, + print_verbose=print_verbose, + user_id=kwargs.get("user", None), + ) + if callback == "lunary": print_verbose("reaches lunary for logging!") model = self.model @@ -3974,12 +4002,10 @@ def calculage_img_tokens( tile_tokens = (base_tokens * 2) * tiles_needed_high_res total_tokens = base_tokens + tile_tokens return total_tokens - + def create_pretrained_tokenizer( - identifier: str, - revision="main", - auth_token: Optional[str] = None + identifier: str, revision="main", auth_token: Optional[str] = None ): """ Creates a tokenizer from an existing file on a HuggingFace repository to be used with `token_counter`. @@ -3993,7 +4019,9 @@ def create_pretrained_tokenizer( dict: A dictionary with the tokenizer and its type. """ - tokenizer = Tokenizer.from_pretrained(identifier, revision=revision, auth_token=auth_token) + tokenizer = Tokenizer.from_pretrained( + identifier, revision=revision, auth_token=auth_token + ) return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} @@ -6973,7 +7001,7 @@ def validate_environment(model: Optional[str] = None) -> dict: def set_callbacks(callback_list, function_id=None): - global sentry_sdk_instance, capture_exception, add_breadcrumb, posthog, slack_app, alerts_channel, traceloopLogger, athinaLogger, heliconeLogger, aispendLogger, berrispendLogger, supabaseClient, liteDebuggerClient, lunaryLogger, promptLayerLogger, langFuseLogger, customLogger, weightsBiasesLogger, langsmithLogger, dynamoLogger, s3Logger, dataDogLogger, prometheusLogger, greenscaleLogger, openMeterLogger + global sentry_sdk_instance, capture_exception, add_breadcrumb, posthog, slack_app, alerts_channel, traceloopLogger, athinaLogger, heliconeLogger, aispendLogger, berrispendLogger, supabaseClient, liteDebuggerClient, lunaryLogger, promptLayerLogger, langFuseLogger, customLogger, weightsBiasesLogger, langsmithLogger, logfireLogger, dynamoLogger, s3Logger, dataDogLogger, prometheusLogger, greenscaleLogger, openMeterLogger try: for callback in callback_list: @@ -7055,6 +7083,8 @@ def set_callbacks(callback_list, function_id=None): weightsBiasesLogger = WeightsBiasesLogger() elif callback == "langsmith": langsmithLogger = LangsmithLogger() + elif callback == "logfire": + logfireLogger = LogfireLogger() elif callback == "aispend": aispendLogger = AISpendLogger() elif callback == "berrispend":