From 53a3ea3d068a72e7b463376b2ee6f24654fa86a2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 22 Jan 2025 22:11:40 -0800 Subject: [PATCH] (Refactor) Langfuse - remove `prepare_metadata`, langfuse python SDK now handles non-json serializable objects (#7925) * test_langfuse_logging_completion_with_langfuse_metadata * fix litellm - remove prepare metadata * test_langfuse_logging_with_non_serializable_metadata * detailed e2e langfuse metadata tests * clean up langfuse logging * fix langfuse * remove unused imports * fix code qa checks * fix _prepare_metadata --- litellm/integrations/langfuse/langfuse.py | 110 ++++---------- .../langfuse/langfuse_prompt_management.py | 3 + .../code_coverage_tests/recursive_detector.py | 1 - .../completion_with_complex_metadata.json | 126 ++++++++++++++++ .../completion_with_langfuse_metadata.json | 104 ++++++++++++++ .../complex_metadata.json | 101 +++++++++++++ .../complex_metadata_2.json | 93 ++++++++++++ .../empty_metadata.json | 87 +++++++++++ .../metadata_with_function.json | 87 +++++++++++ .../metadata_with_lock.json | 87 +++++++++++ .../nested_metadata.json | 93 ++++++++++++ .../simple_metadata.json | 93 ++++++++++++ .../simple_metadata2.json | 97 +++++++++++++ .../simple_metadata3.json | 101 +++++++++++++ .../test_langfuse_e2e_test.py | 135 ++++++++++++++++++ .../test_langfuse_unit_tests.py | 46 ------ 16 files changed, 1231 insertions(+), 133 deletions(-) create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json create mode 100644 tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 6f97fd4f22..20d2befe65 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -3,11 +3,9 @@ import copy import os import traceback -from collections.abc import MutableMapping, MutableSequence, MutableSet from typing import TYPE_CHECKING, Any, Dict, List, Optional, cast from packaging.version import Version -from pydantic import BaseModel import litellm from litellm._logging import verbose_logger @@ -71,8 +69,9 @@ class LangFuseLogger: "flush_interval": self.langfuse_flush_interval, # flush interval in seconds "httpx_client": self.langfuse_client, } + self.langfuse_sdk_version: str = langfuse.version.__version__ - if Version(langfuse.version.__version__) >= Version("2.6.0"): + if Version(self.langfuse_sdk_version) >= Version("2.6.0"): parameters["sdk_integration"] = "litellm" self.Langfuse = Langfuse(**parameters) @@ -360,73 +359,6 @@ class LangFuseLogger: ) ) - def is_base_type(self, value: Any) -> bool: - # Check if the value is of a base type - base_types = (int, float, str, bool, list, dict, tuple) - return isinstance(value, base_types) - - def _prepare_metadata(self, metadata: Optional[dict]) -> Any: - try: - if metadata is None: - return None - - # Filter out function types from the metadata - sanitized_metadata = {k: v for k, v in metadata.items() if not callable(v)} - - return copy.deepcopy(sanitized_metadata) - except Exception as e: - verbose_logger.debug(f"Langfuse Layer Error - {e}, metadata: {metadata}") - - new_metadata: Dict[str, Any] = {} - - # if metadata is not a MutableMapping, return an empty dict since we can't call items() on it - if not isinstance(metadata, MutableMapping): - verbose_logger.debug( - "Langfuse Layer Logging - metadata is not a MutableMapping, returning empty dict" - ) - return new_metadata - - for key, value in metadata.items(): - try: - if isinstance(value, MutableMapping): - new_metadata[key] = self._prepare_metadata(cast(dict, value)) - elif isinstance(value, MutableSequence): - # For lists or other mutable sequences - new_metadata[key] = list( - ( - self._prepare_metadata(cast(dict, v)) - if isinstance(v, MutableMapping) - else copy.deepcopy(v) - ) - for v in value - ) - elif isinstance(value, MutableSet): - # For sets specifically, create a new set by passing an iterable - new_metadata[key] = set( - ( - self._prepare_metadata(cast(dict, v)) - if isinstance(v, MutableMapping) - else copy.deepcopy(v) - ) - for v in value - ) - elif isinstance(value, BaseModel): - new_metadata[key] = value.model_dump() - elif self.is_base_type(value): - new_metadata[key] = value - else: - verbose_logger.debug( - f"Langfuse Layer Error - Unsupported metadata type: {type(value)} for key: {key}" - ) - continue - - except (TypeError, copy.Error): - verbose_logger.debug( - f"Langfuse Layer Error - Couldn't copy metadata key: {key}, type of key: {type(key)}, type of value: {type(value)} - {traceback.format_exc()}" - ) - - return new_metadata - def _log_langfuse_v2( # noqa: PLR0915 self, user_id, @@ -443,27 +375,17 @@ class LangFuseLogger: print_verbose, litellm_call_id, ) -> tuple: - import langfuse - verbose_logger.debug("Langfuse Layer Logging - logging to langfuse v2") try: - metadata = self._prepare_metadata(metadata) - - langfuse_version = Version(langfuse.version.__version__) - - supports_tags = langfuse_version >= Version("2.6.3") - supports_prompt = langfuse_version >= Version("2.7.3") - supports_costs = langfuse_version >= Version("2.7.3") - supports_completion_start_time = langfuse_version >= Version("2.7.3") - + metadata = metadata or {} standard_logging_object: Optional[StandardLoggingPayload] = cast( Optional[StandardLoggingPayload], kwargs.get("standard_logging_object", None), ) tags = ( self._get_langfuse_tags(standard_logging_object=standard_logging_object) - if supports_tags + if self._supports_tags() else [] ) @@ -624,7 +546,7 @@ class LangFuseLogger: if aws_region_name: clean_metadata["aws_region_name"] = aws_region_name - if supports_tags: + if self._supports_tags(): if "cache_hit" in kwargs: if kwargs["cache_hit"] is None: kwargs["cache_hit"] = False @@ -670,7 +592,7 @@ class LangFuseLogger: usage = { "prompt_tokens": _usage_obj.prompt_tokens, "completion_tokens": _usage_obj.completion_tokens, - "total_cost": cost if supports_costs else None, + "total_cost": cost if self._supports_costs() else None, } generation_name = clean_metadata.pop("generation_name", None) if generation_name is None: @@ -713,7 +635,7 @@ class LangFuseLogger: if parent_observation_id is not None: generation_params["parent_observation_id"] = parent_observation_id - if supports_prompt: + if self._supports_prompt(): generation_params = _add_prompt_to_generation_params( generation_params=generation_params, clean_metadata=clean_metadata, @@ -723,7 +645,7 @@ class LangFuseLogger: if output is not None and isinstance(output, str) and level == "ERROR": generation_params["status_message"] = output - if supports_completion_start_time: + if self._supports_completion_start_time(): generation_params["completion_start_time"] = kwargs.get( "completion_start_time", None ) @@ -770,6 +692,22 @@ class LangFuseLogger: tags.append(f"cache_key:{_cache_key}") return tags + def _supports_tags(self): + """Check if current langfuse version supports tags""" + return Version(self.langfuse_sdk_version) >= Version("2.6.3") + + def _supports_prompt(self): + """Check if current langfuse version supports prompt""" + return Version(self.langfuse_sdk_version) >= Version("2.7.3") + + def _supports_costs(self): + """Check if current langfuse version supports costs""" + return Version(self.langfuse_sdk_version) >= Version("2.7.3") + + def _supports_completion_start_time(self): + """Check if current langfuse version supports completion start time""" + return Version(self.langfuse_sdk_version) >= Version("2.7.3") + def _add_prompt_to_generation_params( generation_params: dict, diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index 2bb8d55e6a..1a14968240 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -107,6 +107,9 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge langfuse_host=None, flush_interval=1, ): + import langfuse + + self.langfuse_sdk_version = langfuse.version.__version__ self.Langfuse = langfuse_client_init( langfuse_public_key=langfuse_public_key, langfuse_secret=langfuse_secret, diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 1af0d2708f..71c007e5e4 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -8,7 +8,6 @@ IGNORE_FUNCTIONS = [ "text_completion", "_check_for_os_environ_vars", "clean_message", - "_prepare_metadata", "unpack_defs", "convert_to_nullable", "add_object_type", diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json new file mode 100644 index 0000000000..4c5f345eaa --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json @@ -0,0 +1,126 @@ +{ + "batch": [ + { + "id": "9ee9100b-c4aa-4e40-a10d-bc189f8b4242", + "type": "trace-create", + "body": { + "id": "litellm-test-c414db10-dd68-406e-9d9e-03839bc2f346", + "timestamp": "2025-01-22T17:27:51.702596Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:27:51.702716Z" + }, + { + "id": "f8d20489-ed58-429f-b609-87380e223746", + "type": "generation-create", + "body": { + "traceId": "litellm-test-c414db10-dd68-406e-9d9e-03839bc2f346", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:27:51.150898-08:00", + "metadata": { + "string_value": "hello", + "int_value": 42, + "float_value": 3.14, + "bool_value": true, + "nested_dict": { + "key1": "value1", + "key2": { + "inner_key": "inner_value" + } + }, + "list_value": [ + 1, + 2, + 3 + ], + "set_value": [ + 1, + 2, + 3 + ], + "complex_list": [ + { + "dict_in_list": "value" + }, + "simple_string", + [ + 1, + 2, + 3 + ] + ], + "user": { + "name": "John", + "age": 30, + "tags": [ + "customer", + "active" + ] + }, + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-27-51-150898_chatcmpl-b783291c-dc76-4660-bfef-b79be9d54e57", + "endTime": "2025-01-22T09:27:51.702048-08:00", + "completionStartTime": "2025-01-22T09:27:51.702048-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:27:51.703046Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json new file mode 100644 index 0000000000..d4882c962d --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json @@ -0,0 +1,104 @@ +{ + "batch": [ + { + "id": "872a0a1c-4328-431b-80b6-fd55a8a44477", + "type": "trace-create", + "body": { + "id": "litellm-test-533ffb2d-a0a3-45b5-911c-7940466cdc8e", + "timestamp": "2025-01-22T17:19:11.234960Z", + "name": "test_trace_name", + "userId": "test_user_id", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "sessionId": "test_session_id", + "version": "test_trace_version", + "metadata": { + "test_key": "test_value" + }, + "tags": [ + "test_tag", + "test_tag_2" + ] + }, + "timestamp": "2025-01-22T17:19:11.235169Z" + }, + { + "id": "18d6f044-e522-4376-96e0-7eec765677ed", + "type": "generation-create", + "body": { + "traceId": "litellm-test-533ffb2d-a0a3-45b5-911c-7940466cdc8e", + "name": "test_generation_name", + "startTime": "2025-01-22T09:19:10.957072-08:00", + "metadata": { + "tags": [ + "test_tag", + "test_tag_2" + ], + "parent_observation_id": "test_parent_observation_id", + "version": "test_version", + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "parentObservationId": "test_parent_observation_id", + "version": "test_version", + "id": "time-09-19-10-957072_chatcmpl-4da65aba-32e4-400d-aaa2-6bfe096d8141", + "endTime": "2025-01-22T09:19:11.234200-08:00", + "completionStartTime": "2025-01-22T09:19:11.234200-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:19:11.235541Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json new file mode 100644 index 0000000000..01dcd26488 --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json @@ -0,0 +1,101 @@ +{ + "batch": [ + { + "id": "ddf567e5-a1b5-4e38-8a7c-f48bc847f721", + "type": "trace-create", + "body": { + "id": "litellm-test-46551fc7-c916-4a83-aeef-4274b5582ce1", + "timestamp": "2025-01-22T17:59:39.367430Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:59:39.367707Z" + }, + { + "id": "d3eb2c9e-e123-419d-b27b-c8283a505ae8", + "type": "generation-create", + "body": { + "traceId": "litellm-test-46551fc7-c916-4a83-aeef-4274b5582ce1", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:59:39.362554-08:00", + "metadata": { + "int": 42, + "str": "hello", + "list": [ + 1, + 2, + 3 + ], + "set": [ + 4, + 5 + ], + "dict": { + "nested": "value" + }, + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-59-39-362554_chatcmpl-d20ba1d9-cda6-4773-822e-921ebcd426a0", + "endTime": "2025-01-22T09:59:39.365756-08:00", + "completionStartTime": "2025-01-22T09:59:39.365756-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:59:39.368310Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json new file mode 100644 index 0000000000..1b7b91930e --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json @@ -0,0 +1,93 @@ +{ + "batch": [ + { + "id": "ea3d694a-ce6b-417e-86e3-23ac17c6f6c6", + "type": "trace-create", + "body": { + "id": "litellm-test-38dcf290-8742-4fc5-ad03-c5d47e91dec0", + "timestamp": "2025-01-22T18:06:50.959206Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T18:06:50.959409Z" + }, + { + "id": "5fe03133-5798-4f87-8eec-ae0264f1eccc", + "type": "generation-create", + "body": { + "traceId": "litellm-test-38dcf290-8742-4fc5-ad03-c5d47e91dec0", + "name": "litellm-acompletion", + "startTime": "2025-01-22T10:06:50.957097-08:00", + "metadata": { + "list": [ + "list", + "not", + "a", + "dict" + ], + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-10-06-50-957097_chatcmpl-62d4ad7c-291b-4fc7-a8a4-3ed0fc3912a5", + "endTime": "2025-01-22T10:06:50.958374-08:00", + "completionStartTime": "2025-01-22T10:06:50.958374-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T18:06:50.959850Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json new file mode 100644 index 0000000000..8c1711ee98 --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json @@ -0,0 +1,87 @@ +{ + "batch": [ + { + "id": "28d0c943-284b-4151-bf0d-8acf0f449865", + "type": "trace-create", + "body": { + "id": "litellm-test-d9506624-457c-40bc-9a37-578b896fa22a", + "timestamp": "2025-01-22T17:59:32.888622Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:59:32.888940Z" + }, + { + "id": "384e9fb4-3516-47b2-a4ae-1666337ec4a7", + "type": "generation-create", + "body": { + "traceId": "litellm-test-d9506624-457c-40bc-9a37-578b896fa22a", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:59:32.878577-08:00", + "metadata": { + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-59-32-878577_chatcmpl-1195f870-fd4d-4e38-8dc8-99dd3da5ab0b", + "endTime": "2025-01-22T09:59:32.880691-08:00", + "completionStartTime": "2025-01-22T09:59:32.880691-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:59:32.889548Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json new file mode 100644 index 0000000000..0b1309425e --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json @@ -0,0 +1,87 @@ +{ + "batch": [ + { + "id": "88b1898a-cc5d-4e8e-93bc-3e71300c5e8d", + "type": "trace-create", + "body": { + "id": "litellm-test-a46356d9-ecff-44c8-a3da-fed3588b5128", + "timestamp": "2025-01-22T17:59:36.162545Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:59:36.162702Z" + }, + { + "id": "96bb77a6-a350-431b-bfd8-425491259728", + "type": "generation-create", + "body": { + "traceId": "litellm-test-a46356d9-ecff-44c8-a3da-fed3588b5128", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:59:36.161090-08:00", + "metadata": { + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-59-36-161090_chatcmpl-1ee988c9-9133-4655-bbe4-b97ffb6e3dc9", + "endTime": "2025-01-22T09:59:36.161959-08:00", + "completionStartTime": "2025-01-22T09:59:36.161959-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:59:36.162997Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json new file mode 100644 index 0000000000..8c1711ee98 --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json @@ -0,0 +1,87 @@ +{ + "batch": [ + { + "id": "28d0c943-284b-4151-bf0d-8acf0f449865", + "type": "trace-create", + "body": { + "id": "litellm-test-d9506624-457c-40bc-9a37-578b896fa22a", + "timestamp": "2025-01-22T17:59:32.888622Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:59:32.888940Z" + }, + { + "id": "384e9fb4-3516-47b2-a4ae-1666337ec4a7", + "type": "generation-create", + "body": { + "traceId": "litellm-test-d9506624-457c-40bc-9a37-578b896fa22a", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:59:32.878577-08:00", + "metadata": { + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-59-32-878577_chatcmpl-1195f870-fd4d-4e38-8dc8-99dd3da5ab0b", + "endTime": "2025-01-22T09:59:32.880691-08:00", + "completionStartTime": "2025-01-22T09:59:32.880691-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:59:32.889548Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json new file mode 100644 index 0000000000..bb24688aa5 --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json @@ -0,0 +1,93 @@ +{ + "batch": [ + { + "id": "44f179be-e3b9-486f-986f-030fc50614f0", + "type": "trace-create", + "body": { + "id": "litellm-test-8a04085c-1859-48fa-9fd8-1ec487fe455e", + "timestamp": "2025-01-22T17:55:28.854927Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:55:28.855187Z" + }, + { + "id": "2175ee64-58a3-41ab-96df-405b76695f5f", + "type": "generation-create", + "body": { + "traceId": "litellm-test-8a04085c-1859-48fa-9fd8-1ec487fe455e", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:55:28.852503-08:00", + "metadata": { + "a": { + "nested_a": 1 + }, + "b": { + "nested_b": 2 + }, + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-55-28-852503_chatcmpl-131cf0da-a47b-4cd1-850b-50fa077362ac", + "endTime": "2025-01-22T09:55:28.853979-08:00", + "completionStartTime": "2025-01-22T09:55:28.853979-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:55:28.855732Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json new file mode 100644 index 0000000000..d40ec6bafc --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json @@ -0,0 +1,93 @@ +{ + "batch": [ + { + "id": "02c74119-76b7-4f79-91cb-c55f1495c100", + "type": "trace-create", + "body": { + "id": "litellm-test-e58116c7-ead0-417e-9f86-b35f1e5bc242", + "timestamp": "2025-01-22T17:53:53.754012Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:53:53.754178Z" + }, + { + "id": "097968e0-52e9-46b5-9e8e-e6e08dd00e72", + "type": "generation-create", + "body": { + "traceId": "litellm-test-e58116c7-ead0-417e-9f86-b35f1e5bc242", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:53:53.752422-08:00", + "metadata": { + "a": { + "nested_a": 1 + }, + "b": { + "nested_b": 2 + }, + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-53-53-752422_chatcmpl-e99bc1d3-a393-493f-8afe-4507c0acff15", + "endTime": "2025-01-22T09:53:53.753431-08:00", + "completionStartTime": "2025-01-22T09:53:53.753431-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:53:53.754511Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json new file mode 100644 index 0000000000..610bc461a1 --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json @@ -0,0 +1,97 @@ +{ + "batch": [ + { + "id": "1a55383a-e6fa-41f9-81fe-e7aa58c55f40", + "type": "trace-create", + "body": { + "id": "litellm-test-08fd1578-4a67-49b4-ac23-2dff1c112c80", + "timestamp": "2025-01-22T17:56:35.477276Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:56:35.477571Z" + }, + { + "id": "13ba66e8-f72b-4f57-a6cc-57c0be2829b1", + "type": "generation-create", + "body": { + "traceId": "litellm-test-08fd1578-4a67-49b4-ac23-2dff1c112c80", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:56:35.474752-08:00", + "metadata": { + "a": [ + 1, + 2, + 3 + ], + "b": [ + 4, + 5, + 6 + ], + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-56-35-474752_chatcmpl-9b152610-3d1e-4731-a84e-d0341ea69a0f", + "endTime": "2025-01-22T09:56:35.476236-08:00", + "completionStartTime": "2025-01-22T09:56:35.476236-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:56:35.478171Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json new file mode 100644 index 0000000000..d21c58fdee --- /dev/null +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json @@ -0,0 +1,101 @@ +{ + "batch": [ + { + "id": "7fb1f295-a7af-47af-afbd-e2f2d08280aa", + "type": "trace-create", + "body": { + "id": "litellm-test-c3acc34b-3c06-4868-bcee-87a3c4c1367e", + "timestamp": "2025-01-22T17:56:38.786515Z", + "name": "litellm-acompletion", + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "tags": [] + }, + "timestamp": "2025-01-22T17:56:38.786742Z" + }, + { + "id": "412870bc-fc50-4426-a0dc-9e8b016e14bb", + "type": "generation-create", + "body": { + "traceId": "litellm-test-c3acc34b-3c06-4868-bcee-87a3c4c1367e", + "name": "litellm-acompletion", + "startTime": "2025-01-22T09:56:38.784548-08:00", + "metadata": { + "a": [ + 1, + 2 + ], + "b": [ + 3, + 4 + ], + "c": { + "d": [ + 5, + 6 + ] + }, + "hidden_params": { + "model_id": null, + "cache_key": null, + "api_base": "https://api.openai.com", + "response_cost": 5.4999999999999995e-05, + "additional_headers": {}, + "litellm_overhead_time_ms": null + }, + "litellm_response_cost": 5.4999999999999995e-05, + "cache_hit": false, + "requester_metadata": {} + }, + "input": { + "messages": [ + { + "role": "user", + "content": "Hello!" + } + ] + }, + "output": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "tool_calls": null, + "function_call": null + }, + "level": "DEFAULT", + "id": "time-09-56-38-784548_chatcmpl-438c8727-86b3-44d9-9b46-42330922cf50", + "endTime": "2025-01-22T09:56:38.785762-08:00", + "completionStartTime": "2025-01-22T09:56:38.785762-08:00", + "model": "gpt-3.5-turbo", + "modelParameters": { + "extra_body": "{}" + }, + "usage": { + "input": 10, + "output": 20, + "unit": "TOKENS", + "totalCost": 5.4999999999999995e-05 + } + }, + "timestamp": "2025-01-22T17:56:38.787196Z" + } + ], + "metadata": { + "batch_size": 2, + "sdk_integration": "litellm", + "sdk_name": "python", + "sdk_version": "2.44.1", + "public_key": "pk-lf-e02aaea3-8668-4c9f-8c69-771a4ea1f5c9" + } +} \ No newline at end of file diff --git a/tests/logging_callback_tests/test_langfuse_e2e_test.py b/tests/logging_callback_tests/test_langfuse_e2e_test.py index 4fa6f7d34a..60d25b3340 100644 --- a/tests/logging_callback_tests/test_langfuse_e2e_test.py +++ b/tests/logging_callback_tests/test_langfuse_e2e_test.py @@ -6,6 +6,7 @@ import os import sys from typing import Any, Optional from unittest.mock import MagicMock, patch +import threading logging.basicConfig(level=logging.DEBUG) sys.path.insert(0, os.path.abspath("../..")) @@ -216,3 +217,137 @@ class TestLangfuseLogging: "completion_with_tags_stream.json", setup["trace_id"], ) + + @pytest.mark.asyncio + async def test_langfuse_logging_completion_with_langfuse_metadata(self, mock_setup): + """Test Langfuse logging for chat completion with metadata for langfuse""" + setup = await mock_setup # Await the fixture + with patch("httpx.Client.post", setup["mock_post"]): + await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hello!"}], + mock_response="Hello! How can I assist you today?", + metadata={ + "trace_id": setup["trace_id"], + "tags": ["test_tag", "test_tag_2"], + "generation_name": "test_generation_name", + "parent_observation_id": "test_parent_observation_id", + "version": "test_version", + "trace_user_id": "test_user_id", + "session_id": "test_session_id", + "trace_name": "test_trace_name", + "trace_metadata": {"test_key": "test_value"}, + "trace_version": "test_trace_version", + "trace_release": "test_trace_release", + }, + ) + await self._verify_langfuse_call( + setup["mock_post"], + "completion_with_langfuse_metadata.json", + setup["trace_id"], + ) + + @pytest.mark.asyncio + async def test_langfuse_logging_with_non_serializable_metadata(self, mock_setup): + """Test Langfuse logging with metadata that requires preparation (Pydantic models, sets, etc)""" + from pydantic import BaseModel + from typing import Set + import datetime + + class UserPreferences(BaseModel): + favorite_colors: Set[str] + last_login: datetime.datetime + settings: dict + + setup = await mock_setup + + test_metadata = { + "user_prefs": UserPreferences( + favorite_colors={"red", "blue"}, + last_login=datetime.datetime.now(), + settings={"theme": "dark", "notifications": True}, + ), + "nested_set": { + "inner_set": {1, 2, 3}, + "inner_pydantic": UserPreferences( + favorite_colors={"green", "yellow"}, + last_login=datetime.datetime.now(), + settings={"theme": "light"}, + ), + }, + "trace_id": setup["trace_id"], + } + + with patch("httpx.Client.post", setup["mock_post"]): + response = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hello!"}], + mock_response="Hello! How can I assist you today?", + metadata=test_metadata, + ) + + await self._verify_langfuse_call( + setup["mock_post"], + "completion_with_complex_metadata.json", + setup["trace_id"], + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "test_metadata, response_json_file", + [ + ({"a": 1, "b": 2, "c": 3}, "simple_metadata.json"), + ( + {"a": {"nested_a": 1}, "b": {"nested_b": 2}}, + "nested_metadata.json", + ), + ({"a": [1, 2, 3], "b": {4, 5, 6}}, "simple_metadata2.json"), + ( + {"a": (1, 2), "b": frozenset([3, 4]), "c": {"d": [5, 6]}}, + "simple_metadata3.json", + ), + ({"lock": threading.Lock()}, "metadata_with_lock.json"), + ({"func": lambda x: x + 1}, "metadata_with_function.json"), + ( + { + "int": 42, + "str": "hello", + "list": [1, 2, 3], + "set": {4, 5}, + "dict": {"nested": "value"}, + "non_copyable": threading.Lock(), + "function": print, + }, + "complex_metadata.json", + ), + ( + {"list": ["list", "not", "a", "dict"]}, + "complex_metadata_2.json", + ), + ({}, "empty_metadata.json"), + ], + ) + async def test_langfuse_logging_with_various_metadata_types( + self, mock_setup, test_metadata, response_json_file + ): + """Test Langfuse logging with various metadata types including non-serializable objects""" + import threading + + setup = await mock_setup + + if test_metadata is not None: + test_metadata["trace_id"] = setup["trace_id"] + + with patch("httpx.Client.post", setup["mock_post"]): + await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Hello!"}], + mock_response="Hello! How can I assist you today?", + metadata=test_metadata, + ) + + await self._verify_langfuse_call( + setup["mock_post"], + response_json_file, + setup["trace_id"], + ) diff --git a/tests/logging_callback_tests/test_langfuse_unit_tests.py b/tests/logging_callback_tests/test_langfuse_unit_tests.py index 5a551dbd7d..5096e7b2d7 100644 --- a/tests/logging_callback_tests/test_langfuse_unit_tests.py +++ b/tests/logging_callback_tests/test_langfuse_unit_tests.py @@ -271,52 +271,6 @@ def test_get_langfuse_logger_for_request_with_cached_logger(): mock_cache.get_cache.assert_called_once() -@pytest.mark.parametrize( - "metadata, expected_metadata", - [ - ({"a": 1, "b": 2, "c": 3}, {"a": 1, "b": 2, "c": 3}), - ( - {"a": {"nested_a": 1}, "b": {"nested_b": 2}}, - {"a": {"nested_a": 1}, "b": {"nested_b": 2}}, - ), - ({"a": [1, 2, 3], "b": {4, 5, 6}}, {"a": [1, 2, 3], "b": {4, 5, 6}}), - ( - {"a": (1, 2), "b": frozenset([3, 4]), "c": {"d": [5, 6]}}, - {"a": (1, 2), "b": frozenset([3, 4]), "c": {"d": [5, 6]}}, - ), - ({"lock": threading.Lock()}, {}), - ({"func": lambda x: x + 1}, {}), - ( - { - "int": 42, - "str": "hello", - "list": [1, 2, 3], - "set": {4, 5}, - "dict": {"nested": "value"}, - "non_copyable": threading.Lock(), - "function": print, - }, - { - "int": 42, - "str": "hello", - "list": [1, 2, 3], - "set": {4, 5}, - "dict": {"nested": "value"}, - }, - ), - ( - {"list": ["list", "not", "a", "dict"]}, - {"list": ["list", "not", "a", "dict"]}, - ), - ({}, {}), - (None, None), - ], -) -def test_langfuse_logger_prepare_metadata(metadata, expected_metadata): - result = global_langfuse_logger._prepare_metadata(metadata) - assert result == expected_metadata - - def test_get_langfuse_tags(): """ Test that _get_langfuse_tags correctly extracts tags from the standard logging payload