From e83743f8e1b18b9961a5b4f328c69a1fef22ac7a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 09:59:05 -0700 Subject: [PATCH 1/9] fix langfuse - log metadata on traces --- litellm/integrations/langfuse.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index 1e957dfcf..5e1961b66 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -318,11 +318,11 @@ class LangFuseLogger: else: clean_metadata[key] = value - session_id = clean_metadata.pop("session_id", None) - trace_name = clean_metadata.pop("trace_name", None) - trace_id = clean_metadata.pop("trace_id", None) - existing_trace_id = clean_metadata.pop("existing_trace_id", None) - update_trace_keys = clean_metadata.pop("update_trace_keys", []) + session_id = clean_metadata.get("session_id", None) + trace_name = clean_metadata.get("trace_name", None) + trace_id = clean_metadata.get("trace_id", None) + existing_trace_id = clean_metadata.get("existing_trace_id", None) + update_trace_keys = clean_metadata.get("update_trace_keys", []) if trace_name is None and existing_trace_id is None: # just log `litellm-{call_type}` as the trace name @@ -342,6 +342,10 @@ class LangFuseLogger: if updated_trace_value is not None: trace_params[trace_param_key] = updated_trace_value + if "metadata" in update_trace_keys: + # log metadata in the trace + trace_params["metadata"] = clean_metadata + # Pop the trace specific keys that would have been popped if there were a new trace for key in list( filter(lambda key: key.startswith("trace_"), clean_metadata.keys()) @@ -363,6 +367,7 @@ class LangFuseLogger: "trace_version", clean_metadata.get("version", None) ), # If provided just version, it will applied to the trace as well, if applied a trace version it will take precedence "user_id": user_id, + "metadata": metadata, } for key in list( filter(lambda key: key.startswith("trace_"), clean_metadata.keys()) @@ -426,7 +431,6 @@ class LangFuseLogger: "url": url, "headers": clean_headers, } - trace = self.Langfuse.trace(**trace_params) generation_id = None From ebb5c76e374e63cd0c70407df6f1b5c9541718f4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 10:19:02 -0700 Subject: [PATCH 2/9] fix langfuse log clean metadata --- litellm/integrations/langfuse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index 5e1961b66..bc1d50190 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -367,7 +367,7 @@ class LangFuseLogger: "trace_version", clean_metadata.get("version", None) ), # If provided just version, it will applied to the trace as well, if applied a trace version it will take precedence "user_id": user_id, - "metadata": metadata, + "metadata": clean_metadata, } for key in list( filter(lambda key: key.startswith("trace_"), clean_metadata.keys()) From 97ba230b7ae21bb27ade2893ebd166ed3d0013f1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 10:20:30 -0700 Subject: [PATCH 3/9] fix langfuse test --- litellm/tests/test_alangfuse.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/litellm/tests/test_alangfuse.py b/litellm/tests/test_alangfuse.py index 5a495550c..37737d40b 100644 --- a/litellm/tests/test_alangfuse.py +++ b/litellm/tests/test_alangfuse.py @@ -248,12 +248,14 @@ async def test_langfuse_logging_metadata(langfuse_client): "trace_name", "trace_id", "existing_trace_id", + "update_trace_keys", "trace_user_id", "session_id", "tags", "generation_name", "generation_id", "prompt", + "cache_hit", } trace_metadata = { "trace_actual_metadata_key": "trace_actual_metadata_value" @@ -339,6 +341,13 @@ async def test_langfuse_logging_metadata(langfuse_client): for generation_id, generation in zip(generation_ids, generations): assert generation.id == generation_id assert generation.trace_id == trace_id + print( + "diff in generation keys", + set(generation.metadata.keys()).difference( + expected_filtered_metadata_keys + ), + ) + assert set(generation.metadata.keys()).isdisjoint( expected_filtered_metadata_keys ) From 6577719bf8a78db887c1de254e00f01c83b01ebf Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 10:28:13 -0700 Subject: [PATCH 4/9] fix - langfuse trace --- litellm/tests/test_alangfuse.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/litellm/tests/test_alangfuse.py b/litellm/tests/test_alangfuse.py index 37737d40b..6f9b8d626 100644 --- a/litellm/tests/test_alangfuse.py +++ b/litellm/tests/test_alangfuse.py @@ -247,15 +247,11 @@ async def test_langfuse_logging_metadata(langfuse_client): expected_filtered_metadata_keys = { "trace_name", "trace_id", - "existing_trace_id", - "update_trace_keys", "trace_user_id", - "session_id", "tags", "generation_name", "generation_id", "prompt", - "cache_hit", } trace_metadata = { "trace_actual_metadata_key": "trace_actual_metadata_value" @@ -342,8 +338,8 @@ async def test_langfuse_logging_metadata(langfuse_client): assert generation.id == generation_id assert generation.trace_id == trace_id print( - "diff in generation keys", - set(generation.metadata.keys()).difference( + "common keys in trace", + set(generation.metadata.keys()).intersection( expected_filtered_metadata_keys ), ) From 038522ab24d92e0644834dcc2ee0b9d7d5899417 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 14:02:16 -0700 Subject: [PATCH 5/9] fix - support debugging litellm params --- litellm/integrations/langfuse.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index bc1d50190..b9c8fafc9 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -318,11 +318,12 @@ class LangFuseLogger: else: clean_metadata[key] = value - session_id = clean_metadata.get("session_id", None) - trace_name = clean_metadata.get("trace_name", None) - trace_id = clean_metadata.get("trace_id", None) - existing_trace_id = clean_metadata.get("existing_trace_id", None) - update_trace_keys = clean_metadata.get("update_trace_keys", []) + session_id = clean_metadata.pop("session_id", None) + trace_name = clean_metadata.pop("trace_name", None) + trace_id = clean_metadata.pop("trace_id", None) + existing_trace_id = clean_metadata.pop("existing_trace_id", None) + update_trace_keys = clean_metadata.pop("update_trace_keys", []) + debug = clean_metadata.pop("debug", None) if trace_name is None and existing_trace_id is None: # just log `litellm-{call_type}` as the trace name @@ -342,10 +343,6 @@ class LangFuseLogger: if updated_trace_value is not None: trace_params[trace_param_key] = updated_trace_value - if "metadata" in update_trace_keys: - # log metadata in the trace - trace_params["metadata"] = clean_metadata - # Pop the trace specific keys that would have been popped if there were a new trace for key in list( filter(lambda key: key.startswith("trace_"), clean_metadata.keys()) @@ -367,7 +364,6 @@ class LangFuseLogger: "trace_version", clean_metadata.get("version", None) ), # If provided just version, it will applied to the trace as well, if applied a trace version it will take precedence "user_id": user_id, - "metadata": clean_metadata, } for key in list( filter(lambda key: key.startswith("trace_"), clean_metadata.keys()) @@ -381,6 +377,13 @@ class LangFuseLogger: else: trace_params["output"] = output + if debug == True: + if "metadata" in trace_params: + # log the raw_metadata in the trace + trace_params["metadata"]["metadata_passed_to_litellm"] = metadata + else: + trace_params["metadata"] = {"metadata_passed_to_litellm": metadata} + cost = kwargs.get("response_cost", None) print_verbose(f"trace: {cost}") From 97c81a5c7ed8b25a22c09d1f87177cf40394ae27 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 14:03:40 -0700 Subject: [PATCH 6/9] fix langfuse test --- litellm/tests/test_alangfuse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/tests/test_alangfuse.py b/litellm/tests/test_alangfuse.py index 6f9b8d626..4eeba0721 100644 --- a/litellm/tests/test_alangfuse.py +++ b/litellm/tests/test_alangfuse.py @@ -247,7 +247,9 @@ async def test_langfuse_logging_metadata(langfuse_client): expected_filtered_metadata_keys = { "trace_name", "trace_id", + "existing_trace_id", "trace_user_id", + "session_id", "tags", "generation_name", "generation_id", From 1bf8e7ac75f0d44e911801f7ce23c9944f1d2f79 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 14:08:39 -0700 Subject: [PATCH 7/9] fix langfuse debug mode --- litellm/integrations/langfuse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index b9c8fafc9..90c274a90 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -377,7 +377,7 @@ class LangFuseLogger: else: trace_params["output"] = output - if debug == True: + if debug == True or (isinstance(debug, str) and debug.lower() == "true"): if "metadata" in trace_params: # log the raw_metadata in the trace trace_params["metadata"]["metadata_passed_to_litellm"] = metadata From 360d2840583371b27b03f038d401a2d86db8276f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 14:12:17 -0700 Subject: [PATCH 8/9] docs - debug langfuse --- docs/my-website/docs/observability/langfuse_integration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md index ebf20b633..7ba204497 100644 --- a/docs/my-website/docs/observability/langfuse_integration.md +++ b/docs/my-website/docs/observability/langfuse_integration.md @@ -136,6 +136,7 @@ response = completion( "existing_trace_id": "trace-id22", "trace_metadata": {"key": "updated_trace_value"}, # The new value to use for the langfuse Trace Metadata "update_trace_keys": ["input", "output", "trace_metadata"], # Updates the trace input & output to be this generations input & output also updates the Trace Metadata to match the passed in value + "debug_langfuse": True, # Will log the exact metadata sent to litellm for the trace/generation as `metadata_passed_to_litellm` }, ) From a41bef52971528b1b740f9ccd5e7c3dacb132ea2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 14:12:26 -0700 Subject: [PATCH 9/9] debug langfuse --- litellm/integrations/langfuse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index 90c274a90..5cdf83a7c 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -323,7 +323,7 @@ class LangFuseLogger: trace_id = clean_metadata.pop("trace_id", None) existing_trace_id = clean_metadata.pop("existing_trace_id", None) update_trace_keys = clean_metadata.pop("update_trace_keys", []) - debug = clean_metadata.pop("debug", None) + debug = clean_metadata.pop("debug_langfuse", None) if trace_name is None and existing_trace_id is None: # just log `litellm-{call_type}` as the trace name