Merge pull request #2363 from BerriAI/litellm_handle_circular_ref

(Fix) High Traffic Fix - handle litellm circular ref error

litellm/integrations/langfuse.py

@@ -265,8 +265,14 @@ class LangFuseLogger:
 
             cost = kwargs.get("response_cost", None)
             print_verbose(f"trace: {cost}")
-            if supports_tags:
+
+            # Clean Metadata before logging - never log raw metadata
+            # the raw metadata can contain circular references which leads to infinite recursion
+            # we clean out all extra litellm metadata params before logging
+            clean_metadata = {}
+            if isinstance(metadata, dict):
                 for key, value in metadata.items():
+                    # generate langfuse tags
                     if key in [
                         "user_api_key",
                         "user_api_key_user_id",
@@ -274,6 +280,19 @@ class LangFuseLogger:
                         "semantic-similarity",
                     ]:
                         tags.append(f"{key}:{value}")
+
+                    # clean litellm metadata before logging
+                    if key in [
+                        "headers",
+                        "endpoint",
+                        "caching_groups",
+                        "previous_models",
+                    ]:
+                        continue
+                    else:
+                        clean_metadata[key] = value
+
+            if supports_tags:
                 if "cache_hit" in kwargs:
                     if kwargs["cache_hit"] is None:
                         kwargs["cache_hit"] = False
@@ -301,7 +320,7 @@ class LangFuseLogger:
                 "input": input,
                 "output": output,
                 "usage": usage,
-                "metadata": metadata,
+                "metadata": clean_metadata,
                 "level": level,
             }
 

litellm/integrations/s3.py

@@ -104,6 +104,23 @@ class S3Logger:
             usage = response_obj["usage"]
             id = response_obj.get("id", str(uuid.uuid4()))
 
+            # Clean Metadata before logging - never log raw metadata
+            # the raw metadata can contain circular references which leads to infinite recursion
+            # we clean out all extra litellm metadata params before logging
+            clean_metadata = {}
+            if isinstance(metadata, dict):
+                for key, value in metadata.items():
+                    # clean litellm metadata before logging
+                    if key in [
+                        "headers",
+                        "endpoint",
+                        "caching_groups",
+                        "previous_models",
+                    ]:
+                        continue
+                    else:
+                        clean_metadata[key] = value
+
             # Build the initial payload
             payload = {
                 "id": id,
@@ -117,7 +134,7 @@ class S3Logger:
                 "messages": messages,
                 "response": response_obj,
                 "usage": usage,
-                "metadata": metadata,
+                "metadata": clean_metadata,
             }
 
             # Ensure everything in the payload is converted to str

litellm/proxy/utils.py

@@ -512,7 +512,11 @@ class PrismaClient:
 
         for k, v in db_data.items():
             if isinstance(v, dict):
-                db_data[k] = json.dumps(v)
+                try:
+                    db_data[k] = json.dumps(v)
+                except:
+                    # This avoids Prisma retrying this 5 times, and making 5 clients
+                    db_data[k] = "failed-to-serialize-json"
         return db_data
 
     @backoff.on_exception(
@@ -1641,10 +1645,28 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time):
     if api_key is not None and isinstance(api_key, str) and api_key.startswith("sk-"):
         # hash the api_key
         api_key = hash_token(api_key)
-    if "headers" in metadata and "authorization" in metadata["headers"]:
-        metadata["headers"].pop(
-            "authorization"
-        )  # do not store the original `sk-..` api key in the db
+
+    # clean up litellm metadata
+    if isinstance(metadata, dict):
+        clean_metadata = {}
+        verbose_proxy_logger.debug(
+            f"getting payload for SpendLogs, available keys in metadata: "
+            + str(list(metadata.keys()))
+        )
+        for key in metadata:
+            if key in [
+                "headers",
+                "endpoint",
+                "model_group",
+                "deployment",
+                "model_info",
+                "caching_groups",
+                "previous_models",
+            ]:
+                continue
+            else:
+                clean_metadata[key] = metadata[key]
+
     if litellm.cache is not None:
         cache_key = litellm.cache.get_cache_key(**kwargs)
     else:
@@ -1668,7 +1690,7 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time):
         "team_id": kwargs.get("litellm_params", {})
         .get("metadata", {})
         .get("user_api_key_team_id", ""),
-        "metadata": metadata,
+        "metadata": clean_metadata,
         "cache_key": cache_key,
         "spend": kwargs.get("response_cost", 0),
         "total_tokens": usage.get("total_tokens", 0),