From 7a6cc9c8616b7262e8891861d10af6f0c27e8a7d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 21 Nov 2024 16:20:30 -0800 Subject: [PATCH] fix - add real streaming for anthropic pass through --- .../vertex_ai_endpoints/google_ai_studio_endpoints.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/vertex_ai_endpoints/google_ai_studio_endpoints.py b/litellm/proxy/vertex_ai_endpoints/google_ai_studio_endpoints.py index c4a64fa21..29594a858 100644 --- a/litellm/proxy/vertex_ai_endpoints/google_ai_studio_endpoints.py +++ b/litellm/proxy/vertex_ai_endpoints/google_ai_studio_endpoints.py @@ -180,8 +180,11 @@ async def anthropic_proxy_route( ## check for streaming is_streaming_request = False - if "stream" in str(updated_url): - is_streaming_request = True + # anthropic is streaming when 'stream' = True is in the body + if request.method == "POST": + _request_body = await request.json() + if _request_body.get("stream"): + is_streaming_request = True ## CREATE PASS-THROUGH endpoint_func = create_pass_through_route(