phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

run pass through logging async

Browse source
This commit is contained in:
Ishaan Jaff 2024-11-25 13:17:25 -08:00
parent c60261c3bc
commit 68408c4d77
2 changed files with 30 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

4
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -529,7 +529,8 @@ async def pass_through_request( # noqa: PLR0915
response_body: Optional[dict] = get_response_body(response) response_body: Optional[dict] = get_response_body(response)
passthrough_logging_payload["response_body"] = response_body passthrough_logging_payload["response_body"] = response_body
end_time = datetime.now() end_time = datetime.now()
await pass_through_endpoint_logging.pass_through_async_success_handler( asyncio.create_task(
pass_through_endpoint_logging.pass_through_async_success_handler(
httpx_response=response, httpx_response=response,
response_body=response_body, response_body=response_body,
url_route=str(url), url_route=str(url),
@ -540,6 +541,7 @@ async def pass_through_request( # noqa: PLR0915
cache_hit=False, cache_hit=False,
**kwargs, **kwargs,
) )
)
return Response( return Response(
content=content, content=content,

16
litellm/proxy/pass_through_endpoints/streaming_handler.py
View file

@ -58,7 +58,8 @@ class PassThroughStreamingHandler:
# After all chunks are processed, handle post-processing # After all chunks are processed, handle post-processing
end_time = datetime.now() end_time = datetime.now()
await PassThroughStreamingHandler._route_streaming_logging_to_handler( asyncio.create_task(
PassThroughStreamingHandler._route_streaming_logging_to_handler(
litellm_logging_obj=litellm_logging_obj, litellm_logging_obj=litellm_logging_obj,
passthrough_success_handler_obj=passthrough_success_handler_obj, passthrough_success_handler_obj=passthrough_success_handler_obj,
url_route=url_route, url_route=url_route,
@ -68,6 +69,7 @@ class PassThroughStreamingHandler:
raw_bytes=raw_bytes, raw_bytes=raw_bytes,
end_time=end_time, end_time=end_time,
) )
)
except Exception as e: except Exception as e:
verbose_proxy_logger.error(f"Error in chunk_processor: {str(e)}") verbose_proxy_logger.error(f"Error in chunk_processor: {str(e)}")
raise raise
@ -108,9 +110,9 @@ class PassThroughStreamingHandler:
all_chunks=all_chunks, all_chunks=all_chunks,
end_time=end_time, end_time=end_time,
) )
standard_logging_response_object = anthropic_passthrough_logging_handler_result[ standard_logging_response_object = (
"result" anthropic_passthrough_logging_handler_result["result"]
] )
kwargs = anthropic_passthrough_logging_handler_result["kwargs"] kwargs = anthropic_passthrough_logging_handler_result["kwargs"]
elif endpoint_type == EndpointType.VERTEX_AI: elif endpoint_type == EndpointType.VERTEX_AI:
vertex_passthrough_logging_handler_result = ( vertex_passthrough_logging_handler_result = (
@ -125,9 +127,9 @@ class PassThroughStreamingHandler:
end_time=end_time, end_time=end_time,
) )
) )
standard_logging_response_object = vertex_passthrough_logging_handler_result[ standard_logging_response_object = (
"result" vertex_passthrough_logging_handler_result["result"]
] )
kwargs = vertex_passthrough_logging_handler_result["kwargs"] kwargs = vertex_passthrough_logging_handler_result["kwargs"]
if standard_logging_response_object is None: if standard_logging_response_object is None: