phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

run pass through logging async

Browse source
This commit is contained in:
Ishaan Jaff 2024-11-25 13:17:25 -08:00
parent c60261c3bc
commit 68408c4d77
2 changed files with 30 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

22
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -529,16 +529,18 @@ async def pass_through_request( # noqa: PLR0915
response_body: Optional[dict] = get_response_body(response) response_body: Optional[dict] = get_response_body(response)
passthrough_logging_payload["response_body"] = response_body passthrough_logging_payload["response_body"] = response_body
end_time = datetime.now() end_time = datetime.now()
await pass_through_endpoint_logging.pass_through_async_success_handler( asyncio.create_task(
httpx_response=response, pass_through_endpoint_logging.pass_through_async_success_handler(
response_body=response_body, httpx_response=response,
url_route=str(url), response_body=response_body,
result="", url_route=str(url),
start_time=start_time, result="",
end_time=end_time, start_time=start_time,
logging_obj=logging_obj, end_time=end_time,
cache_hit=False, logging_obj=logging_obj,
**kwargs, cache_hit=False,
**kwargs,
)
) )
return Response( return Response(

32
litellm/proxy/pass_through_endpoints/streaming_handler.py
View file

@ -58,15 +58,17 @@ class PassThroughStreamingHandler:
# After all chunks are processed, handle post-processing # After all chunks are processed, handle post-processing
end_time = datetime.now() end_time = datetime.now()
await PassThroughStreamingHandler._route_streaming_logging_to_handler( asyncio.create_task(
litellm_logging_obj=litellm_logging_obj, PassThroughStreamingHandler._route_streaming_logging_to_handler(
passthrough_success_handler_obj=passthrough_success_handler_obj, litellm_logging_obj=litellm_logging_obj,
url_route=url_route, passthrough_success_handler_obj=passthrough_success_handler_obj,
request_body=request_body or {}, url_route=url_route,
endpoint_type=endpoint_type, request_body=request_body or {},
start_time=start_time, endpoint_type=endpoint_type,
raw_bytes=raw_bytes, start_time=start_time,
end_time=end_time, raw_bytes=raw_bytes,
end_time=end_time,
)
) )
except Exception as e: except Exception as e:
verbose_proxy_logger.error(f"Error in chunk_processor: {str(e)}") verbose_proxy_logger.error(f"Error in chunk_processor: {str(e)}")
@ -108,9 +110,9 @@ class PassThroughStreamingHandler:
all_chunks=all_chunks, all_chunks=all_chunks,
end_time=end_time, end_time=end_time,
) )
standard_logging_response_object = anthropic_passthrough_logging_handler_result[ standard_logging_response_object = (
"result" anthropic_passthrough_logging_handler_result["result"]
] )
kwargs = anthropic_passthrough_logging_handler_result["kwargs"] kwargs = anthropic_passthrough_logging_handler_result["kwargs"]
elif endpoint_type == EndpointType.VERTEX_AI: elif endpoint_type == EndpointType.VERTEX_AI:
vertex_passthrough_logging_handler_result = ( vertex_passthrough_logging_handler_result = (
@ -125,9 +127,9 @@ class PassThroughStreamingHandler:
end_time=end_time, end_time=end_time,
) )
) )
standard_logging_response_object = vertex_passthrough_logging_handler_result[ standard_logging_response_object = (
"result" vertex_passthrough_logging_handler_result["result"]
] )
kwargs = vertex_passthrough_logging_handler_result["kwargs"] kwargs = vertex_passthrough_logging_handler_result["kwargs"]
if standard_logging_response_object is None: if standard_logging_response_object is None: