phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

enforce vertex streaming to use sse for streaming

Browse source
This commit is contained in:
Ishaan Jaff 2024-11-21 18:31:29 -08:00
parent 088532082e
commit 8c68979274
2 changed files with 6 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

15
litellm/proxy/proxy_config.yaml
View file

@ -4,15 +4,6 @@ model_list:
model: openai/gpt-4o model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY api_key: os.environ/OPENAI_API_KEY
default_vertex_config:
router_settings: vertex_project: "adroit-crow-413218"
provider_budget_config: vertex_location: "us-central1"
openai:
budget_limit: 0.000000000001 # float of $ value budget for time period
time_period: 1d # can be 1d, 2d, 30d
azure:
budget_limit: 100
time_period: 1d
litellm_settings:
callbacks: ["prometheus"]

4
litellm/proxy/vertex_ai_endpoints/vertex_endpoints.py
View file

@ -194,14 +194,16 @@ async def vertex_proxy_route(
verbose_proxy_logger.debug("updated url %s", updated_url) verbose_proxy_logger.debug("updated url %s", updated_url)
## check for streaming ## check for streaming
target = str(updated_url)
is_streaming_request = False is_streaming_request = False
if "stream" in str(updated_url): if "stream" in str(updated_url):
is_streaming_request = True is_streaming_request = True
target += "?alt=sse"
## CREATE PASS-THROUGH ## CREATE PASS-THROUGH
endpoint_func = create_pass_through_route( endpoint_func = create_pass_through_route(
endpoint=endpoint, endpoint=endpoint,
target=str(updated_url), target=target,
custom_headers=headers, custom_headers=headers,
) # dynamically construct pass-through endpoint based on incoming path ) # dynamically construct pass-through endpoint based on incoming path
received_value = await endpoint_func( received_value = await endpoint_func(