phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(utils.py): support async streaming for custom llm provider

Browse source
This commit is contained in:
Krrish Dholakia 2024-07-25 17:11:57 -07:00
parent b4e3a77ad0
commit 060249c7e0
3 changed files with 38 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
litellm/utils.py
View file

@ -10132,6 +10132,7 @@ class CustomStreamWrapper:
try:
if self.completion_stream is None:
await self.fetch_stream()
if (
self.custom_llm_provider == "openai"
or self.custom_llm_provider == "azure"
@ -10156,6 +10157,7 @@ class CustomStreamWrapper:
or self.custom_llm_provider == "triton"
or self.custom_llm_provider == "watsonx"
or self.custom_llm_provider in litellm.openai_compatible_endpoints
or self.custom_llm_provider in litellm._custom_providers
):
async for chunk in self.completion_stream:
print_verbose(f"value of async chunk: {chunk}")