phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

Merge pull request #2879 from BerriAI/litellm_async_anthropic_api

Browse source 
[Feat] Async Anthropic API 97.5% lower median latency
This commit is contained in:
Ishaan Jaff 2024-04-07 09:56:52 -07:00 • committed by GitHub
commit a5aef6ec00
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 339 additions and 150 deletions
Download patch file Download diff file Expand all files Collapse all files

5
litellm/utils.py
View file

@ -8764,7 +8764,9 @@ class CustomStreamWrapper:
return hold, curr_chunk
def handle_anthropic_chunk(self, chunk):
str_line = chunk.decode("utf-8") # Convert bytes to string
str_line = chunk
if isinstance(chunk, bytes): # Handle binary data
str_line = chunk.decode("utf-8") # Convert bytes to string
text = ""
is_finished = False
finish_reason = None
@ -10024,6 +10026,7 @@ class CustomStreamWrapper:
or self.custom_llm_provider == "custom_openai"
or self.custom_llm_provider == "text-completion-openai"
or self.custom_llm_provider == "azure_text"
or self.custom_llm_provider == "anthropic"
or self.custom_llm_provider == "huggingface"
or self.custom_llm_provider == "ollama"
or self.custom_llm_provider == "ollama_chat"