From b45bb78bee917a9365e1acfc1b15ffe3404d775e Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Sat, 16 Sep 2023 20:45:16 -0700
Subject: [PATCH] bump pyproject version

---
 litellm/__pycache__/main.cpython-311.pyc  | Bin 34060 -> 34060 bytes
 litellm/__pycache__/utils.cpython-311.pyc | Bin 110884 -> 110911 bytes
 litellm/tests/test_streaming.py           |  47 +++++----
 litellm/utils.py                          | 118 +++++++++++-----------
 pyproject.toml                            |   2 +-
 5 files changed, 86 insertions(+), 81 deletions(-)

diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc
index 593e38f0d537c848251d0faa19a1397c3840e422..5e9be16631ea1fae7099135b56840942e49e2f00 100644
GIT binary patch
delta 21
acmeC_V(RH);#$tj%f$c$-31%Dm|6in=><ao

delta 21
acmeC_V(RH);#$tj%f$c$jA<LWm|6ikdjzil

diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc
index 4f22f8da54eb4b19de6dadf44c70c2cca51844c5..61ed4069b3eed20ecafe57bc2a875be59b93a48f 100644
GIT binary patch
delta 2673
zcmb_cZ%k8H6o2Qnp!_RzeUDNoP@%~9>us?rTJRrbf`ABY2*io<T1P2-sK_<~)@50;
zWJ}gr;$w?!Wl=}6edrdK61POnvW56T-+Q};?gNW4`f;&1O}1nid#|)mTG9{8UT)4k
z@7;5L=bYa;_g#IIeCtuN;>(PTR6=}`&L>;5|9MC8odSLzfW7SB15i_+LNGE$J_wWK
z1Ysv7GwE^GM!zvhL!lFA_2`z>b4iL8a`yNj<aywaY__f*GTC1l(CkQFH9nWN0V4d>
znHLAK%?f3P^~kSoaN81&+#Kg7MP_d>P{Mw11tn|r0_|y6E7+%2P({IQEM#lqLW;~Z
zYp#c^0}~`8GmAC4$&t&^`NZ&&k`Rt<vjH-_mcTzXvHv~EKYN3}Ir@!B3;P>l+3_fY
zxiThQ<GC}VQi%In*X-%+@>x)4uJauth*h2iRpw^Rg$#Dq4wZY{Fye`h9m9koaV1@E
z0BY?N{UMUcXhB3H2zypUA0uMMCNm+f=i8<5<o$IP(%375pnw9;>;k3T0I!O`kBiD^
z0efc<G*!{@>1cy7!dV*2C%erK8mMAtt&oc%!!dF_nh+){oN85^voxxpmUuMKYs53k
zuzk)1;hDAT*|Z@@i*13_$A(E5t%)Qkl9e0L#prnv-L8#j6Qm8ke(Aax=c~0?-h2-@
z^o<kHN&G)htkNO!QO8y|pmcZokX#DxG>HinM$%wZ;quZMN+gVyZW99&#Hlq4w$A~@
zkj}askRx}?KitK#4j2?az%aD1Cx)S|u+dp90q>NAMG~c=D0|s?Owvyh?5$y#Asr;!
zGXm4Ddnp8Q?x~o{MPf;_eU6#e|4`0pEWE~oHJ0}kLGm!96a`6hM)|{{in$JNp<5A9
zsj;deNbFFS`{WoMSvV@7ZVq)XX?0!-Z`;jl_h9WFw<UmfE+X;l=z@hu9T;`E+q|_r
zRVA(ft(ofe?h|NNHAh$b^w`+EFeDgzIAc#difP^NA(exv%Al-FSspN$u%T(;q+saf
z483t)I{l)qPSDlOTX|gr)-|}B0&1Q2&?T4K6(}tJeDc%DPh3}AoGYg~fNIwRh>2mk
zDo7NGwS}@87L0wIu`do^s;yN(tqWEjbz#&M4^bF_7!-`%oN;q{TulOMS}5UBJ4WsC
z7}^MiO)z$G#;#4%XcSQ6?ejcp#i%tNLX!}}CAG=qJAGp-Mh9*yF*+<ZG#(^70`lMK
zP=z75HnA2PXS9WCamCKd2?3qpdS2(zDU42SNMx$_+HPFDkE5kEKZ^HvbA2{pzm40!
z3Kff#VDhfLg7x)3sqw*n7q9PsFkr_6le~WN1wkAnkC4x$t)E;bh?0IXND$el!!nlr
zB~;_He3N=3HJm3G8f<ZXd^rML%h9#54xt-yL=dkG&7)%&9do-@uf)2iMxgg{^xlwV
zvw)g;)PhmVdVnXfm#n1R-S&ja$CQ4q0au!FWeeAJhU;WF%F0t#Oj#GHA%Pm=sbNeF
zbJTDkuZ*M0UL+IBT%4t2IVY65N-M68yA?P~zog1`U+@lMHR4qUtTOlx3Mvz)G6mFm
zSY7E$64WM6Z3-C7(+ckxFWsG;({h|swX|y=x2siL^CPRfyA4z2a|TRRVX9^|<nr4f
z{zd+LF%Cplevp6^T|}@K&q1HbDE^+N52qf_Bz`PDsuj2U#G!O?<1<J}X6AO}%J{PF
z9kfiy1W_00q>fxRbrkLb?SC)|zk!OfKvGeVh>~sopU0pFK<n4N1!qVo_P;w0opPx0
zFJ6GsEzs&;c@I{S;7$L^2k=-1SN)Gaf^Gnx-~2IL1GwpbbP4hS?p~gP8MyENYzjI_
u_{IP1GOQ%SbN{m$sF1@Bf58`UGXu5Dpd?6aE|){!pGTkC{9ApXg?|C!c-aU5

delta 2585
zcmbVMZA?>F7`|^yi^wgv9|*Kuw0yLscv`H227#fhpnQaZ2+Ro~E3F>|SymaF4%hf+
zF`0>-#a*@x*BNyzbA~_5WlNS}%o4K1?B2Vx*gs5X=D#heMw9)}uzO1@7ihY~o0Ipv
z=bZO>-jDO1`_lvU)d%XFuQZyCg!q+u)V{Z#oz3|!2mTp?D(2Y`>|k78$nEv2h82=U
zfb^;-6yD4_QTM_GF&N*KP9_MyQktbNP$OS?Hzcsmv8?5DNs1P#ur6j~j`_gGeCq{c
zW!AFY$?R3=#BQK41=_?=KA>DHj(b)?HzpjnrhpoiL?n~hdmPG`3qH_=OX^CQ+deSJ
z$W~{Ooi>Ze5!hb+iQM~|B9Lj}=A$CKHw|2q4jvB9<}*PbXr(%4F`Y+2ElZ<LPV3D{
z5V=y+#5Cfij4!UpPbSq>yPD*#*au3f4?-~do7}~k#I@-nXRaR;6A!0>>(ar)vp**i
zzjF+X4d!HM1mV{u^>JcNL+R5oid~_Xx}rW`PG(4wjhVw;4|2_?#r2rnU{lgy`6T?N
zKes?q$BBR~MWG86W0g$TB`9QWdLgrX^t!wNUPB^4M+&47i13&Gycf5<fgT-Q1<p-?
z3nqw&F?=hpgt_O3Y-;_>GbzqM_?|xxiEug@zP~7`O(>4QjZN3Z_x;U_Hy+u{+!_Xp
z61s_(Le@hAvos90ENVDI@)Mw$fe|PLia9$11<dpa92fP>6VSe)IpCBGs+i6b&}9rX
zNei1O?B#*iBz-^<BMN5z1WYrYQMjbOBc^!rKJ$DO3WJT1RdRo`ZAL#ec&;N#88B5g
zqY@|wM>(SQs!-0<IIp!v%TXu;Yc26}Ta2jGSZCZAxfikp)Xkyp#R7WjATHX$7gXVb
zs*oy*w%tYh1hj8K#iI_4Iz*mRyw)y`sBY7wY3hAJSHbBjX1qb{Y+l$ZID0r}&w3Wu
z3hur(7i-I7O1s7yb-1vjX`x7P^m2~gbtu|=m#z`$nmG$k@4@t*P*&7HM-&$uLJiTv
zEnn1sR{v?k=M7wLMHE$sg7Uzy&K@I5#N516R1U$}$2t4f@k_b22&iSj!lO=%I#*&8
zbBj}KNmAGaXE*0eU7jWZHO>2iJZi_NeLV-A<gg0PPR`kxGK@w6HO^P_s0E{z^%Tbc
zrEtv#i*M}5sBW$cqXzLp*OTNWNnXAjh6DxI>JgkC&gqfm5?tw*g919p^}NBO!x$Z2
z709G%`dz$b7pIfPTotbE=K4HBt%s|XQ>`$i>AY*1*u4A4K3w0=oBMwsJQBo%r+D+J
zrvz~Z+=?gC)US9%5XJq97(wU<5>gg%+|44YoK=0@dZRTFCl_yQ@$zS07j)H}u6ivX
z{vzB0a?khhs12jG1h8=hc}URh<a9gZmdyfc=8<?a`&S%1T>Hq1t)ZNUS|js!04jp{
z7EWt<swOmfn4}|aTvmn4+zTCCTR%q*@Z<m{2kw$X0y)H!K1}*J(ibhYUEO`I1C!=O
zN*{V9V#fx=Qw~fyW<j7_9OYWvzKh%5B91xd@*K*+T5IHOthHlpWz<m2k;VTmQgn;}
z=A)C~t3=|jX6jXQ7fIZ%X`{vI*Sx*e;>44t1NlncYCWJ=3M7cSpjREpXDnmz3mDkg
z82kYSy&F^|F+wKW*+0kO5P*rTeGfbeC}uC8hVBe-v5!9hTPC!zrVn9B1@Ew?^YE7v
zuCPlNpby|C+xIbC1DIo<UxXrn1*UKkrr{3z{UmfNfM+!fEUDo!tGNtYGr+;tT!9}n
Y+d7nBi4m#gqxk+4`KgC3odp{H17?!IP5=M^

diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py
index 1301b7407c..10d7214780 100644
--- a/litellm/tests/test_streaming.py
+++ b/litellm/tests/test_streaming.py
@@ -214,6 +214,31 @@ def test_completion_cohere_stream():
     except Exception as e:
         pytest.fail(f"Error occurred: {e}")
 
+def test_completion_claude_stream():
+    try:
+        messages = [
+            {"role": "system", "content": "You are a helpful assistant."},
+            {
+                "role": "user",
+                "content": "how does a court case get to the Supreme Court?",
+            },
+        ]
+        response = completion(
+            model="claude-instant-1", messages=messages, stream=True, max_tokens=50
+        )
+        complete_response = ""
+        # Add any assertions here to check the response
+        for idx, chunk in enumerate(response):
+            chunk, finished = streaming_format_tests(idx, chunk)
+            if finished:
+                break
+            complete_response += chunk
+        if complete_response.strip() == "": 
+            raise Exception("Empty response received")
+        print(f"completion_response: {complete_response}")
+    except Exception as e:
+        pytest.fail(f"Error occurred: {e}")
+# test_completion_claude_stream()
 
 def test_completion_bedrock_ai21_stream():
     try:
@@ -327,28 +352,6 @@ def test_together_ai_completion_call_starcoder():
         print(f"error occurred: {traceback.format_exc()}")
         pass
 
-def test_completion_nlp_cloud_streaming():
-    try:
-        messages = [
-            {"role": "system", "content": "You are a helpful assistant."},
-            {
-                "role": "user",
-                "content": "how does a court case get to the Supreme Court?",
-            },
-        ]
-        response = completion(model="dolphin", messages=messages, stream=True, logger_fn=logger_fn)
-        complete_response = "" 
-        # Add any assertions here to check the response
-        for idx, chunk in enumerate(response):
-            chunk, finished = streaming_format_tests(idx, chunk)
-            if finished:
-                break
-            complete_response += chunk
-        if complete_response == "":
-            raise Exception("Empty response received")
-    except Exception as e:
-        pytest.fail(f"Error occurred: {e}")
-
 #### Test Function calling + streaming ####
 
 def test_completion_openai_with_functions():
diff --git a/litellm/utils.py b/litellm/utils.py
index 7e6b961fa3..9431faa3b8 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -2506,69 +2506,71 @@ class CustomStreamWrapper:
                 return chunk_data['outputText']
         return ""
 
+    ## needs to handle the empty string case (even starting chunk can be an empty string)
     def __next__(self):
         model_response = ModelResponse(stream=True, model=self.model)
         try:
-            # return this for all models
-            completion_obj = {"content": ""}
-            if self.sent_first_chunk == False:
-                completion_obj["role"] = "assistant"
-                self.sent_first_chunk = True
-            if self.custom_llm_provider and self.custom_llm_provider == "anthropic":
-                chunk = next(self.completion_stream)
-                completion_obj["content"] = self.handle_anthropic_chunk(chunk)
-            elif self.model == "replicate" or self.custom_llm_provider == "replicate":
-                chunk = next(self.completion_stream)
-                completion_obj["content"] = chunk
-            elif (
-                self.custom_llm_provider and self.custom_llm_provider == "together_ai"):
-                chunk = next(self.completion_stream)
-                text_data = self.handle_together_ai_chunk(chunk)
-                if text_data == "":
-                    return self.__next__()
-                completion_obj["content"] = text_data
-            elif self.custom_llm_provider and self.custom_llm_provider == "huggingface":
-                chunk = next(self.completion_stream)
-                completion_obj["content"] = self.handle_huggingface_chunk(chunk)
-            elif self.custom_llm_provider and self.custom_llm_provider == "baseten": # baseten doesn't provide streaming
-                chunk = next(self.completion_stream)
-                completion_obj["content"] = self.handle_baseten_chunk(chunk)
-            elif self.custom_llm_provider and self.custom_llm_provider == "ai21": #ai21 doesn't provide streaming
-                chunk = next(self.completion_stream)
-                completion_obj["content"] = self.handle_ai21_chunk(chunk)
-            elif self.custom_llm_provider and self.custom_llm_provider == "vllm":
-                chunk = next(self.completion_stream)
-                completion_obj["content"] = chunk[0].outputs[0].text
-            elif self.custom_llm_provider and self.custom_llm_provider == "aleph-alpha": #aleph alpha doesn't provide streaming
-                chunk = next(self.completion_stream)
-                completion_obj["content"] = self.handle_aleph_alpha_chunk(chunk)
-            elif self.custom_llm_provider and self.custom_llm_provider == "text-completion-openai":
-                chunk = next(self.completion_stream)
-                completion_obj["content"] = self.handle_openai_text_completion_chunk(chunk)
-            elif self.model in litellm.nlp_cloud_models or self.custom_llm_provider == "nlp_cloud":
-                chunk = next(self.completion_stream)
-                completion_obj["content"] = self.handle_nlp_cloud_chunk(chunk)
-            elif self.model in (litellm.vertex_chat_models + litellm.vertex_code_chat_models + litellm.vertex_text_models + litellm.vertex_code_text_models):
-                chunk = next(self.completion_stream)
-                completion_obj["content"] = str(chunk)
-            elif self.custom_llm_provider == "cohere":
-                chunk = next(self.completion_stream)
-                completion_obj["content"] = self.handle_cohere_chunk(chunk)
-            elif self.custom_llm_provider == "bedrock":
-                completion_obj["content"] = self.handle_bedrock_stream()
-            else: # openai chat/azure models
-                chunk = next(self.completion_stream)
-                model_response = chunk
+            while True: # loop until a non-empty string is found
+                # return this for all models
+                completion_obj = {"content": ""}
+                if self.custom_llm_provider and self.custom_llm_provider == "anthropic":
+                    chunk = next(self.completion_stream)
+                    completion_obj["content"] = self.handle_anthropic_chunk(chunk)
+                elif self.model == "replicate" or self.custom_llm_provider == "replicate":
+                    chunk = next(self.completion_stream)
+                    completion_obj["content"] = chunk
+                elif (
+                    self.custom_llm_provider and self.custom_llm_provider == "together_ai"):
+                    chunk = next(self.completion_stream)
+                    text_data = self.handle_together_ai_chunk(chunk)
+                    if text_data == "":
+                        return self.__next__()
+                    completion_obj["content"] = text_data
+                elif self.custom_llm_provider and self.custom_llm_provider == "huggingface":
+                    chunk = next(self.completion_stream)
+                    completion_obj["content"] = self.handle_huggingface_chunk(chunk)
+                elif self.custom_llm_provider and self.custom_llm_provider == "baseten": # baseten doesn't provide streaming
+                    chunk = next(self.completion_stream)
+                    completion_obj["content"] = self.handle_baseten_chunk(chunk)
+                elif self.custom_llm_provider and self.custom_llm_provider == "ai21": #ai21 doesn't provide streaming
+                    chunk = next(self.completion_stream)
+                    completion_obj["content"] = self.handle_ai21_chunk(chunk)
+                elif self.custom_llm_provider and self.custom_llm_provider == "vllm":
+                    chunk = next(self.completion_stream)
+                    completion_obj["content"] = chunk[0].outputs[0].text
+                elif self.custom_llm_provider and self.custom_llm_provider == "aleph-alpha": #aleph alpha doesn't provide streaming
+                    chunk = next(self.completion_stream)
+                    completion_obj["content"] = self.handle_aleph_alpha_chunk(chunk)
+                elif self.custom_llm_provider and self.custom_llm_provider == "text-completion-openai":
+                    chunk = next(self.completion_stream)
+                    completion_obj["content"] = self.handle_openai_text_completion_chunk(chunk)
+                elif self.model in litellm.nlp_cloud_models or self.custom_llm_provider == "nlp_cloud":
+                    chunk = next(self.completion_stream)
+                    completion_obj["content"] = self.handle_nlp_cloud_chunk(chunk)
+                elif self.model in (litellm.vertex_chat_models + litellm.vertex_code_chat_models + litellm.vertex_text_models + litellm.vertex_code_text_models):
+                    chunk = next(self.completion_stream)
+                    completion_obj["content"] = str(chunk)
+                elif self.custom_llm_provider == "cohere":
+                    chunk = next(self.completion_stream)
+                    completion_obj["content"] = self.handle_cohere_chunk(chunk)
+                elif self.custom_llm_provider == "bedrock":
+                    completion_obj["content"] = self.handle_bedrock_stream()
+                else: # openai chat/azure models
+                    chunk = next(self.completion_stream)
+                    model_response = chunk
+                    # LOGGING
+                    threading.Thread(target=self.logging_obj.success_handler, args=(completion_obj,)).start()
+                    return model_response
+                
                 # LOGGING
                 threading.Thread(target=self.logging_obj.success_handler, args=(completion_obj,)).start()
-                return model_response
-
-            # LOGGING
-            threading.Thread(target=self.logging_obj.success_handler, args=(completion_obj,)).start()
-            model_response.model = self.model
-            if len(completion_obj["content"]) > 0: # cannot set content of an OpenAI Object to be an empty string
-                model_response.choices[0].delta = Delta(**completion_obj)
-            return model_response
+                model_response.model = self.model
+                if len(completion_obj["content"]) > 0: # cannot set content of an OpenAI Object to be an empty string
+                    if self.sent_first_chunk == False:
+                        completion_obj["role"] = "assistant"
+                        self.sent_first_chunk = True
+                    model_response.choices[0].delta = Delta(**completion_obj)
+                    return model_response
         except StopIteration:
             raise StopIteration
         except Exception as e:
diff --git a/pyproject.toml b/pyproject.toml
index 843bdea018..fb91205e54 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "0.1.685"
+version = "0.1.686"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT License"