From c8d2a5ed5ec2d7d874c3f5e4c59d7726cd4dc466 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Wed, 23 Aug 2023 14:09:06 -0700
Subject: [PATCH] adding support for completion with retries

---
 litellm/__pycache__/main.cpython-311.pyc      | Bin 28633 -> 29055 bytes
 litellm/__pycache__/utils.cpython-311.pyc     | Bin 63736 -> 63523 bytes
 litellm/main.py                               |   4 +
 litellm/tests/test_completion_with_retries.py |  86 ++++++++++++++++++
 pyproject.toml                                |   2 +-
 5 files changed, 91 insertions(+), 1 deletion(-)
 create mode 100644 litellm/tests/test_completion_with_retries.py

diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc
index 3f89800a7ca5efe412363faf32b7808a9dd6c568..932f8294de79877634fdf0854fc770ba2ecec8ce 100644
GIT binary patch
delta 577
zcmcb4pYi`AM!w~|yj%=GFt6lU%G|bzd=iXC8`W<!>ZC}tFhuF5@-s1{3Z_UdV`N}h
z4a5+T&X58WOJPi5Ok+)vYGIswk5N`wFGU)tLqA0ZNE)Ot1~X{NZjNBe4VPvGYT^gt
z&kaCgIs;G}Lk>eNQw`Gs#>tMnqRdPSm?jsph^jN8T9U%NhGiKO&{`mdfLdlIh8pI{
z{wy-}K+{?Mia;_inSn%zCR33;kiNxUlv<RSS)8iLa*LxRH7_wav!wDCM^I`>QDtUc
z`Yoa2lKg`B#I%ysqWHv;lGNOSk|J54K24S)c_3A!03v<?36McRJ*h>-EI^(D5ELnc
zgvFEda|?1(>q|28^Ww`hOETg?>M~P{i~a$nzX2Uo90+7IFx+4fZtw@e8+>Bd`Q$J0
z$uEewuIg|})!~5PMOEi3s?H}uF7U};<O{mO7u4W>gIlP>e5%z2Zv6!;KmvkRutAwD
z`bDZhQ<H%h0yZCtoWR7$JvlgLITI7(<UcXatUSQj`MudCb{9LVAdo4rnK^YEGh_7T
zT^W}dS*3s?Ns}jK#V{I9exK#QD7x4nn^6?x<RXy1B9M-vN)T6SvO|t6mnM)8L=451
Yo6~ciFbNijGBX-{V1N@`lYR4h0IzwO<^TWy

delta 211
zcmezWi1FrqM!w~|yj%=Gpce2f<wom7J_$yhjq0}<)l&JH7*YjOB$hEUFsue*2uNp0
z0g9zCrZA?lrbxChPQJ$|E3BI$)xr>^mm&=$^-~yw88l@!$1&xGZw`;{WnyHX{4{ns
z(=Ud}eR0mLEI^aKZ@w0{i=CAd$mH2PHEkO+WB6vDtjmn7LO_wY$tpQ9j0Te%a~v3X
u7az)D6aZ-{0x2%41iGO}X!7-3SuRB&ABY%=vo`<BeZs_4Bs%$XK@R{^bvYaW

diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc
index 4e7d6c2d1842adcade06e853b8b79ad4b6ba08b8..643e64add7671a3ece2fef2c069ce74c72bd27d6 100644
GIT binary patch
delta 1389
zcmX|=e@v8R9Khf2`yMZFKSTusE)E<@a$&3xg)}s+qb!S9t%j}LkeBlKb?&)a+IFn1
z)}%o5_+qEqwOmbpOu3!C&PD5wCjJqP(Tg3wZh?sdag)UnmX&DF_r2oTd!P6F`+c78
z^S!$<>8l0F(q*w21-=SjpH=o=uvB$}OMR!i65ts2_2h$Jo$mPw#BTwI&pHwXpTOF@
z$#}gn4U?6WH3?n|<;IsHdW{G**?6lnJyB8&N`eb8*80#kMfM3U=mbGH$A9s4KCx5c
zI{(F2GkDYgmjyvd@&P{4n*)HAy?LpkPgIgKSKQ5#Pr~J{bTH!b(;M-Zyo|N43P+?W
z;V{@uFSDaOkbQ#7y^pSL<=<m%gkyx`SQ0AOj$E{-ur>-iMsN^X30V}?L3obTlO%mc
z2oO$TFqC06#A^k~;wOB;;W6-djE1bnGhB0X&9e&chw|UgzO!1$*${M{t`F9?R7{%J
zXlqN(x;gkWY%YzMOEq)pWO`0Vvu3Z*_8-*hd|FlW9YJu2d&MPsRfr3MP$|AeFQ@3D
z*DGS37$aYOJfxnC84YCP<XN08&rg~&wIZt{eDGqUW_9pq*jyelmuu$o@zjR{W$k7D
zT^+kxp8Yk`cB&zgS<v#rc&be^+b}D<)41ocJw?LhqP%CUsuG?}f-mvKzBf%hbllIW
zGu?QhZ>t<4J*3|5drjtpvitiK3H(^uztci3w~Ooq`2!g2-!I2oiTC<5c*wdd#aYxp
zPa?0iKE7BU)eTcdSgpO1ZIJs(?pHtl!6L$-8u)R$2m$r#;0*u`FAZg9{LKAP!`rSV
zg|+isyUcQ_#Hf0E=pWdvQT8z5Dq#y@ocwbXYhaYkA_V<zN4R8{{;QzJs`<mGCFsJ-
zkxh<4vP=-Jb3~Jz&ZgQUUZ<0p^iV3Rx8(*`d2QN9bXASJ@lZpp*X3?<x)0Q`Q7jy>
z!A*7FNH-J?QQk^HMtSTe!A7+nA?W=WAlm@uj(r3JYWG+nz}Gl7{*-)?>;cT2I1Dk>
zKT%^S)K8%w`X%zwDVaW_R+3W5ww|OX2{V{GwY5N>Y%)fZT;5vdtZsTwZ%Z`6bEv`P
zWta4i5JvEWsbcw468-qy)W+mdE=Hk_&Ec)7G7!~G*FKbF{d1!jp56cnYGgW9GU=Pr
zr$EESnd0gho@O_)Kd8{3gq1C@yQJ!u@)t=fm%{$$s@)V#uJiClD%IZV_*mVeD1ArE
zB;6-G06aSTILzTUvpc|~F3p-`2)55}0}GbS?|?MbGv6rYEK=z2ge4C90rm#9Kvvw|
z+NO7TQy=`c8BXHng*iBf>tX}Yf!AWbE%d);`oN#z;qgJ$Cse<p7<omJBY|Kdq~Y;J
M8|10y7x&5k0wxQeqW}N^

delta 1384
zcmYk5eQZ-z6u^7l>mF;nYGawjZC&Xcb9Ko6pqSu*3_%GC!{}znG@&W$Tspdz+&(dq
zHZw8CMENK^e2gGwhR8nRSeB=8OGuE7gluN4le*SwkpZh)GQ)-tK{D~4*Eqd-`Q3BR
zIrrRq&OPt!y!c>2G<F(|dJP}1)Mq>m{l<nam{pp)UI#dggWcs2Rc>~V12{4E<*PbP
zNW(@vLKx6?u4=sy669RknjHstm@tIEBkt9bCTRG-)uY)fkd_^k4LJT?2@alX1`XOf
z?PgK7kR>}j`jFU?hnKKO8xlKoc&WN%c~~3LdJNC}>!JzggmUm<XCdU_Vs$CL)@xs3
z+{r%Xc1CBgPk`f7jsvCmL03UxYrxawbNReMnH?n4A;Mw85mb6B;28eeTTp+T6h(w>
z6yyX+CkgF@Q-m5aBnd%+%;Au-kxrBSGr}1{l+Xb<Rc6Ne^CtZld>!Cr3D%+id^v9D
zslrf?qpdEs+c(cGuvy^KZg^R^uhDE5YK40=)sgZ|!A(=6&`MLi;1?Fi7Rb5;NLjRx
zbewz_XCyamvc`(H50qbcflvIKFxA9OH8E37(qcOqj)sr69czndZ&=Otj@5B%WkjE}
z*kdMp(rk%1aa-Re{g(1Am73woRa-5}k-khWoW=bEZyS2l^eFRg{AFOB)JN++#WGkg
z@jmfgd{TrchK4p7slX0_9VL4^UKx7Jwp7iOiMhSOCf7Rw<_gMg`Q4yn8H-}|u(OB?
z7hJ}}ugng%ijL~1ApJ^YxL7M)Ci$}R^%bK4W6IFTO9Heji{IY@n82yAVu(ktJ`D-Q
za`h2xo*>^J2-gT}3F_0$(5q=#oD@G2)c2j_l0#%AbYz0GKPh#wQzCTX@7JC)jFBcq
zxQVv76>j6kcp)UPA#N3B0B?*GDAD*KSTp{gA4#_fGXyKeEhDH&jgoE@YbQSvMgh|o
zS15Cn`2Y$E$qmvsWNgQ+$@k!{av|BMU8CMmO{i9lNO9CQD<H{Cx|JlYCuFcTy{<y-
zTvDcTy>b(Cx&7~}DW`IR`+Q!ReXAZqn8e<+Q;Lun#oyAUI!ZqUEo>H-U*8C#^2YTK
zMM*s_fj>_@3zG79$}AexOQ}yhf!1kfTZV6Q=-HnX=mBBr3hW^*)q?*;(o(_MBffMP
zQo5F4!0+<7Wp}n`7RgJ!(cdIJCTRGtoieD7{6F^kn^QX4;PQAI<&?QGfU)VPU>0vp
zS3#a)%NQg$gB5cx!V|b>t_t#%&*xeN+aDC^9^pQRV-MR!6_Pvwxyiqu*ZlcAtKkIh
zx-$b`VfFk797p{^Xf6G`Ol|fK?w;*PwH4K8yh~P*-;POyJVHJWE?A*VNi6J;{sB>N
BqJ;ne

diff --git a/litellm/main.py b/litellm/main.py
index 83889f8494..a17dfc8a5c 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -755,6 +755,10 @@ def completion(
             model=model, custom_llm_provider=custom_llm_provider, original_exception=e
         )
 
+def completion_with_retries(*args, **kwargs):
+    import tenacity
+    retryer = tenacity.Retrying(stop=tenacity.stop_after_attempt(3), reraise=True)
+    return retryer(completion, *args, **kwargs)
 
 def batch_completion(*args, **kwargs):
     batch_messages = args[1] if len(args) > 1 else kwargs.get("messages")
diff --git a/litellm/tests/test_completion_with_retries.py b/litellm/tests/test_completion_with_retries.py
new file mode 100644
index 0000000000..4d3d553990
--- /dev/null
+++ b/litellm/tests/test_completion_with_retries.py
@@ -0,0 +1,86 @@
+# import sys, os
+# import traceback
+# from dotenv import load_dotenv
+
+# load_dotenv()
+# import os
+
+# sys.path.insert(
+#     0, os.path.abspath("../..")
+# )  # Adds the parent directory to the system path
+# import pytest
+# import litellm
+# from litellm import completion_with_retries
+# from litellm import (
+#     AuthenticationError,
+#     InvalidRequestError,
+#     RateLimitError,
+#     ServiceUnavailableError,
+#     OpenAIError,
+# )
+
+# user_message = "Hello, whats the weather in San Francisco??"
+# messages = [{"content": user_message, "role": "user"}]
+
+
+# def logger_fn(user_model_dict):
+#     # print(f"user_model_dict: {user_model_dict}")
+#     pass
+
+# # normal call 
+# def test_completion_custom_provider_model_name():
+#     try:
+#         response = completion_with_retries(
+#             model="together_ai/togethercomputer/llama-2-70b-chat",
+#             messages=messages,
+#             logger_fn=logger_fn,
+#         )
+#         # Add any assertions here to check the response
+#         print(response)
+#     except Exception as e:
+#         pytest.fail(f"Error occurred: {e}")
+
+
+# # bad call 
+# # def test_completion_custom_provider_model_name():
+# #     try:
+# #         response = completion_with_retries(
+# #             model="bad-model",
+# #             messages=messages,
+# #             logger_fn=logger_fn,
+# #         )
+# #         # Add any assertions here to check the response
+# #         print(response)
+# #     except Exception as e:
+# #         pytest.fail(f"Error occurred: {e}")
+
+# # impact on exception mapping 
+# def test_context_window():
+#     sample_text = "how does a court case get to the Supreme Court?" * 5000
+#     messages = [{"content": sample_text, "role": "user"}]
+#     try:
+#         model = "chatgpt-test"
+#         response = completion_with_retries(
+#             model=model,
+#             messages=messages,
+#             custom_llm_provider="azure",
+#             logger_fn=logger_fn,
+#         )
+#         print(f"response: {response}")
+#     except InvalidRequestError as e:
+#         print(f"InvalidRequestError: {e.llm_provider}")
+#         return
+#     except OpenAIError as e:
+#         print(f"OpenAIError: {e.llm_provider}")
+#         return
+#     except Exception as e:
+#         print("Uncaught Error in test_context_window")
+#         print(f"Error Type: {type(e).__name__}")
+#         print(f"Uncaught Exception - {e}")
+#         pytest.fail(f"Error occurred: {e}")
+#     return
+
+
+# test_context_window()
+
+# test_completion_custom_provider_model_name()
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index be3dc2a824..fce863ff52 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "0.1.460"
+version = "0.1.461"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT License"