From 73bb1b96e97a52b299c923f8b28b6eb296d3f15a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 4 Sep 2023 11:53:18 -0700 Subject: [PATCH] update exception mapping and get model cost map --- litellm/__init__.py | 105 +++---------------- litellm/__pycache__/__init__.cpython-311.pyc | Bin 8053 -> 7757 bytes litellm/__pycache__/main.cpython-311.pyc | Bin 29341 -> 28706 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 83364 -> 83621 bytes litellm/exceptions.py | 9 +- litellm/tests/test_exceptions.py | 2 +- litellm/tests/test_get_model_cost_map.py | 10 ++ litellm/utils.py | 22 +++- pyproject.toml | 2 +- 9 files changed, 54 insertions(+), 96 deletions(-) create mode 100644 litellm/tests/test_get_model_cost_map.py diff --git a/litellm/__init__.py b/litellm/__init__.py index b5c84bab4..4c89c441f 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1,4 +1,4 @@ -import threading +import threading, requests from typing import Callable, List, Optional, Dict from litellm.caching import Cache @@ -35,94 +35,18 @@ caching = False # deprecated son caching_with_models = False # if you want the caching key to be model + prompt # deprecated soon cache: Optional[Cache] = None # cache object model_alias_map: Dict[str, str] = {} -model_cost = { - "babbage-002": { - "max_tokens": 16384, - "input_cost_per_token": 0.0000004, - "output_cost_per_token": 0.0000004, - }, - "davinci-002": { - "max_tokens": 16384, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000002, - }, - "gpt-3.5-turbo": { - "max_tokens": 4000, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, - }, - "gpt-35-turbo": { - "max_tokens": 4000, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, - }, # azure model name - "gpt-3.5-turbo-0613": { - "max_tokens": 4000, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, - }, - "gpt-3.5-turbo-0301": { - "max_tokens": 4000, - "input_cost_per_token": 0.0000015, - "output_cost_per_token": 0.000002, - }, - "gpt-3.5-turbo-16k": { - "max_tokens": 16000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004, - }, - "gpt-35-turbo-16k": { - "max_tokens": 16000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004, - }, # azure model name - "gpt-3.5-turbo-16k-0613": { - "max_tokens": 16000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.000004, - }, - "gpt-4": { - "max_tokens": 8000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.00006, - }, - "gpt-4-0613": { - "max_tokens": 8000, - "input_cost_per_token": 0.000003, - "output_cost_per_token": 0.00006, - }, - "gpt-4-32k": { - "max_tokens": 8000, - "input_cost_per_token": 0.00006, - "output_cost_per_token": 0.00012, - }, - "claude-instant-1": { - "max_tokens": 100000, - "input_cost_per_token": 0.00000163, - "output_cost_per_token": 0.00000551, - }, - "claude-2": { - "max_tokens": 100000, - "input_cost_per_token": 0.00001102, - "output_cost_per_token": 0.00003268, - }, - "text-bison-001": { - "max_tokens": 8192, - "input_cost_per_token": 0.000004, - "output_cost_per_token": 0.000004, - }, - "chat-bison-001": { - "max_tokens": 4096, - "input_cost_per_token": 0.000002, - "output_cost_per_token": 0.000002, - }, - "command-nightly": { - "max_tokens": 4096, - "input_cost_per_token": 0.000015, - "output_cost_per_token": 0.000015, - }, -} - +def get_model_cost_map(): + url = "https://raw.githubusercontent.com/BerriAI/litellm/main/cookbook/community-resources/max_tokens.json" + + try: + response = requests.get(url) + response.raise_for_status() # Raise an exception if request is unsuccessful + content = response.json() + return content + except requests.exceptions.RequestException as e: + print("Error occurred:", e) + return None +model_cost = get_model_cost_map() ####### THREAD-SPECIFIC DATA ################### class MyLocal(threading.local): @@ -298,7 +222,8 @@ from .utils import ( Logging, acreate, get_model_list, - completion_with_split_tests + completion_with_split_tests, + get_max_tokens ) from .main import * # type: ignore from .integrations import * diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc index c2b76dce9e6b01602ba0a442589af246a681b757..e68c4da03434062d30b612cd17ca95e969019102 100644 GIT binary patch delta 2693 zcmc(f?N1a(9LHyO@3_4^?+!TRKoAZOA5J*#fd>^4K}BmCZ6aEAd?3focr5JR;_jZv z1$w0Mg)!+1yC%MA8q*gxjW78NO#8NpHY6q6nDkZCv~Ng^jqyd(-z*D=*S_w+e80b$ z&u?buW@q-z@Q*{GZ-YSzLDBaAMSf{{7;36q_aO8bT|xvAPClMJ*qo0ExJywTd*jWOo(W>5{f8(&&=*_X+GjY=6{6vGXQI}72p`}PJ@6EgpFv$O zIKiVci0r*oz&pVE z!S{dx*2Hy*QgzO{x3p^eq2S@_^DD)qM{vq(L3IxX>=msAKPl6u@?*Sj7 zeYC#}Yo`N_9&+@sqemPaa&*wqqmCYPblA}mM~^ebqcCR*<~*@ACp|Dyuiz`{ReV*&7^~OtHT62auHL{mAfFpYi9^d5y_l=y-8bZz7gT`y`gf4W zkhh5Z$mAU6$n<8sfSyA9Y_q+{EpSixt<0@x`y1vE-~Phly~HkZSptMJ8J6h0FO8h4{PZVufQDp%1Q zmk`dE^HZidYYe7RdTy?#Ae&RA$&x|!yjC=6(d@}kDxI1BLGqh;*OdDFU93InkO^<2$d`Z_S8BF*pwy4wF zC2E+4EflC}*XlXhpcA(=eZnwvX34NUY^W{KyLmcm%39H|YcH%18QbWxy|cPpG!q_M zDCr8D(wK!rYWv*v+Z;96-o>E2JDR!#zsyL@z|XOo{h3NVJa|IY2vsKXngL;MR_}uB zeNc_hL6*>}q%ECX^Z2|CuOcmvhMov7Bb_Ue&imr3So^cs@srr`QjD*N@fTuzwSL#* zwyzmJ^&77 zL)IfHlV-Cd0%dPu)=c)5CC8HKflTja+?(ku``7z>voqFjQf#suWbNLb{-jybC$)0@ zwq`Qjo6c1Au_^1e=Vbqc@Bj$zSEW|o%&lCS4D{9b*8$taT(OinU6 z#pELep!crRMci7U_ zwXyzac);sj*6p3qfb;$ivv~)Xt({M~_{R|2J`?T^jJLfd6W*78jwPID@(GA_E}E^_ z9(Ggh9dJWvLC-mNNRP7=%;XxA>r8GixyfXL$pDkh^Ycy|f-6!|=&=3(06Q1P*YFiu z6F81rMY~@7$N#DoZi-s5rkdqgQ;2VuI5_{C(uhK_7s2+I!H$(+$C9*GvwWtti?4(@ VehvM968-n5w^@I;86LJF{{ZLIV#NRe delta 2962 zcmc(gOKcle6o&6Ob{s#FIEn2zao%a0IEj-u&O6Se&ATmq=F#L96r9|tJH%s$@dUL2 zA-EKkXEwqFPB3v9{YE5xYGGUWcgo`9I zHcJ+4;b^=lUMyL$l|wOvON1i9Rzq8{k6GqDP6H{9q@jpdaVg<&nPC1$ z%Mw}$q})JkjM%Y@IB*5A<4VY?GGsaRdbfd;>co>apfVkK4WwKr)du3yiO)bPbW&p= zm5ii0Tx&p8dXe8i+&ZZ<5RXpk4aBRH1_P;%AUJ#Uy)kU=t}d&A(3kWt+m zBN3Qxf5LIX=WSvFyu)y@E#Wp3nnUEsRYWEQ^bFUo-ZNFF112N{?!a-~g`s_L2bsEx za3>sX7mOf`ZWui*X+!o%bV zasuoK#7{ze6zr)q+>F7^X|NHnXTXkwJqwl>`h@TftV!r+I>2<0=@8RVro&7}n2s@x zFy)z!Q^ga|a{+oD{=26zAPfrps1Ol&VMrJj^af<{mK`4C21KiOXu)8X$7xHt9Vty7(+g{ zfCA@tubDBI&N+8{&5bW*k6VYM#DVu-))0@9NxXA~KvUVwYX-!FP3W-$KEhe_M zm9>pzN?F?yWG$TaB)1h~IOibI=fuQ@NV~?a|NO!2M_+u#=SpK+N-*3#5LCA1^`v;a z9KMFJ#UEYIraC_2cbtssFjrZS8|)nng$=QAZzxw$m=_w{6tD2G3$5*!ygvIqe#e#H zKM09AZ!Dd^AR5yqve49i@o*zF66=>vxVFnDa&{Kf5^UOixNlSEjT1T7Mm)MrgrJy6 zDba)y42id2f&UqAerx;VtCdgq*h7AE`b^yeK3AeA_Qf79ve-|2?1!;O6Ok5fVy8?I zE-ArvF_lcf;e|xhgwRge#>Hqs(1lR0d?P7I(F6%5#Mnh8epz41ue|zA$#wOndiTM|IlI)r5cqOxf}^<)SP^2|VxwU6LG9 zU$Fb#v^ZNL5+R|87cR@=;9gVj+C$B>L%<=ogOLLuJNw5K4)x0&5Kx_trW(1G@`F?k zP&q`U4Me@+7^sjNsM`V(u*g9g-E&Oaj#HVZ za+Jz3^;%VkbE+R#eXibcr=91sHf>*GBBnNYE?Z=H#z=j~^QzkCbvxLxjM2jJ-80@h zocko)XRV5mQlKDf*;d4OO0D(vb;y`zou}9=Ma4wcDZ)9$|Ra! zJTp(z>?oBHDvuvTt|THQ$#z`e<<|gkFj6mD5!x%_I4*-+_wxQS>V03m`n|7s_ZMG@ zDZR{FRcZs zX0!=es`9CaQjlgwJCLO&osaz#gLE?75BT5c0HZvz`0|%kOKU+U7@g;kr8*ziPz-XE zWBe@eddkyH`nDQr0?2}v@FXNS+3djEVPj>KgFL123&7#fVB| zO3Fc=)%Xr@SYTZ`)6|h^?t}^USNOb^Fl|DXa!7d0D`*3d857&VN#LIS1<;ePrX1vy z##jD}uWEc9xHsdi*LFY~_-F%=4VG}lzNeUWJ^oHpx|B2Rq5n@x^JlM9zpGD~>5Kmt D48Pd) diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc index 5be8c1fc544bb7f729b3c59ac711a3acd566f4f0..4e3285a90e09b54c4a536b6aa0d29e799a8e7456 100644 GIT binary patch delta 7473 zcmcIp3v650c|P}2q^JiaKE#(qN~9=?qTVv~v}C**JQ>5{L(umyz!@BPO zFCP+ZB~I6s(*4hW&iT*d|IYt7ynN?;k}CT$GivbkJ;X}~^dm-sw?dB8F0kobImMZh`flz5fDGT@qY0Z%x_Uo}|; zWr5!vsGh852uplb3pI2PV{WL0cD3IWFi)BT#goN>lF5=l-DF+BGHH>E&Y(dV* zkvYPdmNlH&S9hG{iZ5hehAFPl|8N=$Cbw}Vu!e`T0BxrG*&$sER|b45X9L>C*@3px z7ucKjoavx&f?o>F&$^jdp0G6M6HfSe@AN!Wv!>^_HPLU&s@Fb}eabQ#`o6+cnKvDp z_n!9g;rXRtOm%#TpY?f#`G9Zfln~1ao(gzpeA9t2Rb)3YO4^gXU@T0krhzga9vM8J zv^(i**=04h93mD+>|)s*S;Th2=L=GaJR#iKc_P}LW1eIV5ZbLYP@A$P+r}|n&V*B;S8bWQjW|W;0cfYk)F!%YRpq4-5uLPK zW1tH<6)abww_nZzE#XOg7;>>NxJutsSv^LsJfUq^&N<)=2xvXWReY}2mW0+7A+=DF znhwmsndu}UhN}cbRgc+` z$kfxl)lON`$N(`HAhx14&>t5KFb(u+U0Bn2x6M#Xk=xO<%%;%$J?!qH%`5{%QY)To zSk79#$JPL`ZJX>_zKQp2@R2I9N-Y0e$>t47t_7nD%-Xuq1|wU$2$%r4NjX=P6llv3 zXp>sYBhHKYK?SFf4k;T#%0KG8Ogk+%xS0+*X0YnoGYvAN0!T%}cy>FEPr|+(V6Uj+ zIz`t!6RF_}B+tzibHp01OIilG9TJxc0EH%nyhhC7y6Jvnle_0KM7}Fh^h3Li)PB(^ zbkxFjKgavIUWfvn&H02Yn&uG!~15J*hgZ*a$ig+!evgzX+#+@-K*zcuFn&6hYt?s;>4QCEd z;uHBoT^#qTCL13 zdemB?O3Q3cV48I6>h6U7(uI8Q;W4m*WDW}sWXw2z*XesQdiUJb+bnv{v)n|)BQ`_1 zH_}X>$gebrQ04Y5v)ra-Hd3BFy(Q7Px<512L7K}BK1Oes4#---9JiMpl(&j4EA4np zdLp*owR~H`x9PiOm8pO_fdREOrEKcp@U|$nEp(*n+*vqs7x(`bQjM)!=>lQ)HQRUp zfUc_uGW0XMRb%81B@Bwc${mUr0lCWZZ_3(*N6{6E#dfg%$xNUl9b!kq`p+e--?Wu) zuReTP3HA?9;*++2``tI5 zjPd+AZc5VPTaJUW?ZP<>Kd{X$T{I*7Y~M2cOiQ62w>-U(UAVH(-SX5{Th5p6ZMf&G z;LG%-3WvUvlXElv{=h(_D=-+@0r$|(l_BZk_?wDeM#}Z{pfhclW;8f*uVIQ^^fi}D z-Ys^oj7p=Qsw=cbdca9p_%+&de*O8>5^B>Ox!C)vwSs}9Nfg}?mL z_R_zly(BL6U*5#$wz8Lkw7sywa=5a5=I)f6A_K{QHBp@g6TYp1U2u!y0R59{d&vL> zrn`J~P#lEs3`V+MT_*$Q{zlCq*^uyj!ncQD@wT4nOb|^FxVsUABv15A^WS^G-M75 zGI5v-a{^qDteJ3u^^UAOgsvnb;xH|)E0H;+Yn2IJEr`yA?P=Xo6Krw+T{)ceMceP< zVmr2Uv7lCXCAB8pzs`uZKPS(ynYjKD1IePR-U;8bXX-ncHo9E@^4@kB#dia7=kkrn zHzDyLX+hG8L=wWp#-n2V0Fan=b}10@`viEM;GLcJ`@Qqr^D_PbU1^xk z+-zb_0Xwp1Pb|&P`ogg+!FNXBmvEg$dUTs$SwPm0Bmg9gcXXSLuf~QDJ=Wq-``~2| ze-i7pNCay4v?<4dbu*yMG0z@lZgTVVV~?X7&7VJn1aXcjeZko!4%AO5GmUM!zOkXb zBi_ETy(4|zzP+Q~-9v`~nXf`0ebrOpMj?0hCPW^C2L1w)%-LyU^SE&e+Y#|32uTp> zW6cldrTh39`t#;v_f=Mr1^V+R>EkVJqqv=oO}mww_9L0h=WPmbm+T6);B`Q#ruBRG zUwJ06azWce_m$OsMUut&r6a+2HuTlDiTe&zyqH*Iy!{{UFB?n9?(5L&Bd~y57E{az zL#Ko=k5%x9M?3bFNmh&M#kFa(@P2x;!@>+u<@Vub4hMyiNa*34 zH_4r1In!rP@jmaO?~#}?1PnS|0{4gMW843l8KO;{4a`^RbZ5Jw3`bp{&vn+Z{5XB9 zvz2*@%Dc*$)3mC~U5u|-W7^a^!|8c1x(Ce5>*!+FCrlCb?l3S0dU{8nax`VKZ|_f7j=5p`{`Nd=HtcG8!5>G0FW7*q1TaA+wQ_Qe<)=BniqTl{v5WSNAf0;e*qGszv%8RdZ-1iw{9v3tuy(Zf-x*eKr5*j_RdEH;nB#=iG~Un^U)MR} zGGIT{S=USJ-_bQ(*EPhmiAsM*P5-ukK@P&IBI+FI=`@vIH`ZP^dg3J8R1h^u6RDtmB&@7>eoi%YHHirL$!l`#!Sx**2wSV6Sa~4VDKO<9%^=J zRi+yy_Uk2$*Gt;sL}}4Ri_4-`=M9Tvc>>C)#SyjD#8uR~$1K07{Opc)?T16la(Sui zy0wX3+NZbNFx#Uhl#A>0N(*igAh(fxZBN`p;}2Nr(__xHPj`Pmi#6NnD|>$+FLhtH zdNzi-wjR3n>b{TJtkUvj?}n&tOVMIj#g}&?E(6#AFN2x(4O`bD2Yza)PUu$$WSwuw_Ey^SpK?umj=si*ZS0b4ai*^ApI>8Gt@dLEqZOV zJ-2U{S$1s~tL@uW1Z9yN>h!mBMyfviO=gTq}ys zkP&vAg<^(1iAAq$R0hRSW>>0MSJvJlBxkqmwnU*5__f=)WNf831A8DbZ4cJ&LUZTQPI-iuN6iu!7HPkzMZ2pfD>w4yYlo-}C|EVQZNsou# zfcY9u?l@PdiRR=*wT8I;(Of1sUP-jZXudkCE{baP(SpKgfi{}2iRMCRwnSB$Xpsq- z!lGz)9)zbnRkR4C)bZVnG@5LpV+>K~;^X&p+{$KI_BH_mnz8xH3x>E-8Yb%L?UT>{ EFZ!90AOHXW delta 7563 zcmb_h32akKh2R2$8%RoAL{b75iKHk>RAkYj&A<>pM1jOYe}EoK z{)lXk+$`fo_DyHpRG!L>D?1~$j!V~e(!{Y+r*WN3O}Uy1##B?avAVcvN{ys8ZtF?! z`yfa{t26Z<;J)qN_wGCIE)O4nc!|7lnbf?jRx22M{`31+xq+t`<`=Y)d-SW7tH+dLqmY=};gUj)vk3Nia)&GR^ctA{mbcU_?4=Dp}o^21^xwCHTGiZZ-T& zYOk`(8Nrfr2o*ET9PMX#RZ=Wzong46VMefnMmA3D5R?tdoq~avaBRv5ugbnojPRPw z3!J<}Z=7Q(Kd<5>0`m~zWr7jDDxdPJfp9vZ7^z;+AT1Rbqy`)%oKYwT@-#`gl3FU8 zeS{K0E9eBhP%0P&lVFA>nb9a%7fq+p%+83WMj2C;f&<1CmMWJ}xoAF(DxX2=6Ft;8m)L3j8_){8F)>XrW)}E7Zmuf)a55|sC%yc|BVs=%&t!twHb>WY&R!Sxc|jyq-npRL->*-x^$E>0!5 z9%6Wh;Nn!4wG}uAyl6Do-2yJK=*-_l$}PCRL>D&Z7FMpS&`VKM!Oe+Nn(34`Azd7s zfnbSE-$2ntlp95qSzg0Yly{ICgs_pt{oWBn=Up)bzjd^LYdU2EVla?q_@K55cdmtJ z^(H(ZQfz_|T_5qmQ@VtZo7elf_HMK&l;~LRL%t5Nm%&=;>Ud|~c&FIwOu0{}Vt%ej zL|=-L<;~In=>n6%L$-k5VAFTy8|SGq`X|Q=&%Yw zE_k_KaUojUD%L3ISB#cIuOR2P!49L(<3Ekz?~E7%sc)p}?*Ol5(`}rc>qDonA7G}W zsRqFh1<|oTwpN0Dp`e5od+XKIXub@z`?T)y{{kr$H}dO zrHCyEOmZz$FwATn)tNpRqMk=^rp-Ix9ZLy(->fc3jxmfFFn{1uSeh|T4d&Z_$|v|v zHDZi1Z6$&;`Q_|HlnNM{1^wdo(^7;6zu70{yaP0bapxUkAw6k%?1LMWZ_dq}ReruM zZ)-ky#?$WHk#7O1mSiX=#V~n0Y`Mh_TZPsyZikh3-=POyF0*Nj+|XJIdvG<94AWq& z*Yaq=`H2c%v<|N4hgdCW%FCgtLPK_fO=(rHXO0btY@L5`6cXj30~-Tve_oK^bpw8F zd81=DjBXOz9%Z@RsWxE~mU~j0Fdf#Iu*Hp^WVsEiY|L)XbsgP{d-J0gW;1E*+puKo zmuyD!{M2@jq+Qqy;}s@D`weS%Z-YX|Y-i32xT9!yHQW~# zA5HDTEp~x1^O*h3&+oeW8(p_3=7cvLHm#AH$l3<~U2Y;}L`*I_ewTE@BTkE@6D@c$ zAHP(W(3Q2|V+8M1xAM~aPRr0s@tb|tywrK)jb(LRL(-e$MNdbY<|c66`)`O1**gsN z+rrosUdpCcuZf8Q?#nb>^UrWoqWTA1_ei!Rk3q_5vTV7<2D*jrwGG@#{hNbKP>kPH z{os1i;2gJAf9j6v7sTGph8>uwv>AH1LhfLsFWR5#jSi%?;(fE{)F6hyQh1?qt7J7n z!?nts0UC`QbF&2sy%4CbDcmM(J2fl{YwC*~DL)#fe|97Xu26G`;1xW3v!q$z8NC@k zaJSyxh%6i8|7zl-2Ox>P33FAITB>^}G8)~L>bs5Jit4-Q-ADDd_oJfyvzQ&S&ljdB z8+!*1)L5k_xEawj9;um-3?#oTn#RB_Ok+?O{4Gr*dl&!wmOXPv(_mvRwCwO43y;*c zOHHZmYZ2(LYuA7*!e5kg!gkPmoYw6$)ZE~pJA@s0**C&&PlJTI^D)naWGML^annP$ z*z~Y4{M&6hc_w#InD6)J>Vh*rd?rd(lk?{W$HhhNgih=;+|=1Rzj_4Cj)_d9cCMl0 z_O78*!7;)Jm*A4O`XM0fgvY!lG6KKnwMyJs52!~0_iWeNHgdu@-OUB}sROINWxu`4 z^xV>AV!Gs?x*Y|8jDV|;wJqn)Q zPIwB z3B(W1dJ6da;ZkEcnS#$6=V~cCKS^b9!JJHPk_{aIO@E3%J+?RZ4X=r0wMibC53LGk zispITNit79fwN|$(}&~X=};n5kPJPXGX33`=wICy$9<>L_a zRhC5Y{R}@(+x3(r;G4d7*-or_2=3%>d}GRjj4TwJj&ry}_(qFc{Qw=5u5JfBEtQpD zqLTY4p$C!C`$q@&3s>Cg02jJE zL=R+(8GX2H%T;24;%*};g{p49Zgj(m4N&nxh;@5S8F_3W8aNmVMiV?ufc$Ps_Q0#% zyN|iCnvt$L8ZVyk+Tmtod?+y=k0nClW-CQ`28A3Kb`X@lu#4Q6yd8qge+4K zPKKiVS7`q+O5Ue~CBfW_M0sF35)Q?Z{LkS?PoL&#%CbyZ*y%Wb9^UV%HD+`hEGox- z0j~A54*r62Dya?R`u~Xb$Y(;a5Fbp&dH!SCE~b$u8NwiyU5n-x{-2Q~Mv*|UcL&V( z7Q!pNrG`&U4D%>+eLrDTy03RU+41PW3OluQuJ0o*5>op_7$eET$i>wAiA@*q1=`0WW~-`I#wiDTVZIkefxUjglwei{ zdHWVz-(gHwRHmJti;kL?G%HeUi=8Vn##*-`&oxgBofx`a%s8rZa;zx#%3;)sO3-H4 z@OtCK(64_@(Xy>DGPQYyS$ezwxx)R9`imCl#fG-?4O`RZs&siJU7+|1Dp7oml4Vsn zZlt2|g0162TSwYxOm2_F+{T?T6Bwg=nU_XfwD_5H zz(>muI{k|A0_L2A(AK#E#a^l8Tyfi8R`MZRi1iPp)pX#)#_ip-{K&iANy|&>K`SjU zd-|$q`7fGbBQ38~54O?rzpQ;`T7KF#Lit;nvjawd6YV*>y{K=KSQ+Xb5Enf=+N<#I zvP;hHlIi@r%CIae#5TORe-W=LV-rKHguwF?t%XudTgs*7iQOy-fMar>m*zOk#YO1o z3y~y-`Y3i}iViK{tNqXodG8PK{|^`Z)ns$gUm_F#F8pTlC{e=E`vwY4IInK$FYbGo zDCn;?`ttkgEDTQ781d(VL|}yip@<{+W{y3L!S@WibbM-aK}@`aY@&0Cw+@;GQG>Mr zwT~Ca?x76ylE_a`2FZr_OG7VzAMItuxRkwMh(VgZQsED67#auX!RBN5pU~Q+f}W_D z+B`b`by^n%#po^ub}__@OZe#M8%Q$p!RWydN0U1L7VUhEcADe!7=D7`KqxmDh{o}E zO@jX}<@tn??@{u@eoEh_gua6ED6RIu3bYs}sa&xvtG~i9@~^PhM2dr0UwUEs zIx(7dlrcXoQw+68e(LtihZ-dBH53lDp!t~KN1>w~Vp=Fl97$lX_pq4)J{}42vs4*+ zm%|>(e0*NaXm~oJl2zg$Jb1`ei2t!Mb4$-3I!MTC@IN!XiYMq8B@0XahaVn)j0E-K|CKf`!_%{E z4*c(q;cc}29wiPWnS$x~{1Lv8wqJvPoE^~?M^gWe_oTh@(s zvJeUN@;x|;A13juB~K(oN<7ON+H9_jvH4LqSTlXj>Nz7e}8oPAOHXW diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc index 2826ca27fd6bbbeb5fc03e8ca28e312cbbe002a9..309ff5b2e133cb1ff0d779f9fe93e5d4c586e3b3 100644 GIT binary patch delta 5783 zcmai234B!5)qm&BmW50fGLvPpO!iDjCJO|^P7-8M2q*zGEetVlfC3yKG;l`{m<0KdGMdQU$)l@g0G0rE{S)(z$Ta5b2IJx`sw+ zj#RY<_&H5h=lC^3!xBtSSObkNX(FbYrE0{b(p;$q>uix~G1V$fLUc=aB6>t&iXcs1 zBS=$rh=T9}zM-d~O`3}N%OVA7o-`fHGHC{;y;2>b>=G6WQjt`&NGr|6ymo08YM1LJ zm@U}u^RLnNxgdi3++T=^X@2H;FhytXs9Aeu>L#ZS_db7OTtaE|9IQjcsjC$-2@h+pX1-WOd2bB~IqFmROg_ za+`N_X(?-5VPEW$m$X07?sc*zk6U)SWqXsSwY0{`n5%kPX^TsCwzRaCwl=!lrA;2s z(g*NGp|!Q$?UGlPFsIkk&N`c%UNm{oA$yiO-CldM*WGzAF;I?GVqT2~ZN^}3m(V9_bj{*NvF7gBtgU(7v4w{t3a|EwKuTj$pCF{T z#LI#Z+ng?bK|dm@BAdmlSFd^_kYIIuqVQCFp2mOCH&ujre0b;kFp`U1RWO#1=_=Mu z#@eRwmM$im(*38q%A#Nzk6*Vit`7BVCc#{SIsD;uXEUau7>xH~|FD$~nKd>!9jsAy zhV0N}eJ;rU9qXTshk9P~^J@K6Dq6=M`}rgA0FT+y1dV*zmO5C#_iw2OA2)413NGHe z)e_fChCf4h+5H4{JaJo=X&EVZQo4<#T_m;gySDjZHUD~B4ZP{Eer`vCtzE5zz(-I| zpe0yNu!3ME!F(RKH%H$=;sQQuZ|<}%5+6ngYMgF~tw4TT6kAQ&M+j)3*uw-W@ivm$ z37#WZLnil<MO6>E0ZBBWWKZ=p^_d<^PD_DT2|YTT9XpN%|2V_40(ywIr&u zc%}&(7RP`X>mUB=`xzIsz}5t>&(P9hUl^4t%WFJ&WqA{;qwO0lwyJ z|3SF!uRL%NU^Ksa@R^)#nCqpGVC|pX(aRfKT#`fmB-t*^jrPCv<|Cl{Es7TZ*#G}P zo4g3cpdP;hr+ay*Bbrdd?&n2^rlu_>(MYg?TzLWk2k952c$sfFG)n&}iR<|v4^`>j zMKRAG_1^p1xILtOk-(3zDpxsa1FNaU<(1h>q}@yKGC_c+AHKWuH56@TMy{~e3Df}z z`9aB-sPeZ+QakPqlJ*lEAUH^%js<>#LQvO^F^4hL^;1;vGy!?X&X6?F5<|#`w__7a6zlzCj=Tk8ceGz~^iI&cD97;lyHa`YTSo5fo&zcT-R?(VFQ0L& z7Si~W$8w{5!tR6}7(HIl@WvBKe8=%r{@Jk~8`31PLnG($10Q5q2Tafz^;r!YHSU#? z$){Z|FqLQeAV%{fvrj;ybyAXQoTT#Rgoy+}O7>}=6*h}xchi%C4<8;AvdsI>mf_sh z;L}NIOVzc7nfRFx&B+6F14bJ~_d7hM#}dCtN+)-`S}CKy4WoK)=?iDiJvaMcLQh8S zAZ+zMQOfDBb_>7JGZWf*<_Qz_ZPW2v_(FX<7c*`g&%HexxPOam`TcCK;z(s`@}vU( zSTG9;`GMm@`MzKU$i!b78lG==W1$b4jp3H69hyJjGYzf9*3TR&8y&XAAqcY%BEo2{}y6nf zGs2QQJ}e1>&(LfLrxwBPk+;s4tyX^IWOddZb(8B|n=vv4qdaX+H}kaP`p=%Ds1--B zgAYHo-l$=tkSP?VYX$#1r+xz5ai+xR+JQDUu|Jn@K65Nujnp5awJnQxoV^=cr34H%2B^os08!jbt z-G!f}3}W6lgzveKJ}8+^Fx}B-Ur+$5X$Y15$Q2EjBQ-P`wRoyHWO%G=JT@WS4> zo2|?1jpyyXlkx^>-9P)HgoM1E`ai|n569d2-+Nsu!^VqYh9oh$h_AYs%k39a_}|Va zbK|A=`~Aw{-&}gVzoI05^e^w26XuwOW9F!N1)5{l8gX8>=EH2=yaK-Gub1FezWFl? zJnDbtGra~D@%@+g=&s}FZS*hy{O=;>HeVT^NUJ~2!dGZ$X)4(a>^gt#N}4{juJe;u zlFQW9_!?#g4Q-7pX{xceG3C|bgt6kIZr<5Nq9EAm&;RljfM@vWe^l#3(~lQiEsoOS z8pa3?_~&1(5Yb@cwM?Ae|9j02L-?s{lc1aDe)SvFAN{HZ&T#Yf%IIf^lokVKCfp=jN?yT_n?{f#tgXNzxT#0(39)zJm2|s2VC;!f3pDYFk?llBC=)YgL3xh>{ViR zNAEZIsQY0&i=<$7P`k*5yWQU-y-m+-REy=Ozk|o&b7htW(gIF^d!U+#SQ0@p!TnSd z?QKF#T9t{VQdYUlsFAcfExiMo(IXq8H~nbQ(KEuc6r=hh6>#F zC~s(Cw26EhAY+WO3KgRc?h*}#K!Xk@L8j`}Ffvq2E+(5|N|q25Ql0ndV=;(%7bpvK zFiNR6!0U-*f#)t_+$Hw=sI*2xrqU4sCguJJ7zIfRkAUM)LHbrm51fdE3eES``qZih z>x$qIb!Y3qFxb{EhqiDx-Lee{^%M2=zjYQnrMP0i289Zbfmx6fI2Qwt8ep{2YJ%*z z)9FG(6s z(nv*bhN)mtW||>KKOR-9()hCLW@UpJ^e`dtQ!_lIiJ^Ijvt}HXn4nlwAsLLys8pze zR)wX4t09(h5(p9rRNu%grn;cku2vkXlQLAds6IA{K%L|z)J8fAvq;hr91T=jKm!mN z7?lQgfWInB(xG%HeOKy9p=pK0KBv|kP@YeRLRh36O@}(&bS&B)$j<*yaZrhs8tf2C!hnp zdL9Vsnp!+wr_IP>NMB2OJxW1?7ms5&)!F}%{#9j04&+8!(90k+vx&;892lqjJ|@=& zUdsWmMz1<(Q7Q`{N9REOuE6{P_<;z1<+u&XGH)JTFnH0CA*feppO*!Q{yn9z2+H!+ zqfFd?JFzGa6+wF0?am~6)`&H${A$DtP3i|o*QC5x1VuUtZw#gGmwAnaU6BFg!Z?rXq_JrOKyvNF7Ggy8mgWos0TNF+PJ2 zaKk-h6ZXt@*g#ibMx~?_u4{2T874L%X`ev9J<+inmmiK! z$N#P7o!aebhqEd_vSe(QH?4Zcx5;fI|JD-iEvp7%>JG0Gw9%SXfczq+Ipx=p8Di z-k-Qgr{0jz

+N`%rI2+DNxdsTl`V`nO2lr*w^jhR#ZoUF73rlHMk|j|t8a(8UMa zOVTcq=mH~jrl)hdxEu2W0 eqJ_LZ;afOY***#0OU9T9eZqIJURir5r2h--CdbGC delta 5651 zcmai23w#vS)t|GkY_fSQ$!7D|glrxJHjm8{SttY&9`X?a;UyT9P0TD|A!Ot1Bq%Oo zfL3eE>$`0Se_{otfIqR7Z^5P3Ck1WMYTyGS3u!585vXNRt3?5O&fP2tCD_St|2cEc zJ^y>|nLFp+xxZSk-SHD`{I0mT7y+-}z43*(t#NaFDlFJL+^kNh7l*Hdh6Zt#Xv6;t zS-m=yHL5i#aW-43u5m;Qkw0;cSn9e{EW=+p{wnZS>2iuAUCza^;vMU>4UOVlaTIbi ziH>ywyP(cqJy)z+2Te=F@#};Jw>SZDsW?xp#x~2u8oX;2Cn7EvCn2_}ggXUs@;X7B zvQs4p&*LwAHF(53(cT^{hDx=W9Y)xql+{w27w9z;n^->+dT!J|)cKum2U0RFz-;!Xr>k{S$Z0w=ZCeYCY|FN0aI&l0X23nHcKcjd$+m6(|DqOZavr^d#^xnW zva{XemFrwyZ&Qm)T0*v85cAyx)7jI{WG8z_xts3WNP3Q>W$eT=et3la_}LnG#sBWJ zI}>eQGV<_Ok~Et@Lm(4$5Ue0r$ew&D*YFUD_aLks%Iddg)@Dl&lV&wSQ0;0Hr4`tX zEk=5Tv|R)fsoyRF&cB_c4uWS0)=-n#B;Cu(0@bjHc>}oyCuw~ItJwBHlF>opqtt3G zN$c35!1&caB(aW^y9oFY9wXgJx(yPXBDj$_=~1%sT=5>4;%(5->S=L$JDQs@mC`b5 zz*EM^^JfIDq+3qF(>RlKn-SQW19S2=kbXVEMuJTQKPK=}GoA#6{dz@~K{`tc0@0uQ z+mCeGC$QCJzu}F`0AI6fZytny`rkNs5WvCK9(n?vW|@aGwY#v5#y|e>TF}0LqRIdE zTOVjrMHGX&7MJX5>kN;-2{qDuw*Q@}87>mz2{urN8xinnY$wG&miF#w-2oCGXA9nS zXx~P0sQ>5hzNaz#hO{pd>}7373yl9qqMu+N!G5;;Xzl8kQM8#Q>RG&Eww?)kL_HmTbbG1;f-6eR9Hb?52^8Fa+ygU{(Wu{8%CyTyWkUK z4cj~0WWv5hF_ZfCYQ&*^;dY-KFx=}WXJH2FgLyfT2yUW!z^4*(m+^usJ;k;JXQp_4 zu+$i7BO3c`7Ir;2fHj=NyLbYhKb2HE2sgByK<@lL|Fd#ogcGweT*;0GQyvJirC;j< z(QtElmxhTWnC5gg*jWDQ$w`mj#8E(TvLUq@OxKfKY`G=#Hnhh_*!eTak3>mx01DZQ zJ!zadpM`poX($!n;}c<818tw-xQhD5b&kE}FtgF04@uC8#XcQAkKv--hcesJo3*-x zq_l{uKvDPvv1pHWry7?r{DrSI(C3>kmfq}vd>OZlY!FuD?3>>MIal`uRCWt#CMPvS z8sj4wg!DF{mUC7ca+dcGO|0lk$Ux>y&VM^HKYdgS+gaMjIk?b|{Wvx27-uH3M1mxO64LQ9{W@#=IDI6q;=Hoo zt|Hs?;iddEc^)N>Q|#4`r<8p{_XM3RuOYeHJmQ7o zqXx%ugCiukSmh^&U?#ir$qvY4zx>-2b4aVsO1~^1d`S=z)M@DWb@@{NX`x!nd#GF>f#nVcG2+ZmcdOi%y2$) za8|95JSr5+rko!?xT~K1^1N$s*C-*m3Ioo*FawfU`-Q^q8~Kq7^TrG|>MJ3=kT`6& z?${7}_Yixp5tCcmona?5mmW2i9XFPB8_U>)za_J~FV3~}n$o({r*#*MJzg-Cg!E}g zO|{2OwcVy#_QFLQYxuiK71H}JUCf8u4aff3k=q42C_;y?>^tF{RSt>!~XqWe4#??b6-`NXp>=6NdcNg zv`?@_N#C$(o-d=>H3i!Kn782U|T&qI4MVyc(Q^R=lJlnvS0m1;H=; zuCIR!ps=hf)w=MUW{a;B#i($BlL&s}-+HB7g$>^SHVfa8OW)cdh7G+s2@bO6tG`Bl z;crGwE5)5EJKYGDJ;_?#*-!2PP$=f+B))BD0h9EJcnoNF?CFQCt#_fp^b)4tGih9v5{e z)e!JGPIEf^GDVS&;LU)X0cy~Z+zf11$Q0O|3CRE#m0d%jB#GWH{x#CPLX<)@uGf|G zLtzAnN@5nw&`u-2fGZ1}8atnm{GsvBN)5H97g@4WzYdb}NvbBOP(IIvkuLrLLE50+ zjmxC7sqMk=+oJWhx4LENbMm?|)1~#~O6yus+uZ8$x@_@5eN(&Jd0&&)6<)t|=n#zZ z;^!Vtbm=8>KcGCC2YJzH7-|rfN>$3fJQ$~4h_@YqukygF*6}FQ6we6A)hQ6K zLn_@~bYEG(-aC?Bx*V6|Rwob2DZ9L~bmJQe%Nu>W1a-WLSK3c~{!zKF5K8m;H0~xw*d8 zg_FuEH?>&hC2rb}MarkOxv90a$m(8febCb}T(q{jmti}PwWD2Zl3iA}Jlt#L6U9ms z#oF1_+Trpd3GM`2mf(Mvb>9Ejt;JSXi{09&RFy$SqTP;s;<&Q%ib`9f9cNUdvat+y zW1Vu8L*4?TRF6T2OEW$2@GEqfbax{Jb>Tk3U)2t3z>kA|PYr%Z)3FpGsCKu>;g5Q% zoOo`~sQkVhYvX;&XXVfmvmE_i#x1V5S6Ng6>Cqak3$~Pm1dY;F0jZgZ86iARxL?L!h>nN`oB;9jA$|#qSGB|KN0RCiB50AXB%{0!FS8jakA2OD77!R2-KS3GY1aDDb)_B;iQJGSeFDHVmmo$Pk=R-@TvS||JtITQ2 kfk|*G(Wi!CAz=VcP<}NT-boz}iy trigger an exception from an llm provider -> assert if output is of the expected type diff --git a/litellm/tests/test_get_model_cost_map.py b/litellm/tests/test_get_model_cost_map.py new file mode 100644 index 000000000..e8d751af8 --- /dev/null +++ b/litellm/tests/test_get_model_cost_map.py @@ -0,0 +1,10 @@ +import sys, os +import traceback + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import time +from litellm import get_max_tokens + +print(get_max_tokens("gpt-3.5-turbo")) \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index 2ca755b94..d611c05ed 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -846,6 +846,13 @@ def get_optional_params( # use the openai defaults return optional_params +def get_max_tokens(model: str): + try: + return litellm.model_cost[model] + except: + raise Exception("This model isn't mapped yet. Add it here - https://raw.githubusercontent.com/BerriAI/litellm/main/cookbook/community-resources/max_tokens.json") + + def load_test_model( model: str, custom_llm_provider: str = "", @@ -1458,6 +1465,7 @@ def exception_type(model, original_exception, custom_llm_provider): raise AuthenticationError( message=f"AnthropicException - {original_exception.message}", llm_provider="anthropic", + model=model ) elif original_exception.status_code == 400: exception_mapping_worked = True @@ -1478,6 +1486,7 @@ def exception_type(model, original_exception, custom_llm_provider): raise RateLimitError( message=f"AnthropicException - {original_exception.message}", llm_provider="anthropic", + model=model ) elif ( "Could not resolve authentication method. Expected either api_key or auth_token to be set." @@ -1487,6 +1496,7 @@ def exception_type(model, original_exception, custom_llm_provider): raise AuthenticationError( message=f"AnthropicException - {original_exception.message}", llm_provider="anthropic", + model=model ) elif "replicate" in model: if "Incorrect authentication token" in error_str: @@ -1494,6 +1504,7 @@ def exception_type(model, original_exception, custom_llm_provider): raise AuthenticationError( message=f"ReplicateException - {error_str}", llm_provider="replicate", + model=model ) elif "input is too long" in error_str: exception_mapping_worked = True @@ -1514,6 +1525,7 @@ def exception_type(model, original_exception, custom_llm_provider): raise RateLimitError( message=f"ReplicateException - {error_str}", llm_provider="replicate", + model=model ) elif ( exception_type == "ReplicateError" @@ -1521,6 +1533,7 @@ def exception_type(model, original_exception, custom_llm_provider): raise ServiceUnavailableError( message=f"ReplicateException - {error_str}", llm_provider="replicate", + model=model ) elif model in litellm.cohere_models: # Cohere if ( @@ -1531,6 +1544,7 @@ def exception_type(model, original_exception, custom_llm_provider): raise AuthenticationError( message=f"CohereException - {original_exception.message}", llm_provider="cohere", + model=model ) elif "too many tokens" in error_str: exception_mapping_worked = True @@ -1546,6 +1560,7 @@ def exception_type(model, original_exception, custom_llm_provider): raise RateLimitError( message=f"CohereException - {original_exception.message}", llm_provider="cohere", + model=model ) elif custom_llm_provider == "huggingface": if "length limit exceeded" in error_str: @@ -1561,6 +1576,7 @@ def exception_type(model, original_exception, custom_llm_provider): raise AuthenticationError( message=f"HuggingfaceException - {original_exception.message}", llm_provider="huggingface", + model=model ) elif original_exception.status_code == 400: exception_mapping_worked = True @@ -1574,6 +1590,7 @@ def exception_type(model, original_exception, custom_llm_provider): raise RateLimitError( message=f"HuggingfaceException - {original_exception.message}", llm_provider="huggingface", + model=model ) elif custom_llm_provider == "ai21": if hasattr(original_exception, "message"): @@ -1590,6 +1607,7 @@ def exception_type(model, original_exception, custom_llm_provider): raise AuthenticationError( message=f"AI21Exception - {original_exception.message}", llm_provider="ai21", + model=model ) if original_exception.status_code == 422: exception_mapping_worked = True @@ -1617,7 +1635,8 @@ def exception_type(model, original_exception, custom_llm_provider): exception_mapping_worked = True raise AuthenticationError( message=f"TogetherAIException - {error_response['error']}", - llm_provider="together_ai" + llm_provider="together_ai", + model=model ) elif "error" in error_response and "INVALID_ARGUMENT" in error_response["error"]: exception_mapping_worked = True @@ -1638,6 +1657,7 @@ def exception_type(model, original_exception, custom_llm_provider): raise RateLimitError( message=f"TogetherAIException - {original_exception.message}", llm_provider="together_ai", + model=model ) raise original_exception # base case - return the original exception else: diff --git a/pyproject.toml b/pyproject.toml index 7a2d0b33b..cf6b93ce8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "0.1.525" +version = "0.1.526" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT License"