From c6d36fb59d908aa4e911b187029f79a8ded85afb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 7 Oct 2023 13:49:19 -0700 Subject: [PATCH] docs(completion-docs): adds more details on provider-specific params --- docs/my-website/docs/completion/input.md | 455 ++++++++++++++++-- .../tutorials/provider_specific_params.md | 34 ++ litellm/__init__.py | 1 + litellm/__pycache__/__init__.cpython-311.pyc | Bin 13075 -> 13216 bytes litellm/__pycache__/main.cpython-311.pyc | Bin 49848 -> 50546 bytes litellm/llms/openai.py | 184 +++++++ litellm/main.py | 20 +- .../tests/test_provider_specific_config.py | 90 +++- 8 files changed, 749 insertions(+), 35 deletions(-) create mode 100644 docs/my-website/docs/tutorials/provider_specific_params.md create mode 100644 litellm/llms/openai.py diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index 2a7bcb6459..8ccd6a256b 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -1,5 +1,426 @@ -# Input Format -The Input params are **exactly the same** as the OpenAI Create chat completion, and let you call 100+ models in the same format. +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Input Params + +## Common Params +LiteLLM accepts and translates the [OpenAI Chat Completion params](https://platform.openai.com/docs/api-reference/chat/create) across all providers. + +### usage +```python +import litellm + +# set env variables +os.environ["OPENAI_API_KEY"] = "your-openai-key" + +## SET MAX TOKENS - via completion() +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[{ "content": "Hello, how are you?","role": "user"}], + max_tokens=10 + ) + +print(response) +``` + +### translated OpenAI params +This is a list of openai params we translate across providers. + +This list is constantly being updated. + +| Provider | temperature | max_tokens | top_p | stream | stop | n | presence_penalty | frequency_penalty | functions | function_call | +|---|---|---|---|---|---|---|---|---|---|---| +|Anthropic| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | +|OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +|Replicate | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | +|Cohere| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | +|Huggingface| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | +|Openrouter| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +|AI21| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | +|VertexAI| ✅ | ✅ | | ✅ | | | | | | | +|Bedrock| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | +|Sagemaker| ✅ | ✅ | | ✅ | | | | | | | +|TogetherAI| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | +|AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | +|Palm| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | +|NLP Cloud| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | +|Petals| ✅ | ✅ | | ✅ | | | | | | | +|Ollama| ✅ | ✅ | ✅ | ✅ | ✅ | | | ✅ | | |n + +:::note + +By default, LiteLLM raises an exception if the openai param being passed in isn't supported. + +To drop the param instead, set `litellm.drop_params = True`. + +::: + +## Provider-specific Params +Providers might offer params not supported by OpenAI (e.g. top_k). You can pass those in 2 ways: +- via completion(): We'll pass the non-openai param, straight to the provider as part of the request body. + - e.g. `completion(model="claude-instant-1", top_k=3)` +- via provider-specific config variable (e.g. `litellm.OpenAIConfig()`). + + + + +```python +import litellm, os + +# set env variables +os.environ["OPENAI_API_KEY"] = "your-openai-key" + +## SET MAX TOKENS - via completion() +response_1 = litellm.completion( + model="gpt-3.5-turbo", + messages=[{ "content": "Hello, how are you?","role": "user"}], + max_tokens=10 + ) + +response_1_text = response_1.choices[0].message.content + +## SET MAX TOKENS - via config +litellm.OpenAIConfig(max_tokens=10) + +response_2 = litellm.completion( + model="gpt-3.5-turbo", + messages=[{ "content": "Hello, how are you?","role": "user"}], + ) + +response_2_text = response_2.choices[0].message.content + +## TEST OUTPUT +assert len(response_2_text) > len(response_1_text) +``` + + + + +```python +import litellm, os + +# set env variables +os.environ["OPENAI_API_KEY"] = "your-openai-key" + + +## SET MAX TOKENS - via completion() +response_1 = litellm.completion( + model="text-davinci-003", + messages=[{ "content": "Hello, how are you?","role": "user"}], + max_tokens=10 + ) + +response_1_text = response_1.choices[0].message.content + +## SET MAX TOKENS - via config +litellm.OpenAITextCompletionConfig(max_tokens=10) +response_2 = litellm.completion( + model="text-davinci-003", + messages=[{ "content": "Hello, how are you?","role": "user"}], + ) + +response_2_text = response_2.choices[0].message.content + +## TEST OUTPUT +assert len(response_2_text) > len(response_1_text) +``` + + + + +```python +import litellm, os + +# set env variables +os.environ["AZURE_API_BASE"] = "your-azure-api-base" +os.environ["AZURE_API_TYPE"] = "azure" # [OPTIONAL] +os.environ["AZURE_API_VERSION"] = "2023-07-01-preview" # [OPTIONAL] + +## SET MAX TOKENS - via completion() +response_1 = litellm.completion( + model="azure/chatgpt-v-2", + messages=[{ "content": "Hello, how are you?","role": "user"}], + max_tokens=10 + ) + +response_1_text = response_1.choices[0].message.content + +## SET MAX TOKENS - via config +litellm.AzureOpenAIConfig(max_tokens=10) +response_2 = litellm.completion( + model="azure/chatgpt-v-2", + messages=[{ "content": "Hello, how are you?","role": "user"}], + ) + +response_2_text = response_2.choices[0].message.content + +## TEST OUTPUT +assert len(response_2_text) > len(response_1_text) +``` + + + + +```python +import litellm, os + +# set env variables +os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key" + +## SET MAX TOKENS - via completion() +response_1 = litellm.completion( + model="claude-instant-1", + messages=[{ "content": "Hello, how are you?","role": "user"}], + max_tokens=10 + ) + +response_1_text = response_1.choices[0].message.content + +## SET MAX TOKENS - via config +litellm.AnthropicConfig(max_tokens_to_sample=200) +response_2 = litellm.completion( + model="claude-instant-1", + messages=[{ "content": "Hello, how are you?","role": "user"}], + ) + +response_2_text = response_2.choices[0].message.content + +## TEST OUTPUT +assert len(response_2_text) > len(response_1_text) +``` + + + + + +```python +import litellm, os + +# set env variables +os.environ["HUGGINGFACE_API_KEY"] = "your-huggingface-key" #[OPTIONAL] + +## SET MAX TOKENS - via completion() +response_1 = litellm.completion( + model="huggingface/mistralai/Mistral-7B-Instruct-v0.1", + messages=[{ "content": "Hello, how are you?","role": "user"}], + api_base="https://your-huggingface-api-endpoint", + max_tokens=10 + ) + +response_1_text = response_1.choices[0].message.content + +## SET MAX TOKENS - via config +litellm.HuggingfaceConfig(max_new_tokens=200) +response_2 = litellm.completion( + model="huggingface/mistralai/Mistral-7B-Instruct-v0.1", + messages=[{ "content": "Hello, how are you?","role": "user"}], + api_base="https://your-huggingface-api-endpoint" + ) + +response_2_text = response_2.choices[0].message.content + +## TEST OUTPUT +assert len(response_2_text) > len(response_1_text) +``` + + + + + + +```python +import litellm, os + +# set env variables +os.environ["TOGETHERAI_API_KEY"] = "your-togetherai-key" + +## SET MAX TOKENS - via completion() +response_1 = litellm.completion( + model="together_ai/togethercomputer/llama-2-70b-chat", + messages=[{ "content": "Hello, how are you?","role": "user"}], + max_tokens=10 + ) + +response_1_text = response_1.choices[0].message.content + +## SET MAX TOKENS - via config +litellm.TogetherAIConfig(max_tokens_to_sample=200) +response_2 = litellm.completion( + model="together_ai/togethercomputer/llama-2-70b-chat", + messages=[{ "content": "Hello, how are you?","role": "user"}], + ) + +response_2_text = response_2.choices[0].message.content + +## TEST OUTPUT +assert len(response_2_text) > len(response_1_text) +``` + + + + + +```python +import litellm, os + +# set env variables +os.environ["REPLICATE_API_KEY"] = "your-replicate-key" + +## SET MAX TOKENS - via completion() +response_1 = litellm.completion( + model="replicate/meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3", + messages=[{ "content": "Hello, how are you?","role": "user"}], + max_tokens=10 + ) + +response_1_text = response_1.choices[0].message.content + +## SET MAX TOKENS - via config +litellm.ReplicateConfig(max_new_tokens=200) +response_2 = litellm.completion( + model="replicate/meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3", + messages=[{ "content": "Hello, how are you?","role": "user"}], + ) + +response_2_text = response_2.choices[0].message.content + +## TEST OUTPUT +assert len(response_2_text) > len(response_1_text) +``` + + + + + + +```python +import litellm + +## SET MAX TOKENS - via completion() +response_1 = litellm.completion( + model="petals/petals-team/StableBeluga2", + messages=[{ "content": "Hello, how are you?","role": "user"}], + api_base="https://chat.petals.dev/api/v1/generate", + max_tokens=10 + ) + +response_1_text = response_1.choices[0].message.content + +## SET MAX TOKENS - via config +litellm.PetalsConfig(max_new_tokens=10) +response_2 = litellm.completion( + model="petals/petals-team/StableBeluga2", + messages=[{ "content": "Hello, how are you?","role": "user"}], + api_base="https://chat.petals.dev/api/v1/generate", + ) + +response_2_text = response_2.choices[0].message.content + +## TEST OUTPUT +assert len(response_2_text) > len(response_1_text) +``` + + + + + +```python +import litellm, os + +# set env variables +os.environ["PALM_API_KEY"] = "your-palm-key" + +## SET MAX TOKENS - via completion() +response_1 = litellm.completion( + model="palm/chat-bison", + messages=[{ "content": "Hello, how are you?","role": "user"}], + max_tokens=10 + ) + +response_1_text = response_1.choices[0].message.content + +## SET MAX TOKENS - via config +litellm.PalmConfig(maxOutputTokens=10) +response_2 = litellm.completion( + model="palm/chat-bison", + messages=[{ "content": "Hello, how are you?","role": "user"}], + ) + +response_2_text = response_2.choices[0].message.content + +## TEST OUTPUT +assert len(response_2_text) > len(response_1_text) +``` + + + + +```python +import litellm, os + +# set env variables +os.environ["AI21_API_KEY"] = "your-ai21-key" + +## SET MAX TOKENS - via completion() +response_1 = litellm.completion( + model="j2-mid", + messages=[{ "content": "Hello, how are you?","role": "user"}], + max_tokens=10 + ) + +response_1_text = response_1.choices[0].message.content + +## SET MAX TOKENS - via config +litellm.AI21Config(maxOutputTokens=10) +response_2 = litellm.completion( + model="j2-mid", + messages=[{ "content": "Hello, how are you?","role": "user"}], + ) + +response_2_text = response_2.choices[0].message.content + +## TEST OUTPUT +assert len(response_2_text) > len(response_1_text) +``` + + + + + +```python +import litellm, os + +# set env variables +os.environ["COHERE_API_KEY"] = "your-cohere-key" + +## SET MAX TOKENS - via completion() +response_1 = litellm.completion( + model="command-nightly", + messages=[{ "content": "Hello, how are you?","role": "user"}], + max_tokens=10 + ) + +response_1_text = response_1.choices[0].message.content + +## SET MAX TOKENS - via config +litellm.CohereConfig(max_tokens=200) +response_2 = litellm.completion( + model="command-nightly", + messages=[{ "content": "Hello, how are you?","role": "user"}], + ) + +response_2_text = response_2.choices[0].message.content + +## TEST OUTPUT +assert len(response_2_text) > len(response_1_text) +``` + + + + + + +[**Check out the tutorial!**](../tutorials/provider_specific_params.md) ## Input - Request Body # Request Body @@ -53,32 +474,4 @@ The Input params are **exactly the same** as the Config`. + +All provider configs are typed and have docstrings, so you should see them autocompleted for you in VSCode with an explanation of what it means. + +Here's an example of setting max tokens through provider configs. + diff --git a/litellm/__init__.py b/litellm/__init__.py index 7b79b69b22..f59a988f3e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -325,6 +325,7 @@ from .llms.vertex_ai import VertexAIConfig from .llms.sagemaker import SagemakerConfig from .llms.ollama import OllamaConfig from .llms.bedrock import AmazonTitanConfig, AmazonAI21Config, AmazonAnthropicConfig, AmazonCohereConfig +from .llms.openai import OpenAIConfig, OpenAITextCompletionConfig, AzureOpenAIConfig from .main import * # type: ignore from .integrations import * from .exceptions import ( diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc index 31a82ea237d6f552ef52e6d9fc1a1497dc869d71..bff4121c2c92b5e53f4e56cc42e3a6e42ff1ff3a 100644 GIT binary patch delta 361 zcmbQ7wjiB%IWI340}w>*QcQh2kyn!O$wu`y?&R{ENx74wCg)Cxnvy#;YAPc`icLy- z&a@B+JsnI}G+fK}a`4*3VL290(r*nQ@T4wq!DF`nlwW7p1Ker$!wInk? z4=N|Qv_40wnBxV-P270Dw4v3fwByO?fCT8Z{ z;>pY_NlhPkyn!O!bbHrZvXV0Ng)g=HYw>jlfiU(&XnA#QB!lL zMNP|{9yL98M%0YlnNc%yXGP7*ogFos5u~;vXHM?isJUQvWzM|Z`BC$MY{p;)&8p1; zJa%G}dzCgZF4*j#tiz(hlAD;BcZ(-8uOv0SD6u3nKd-n*8|aLp*&t#Lh?qNhuDbZ- z!|H9=<3Wz6XT?NlBoW)(wS#^a4>E`$rU3_j7IkLLy&fGL4Ef#v*&1q-;`Dgye z{PX?)%zfcL`s)X2`5~DsnZmzyCrs_P{P>*Qkcv8j?oZRHTE~O&X0Zs+z+AivAYs;CVwK z(!&9r2A(w;QVh|V|2Y_y#v29$hpW%1)jIr*W{RtgTBWF5imeDZqZ?N6D$X05Y7Qq})n3|CbT1_>vIKpSSNuujQn13@o4RcR8HuEUQ7Il%S@6UzV52`_* zlPhrvctm^DbuJvYc$mbr#B*bK%!uDoXAV4)W1Jd~WJ}auXJ(XQ26&2h4YLx{hw;t$ z_DkZi?H-10o?;sLoWo^LiYe+D{2%!RVra*^D8{=O0r#8igLA>g3YC1sW?nKIkpqrZ zWu-M|MJ-VQH}8g{RW1?Ev*$Ky=xN+^nB`W%)hs#qb4^$PbKy{~4pXOdt!8UfaxKL^ zgmIV8tGY{v?cN4*OBTK zs&I1T7RE51G!V^=j1t6JGm&c$CiI9Sa3Q7;@)ai#vQ z5!Im)Hk4>VUz#)?KLB(iCMQP#Od#ANhfhnj@Q)HXXv>n}kWLOil_=6GhDCiU7)q^l zCHFAdhii0rN7dr{qLQz`9tWV>UWkKoyfnU+N&L@_aA-PwS8B?d5taYhOQNOBMptnM z#~#Y6xZ3??nOPgF-4ef=&t^8=j$g+V#ddO!5v4as+h0{!pm*Xl@Ab#B>4wK5_CJB` zY#y@H#tn?Y&f@ay`1c_91l(U<80Wj#UypTe8PYk6J9JZ>1oEDWcvP;g@FEW!t;q6v zq9Nz0xaUPR_f(XRkFuKC8k@tTZSf;WtTako{&emca8;UOuQ;U(r(DJ`6nS7nWy{hd zG0mOxqcr}a!0$2MGU7#5!`HM>T)f1!2goQAwY)`m`}nGK%Gwr+*%_$gYw^p~81V3Q zyoWrbU`LfsQn!6PUx%MiZWMbt{aBTi6^wdBQSpzR{#L+mI;l&zY{36)rq;{dcxm0W zrTTi_%M1S-uySTMzUa)%^lD7=r1HOB?aR6~YV~J;-$XZXFOD7b4REYREjtlaNKu`O z;A{;kney6r2vpV@(PC(=b)!>6Ij#`+$u~+ph8Jqvny$zGuWyKb)^FI@^vt&e>ph=iH{%! z6rQP)M!vD1Az!_Tfu6E4DQbW*QErK9c&de7`(QiDXWp*)8rveT9Qc z)xMc;D4qCk(C+A)+3!80%DWnt2G27GP*k1ug0kqWvgnMmn2@8uFf{pHeR&PiXcvvPU#H+d3Ff8!L?Ur{y~$?~hk-Yn98C#%mP z{eKD~>VK@QuOs7A-r9O$!71$g1cB3-=oOZpYI1tj0;d@@Bn!l})S^gVQ0Ei8IyJ{u zFBVvH-IF8&v7k8FyHHJ^T2Q0(E>zH`7Ahoo9BjO22|`vl)LSAUv0E2>ulJA`iDBE8 zW)uvb-J(P2Pr-lPXGW+9lv|roIkau9l}yG@Ak9?5 zD<{+x=;)guX~EH-BSO~ik!;ySRH)cNtM~?i6=ELEm zPINptc$7sb1*RUGD7gzy{sFcgyA2fuFCGhur`}F9%^~gzr!$jKe<96EJn_j-_{VmT zC9Ql+tq9lp(AsIidk6+_0KO<8VQpa z#Km+~SI6>pTt|;S&qy|7{wqPv@81%sza)w`ktKA``X2TS874sDaUZIN8OJ|I^PuX) zYz)>NC-O=N=ve}1mf(sdM<|3?h!abq!VVC55`vI;bw>}!ZiauJ(4##;$xCt?C4%aY zyL(&HP;!_e?O=*LObbIPm8J|Q2xKCqP=%8OGMQ3o!YKloN~LG-v+Oa3Wx`NSrK=S2#_M~zd5^#W<2u!BZ{%%L(2`h?KtpmA1SBt&o`Zq}*j+@@2vpg51t<96>$Uz!GiZs#oL z`i)V@1Y?#gi$TVPG{Gbog#vcgA+?bIFxrbk4rt8Hb$etEiV_?G#byh6 zV^znL6!kd%M}8rP;OL;R<_x0QfUqnFFBuhC?>E5m9J?VUa8#hC&@*8u$BpK}nVfkZ zvn-7>P2q{sd8f=lQLD_*pI3&m;DPLP5c5*v&2F?^)9mU=vrR%8Hp$2~PMRHPvr~M{ zq_UY5R~vH5=A~fs`1;9J!3klT1-rVrEL$r$w;@&;a^B^HGo?lUz3+^Y@7Sg(9w4GE zbM=AxIPc}ihq#RjYS?Ch!ZX8q$vlL(_u1?z9<$2a*xQXtR>?czmb}?f7v|JQ9X-7O z$l_Svs15c#WrQR4nT$(LzZ4htQfvyChx3h@F>a4NTfY@-Gp+Mt^E6i$-*1v3_J}|U zt|>w%<>BmQ{fC%qW^UVcGZ!B^E_2DTXGGOtpV{(_?wCaEkGhp@J)|6~o>~|Pmkiiy zGhuUnB3&pHLQbty)qoAcK&(0gVf53mlq zCsM)okYh*}I2@N}(YRpcyRqUbUJK2xH2q*9zhqKee=&?Z+Ua6`#q>{5JaK{%r`-~J z0w(SRCGeWlg-`m8vx#=|-P2d#o=|Yy6{{dIaC@ABG6=gYs2q|CJLwXBEm4qtEgzDs zsFZCNO5jW(apdJfyKd5q8G+xyMVHwYR10Qy#jOg$j_WcMUzb=Gxbrk16j}0PGHX?m zET7b>4CB_9j5jp>kHspaaXMU>PDm}bgRW4c;L?W+O$tm{R@|2vt3@wslq;G>m}|5k z4ODKcJz5c5J|9gFoWXI|i!sW_b{(Rl#d(L@ioJEfZMmsb@Z7CnSBE@8DW2I->D8pk zr;(Zf5->m8dq@#0RqoI7ITg6Z%1iQ5H7qN!pc)+7bofC@8(qc+r=LG%6ZzBTv1_7{ z1n=Kpm8dEv%l zjec-$T*=>GR;G(@0#^8<%t@E?w-Z0tNM&TYZZj$u@}~5&1_+eb<1G0}x!oG|{E#~E z6)q+&QcT<;mJ6Br%|uO3q*?`2e9vQ*r0YRWMH3cwOGRuLcUL&IW7j!Rhu3oQi&m7G^kIQJCIl|vAH1Hxi#7RhU%46k_l3IHRQt5fyMjD7q#q7z z&6Kc4R|G0kW}-s1{6kTM5^C9-{v%+AbwTvmoTV%}4u zOUTy`a=}ZpEm@8HwYeGym!(N)6r}(6ut;_vz9Q}HjnexNS28SW^k*)Wb$!cY%L?Zk zopdvQV)`}FjB6q-i5F$5*xAb=sV)Nso9sAyMw()?=eZ^;S^*cE+~^e15P1+?rISNr zb7y;8nc!CS!)vxG6rgaP@1Mx?57;@Za(!pW3%0y;J1v+75ss5j@UhAWM^J^~y%XHY z{AtMc=Ay{AM~9budw54zkr3#7Bx}|pK_e&y2K4Ef@RYZZX%SjRyC8ROo+{L;p@ddQ z@|Dqk!GEoA5BrRa11I=t81va%@JH?yH*!YE$)B5aC)R8hoI=XjkEiCW{8=cbwtSh0 z{wyXgI7Fjo#K$_6YE%zDUH%g4fz2zngJtC=7+qN^e!TKo1!@QHJKHM zv9o6zQf$_V@2-Abp$O}rT>Aimqg;yk&vh4&n0)KGv@L~L!t-SY@8VMG1!j(!%rDzo z(#d?j#Dxsa>vm`PtKs_d}RuUZ)lzdF-F7G6vA&m{93W&SKOzoYX>yWh?C z=?MO(3STan|6-iuB=g@Jy(Th;joE|c)Ru6WBtBfO-RPvl4UUa=`jVZ&@Fk}t{!@L6 zm9T$a>dhqc7g}E)nZI;L#Q&?_*F^Ad>V4AsS+CLSA%wFnj9213+v>zLLh(1YXeD5- zraUq?c>L0#&fb*oQ%a2+86r$^T+9h{?>73p})hw zh$D)|HQ1cNfj|%&MGiJj_px`#r-OqBo0(;p`i6M%;O83CoA}V<8vll#UY6|(tmWP! zM!XFl9c@6Lf%907`7$ApJBoXq3~IcUBkxo0J?J{to-u>qQq4$y(P1=5zI0&plyL@k z3t1(DT72i&DW<3$gX6k2eSyBa0ncW@x!QgROxane%4`hm5B+(1u|W8vJ`e!b)@ z5(#c0k)($HC%oF#1SEqX{``qW3iK{$kGs%YPRVz`Y2UIT}ot}7s6;@-!&%gDJ299bjKq?R7|`swc)%!Ei3j;mI$8|Yiz%iRX+ zPO;2gn0`_`f9j74j26J>r*B5@LHn74N>V2r z$#{-r9Y+EjalYg%;uY=)*-u zh~5zjr$}fjr85pK+A}|FkWdqql6k;5k{nJa=$3E>m2N$-X=EUrDJ`3+4BIF(vL&1) zq1jZ5F>H~*910)MDuH>FDd)iIk@bBB TB$h=vYE&AgB7@j@&Zzhg#{}Q# diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py new file mode 100644 index 0000000000..813dd9dc9c --- /dev/null +++ b/litellm/llms/openai.py @@ -0,0 +1,184 @@ +from typing import Optional, Union +import types + +# This file just has the openai config classes. +# For implementation check out completion() in main.py + +class OpenAIConfig(): + """ + Reference: https://platform.openai.com/docs/api-reference/chat/create + + The class `OpenAIConfig` provides configuration for the OpenAI's Chat API interface. Below are the parameters: + + - `frequency_penalty` (number or null): Defaults to 0. Allows a value between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, thereby minimizing repetition. + + - `function_call` (string or object): This optional parameter controls how the model calls functions. + + - `functions` (array): An optional parameter. It is a list of functions for which the model may generate JSON inputs. + + - `logit_bias` (map): This optional parameter modifies the likelihood of specified tokens appearing in the completion. + + - `max_tokens` (integer or null): This optional parameter helps to set the maximum number of tokens to generate in the chat completion. + + - `n` (integer or null): This optional parameter helps to set how many chat completion choices to generate for each input message. + + - `presence_penalty` (number or null): Defaults to 0. It penalizes new tokens based on if they appear in the text so far, hence increasing the model's likelihood to talk about new topics. + + - `stop` (string / array / null): Specifies up to 4 sequences where the API will stop generating further tokens. + + - `temperature` (number or null): Defines the sampling temperature to use, varying between 0 and 2. + + - `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling. + """ + frequency_penalty: Optional[int]=None + function_call: Optional[Union[str, dict]]=None + functions: Optional[list]=None + logit_bias: Optional[dict]=None + max_tokens: Optional[int]=None + n: Optional[int]=None + presence_penalty: Optional[int]=None + stop: Optional[Union[str, list]]=None + temperature: Optional[int]=None + top_p: Optional[int]=None + + def __init__(self, + frequency_penalty: Optional[int]=None, + function_call: Optional[Union[str, dict]]=None, + functions: Optional[list]=None, + logit_bias: Optional[dict]=None, + max_tokens: Optional[int]=None, + n: Optional[int]=None, + presence_penalty: Optional[int]=None, + stop: Optional[Union[str, list]]=None, + temperature: Optional[int]=None, + top_p: Optional[int]=None,) -> None: + + locals_ = locals() + for key, value in locals_.items(): + if key != 'self' and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return {k: v for k, v in cls.__dict__.items() + if not k.startswith('__') + and not isinstance(v, (types.FunctionType, types.BuiltinFunctionType, classmethod, staticmethod)) + and v is not None} + +class OpenAITextCompletionConfig(): + """ + Reference: https://platform.openai.com/docs/api-reference/completions/create + + The class `OpenAITextCompletionConfig` provides configuration for the OpenAI's text completion API interface. Below are the parameters: + + - `best_of` (integer or null): This optional parameter generates server-side completions and returns the one with the highest log probability per token. + + - `echo` (boolean or null): This optional parameter will echo back the prompt in addition to the completion. + + - `frequency_penalty` (number or null): Defaults to 0. It is a numbers from -2.0 to 2.0, where positive values decrease the model's likelihood to repeat the same line. + + - `logit_bias` (map): This optional parameter modifies the likelihood of specified tokens appearing in the completion. + + - `logprobs` (integer or null): This optional parameter includes the log probabilities on the most likely tokens as well as the chosen tokens. + + - `max_tokens` (integer or null): This optional parameter sets the maximum number of tokens to generate in the completion. + + - `n` (integer or null): This optional parameter sets how many completions to generate for each prompt. + + - `presence_penalty` (number or null): Defaults to 0 and can be between -2.0 and 2.0. Positive values increase the model's likelihood to talk about new topics. + + - `stop` (string / array / null): Specifies up to 4 sequences where the API will stop generating further tokens. + + - `suffix` (string or null): Defines the suffix that comes after a completion of inserted text. + + - `temperature` (number or null): This optional parameter defines the sampling temperature to use. + + - `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling. + """ + best_of: Optional[int]=None + echo: Optional[bool]=None + frequency_penalty: Optional[int]=None + logit_bias: Optional[dict]=None + logprobs: Optional[int]=None + max_tokens: Optional[int]=None + n: Optional[int]=None + presence_penalty: Optional[int]=None + stop: Optional[Union[str, list]]=None + suffix: Optional[str]=None + temperature: Optional[float]=None + top_p: Optional[float]=None + + def __init__(self, + best_of: Optional[int]=None, + echo: Optional[bool]=None, + frequency_penalty: Optional[int]=None, + logit_bias: Optional[dict]=None, + logprobs: Optional[int]=None, + max_tokens: Optional[int]=None, + n: Optional[int]=None, + presence_penalty: Optional[int]=None, + stop: Optional[Union[str, list]]=None, + suffix: Optional[str]=None, + temperature: Optional[float]=None, + top_p: Optional[float]=None) -> None: + locals_ = locals() + for key, value in locals_.items(): + if key != 'self' and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return {k: v for k, v in cls.__dict__.items() + if not k.startswith('__') + and not isinstance(v, (types.FunctionType, types.BuiltinFunctionType, classmethod, staticmethod)) + and v is not None} + + +class AzureOpenAIConfig(OpenAIConfig): + """ + Reference: https://platform.openai.com/docs/api-reference/chat/create + + The class `AzureOpenAIConfig` provides configuration for the OpenAI's Chat API interface, for use with Azure. It inherits from `OpenAIConfig`. Below are the parameters:: + + - `frequency_penalty` (number or null): Defaults to 0. Allows a value between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, thereby minimizing repetition. + + - `function_call` (string or object): This optional parameter controls how the model calls functions. + + - `functions` (array): An optional parameter. It is a list of functions for which the model may generate JSON inputs. + + - `logit_bias` (map): This optional parameter modifies the likelihood of specified tokens appearing in the completion. + + - `max_tokens` (integer or null): This optional parameter helps to set the maximum number of tokens to generate in the chat completion. + + - `n` (integer or null): This optional parameter helps to set how many chat completion choices to generate for each input message. + + - `presence_penalty` (number or null): Defaults to 0. It penalizes new tokens based on if they appear in the text so far, hence increasing the model's likelihood to talk about new topics. + + - `stop` (string / array / null): Specifies up to 4 sequences where the API will stop generating further tokens. + + - `temperature` (number or null): Defines the sampling temperature to use, varying between 0 and 2. + + - `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling. + """ + + def __init__(self, + frequency_penalty: int | None = None, + function_call: str | dict | None = None, + functions: list | None = None, + logit_bias: dict | None = None, + max_tokens: int | None = None, + n: int | None = None, + presence_penalty: int | None = None, + stop: str | list | None = None, + temperature: int | None = None, + top_p: int | None = None) -> None: + super().__init__(frequency_penalty, + function_call, + functions, + logit_bias, + max_tokens, + n, + presence_penalty, + stop, + temperature, + top_p) \ No newline at end of file diff --git a/litellm/main.py b/litellm/main.py index c84e0cb975..759d45c17a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -66,7 +66,6 @@ from litellm.utils import ( ####### ENVIRONMENT VARIABLES ################### dotenv.load_dotenv() # Loading env variables using dotenv - ####### COMPLETION ENDPOINTS ################ async def acompletion(*args, **kwargs): @@ -310,6 +309,12 @@ def completion( get_secret("AZURE_API_KEY") ) + ## LOAD CONFIG - if set + config=litellm.AzureOpenAIConfig.get_config() + for k, v in config.items(): + if k not in optional_params: # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + ## LOGGING logging.pre_call( input=messages, @@ -368,6 +373,13 @@ def completion( litellm.openai_key or get_secret("OPENAI_API_KEY") ) + + ## LOAD CONFIG - if set + config=litellm.OpenAIConfig.get_config() + for k, v in config.items(): + if k not in optional_params: # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + ## LOGGING logging.pre_call( input=messages, @@ -436,6 +448,12 @@ def completion( get_secret("OPENAI_API_KEY") ) + ## LOAD CONFIG - if set + config=litellm.OpenAITextCompletionConfig.get_config() + for k, v in config.items(): + if k not in optional_params: # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + if litellm.organization: openai.organization = litellm.organization diff --git a/litellm/tests/test_provider_specific_config.py b/litellm/tests/test_provider_specific_config.py index de3f428707..bc4bef506c 100644 --- a/litellm/tests/test_provider_specific_config.py +++ b/litellm/tests/test_provider_specific_config.py @@ -50,7 +50,7 @@ def claude_test_completion(): try: # OVERRIDE WITH DYNAMIC MAX TOKENS response_1 = litellm.completion( - model="claude-instant-1", + model="together_ai/togethercomputer/llama-2-70b-chat", messages=[{ "content": "Hello, how are you?","role": "user"}], max_tokens=10 ) @@ -60,7 +60,7 @@ def claude_test_completion(): # USE CONFIG TOKENS response_2 = litellm.completion( - model="claude-instant-1", + model="together_ai/togethercomputer/llama-2-70b-chat", messages=[{ "content": "Hello, how are you?","role": "user"}], ) # Add any assertions here to check the response @@ -392,4 +392,88 @@ def bedrock_test_completion(): except Exception as e: pytest.fail(f"Error occurred: {e}") -# bedrock_test_completion() \ No newline at end of file +# bedrock_test_completion() + +# OpenAI Chat Completion +def openai_test_completion(): + litellm.OpenAIConfig(max_tokens=10) + # litellm.set_verbose=True + try: + # OVERRIDE WITH DYNAMIC MAX TOKENS + response_1 = litellm.completion( + model="gpt-3.5-turbo", + messages=[{ "content": "Hello, how are you? Be as verbose as possible","role": "user"}], + max_tokens=100 + ) + response_1_text = response_1.choices[0].message.content + print(f"response_1_text: {response_1_text}") + + # USE CONFIG TOKENS + response_2 = litellm.completion( + model="gpt-3.5-turbo", + messages=[{ "content": "Hello, how are you? Be as verbose as possible","role": "user"}], + ) + response_2_text = response_2.choices[0].message.content + print(f"response_2_text: {response_2_text}") + + assert len(response_2_text) < len(response_1_text) + except Exception as e: + pytest.fail(f"Error occurred: {e}") + +# openai_test_completion() + +# OpenAI Text Completion +def openai_text_completion_test(): + litellm.OpenAITextCompletionConfig(max_tokens=10) + # litellm.set_verbose=True + try: + # OVERRIDE WITH DYNAMIC MAX TOKENS + response_1 = litellm.completion( + model="text-davinci-003", + messages=[{ "content": "Hello, how are you? Be as verbose as possible","role": "user"}], + max_tokens=100 + ) + response_1_text = response_1.choices[0].message.content + print(f"response_1_text: {response_1_text}") + + # USE CONFIG TOKENS + response_2 = litellm.completion( + model="text-davinci-003", + messages=[{ "content": "Hello, how are you? Be as verbose as possible","role": "user"}], + ) + response_2_text = response_2.choices[0].message.content + print(f"response_2_text: {response_2_text}") + + assert len(response_2_text) < len(response_1_text) + except Exception as e: + pytest.fail(f"Error occurred: {e}") + +# openai_text_completion_test() + +# Azure OpenAI +def azure_openai_test_completion(): + litellm.AzureOpenAIConfig(max_tokens=10) + # litellm.set_verbose=True + try: + # OVERRIDE WITH DYNAMIC MAX TOKENS + response_1 = litellm.completion( + model="azure/chatgpt-v-2", + messages=[{ "content": "Hello, how are you? Be as verbose as possible","role": "user"}], + max_tokens=100 + ) + response_1_text = response_1.choices[0].message.content + print(f"response_1_text: {response_1_text}") + + # USE CONFIG TOKENS + response_2 = litellm.completion( + model="azure/chatgpt-v-2", + messages=[{ "content": "Hello, how are you? Be as verbose as possible","role": "user"}], + ) + response_2_text = response_2.choices[0].message.content + print(f"response_2_text: {response_2_text}") + + assert len(response_2_text) < len(response_1_text) + except Exception as e: + pytest.fail(f"Error occurred: {e}") + +# azure_openai_test_completion() \ No newline at end of file