phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

new v bump

Browse source
This commit is contained in:
ishaan-jaff 2023-08-14 10:57:14 -07:00
parent 1b4aadbb25
commit 828f4e7bf8
4 changed files with 175 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

124
cookbook/liteLLM_VertextAI_Example.ipynb
View file

@ -6,16 +6,45 @@
"metadata": {}, "metadata": {},
"source": [ "source": [
"## Using Google Palm (VertexAI) with liteLLM \n", "## Using Google Palm (VertexAI) with liteLLM \n",
"### chat-bison & chat-bison@001" "### chat-bison, chat-bison@001, text-bison, text-bison@001"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 1,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Defaulting to user installation because normal site-packages is not writeable\n",
"Requirement already satisfied: litellm==0.1.387 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (0.1.387)\n",
"Requirement already satisfied: openai<0.28.0,>=0.27.8 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from litellm==0.1.387) (0.27.8)\n",
"Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from litellm==0.1.387) (1.0.0)\n",
"Requirement already satisfied: tiktoken<0.5.0,>=0.4.0 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from litellm==0.1.387) (0.4.0)\n",
"Requirement already satisfied: requests>=2.20 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.387) (2.28.2)\n",
"Requirement already satisfied: tqdm in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.387) (4.65.0)\n",
"Requirement already satisfied: aiohttp in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.387) (3.8.4)\n",
"Requirement already satisfied: regex>=2022.1.18 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from tiktoken<0.5.0,>=0.4.0->litellm==0.1.387) (2023.6.3)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.387) (3.1.0)\n",
"Requirement already satisfied: idna<4,>=2.5 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.387) (3.4)\n",
"Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.387) (1.26.6)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.387) (2023.5.7)\n",
"Requirement already satisfied: attrs>=17.3.0 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.387) (23.1.0)\n",
"Requirement already satisfied: multidict<7.0,>=4.5 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.387) (6.0.4)\n",
"Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.387) (4.0.2)\n",
"Requirement already satisfied: yarl<2.0,>=1.0 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.387) (1.9.2)\n",
"Requirement already satisfied: frozenlist>=1.1.1 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.387) (1.3.3)\n",
"Requirement already satisfied: aiosignal>=1.1.2 in /Users/ishaanjaffer/Library/Python/3.9/lib/python/site-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.387) (1.3.1)\n",
"\n",
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n",
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49m/Library/Developer/CommandLineTools/usr/bin/python3 -m pip install --upgrade pip\u001b[0m\n"
]
}
],
"source": [ "source": [
"!pip install litellm==0.1.379" "!pip install litellm==0.1.387"
] ]
}, },
{ {
@ -31,12 +60,13 @@
"\n", "\n",
"VertexAI uses Application Default Credentials, see https://cloud.google.com/docs/authentication/external/set-up-adc for more information on setting this up\n", "VertexAI uses Application Default Credentials, see https://cloud.google.com/docs/authentication/external/set-up-adc for more information on setting this up\n",
"\n", "\n",
"NOTE: VertexAI requires you to set `application_default_credentials.json` \n" "NOTE: VertexAI requires you to set `application_default_credentials.json`, this can be set by running `gcloud auth application-default login` in your terminal\n",
"\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 14, "execution_count": 2,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -53,14 +83,22 @@
"cell_type": "markdown", "cell_type": "markdown",
"metadata": {}, "metadata": {},
"source": [ "source": [
"## Call VertexAI using liteLLM" "## Call VertexAI - chat-bison using liteLLM"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': LiteLLM LiteLLM is a large language model from Google AI that is designed to be lightweight and efficient. It is based on the Transformer architecture and has been trained on a massive dataset of text. LiteLLM is available as a pre-trained model that can be used for a variety of natural language processing tasks, such as text classification, question answering, and summarization.}}], 'created': 1692035587.573185, 'model': 'chat-bison'}\n"
]
}
],
"source": [ "source": [
"user_message = \"what is liteLLM \"\n", "user_message = \"what is liteLLM \"\n",
"messages = [{ \"content\": user_message,\"role\": \"user\"}]\n", "messages = [{ \"content\": user_message,\"role\": \"user\"}]\n",
@ -69,6 +107,74 @@
"response = completion(model=\"chat-bison\", messages=messages)\n", "response = completion(model=\"chat-bison\", messages=messages)\n",
"print(response)" "print(response)"
] ]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"## Call VertexAI - text-bison using liteLLM"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['claude-2', 'claude-instant-1', 'claude-instant-1.2']\n"
]
}
],
"source": [
"print(litellm.anthropic_models)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"ename": "ValueError",
"evalue": "Invalid completion model args passed in. Check your input - {'model': 'text-bison@001', 'messages': [{'content': 'what is liteLLM ', 'role': 'user'}], 'functions': [], 'function_call': '', 'temperature': 1, 'top_p': 1, 'n': 1, 'stream': False, 'stop': None, 'max_tokens': inf, 'presence_penalty': 0, 'frequency_penalty': 0, 'logit_bias': {}, 'user': '', 'deployment_id': None, 'return_async': False, 'api_key': None, 'force_timeout': 600, 'logger_fn': None, 'verbose': False, 'azure': False, 'custom_llm_provider': None, 'custom_api_base': None, 'args': {...}, 'model_response': {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant'}}]}, 'optional_params': {}, 'litellm_params': {'return_async': False, 'api_key': None, 'force_timeout': 600, 'logger_fn': None, 'verbose': False, 'custom_llm_provider': None, 'custom_api_base': None}}",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[4], line 5\u001b[0m\n\u001b[1;32m 2\u001b[0m messages \u001b[39m=\u001b[39m [{ \u001b[39m\"\u001b[39m\u001b[39mcontent\u001b[39m\u001b[39m\"\u001b[39m: user_message,\u001b[39m\"\u001b[39m\u001b[39mrole\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m\"\u001b[39m\u001b[39muser\u001b[39m\u001b[39m\"\u001b[39m}]\n\u001b[1;32m 4\u001b[0m \u001b[39m# text-bison or text-bison@001 supported by Vertex AI (As of Aug 2023)\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m response \u001b[39m=\u001b[39m completion(model\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mtext-bison@001\u001b[39;49m\u001b[39m\"\u001b[39;49m, messages\u001b[39m=\u001b[39;49mmessages)\n\u001b[1;32m 6\u001b[0m \u001b[39mprint\u001b[39m(response)\n",
"File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/litellm/utils.py:158\u001b[0m, in \u001b[0;36mclient.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 156\u001b[0m my_thread \u001b[39m=\u001b[39m threading\u001b[39m.\u001b[39mThread(target\u001b[39m=\u001b[39mhandle_failure, args\u001b[39m=\u001b[39m(e, traceback_exception, start_time, end_time, args, kwargs)) \u001b[39m# don't interrupt execution of main thread\u001b[39;00m\n\u001b[1;32m 157\u001b[0m my_thread\u001b[39m.\u001b[39mstart()\n\u001b[0;32m--> 158\u001b[0m \u001b[39mraise\u001b[39;00m e\n",
"File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/litellm/utils.py:145\u001b[0m, in \u001b[0;36mclient.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[39m## MODEL CALL\u001b[39;00m\n\u001b[1;32m 144\u001b[0m start_time \u001b[39m=\u001b[39m datetime\u001b[39m.\u001b[39mdatetime\u001b[39m.\u001b[39mnow()\n\u001b[0;32m--> 145\u001b[0m result \u001b[39m=\u001b[39m original_function(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 146\u001b[0m end_time \u001b[39m=\u001b[39m datetime\u001b[39m.\u001b[39mdatetime\u001b[39m.\u001b[39mnow()\n\u001b[1;32m 147\u001b[0m \u001b[39m## LOG SUCCESS \u001b[39;00m\n",
"File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/litellm/timeout.py:44\u001b[0m, in \u001b[0;36mtimeout.<locals>.decorator.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 42\u001b[0m local_timeout_duration \u001b[39m=\u001b[39m kwargs[\u001b[39m\"\u001b[39m\u001b[39mforce_timeout\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[1;32m 43\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m---> 44\u001b[0m result \u001b[39m=\u001b[39m future\u001b[39m.\u001b[39;49mresult(timeout\u001b[39m=\u001b[39;49mlocal_timeout_duration)\n\u001b[1;32m 45\u001b[0m \u001b[39mexcept\u001b[39;00m futures\u001b[39m.\u001b[39mTimeoutError:\n\u001b[1;32m 46\u001b[0m thread\u001b[39m.\u001b[39mstop_loop()\n",
"File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:445\u001b[0m, in \u001b[0;36mFuture.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 443\u001b[0m \u001b[39mraise\u001b[39;00m CancelledError()\n\u001b[1;32m 444\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_state \u001b[39m==\u001b[39m FINISHED:\n\u001b[0;32m--> 445\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m__get_result()\n\u001b[1;32m 446\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 447\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mTimeoutError\u001b[39;00m()\n",
"File \u001b[0;32m/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/concurrent/futures/_base.py:390\u001b[0m, in \u001b[0;36mFuture.__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 388\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_exception:\n\u001b[1;32m 389\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 390\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_exception\n\u001b[1;32m 391\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 392\u001b[0m \u001b[39m# Break a reference cycle with the exception in self._exception\u001b[39;00m\n\u001b[1;32m 393\u001b[0m \u001b[39mself\u001b[39m \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n",
"File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/litellm/timeout.py:35\u001b[0m, in \u001b[0;36mtimeout.<locals>.decorator.<locals>.wrapper.<locals>.async_func\u001b[0;34m()\u001b[0m\n\u001b[1;32m 34\u001b[0m \u001b[39masync\u001b[39;00m \u001b[39mdef\u001b[39;00m \u001b[39masync_func\u001b[39m():\n\u001b[0;32m---> 35\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
"File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/litellm/main.py:414\u001b[0m, in \u001b[0;36mcompletion\u001b[0;34m(model, messages, functions, function_call, temperature, top_p, n, stream, stop, max_tokens, presence_penalty, frequency_penalty, logit_bias, user, deployment_id, return_async, api_key, force_timeout, logger_fn, verbose, azure, custom_llm_provider, custom_api_base)\u001b[0m\n\u001b[1;32m 412\u001b[0m logging(model\u001b[39m=\u001b[39mmodel, \u001b[39minput\u001b[39m\u001b[39m=\u001b[39mmessages, custom_llm_provider\u001b[39m=\u001b[39mcustom_llm_provider, additional_args\u001b[39m=\u001b[39m{\u001b[39m\"\u001b[39m\u001b[39mmax_tokens\u001b[39m\u001b[39m\"\u001b[39m: max_tokens}, logger_fn\u001b[39m=\u001b[39mlogger_fn, exception\u001b[39m=\u001b[39me)\n\u001b[1;32m 413\u001b[0m \u001b[39m## Map to OpenAI Exception\u001b[39;00m\n\u001b[0;32m--> 414\u001b[0m \u001b[39mraise\u001b[39;00m exception_type(model\u001b[39m=\u001b[39;49mmodel, original_exception\u001b[39m=\u001b[39;49me)\n",
"File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/litellm/utils.py:644\u001b[0m, in \u001b[0;36mexception_type\u001b[0;34m(model, original_exception)\u001b[0m\n\u001b[1;32m 642\u001b[0m \u001b[39mraise\u001b[39;00m e\n\u001b[1;32m 643\u001b[0m \u001b[39melse\u001b[39;00m: \u001b[39m# don't let an error with mapping interrupt the user from receiving an error from the llm api calls \u001b[39;00m\n\u001b[0;32m--> 644\u001b[0m \u001b[39mraise\u001b[39;00m original_exception\n",
"File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/litellm/utils.py:635\u001b[0m, in \u001b[0;36mexception_type\u001b[0;34m(model, original_exception)\u001b[0m\n\u001b[1;32m 633\u001b[0m exception_mapping_worked \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[1;32m 634\u001b[0m \u001b[39mraise\u001b[39;00m RateLimitError(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mCohereException - \u001b[39m\u001b[39m{\u001b[39;00moriginal_exception\u001b[39m.\u001b[39mmessage\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 635\u001b[0m \u001b[39mraise\u001b[39;00m original_exception \u001b[39m# base case - return the original exception\u001b[39;00m\n\u001b[1;32m 636\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 637\u001b[0m \u001b[39mraise\u001b[39;00m original_exception\n",
"File \u001b[0;32m~/Library/Python/3.9/lib/python/site-packages/litellm/main.py:408\u001b[0m, in \u001b[0;36mcompletion\u001b[0;34m(model, messages, functions, function_call, temperature, top_p, n, stream, stop, max_tokens, presence_penalty, frequency_penalty, logit_bias, user, deployment_id, return_async, api_key, force_timeout, logger_fn, verbose, azure, custom_llm_provider, custom_api_base)\u001b[0m\n\u001b[1;32m 406\u001b[0m logging(model\u001b[39m=\u001b[39mmodel, \u001b[39minput\u001b[39m\u001b[39m=\u001b[39mmessages, custom_llm_provider\u001b[39m=\u001b[39mcustom_llm_provider, logger_fn\u001b[39m=\u001b[39mlogger_fn)\n\u001b[1;32m 407\u001b[0m args \u001b[39m=\u001b[39m \u001b[39mlocals\u001b[39m()\n\u001b[0;32m--> 408\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mInvalid completion model args passed in. Check your input - \u001b[39m\u001b[39m{\u001b[39;00margs\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 409\u001b[0m \u001b[39mreturn\u001b[39;00m response\n\u001b[1;32m 410\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 411\u001b[0m \u001b[39m## LOGGING\u001b[39;00m\n",
"\u001b[0;31mValueError\u001b[0m: Invalid completion model args passed in. Check your input - {'model': 'text-bison@001', 'messages': [{'content': 'what is liteLLM ', 'role': 'user'}], 'functions': [], 'function_call': '', 'temperature': 1, 'top_p': 1, 'n': 1, 'stream': False, 'stop': None, 'max_tokens': inf, 'presence_penalty': 0, 'frequency_penalty': 0, 'logit_bias': {}, 'user': '', 'deployment_id': None, 'return_async': False, 'api_key': None, 'force_timeout': 600, 'logger_fn': None, 'verbose': False, 'azure': False, 'custom_llm_provider': None, 'custom_api_base': None, 'args': {...}, 'model_response': {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant'}}]}, 'optional_params': {}, 'litellm_params': {'return_async': False, 'api_key': None, 'force_timeout': 600, 'logger_fn': None, 'verbose': False, 'custom_llm_provider': None, 'custom_api_base': None}}"
]
}
],
"source": [
"user_message = \"what is liteLLM \"\n",
"messages = [{ \"content\": user_message,\"role\": \"user\"}]\n",
"\n",
"# text-bison or text-bison@001 supported by Vertex AI (As of Aug 2023)\n",
"response = completion(model=\"text-bison@001\", messages=messages)\n",
"print(response)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
} }
], ],
"metadata": { "metadata": {

29
docs/my-website/docs/completion/supported.md
View file

@ -22,12 +22,27 @@
|------------------|--------------------------------------------|--------------------------------------| |------------------|--------------------------------------------|--------------------------------------|
| text-davinci-003 | `completion('text-davinci-003', messages)` | `os.environ['OPENAI_API_KEY']` | | text-davinci-003 | `completion('text-davinci-003', messages)` | `os.environ['OPENAI_API_KEY']` |
### Cohere Models
| Model Name | Function Call | Required OS Variables | ### Google VertexAI Models
|------------------|--------------------------------------------|--------------------------------------| Sample notebook for calling VertexAI models: https://github.com/BerriAI/litellm/blob/main/cookbook/liteLLM_VertextAI_Example.ipynb
| command-nightly | `completion('command-nightly', messages)` | `os.environ['COHERE_API_KEY']` |
All calls using Vertex AI require the following parameters:
* Your Project ID
`litellm.vertex_project` = "hardy-device-38811" Your Project ID
* Your Project Location
`litellm.vertex_location` = "us-central1"
Authentication:
VertexAI uses Application Default Credentials, see https://cloud.google.com/docs/authentication/external/set-up-adc for more information on setting this up
VertexAI requires you to set `application_default_credentials.json`, this can be set by running `gcloud auth application-default login` in your terminal
| Model Name | Function Call |
|------------------|----------------------------------------------------------|
| chat-bison | `completion('chat-bison', messages)` |
| chat-bison@001 | `completion('chat-bison@001', messages)` |
| text-bison | `completion('text-bison', messages)` |
| text-bison@001 | `completion('text-bison@001', messages)` |
### Anthropic Models ### Anthropic Models
@ -54,6 +69,12 @@ Here are some examples of supported models:
| [google/flan-t5-xxl](https://huggingface.co/google/flan-t5-xxl) | `completion(model="google/flan-t5-xxl", messages=messages, custom_llm_provider="huggingface")` | `os.environ['HF_TOKEN']` | | [google/flan-t5-xxl](https://huggingface.co/google/flan-t5-xxl) | `completion(model="google/flan-t5-xxl", messages=messages, custom_llm_provider="huggingface")` | `os.environ['HF_TOKEN']` |
| [google/flan-t5-large](https://huggingface.co/google/flan-t5-large) | `completion(model="google/flan-t5-large", messages=messages, custom_llm_provider="huggingface")` | `os.environ['HF_TOKEN']` | | [google/flan-t5-large](https://huggingface.co/google/flan-t5-large) | `completion(model="google/flan-t5-large", messages=messages, custom_llm_provider="huggingface")` | `os.environ['HF_TOKEN']` |
### Cohere Models
| Model Name | Function Call | Required OS Variables |
|------------------|--------------------------------------------|--------------------------------------|
| command-nightly | `completion('command-nightly', messages)` | `os.environ['COHERE_API_KEY']` |
### OpenRouter Completion Models ### OpenRouter Completion Models
All the text models from [OpenRouter](https://openrouter.ai/docs) are supported by liteLLM. All the text models from [OpenRouter](https://openrouter.ai/docs) are supported by liteLLM.

65
litellm/tests/test_vertex.py
View file

@ -1,49 +1,52 @@
import sys, os # import sys, os
import traceback # import traceback
from dotenv import load_dotenv # from dotenv import load_dotenv
load_dotenv() # load_dotenv()
import os # import os
sys.path.insert(0, os.path.abspath('../..')) # Adds the parent directory to the system path # sys.path.insert(0, os.path.abspath('../..')) # Adds the parent directory to the system path
import pytest # import pytest
import litellm # import litellm
from litellm import embedding, completion # from litellm import embedding, completion
litellm.vertex_project = "hardy-device-386718" # litellm.vertex_project = "hardy-device-386718"
litellm.vertex_location = "us-central1" # litellm.vertex_location = "us-central1"
litellm.set_verbose = True # litellm.set_verbose = True
user_message = "what's the weather in SF "
messages = [{ "content": user_message,"role": "user"}]
# user_message = "what's the weather in SF "
# messages = [{ "content": user_message,"role": "user"}]
# def logger_fn(user_model_dict): # def logger_fn(user_model_dict):
# print(f"user_model_dict: {user_model_dict}") # print(f"user_model_dict: {user_model_dict}")
# chat-bison
# response = completion(model="chat-bison", messages=messages, temperature=0.5, top_p=0.1)
# print(response)
# text-bison # # chat-bison
# # response = completion(model="chat-bison", messages=messages, temperature=0.5, top_p=0.1)
# # print(response)
# response = completion(model="text-bison@001", messages=messages) # #text-bison
# response = completion(model="text-bison", messages=messages)
# print(response) # print(response)
# response = completion(model="text-bison@001", messages=messages, temperature=0.1, logger_fn=logger_fn) # response = completion(model="text-bison@001", messages=messages, temperature=0.1, logger_fn=logger_fn)
# print(response) # print(response)
# response = completion(model="text-bison@001", messages=messages, temperature=0.4, top_p=0.1, logger_fn=logger_fn) # response = completion(model="text-bison", messages=messages, temperature=0.4, top_p=0.1, logger_fn=logger_fn)
# print(response)
# response = completion(model="text-bison", messages=messages, temperature=0.8, top_p=0.4, top_k=30, logger_fn=logger_fn)
# print(response) # print(response)
# response = completion(model="text-bison@001", messages=messages, temperature=0.8, top_p=0.4, top_k=30, logger_fn=logger_fn) # response = completion(model="text-bison@001", messages=messages, temperature=0.8, top_p=0.4, top_k=30, logger_fn=logger_fn)
# print(response) # print(response)
# chat_model = ChatModel.from_pretrained("chat-bison@001") # # chat_model = ChatModel.from_pretrained("chat-bison@001")
# parameters = { # # parameters = {
# "temperature": 0.2, # # "temperature": 0.2,
# "max_output_tokens": 256, # # "max_output_tokens": 256,
# "top_p": 0.8, # # "top_p": 0.8,
# "top_k": 40 # # "top_k": 40
# } # # }
# chat = chat_model.start_chat() # # chat = chat_model.start_chat()
# response = chat.send_message("who are u? write a sentence", **parameters) # # response = chat.send_message("who are u? write a sentence", **parameters)
# print(f"Response from Model: {response.text}") # # print(f"Response from Model: {response.text}")

2
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "litellm" name = "litellm"
version = "0.1.387" version = "0.1.388"
description = "Library to easily interface with LLM API providers" description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"] authors = ["BerriAI"]
license = "MIT License" license = "MIT License"