mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-24 18:24:20 +00:00
baseten docs
This commit is contained in:
ishaan-jaff
parent
a5aa6228e4
commit
b49583893c
2 changed files with 253 additions and 6 deletions
238
cookbook/liteLLM_Baseten.ipynb
Normal file
238
cookbook/liteLLM_Baseten.ipynb
Normal file
|
|
@ -0,0 +1,238 @@
|
||||||
|
{
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 0,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"provenance": []
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"name": "python3",
|
||||||
|
"display_name": "Python 3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"name": "python"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"source": [
|
||||||
|
"# Use liteLLM to call Falcon, Wizard, MPT 7B using OpenAI chatGPT Input/output\n",
|
||||||
|
"\n",
|
||||||
|
"* Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
|
||||||
|
"* Wizard LM: https://app.baseten.co/explore/wizardlm\n",
|
||||||
|
"* MPT 7B Base: https://app.baseten.co/explore/mpt_7b_instruct\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"## Call all baseten llm models using OpenAI chatGPT Input/Output using liteLLM\n",
|
||||||
|
"Example call\n",
|
||||||
|
"```python\n",
|
||||||
|
"model = \"q841o8w\" # baseten model version ID\n",
|
||||||
|
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
|
||||||
|
"```"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"id": "gZx-wHJapG5w"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"id": "4JSRa0QVogPo"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"!pip install litellm==0.1.399\n",
|
||||||
|
"!pip install baseten urllib3"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"import litellm\n",
|
||||||
|
"from litellm import completion"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"id": "VEukLhDzo4vw"
|
||||||
|
},
|
||||||
|
"execution_count": 2,
|
||||||
|
"outputs": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"source": [
|
||||||
|
"## Setup"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"id": "4STYM2OHFNlc"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"source": [
|
||||||
|
"os.environ['BASETEN_API_KEY'] = \"\" #@param\n",
|
||||||
|
"messages = [{ \"content\": \"what does Baseten do? \",\"role\": \"user\"}]"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"id": "DorpLxw1FHbC"
|
||||||
|
},
|
||||||
|
"execution_count": 21,
|
||||||
|
"outputs": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"source": [
|
||||||
|
"## Calling Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
|
||||||
|
"### Pass Your Baseten model `Version ID` as `model`"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"id": "syF3dTdKFSQQ"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"source": [
|
||||||
|
"model = \"qvv0xeq\"\n",
|
||||||
|
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
|
||||||
|
"response"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"base_uri": "https://localhost:8080/"
|
||||||
|
},
|
||||||
|
"id": "rPgSoMlsojz0",
|
||||||
|
"outputId": "81d6dc7b-1681-4ae4-e4c8-5684eb1bd050"
|
||||||
|
},
|
||||||
|
"execution_count": 18,
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"output_type": "stream",
|
||||||
|
"name": "stderr",
|
||||||
|
"text": [
|
||||||
|
"\u001b[32mINFO\u001b[0m API key set.\n",
|
||||||
|
"INFO:baseten:API key set.\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"output_type": "execute_result",
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"{'choices': [{'finish_reason': 'stop',\n",
|
||||||
|
" 'index': 0,\n",
|
||||||
|
" 'message': {'role': 'assistant',\n",
|
||||||
|
" 'content': \"what does Baseten do? \\nI'm sorry, I cannot provide a specific answer as\"}}],\n",
|
||||||
|
" 'created': 1692135883.699066,\n",
|
||||||
|
" 'model': 'qvv0xeq'}"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"execution_count": 18
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"source": [
|
||||||
|
"## Calling Wizard LM https://app.baseten.co/explore/wizardlm\n",
|
||||||
|
"### Pass Your Baseten model `Version ID` as `model`"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"id": "7n21UroEGCGa"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"source": [
|
||||||
|
"model = \"q841o8w\"\n",
|
||||||
|
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
|
||||||
|
"response"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"base_uri": "https://localhost:8080/"
|
||||||
|
},
|
||||||
|
"id": "uLVWFH899lAF",
|
||||||
|
"outputId": "61c2bc74-673b-413e-bb40-179cf408523d"
|
||||||
|
},
|
||||||
|
"execution_count": 19,
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"output_type": "stream",
|
||||||
|
"name": "stderr",
|
||||||
|
"text": [
|
||||||
|
"\u001b[32mINFO\u001b[0m API key set.\n",
|
||||||
|
"INFO:baseten:API key set.\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"output_type": "execute_result",
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"{'choices': [{'finish_reason': 'stop',\n",
|
||||||
|
" 'index': 0,\n",
|
||||||
|
" 'message': {'role': 'assistant',\n",
|
||||||
|
" 'content': 'As an AI language model, I do not have personal beliefs or practices, but based on the information available online, Baseten is a popular name for a traditional Ethiopian dish made with injera, a spongy flatbread, and wat, a spicy stew made with meat or vegetables. It is typically served for breakfast or dinner and is a staple in Ethiopian cuisine. The name Baseten is also used to refer to a traditional Ethiopian coffee ceremony, where coffee is brewed and served in a special ceremony with music and food.'}}],\n",
|
||||||
|
" 'created': 1692135900.2806294,\n",
|
||||||
|
" 'model': 'q841o8w'}"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"execution_count": 19
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"source": [
|
||||||
|
"## Calling mosaicml/mpt-7b https://app.baseten.co/explore/mpt_7b_instruct\n",
|
||||||
|
"### Pass Your Baseten model `Version ID` as `model`"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"id": "6-TFwmPAGPXq"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"source": [
|
||||||
|
"model = \"31dxrj3\"\n",
|
||||||
|
"response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
|
||||||
|
"response"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"base_uri": "https://localhost:8080/"
|
||||||
|
},
|
||||||
|
"id": "gbeYZOrUE_Bp",
|
||||||
|
"outputId": "838d86ea-2143-4cb3-bc80-2acc2346c37a"
|
||||||
|
},
|
||||||
|
"execution_count": 20,
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"output_type": "stream",
|
||||||
|
"name": "stderr",
|
||||||
|
"text": [
|
||||||
|
"\u001b[32mINFO\u001b[0m API key set.\n",
|
||||||
|
"INFO:baseten:API key set.\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"output_type": "execute_result",
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"{'choices': [{'finish_reason': 'stop',\n",
|
||||||
|
" 'index': 0,\n",
|
||||||
|
" 'message': {'role': 'assistant',\n",
|
||||||
|
" 'content': \"\\n===================\\n\\nIt's a tool to build a local version of a game on your own machine to host\\non your website.\\n\\nIt's used to make game demos and show them on Twitter, Tumblr, and Facebook.\\n\\n\\n\\n## What's built\\n\\n- A directory of all your game directories, named with a version name and build number, with images linked to.\\n- Includes HTML to include in another site.\\n- Includes images for your icons and\"}}],\n",
|
||||||
|
" 'created': 1692135914.7472186,\n",
|
||||||
|
" 'model': '31dxrj3'}"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"execution_count": 20
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -68,21 +68,30 @@ Here are some examples of supported models:
|
||||||
| [bigcode/starcoder](https://huggingface.co/bigcode/starcoder) | `completion(model="bigcode/starcoder", messages=messages, custom_llm_provider="huggingface")` | `os.environ['HF_TOKEN']` |
|
| [bigcode/starcoder](https://huggingface.co/bigcode/starcoder) | `completion(model="bigcode/starcoder", messages=messages, custom_llm_provider="huggingface")` | `os.environ['HF_TOKEN']` |
|
||||||
| [google/flan-t5-xxl](https://huggingface.co/google/flan-t5-xxl) | `completion(model="google/flan-t5-xxl", messages=messages, custom_llm_provider="huggingface")` | `os.environ['HF_TOKEN']` |
|
| [google/flan-t5-xxl](https://huggingface.co/google/flan-t5-xxl) | `completion(model="google/flan-t5-xxl", messages=messages, custom_llm_provider="huggingface")` | `os.environ['HF_TOKEN']` |
|
||||||
| [google/flan-t5-large](https://huggingface.co/google/flan-t5-large) | `completion(model="google/flan-t5-large", messages=messages, custom_llm_provider="huggingface")` | `os.environ['HF_TOKEN']` |
|
| [google/flan-t5-large](https://huggingface.co/google/flan-t5-large) | `completion(model="google/flan-t5-large", messages=messages, custom_llm_provider="huggingface")` | `os.environ['HF_TOKEN']` |
|
||||||
|
|
||||||
### AI21 Models
|
### AI21 Models
|
||||||
|
|
||||||
| Model Name | Function Call | Required OS Variables |
|
| Model Name | Function Call | Required OS Variables |
|
||||||
|------------------|--------------------------------------------|--------------------------------------|
|
|------------------|--------------------------------------------|--------------------------------------|
|
||||||
| j2-light | `completion('j2-light', messages)` | `os.environ['AI21_API_KEY']` |
|
| j2-light | `completion('j2-light', messages)` | `os.environ['AI21_API_KEY']` |
|
||||||
| j2-mid | `completion('j2-mid', messages)` | `os.environ['AI21_API_KEY']` |
|
| j2-mid | `completion('j2-mid', messages)` | `os.environ['AI21_API_KEY']` |
|
||||||
| j2-ultra | `completion('j2-ultra', messages)` | `os.environ['AI21_API_KEY']` |
|
| j2-ultra | `completion('j2-ultra', messages)` | `os.environ['AI21_API_KEY']` |
|
||||||
|
|
||||||
|
|
||||||
### Cohere Models
|
### Cohere Models
|
||||||
|
|
||||||
| Model Name | Function Call | Required OS Variables |
|
| Model Name | Function Call | Required OS Variables |
|
||||||
|------------------|--------------------------------------------|--------------------------------------|
|
|------------------|--------------------------------------------|--------------------------------------|
|
||||||
| command-nightly | `completion('command-nightly', messages)` | `os.environ['COHERE_API_KEY']` |
|
| command-nightly | `completion('command-nightly', messages)` | `os.environ['COHERE_API_KEY']` |
|
||||||
|
|
||||||
|
### BaseTen Models
|
||||||
|
Baseten provides infrastructure to deploy and serve ML models https://www.baseten.co/. Use liteLLM to easily call models deployed on Baseten.
|
||||||
|
|
||||||
|
|
||||||
|
| Model Name | Function Call | Required OS Variables |
|
||||||
|
|------------------|--------------------------------------------|------------------------------------|
|
||||||
|
| Falcon 7B | `completion(model='<your model version id>', messages=messages, custom_llm_provider="baseten")` | `os.environ['BASETEN_API_KEY']` |
|
||||||
|
| Wizard LM | `completion(model='<your model version id>', messages=messages, custom_llm_provider="baseten")` | `os.environ['BASETEN_API_KEY']` |
|
||||||
|
| MPT 7B Base | `completion(model='<your model version id>', messages=messages, custom_llm_provider="baseten")` | `os.environ['BASETEN_API_KEY']` |
|
||||||
|
|
||||||
|
|
||||||
### OpenRouter Completion Models
|
### OpenRouter Completion Models
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue