mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-24 18:24:20 +00:00
update cookbook
This commit is contained in:
ishaan-jaff
parent
b1ab1191e6
commit
361bf02e53
1 changed files with 138 additions and 71 deletions
209
cookbook/LiteLLM_HuggingFace.ipynb
vendored
209
cookbook/LiteLLM_HuggingFace.ipynb
vendored
|
|
@ -35,6 +35,142 @@
|
||||||
"!pip install litellm"
|
"!pip install litellm"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"source": [
|
||||||
|
"## HuggingFace TGI Model - Deployed Inference Endpoints\n",
|
||||||
|
"Steps to use\n",
|
||||||
|
"* set `api_base` to your deployed api base\n",
|
||||||
|
"* Add the `huggingface/` prefix to your model so litellm knows it's a huggingface Deployed Inference Endpoint"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"id": "-klhAhjLtclv"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"import litellm\n",
|
||||||
|
"\n",
|
||||||
|
"os.environ[\"HUGGINGFACE_API_KEY\"] = \"\"\n",
|
||||||
|
"\n",
|
||||||
|
"# TGI model: Call https://huggingface.co/glaiveai/glaive-coder-7b\n",
|
||||||
|
"# add the 'huggingface/' prefix to the model to set huggingface as the provider\n",
|
||||||
|
"# set api base to your deployed api endpoint from hugging face\n",
|
||||||
|
"response = litellm.completion(\n",
|
||||||
|
" model=\"huggingface/glaiveai/glaive-coder-7b\",\n",
|
||||||
|
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
|
||||||
|
" api_base=\"https://wjiegasee9bmqke2.us-east-1.aws.endpoints.huggingface.cloud\"\n",
|
||||||
|
")\n",
|
||||||
|
"print(response)"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"base_uri": "https://localhost:8080/"
|
||||||
|
},
|
||||||
|
"id": "Lbmw8Gl_pHns",
|
||||||
|
"outputId": "ea8408bf-1cc3-4670-ecea-f12666d204a8"
|
||||||
|
},
|
||||||
|
"execution_count": 9,
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"output_type": "stream",
|
||||||
|
"name": "stdout",
|
||||||
|
"text": [
|
||||||
|
"{\n",
|
||||||
|
" \"object\": \"chat.completion\",\n",
|
||||||
|
" \"choices\": [\n",
|
||||||
|
" {\n",
|
||||||
|
" \"finish_reason\": \"length\",\n",
|
||||||
|
" \"index\": 0,\n",
|
||||||
|
" \"message\": {\n",
|
||||||
|
" \"content\": \"\\n\\nI am doing well, thank you for asking. How about you?\\nI am doing\",\n",
|
||||||
|
" \"role\": \"assistant\",\n",
|
||||||
|
" \"logprobs\": -8.9481967812\n",
|
||||||
|
" }\n",
|
||||||
|
" }\n",
|
||||||
|
" ],\n",
|
||||||
|
" \"id\": \"chatcmpl-74dc9d89-3916-47ce-9bea-b80e66660f77\",\n",
|
||||||
|
" \"created\": 1695871068.8413374,\n",
|
||||||
|
" \"model\": \"glaiveai/glaive-coder-7b\",\n",
|
||||||
|
" \"usage\": {\n",
|
||||||
|
" \"prompt_tokens\": 6,\n",
|
||||||
|
" \"completion_tokens\": 18,\n",
|
||||||
|
" \"total_tokens\": 24\n",
|
||||||
|
" }\n",
|
||||||
|
"}\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"source": [
|
||||||
|
"## HuggingFace Non TGI/Non Conversational Model - Deployed Inference Endpoints\n",
|
||||||
|
"* set `api_base` to your deployed api base\n",
|
||||||
|
"* Add the `huggingface/` prefix to your model so litellm knows it's a huggingface Deployed Inference Endpoint"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"id": "WZNyq76syYyh"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"import litellm\n",
|
||||||
|
"\n",
|
||||||
|
"os.environ[\"HUGGINGFACE_API_KEY\"] = \"\"\n",
|
||||||
|
"# model: https://huggingface.co/roneneldan/TinyStories-3M\n",
|
||||||
|
"# add the 'huggingface/' prefix to the model to set huggingface as the provider\n",
|
||||||
|
"# set api base to your deployed api endpoint from hugging face\n",
|
||||||
|
"response = litellm.completion(\n",
|
||||||
|
" model=\"huggingface/roneneldan/TinyStories-3M\",\n",
|
||||||
|
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
|
||||||
|
" api_base=\"https://p69xlsj6rpno5drq.us-east-1.aws.endpoints.huggingface.cloud\",\n",
|
||||||
|
" )\n",
|
||||||
|
"print(response)\n"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"base_uri": "https://localhost:8080/"
|
||||||
|
},
|
||||||
|
"id": "W8kMlXd6yRXu",
|
||||||
|
"outputId": "63e2cd7a-8759-4ee6-bac4-fe34ce8f0ca0"
|
||||||
|
},
|
||||||
|
"execution_count": 6,
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"output_type": "stream",
|
||||||
|
"name": "stdout",
|
||||||
|
"text": [
|
||||||
|
"{\n",
|
||||||
|
" \"object\": \"chat.completion\",\n",
|
||||||
|
" \"choices\": [\n",
|
||||||
|
" {\n",
|
||||||
|
" \"finish_reason\": \"stop\",\n",
|
||||||
|
" \"index\": 0,\n",
|
||||||
|
" \"message\": {\n",
|
||||||
|
" \"content\": \"Hello, how are you? I have a surprise for you. I have a surprise for you.\",\n",
|
||||||
|
" \"role\": \"assistant\",\n",
|
||||||
|
" \"logprobs\": null\n",
|
||||||
|
" }\n",
|
||||||
|
" }\n",
|
||||||
|
" ],\n",
|
||||||
|
" \"id\": \"chatcmpl-6035abd6-7753-4a7d-ba0a-8193522e23cf\",\n",
|
||||||
|
" \"created\": 1695871015.0468287,\n",
|
||||||
|
" \"model\": \"roneneldan/TinyStories-3M\",\n",
|
||||||
|
" \"usage\": {\n",
|
||||||
|
" \"prompt_tokens\": 6,\n",
|
||||||
|
" \"completion_tokens\": 20,\n",
|
||||||
|
" \"total_tokens\": 26\n",
|
||||||
|
" }\n",
|
||||||
|
"}\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"source": [
|
"source": [
|
||||||
|
|
@ -84,7 +220,7 @@
|
||||||
"id": "Pi5Oww8gpCUm",
|
"id": "Pi5Oww8gpCUm",
|
||||||
"outputId": "659a67c7-f90d-4c06-b94e-2c4aa92d897a"
|
"outputId": "659a67c7-f90d-4c06-b94e-2c4aa92d897a"
|
||||||
},
|
},
|
||||||
"execution_count": 8,
|
"execution_count": null,
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
"output_type": "stream",
|
"output_type": "stream",
|
||||||
|
|
@ -138,75 +274,6 @@
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"source": [
|
|
||||||
"## HuggingFace - Deployed Inference Endpoints\n",
|
|
||||||
"Steps to use\n",
|
|
||||||
"* set `api_base` to your deployed api base\n",
|
|
||||||
"* Add the `huggingface/` prefix to your model so litellm knows it's a huggingface Deployed Inference Endpoint"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"id": "-klhAhjLtclv"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"source": [
|
|
||||||
"import os\n",
|
|
||||||
"import litellm\n",
|
|
||||||
"\n",
|
|
||||||
"os.environ[\"HUGGINGFACE_API_KEY\"] = \"\"\n",
|
|
||||||
"\n",
|
|
||||||
"# Call https://huggingface.co/glaiveai/glaive-coder-7b\n",
|
|
||||||
"# add the 'huggingface/' prefix to the model to set huggingface as the provider\n",
|
|
||||||
"# set api base to your deployed api endpoint from hugging face\n",
|
|
||||||
"response = litellm.completion(\n",
|
|
||||||
" model=\"huggingface/aws-glaive-coder-7b-0998\",\n",
|
|
||||||
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
|
|
||||||
" api_base=\"https://wjiegasee9bmqke2.us-east-1.aws.endpoints.huggingface.cloud\"\n",
|
|
||||||
")\n",
|
|
||||||
"print(response)\n"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"colab": {
|
|
||||||
"base_uri": "https://localhost:8080/"
|
|
||||||
},
|
|
||||||
"id": "Lbmw8Gl_pHns",
|
|
||||||
"outputId": "60b3067f-4569-4606-bb08-a186bfdbaecc"
|
|
||||||
},
|
|
||||||
"execution_count": 11,
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"output_type": "stream",
|
|
||||||
"name": "stdout",
|
|
||||||
"text": [
|
|
||||||
"{\n",
|
|
||||||
" \"object\": \"chat.completion\",\n",
|
|
||||||
" \"choices\": [\n",
|
|
||||||
" {\n",
|
|
||||||
" \"finish_reason\": \"length\",\n",
|
|
||||||
" \"index\": 0,\n",
|
|
||||||
" \"message\": {\n",
|
|
||||||
" \"content\": \"\\n\\nI am doing well, thank you for asking. How about you?\\nI am doing\",\n",
|
|
||||||
" \"role\": \"assistant\",\n",
|
|
||||||
" \"logprobs\": -8.9481967812\n",
|
|
||||||
" }\n",
|
|
||||||
" }\n",
|
|
||||||
" ],\n",
|
|
||||||
" \"id\": \"chatcmpl-b4fb54a7-e4f3-476a-8e2b-3d1745f5c8a5\",\n",
|
|
||||||
" \"created\": 1695837218.8811502,\n",
|
|
||||||
" \"model\": \"aws-glaive-coder-7b-0998\",\n",
|
|
||||||
" \"usage\": {\n",
|
|
||||||
" \"prompt_tokens\": 6,\n",
|
|
||||||
" \"completion_tokens\": 18,\n",
|
|
||||||
" \"total_tokens\": 24\n",
|
|
||||||
" }\n",
|
|
||||||
"}\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"source": [
|
"source": [
|
||||||
|
|
@ -246,7 +313,7 @@
|
||||||
"id": "y-QfIvA-uJKX",
|
"id": "y-QfIvA-uJKX",
|
||||||
"outputId": "b007bb98-00d0-44a4-8264-c8a2caed6768"
|
"outputId": "b007bb98-00d0-44a4-8264-c8a2caed6768"
|
||||||
},
|
},
|
||||||
"execution_count": 12,
|
"execution_count": null,
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
"output_type": "stream",
|
"output_type": "stream",
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue