mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-24 10:14:26 +00:00
add bedrock cookbook
This commit is contained in:
ishaan-jaff
parent
a5700861e7
commit
6713134247
1 changed files with 303 additions and 0 deletions
303
cookbook/LiteLLM_Bedrock.ipynb
vendored
Normal file
303
cookbook/LiteLLM_Bedrock.ipynb
vendored
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
{
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"name": "python3",
|
||||
"display_name": "Python 3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"# LiteLLM Bedrock Usage"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "fNkMBurtxawJ"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Pre-Requisites"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "htAufI28xeSy"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "jT5GbPjAuDTp"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install litellm\n",
|
||||
"!pip install boto3>=1.28.57 # this version onwards has bedrock support"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Set Bedrock/AWS Credentials"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "H4Vu4er2xnfI"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"import os\n",
|
||||
"os.environ[\"AWS_ACCESS_KEY_ID\"] = \"\" # Access key\n",
|
||||
"os.environ[\"AWS_SECRET_ACCESS_KEY\"] = \"\" # Secret access key\n",
|
||||
"os.environ[\"AWS_REGION_NAME\"] = \"\""
|
||||
],
|
||||
"metadata": {
|
||||
"id": "CtTrBthWxp-t"
|
||||
},
|
||||
"execution_count": 9,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Anthropic Requests"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "ycRK9NUdx1EI"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"from litellm import completion\n",
|
||||
"\n",
|
||||
"response = completion(\n",
|
||||
" model=\"bedrock/anthropic.claude-instant-v1\",\n",
|
||||
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
|
||||
")\n",
|
||||
"print(\"Claude instant 1, response\")\n",
|
||||
"print(response)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"response = completion(\n",
|
||||
" model=\"bedrock/anthropic.claude-v2\",\n",
|
||||
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
|
||||
")\n",
|
||||
"print(\"Claude v2, response\")\n",
|
||||
"print(response)"
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "tgkuoHa5uLOy",
|
||||
"outputId": "27a78e86-c6a7-4bcc-8559-0813cb978426"
|
||||
},
|
||||
"execution_count": 15,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"Claude instant 1, response\n",
|
||||
"{\n",
|
||||
" \"object\": \"chat.completion\",\n",
|
||||
" \"choices\": [\n",
|
||||
" {\n",
|
||||
" \"finish_reason\": \"stop\",\n",
|
||||
" \"index\": 0,\n",
|
||||
" \"message\": {\n",
|
||||
" \"content\": \" I'm doing well, thanks for asking!\",\n",
|
||||
" \"role\": \"assistant\",\n",
|
||||
" \"logprobs\": null\n",
|
||||
" }\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" \"id\": \"chatcmpl-4f2e64a1-56d2-43f2-90d3-60ffd6f5086d\",\n",
|
||||
" \"created\": 1696256761.3265705,\n",
|
||||
" \"model\": \"anthropic.claude-instant-v1\",\n",
|
||||
" \"usage\": {\n",
|
||||
" \"prompt_tokens\": 11,\n",
|
||||
" \"completion_tokens\": 9,\n",
|
||||
" \"total_tokens\": 20\n",
|
||||
" },\n",
|
||||
" \"finish_reason\": \"stop_sequence\"\n",
|
||||
"}\n",
|
||||
"Claude v2, response\n",
|
||||
"{\n",
|
||||
" \"object\": \"chat.completion\",\n",
|
||||
" \"choices\": [\n",
|
||||
" {\n",
|
||||
" \"finish_reason\": \"stop\",\n",
|
||||
" \"index\": 0,\n",
|
||||
" \"message\": {\n",
|
||||
" \"content\": \" I'm doing well, thanks for asking!\",\n",
|
||||
" \"role\": \"assistant\",\n",
|
||||
" \"logprobs\": null\n",
|
||||
" }\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" \"id\": \"chatcmpl-34f59b33-f94e-40c2-8bdb-f4af0813405e\",\n",
|
||||
" \"created\": 1696256762.2137017,\n",
|
||||
" \"model\": \"anthropic.claude-v2\",\n",
|
||||
" \"usage\": {\n",
|
||||
" \"prompt_tokens\": 11,\n",
|
||||
" \"completion_tokens\": 9,\n",
|
||||
" \"total_tokens\": 20\n",
|
||||
" },\n",
|
||||
" \"finish_reason\": \"stop_sequence\"\n",
|
||||
"}\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Anthropic Requests - With Streaming"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "HnM-HtM3yFMT"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"from litellm import completion\n",
|
||||
"\n",
|
||||
"response = completion(\n",
|
||||
" model=\"bedrock/anthropic.claude-instant-v1\",\n",
|
||||
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
|
||||
" stream=True,\n",
|
||||
")\n",
|
||||
"print(\"Claude instant 1, response\")\n",
|
||||
"for chunk in response:\n",
|
||||
" print(chunk)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"response = completion(\n",
|
||||
" model=\"bedrock/anthropic.claude-v2\",\n",
|
||||
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
|
||||
" stream=True\n",
|
||||
")\n",
|
||||
"print(\"Claude v2, response\")\n",
|
||||
"print(response)\n",
|
||||
"for chunk in response:\n",
|
||||
" print(chunk)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "_JZvg2yovRsU"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## A121 Requests"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "zj1U1mh9zEhP"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"response = completion(\n",
|
||||
" model=\"bedrock/ai21.j2-ultra\",\n",
|
||||
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
|
||||
")\n",
|
||||
"print(\"J2 ultra response\")\n",
|
||||
"print(response)\n",
|
||||
"\n",
|
||||
"response = completion(\n",
|
||||
" model=\"bedrock/ai21.j2-mid\",\n",
|
||||
" messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
|
||||
")\n",
|
||||
"print(\"J2 mid response\")\n",
|
||||
"print(response)"
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "6wK6MZLovU7r",
|
||||
"outputId": "4cf80c04-f15d-4066-b4c7-113b551538de"
|
||||
},
|
||||
"execution_count": 19,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"J2 ultra response\n",
|
||||
"{\n",
|
||||
" \"object\": \"chat.completion\",\n",
|
||||
" \"choices\": [\n",
|
||||
" {\n",
|
||||
" \"finish_reason\": \"stop\",\n",
|
||||
" \"index\": 0,\n",
|
||||
" \"message\": {\n",
|
||||
" \"content\": \"\\nHi, I'm doing well, thanks for asking! How about you?\",\n",
|
||||
" \"role\": \"assistant\",\n",
|
||||
" \"logprobs\": null\n",
|
||||
" }\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" \"id\": \"chatcmpl-f2de678f-0e70-4e36-a01f-8b184c2e4d50\",\n",
|
||||
" \"created\": 1696257116.044311,\n",
|
||||
" \"model\": \"ai21.j2-ultra\",\n",
|
||||
" \"usage\": {\n",
|
||||
" \"prompt_tokens\": 6,\n",
|
||||
" \"completion_tokens\": 16,\n",
|
||||
" \"total_tokens\": 22\n",
|
||||
" }\n",
|
||||
"}\n",
|
||||
"J2 mid response\n",
|
||||
"{\n",
|
||||
" \"object\": \"chat.completion\",\n",
|
||||
" \"choices\": [\n",
|
||||
" {\n",
|
||||
" \"finish_reason\": \"stop\",\n",
|
||||
" \"index\": 0,\n",
|
||||
" \"message\": {\n",
|
||||
" \"content\": \"\\nGood. And you?\",\n",
|
||||
" \"role\": \"assistant\",\n",
|
||||
" \"logprobs\": null\n",
|
||||
" }\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" \"id\": \"chatcmpl-420d6bf9-36d8-484b-93b4-4c9e00f7ce2e\",\n",
|
||||
" \"created\": 1696257116.5756805,\n",
|
||||
" \"model\": \"ai21.j2-mid\",\n",
|
||||
" \"usage\": {\n",
|
||||
" \"prompt_tokens\": 6,\n",
|
||||
" \"completion_tokens\": 6,\n",
|
||||
" \"total_tokens\": 12\n",
|
||||
" }\n",
|
||||
"}\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [],
|
||||
"metadata": {
|
||||
"id": "Y5gGZIwzzSON"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue