From 6713134247db8ec1db3b063255f58b9f0a3d3b9f Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 2 Oct 2023 07:34:04 -0700 Subject: [PATCH] add bedrock cookbook --- cookbook/LiteLLM_Bedrock.ipynb | 303 +++++++++++++++++++++++++++++++++ 1 file changed, 303 insertions(+) create mode 100644 cookbook/LiteLLM_Bedrock.ipynb diff --git a/cookbook/LiteLLM_Bedrock.ipynb b/cookbook/LiteLLM_Bedrock.ipynb new file mode 100644 index 0000000000..b47ffef67c --- /dev/null +++ b/cookbook/LiteLLM_Bedrock.ipynb @@ -0,0 +1,303 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# LiteLLM Bedrock Usage" + ], + "metadata": { + "id": "fNkMBurtxawJ" + } + }, + { + "cell_type": "markdown", + "source": [ + "## Pre-Requisites" + ], + "metadata": { + "id": "htAufI28xeSy" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jT5GbPjAuDTp" + }, + "outputs": [], + "source": [ + "!pip install litellm\n", + "!pip install boto3>=1.28.57 # this version onwards has bedrock support" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Set Bedrock/AWS Credentials" + ], + "metadata": { + "id": "H4Vu4er2xnfI" + } + }, + { + "cell_type": "code", + "source": [ + "import os\n", + "os.environ[\"AWS_ACCESS_KEY_ID\"] = \"\" # Access key\n", + "os.environ[\"AWS_SECRET_ACCESS_KEY\"] = \"\" # Secret access key\n", + "os.environ[\"AWS_REGION_NAME\"] = \"\"" + ], + "metadata": { + "id": "CtTrBthWxp-t" + }, + "execution_count": 9, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Anthropic Requests" + ], + "metadata": { + "id": "ycRK9NUdx1EI" + } + }, + { + "cell_type": "code", + "source": [ + "from litellm import completion\n", + "\n", + "response = completion(\n", + " model=\"bedrock/anthropic.claude-instant-v1\",\n", + " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", + ")\n", + "print(\"Claude instant 1, response\")\n", + "print(response)\n", + "\n", + "\n", + "response = completion(\n", + " model=\"bedrock/anthropic.claude-v2\",\n", + " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", + ")\n", + "print(\"Claude v2, response\")\n", + "print(response)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "tgkuoHa5uLOy", + "outputId": "27a78e86-c6a7-4bcc-8559-0813cb978426" + }, + "execution_count": 15, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Claude instant 1, response\n", + "{\n", + " \"object\": \"chat.completion\",\n", + " \"choices\": [\n", + " {\n", + " \"finish_reason\": \"stop\",\n", + " \"index\": 0,\n", + " \"message\": {\n", + " \"content\": \" I'm doing well, thanks for asking!\",\n", + " \"role\": \"assistant\",\n", + " \"logprobs\": null\n", + " }\n", + " }\n", + " ],\n", + " \"id\": \"chatcmpl-4f2e64a1-56d2-43f2-90d3-60ffd6f5086d\",\n", + " \"created\": 1696256761.3265705,\n", + " \"model\": \"anthropic.claude-instant-v1\",\n", + " \"usage\": {\n", + " \"prompt_tokens\": 11,\n", + " \"completion_tokens\": 9,\n", + " \"total_tokens\": 20\n", + " },\n", + " \"finish_reason\": \"stop_sequence\"\n", + "}\n", + "Claude v2, response\n", + "{\n", + " \"object\": \"chat.completion\",\n", + " \"choices\": [\n", + " {\n", + " \"finish_reason\": \"stop\",\n", + " \"index\": 0,\n", + " \"message\": {\n", + " \"content\": \" I'm doing well, thanks for asking!\",\n", + " \"role\": \"assistant\",\n", + " \"logprobs\": null\n", + " }\n", + " }\n", + " ],\n", + " \"id\": \"chatcmpl-34f59b33-f94e-40c2-8bdb-f4af0813405e\",\n", + " \"created\": 1696256762.2137017,\n", + " \"model\": \"anthropic.claude-v2\",\n", + " \"usage\": {\n", + " \"prompt_tokens\": 11,\n", + " \"completion_tokens\": 9,\n", + " \"total_tokens\": 20\n", + " },\n", + " \"finish_reason\": \"stop_sequence\"\n", + "}\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Anthropic Requests - With Streaming" + ], + "metadata": { + "id": "HnM-HtM3yFMT" + } + }, + { + "cell_type": "code", + "source": [ + "from litellm import completion\n", + "\n", + "response = completion(\n", + " model=\"bedrock/anthropic.claude-instant-v1\",\n", + " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", + " stream=True,\n", + ")\n", + "print(\"Claude instant 1, response\")\n", + "for chunk in response:\n", + " print(chunk)\n", + "\n", + "\n", + "response = completion(\n", + " model=\"bedrock/anthropic.claude-v2\",\n", + " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", + " stream=True\n", + ")\n", + "print(\"Claude v2, response\")\n", + "print(response)\n", + "for chunk in response:\n", + " print(chunk)" + ], + "metadata": { + "id": "_JZvg2yovRsU" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## A121 Requests" + ], + "metadata": { + "id": "zj1U1mh9zEhP" + } + }, + { + "cell_type": "code", + "source": [ + "response = completion(\n", + " model=\"bedrock/ai21.j2-ultra\",\n", + " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", + ")\n", + "print(\"J2 ultra response\")\n", + "print(response)\n", + "\n", + "response = completion(\n", + " model=\"bedrock/ai21.j2-mid\",\n", + " messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n", + ")\n", + "print(\"J2 mid response\")\n", + "print(response)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "6wK6MZLovU7r", + "outputId": "4cf80c04-f15d-4066-b4c7-113b551538de" + }, + "execution_count": 19, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "J2 ultra response\n", + "{\n", + " \"object\": \"chat.completion\",\n", + " \"choices\": [\n", + " {\n", + " \"finish_reason\": \"stop\",\n", + " \"index\": 0,\n", + " \"message\": {\n", + " \"content\": \"\\nHi, I'm doing well, thanks for asking! How about you?\",\n", + " \"role\": \"assistant\",\n", + " \"logprobs\": null\n", + " }\n", + " }\n", + " ],\n", + " \"id\": \"chatcmpl-f2de678f-0e70-4e36-a01f-8b184c2e4d50\",\n", + " \"created\": 1696257116.044311,\n", + " \"model\": \"ai21.j2-ultra\",\n", + " \"usage\": {\n", + " \"prompt_tokens\": 6,\n", + " \"completion_tokens\": 16,\n", + " \"total_tokens\": 22\n", + " }\n", + "}\n", + "J2 mid response\n", + "{\n", + " \"object\": \"chat.completion\",\n", + " \"choices\": [\n", + " {\n", + " \"finish_reason\": \"stop\",\n", + " \"index\": 0,\n", + " \"message\": {\n", + " \"content\": \"\\nGood. And you?\",\n", + " \"role\": \"assistant\",\n", + " \"logprobs\": null\n", + " }\n", + " }\n", + " ],\n", + " \"id\": \"chatcmpl-420d6bf9-36d8-484b-93b4-4c9e00f7ce2e\",\n", + " \"created\": 1696257116.5756805,\n", + " \"model\": \"ai21.j2-mid\",\n", + " \"usage\": {\n", + " \"prompt_tokens\": 6,\n", + " \"completion_tokens\": 6,\n", + " \"total_tokens\": 12\n", + " }\n", + "}\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "Y5gGZIwzzSON" + }, + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file