litellm/cookbook/LiteLLM_Bedrock.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "fNkMBurtxawJ"
      },
      "source": [
        "# LiteLLM Bedrock Usage\n",
        "Important Note: For Bedrock Requests you need to ensure you have `pip install boto3>=1.28.57`, boto3 supports bedrock from `boto3>=1.28.57` and higher "
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "htAufI28xeSy"
      },
      "source": [
        "## Pre-Requisites"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jT5GbPjAuDTp"
      },
      "outputs": [],
      "source": [
        "!pip install litellm\n",
        "!pip install boto3>=1.28.57 # this version onwards has bedrock support"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "H4Vu4er2xnfI"
      },
      "source": [
        "## Set Bedrock/AWS Credentials"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "CtTrBthWxp-t"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "os.environ[\"AWS_ACCESS_KEY_ID\"] = \"\" # Access key\n",
        "os.environ[\"AWS_SECRET_ACCESS_KEY\"] = \"\" # Secret access key\n",
        "os.environ[\"AWS_REGION_NAME\"] = \"\""
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "ycRK9NUdx1EI"
      },
      "source": [
        "## Anthropic Requests"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tgkuoHa5uLOy",
        "outputId": "27a78e86-c6a7-4bcc-8559-0813cb978426"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Claude instant 1, response\n",
            "{\n",
            "  \"object\": \"chat.completion\",\n",
            "  \"choices\": [\n",
            "    {\n",
            "      \"finish_reason\": \"stop\",\n",
            "      \"index\": 0,\n",
            "      \"message\": {\n",
            "        \"content\": \" I'm doing well, thanks for asking!\",\n",
            "        \"role\": \"assistant\",\n",
            "        \"logprobs\": null\n",
            "      }\n",
            "    }\n",
            "  ],\n",
            "  \"id\": \"chatcmpl-4f2e64a1-56d2-43f2-90d3-60ffd6f5086d\",\n",
            "  \"created\": 1696256761.3265705,\n",
            "  \"model\": \"anthropic.claude-instant-v1\",\n",
            "  \"usage\": {\n",
            "    \"prompt_tokens\": 11,\n",
            "    \"completion_tokens\": 9,\n",
            "    \"total_tokens\": 20\n",
            "  },\n",
            "  \"finish_reason\": \"stop_sequence\"\n",
            "}\n",
            "Claude v2, response\n",
            "{\n",
            "  \"object\": \"chat.completion\",\n",
            "  \"choices\": [\n",
            "    {\n",
            "      \"finish_reason\": \"stop\",\n",
            "      \"index\": 0,\n",
            "      \"message\": {\n",
            "        \"content\": \" I'm doing well, thanks for asking!\",\n",
            "        \"role\": \"assistant\",\n",
            "        \"logprobs\": null\n",
            "      }\n",
            "    }\n",
            "  ],\n",
            "  \"id\": \"chatcmpl-34f59b33-f94e-40c2-8bdb-f4af0813405e\",\n",
            "  \"created\": 1696256762.2137017,\n",
            "  \"model\": \"anthropic.claude-v2\",\n",
            "  \"usage\": {\n",
            "    \"prompt_tokens\": 11,\n",
            "    \"completion_tokens\": 9,\n",
            "    \"total_tokens\": 20\n",
            "  },\n",
            "  \"finish_reason\": \"stop_sequence\"\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "from litellm import completion\n",
        "\n",
        "response = completion(\n",
        "            model=\"bedrock/anthropic.claude-instant-v1\",\n",
        "            messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
        ")\n",
        "print(\"Claude instant 1, response\")\n",
        "print(response)\n",
        "\n",
        "\n",
        "response = completion(\n",
        "            model=\"bedrock/anthropic.claude-v2\",\n",
        "            messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
        ")\n",
        "print(\"Claude v2, response\")\n",
        "print(response)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "HnM-HtM3yFMT"
      },
      "source": [
        "## Anthropic Requests - With Streaming"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_JZvg2yovRsU"
      },
      "outputs": [],
      "source": [
        "from litellm import completion\n",
        "\n",
        "response = completion(\n",
        "            model=\"bedrock/anthropic.claude-instant-v1\",\n",
        "            messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
        "            stream=True,\n",
        ")\n",
        "print(\"Claude instant 1, response\")\n",
        "for chunk in response:\n",
        "  print(chunk)\n",
        "\n",
        "\n",
        "response = completion(\n",
        "            model=\"bedrock/anthropic.claude-v2\",\n",
        "            messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
        "            stream=True\n",
        ")\n",
        "print(\"Claude v2, response\")\n",
        "print(response)\n",
        "for chunk in response:\n",
        "  print(chunk)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "zj1U1mh9zEhP"
      },
      "source": [
        "## A121 Requests"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6wK6MZLovU7r",
        "outputId": "4cf80c04-f15d-4066-b4c7-113b551538de"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "J2 ultra response\n",
            "{\n",
            "  \"object\": \"chat.completion\",\n",
            "  \"choices\": [\n",
            "    {\n",
            "      \"finish_reason\": \"stop\",\n",
            "      \"index\": 0,\n",
            "      \"message\": {\n",
            "        \"content\": \"\\nHi, I'm doing well, thanks for asking! How about you?\",\n",
            "        \"role\": \"assistant\",\n",
            "        \"logprobs\": null\n",
            "      }\n",
            "    }\n",
            "  ],\n",
            "  \"id\": \"chatcmpl-f2de678f-0e70-4e36-a01f-8b184c2e4d50\",\n",
            "  \"created\": 1696257116.044311,\n",
            "  \"model\": \"ai21.j2-ultra\",\n",
            "  \"usage\": {\n",
            "    \"prompt_tokens\": 6,\n",
            "    \"completion_tokens\": 16,\n",
            "    \"total_tokens\": 22\n",
            "  }\n",
            "}\n",
            "J2 mid response\n",
            "{\n",
            "  \"object\": \"chat.completion\",\n",
            "  \"choices\": [\n",
            "    {\n",
            "      \"finish_reason\": \"stop\",\n",
            "      \"index\": 0,\n",
            "      \"message\": {\n",
            "        \"content\": \"\\nGood. And you?\",\n",
            "        \"role\": \"assistant\",\n",
            "        \"logprobs\": null\n",
            "      }\n",
            "    }\n",
            "  ],\n",
            "  \"id\": \"chatcmpl-420d6bf9-36d8-484b-93b4-4c9e00f7ce2e\",\n",
            "  \"created\": 1696257116.5756805,\n",
            "  \"model\": \"ai21.j2-mid\",\n",
            "  \"usage\": {\n",
            "    \"prompt_tokens\": 6,\n",
            "    \"completion_tokens\": 6,\n",
            "    \"total_tokens\": 12\n",
            "  }\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "response = completion(\n",
        "            model=\"bedrock/ai21.j2-ultra\",\n",
        "            messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
        ")\n",
        "print(\"J2 ultra response\")\n",
        "print(response)\n",
        "\n",
        "response = completion(\n",
        "            model=\"bedrock/ai21.j2-mid\",\n",
        "            messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}],\n",
        ")\n",
        "print(\"J2 mid response\")\n",
        "print(response)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Y5gGZIwzzSON"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}