From e20df68413050b4dc6474c3664a70e321613a24e Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 6 Sep 2023 13:17:48 -0700 Subject: [PATCH] add LiteLLM_batch_completions cookbook --- cookbook/LiteLLM_batch_completion.ipynb | 166 ++++++++++++++++++++ docs/my-website/docs/completion/batching.md | 9 +- 2 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 cookbook/LiteLLM_batch_completion.ipynb diff --git a/cookbook/LiteLLM_batch_completion.ipynb b/cookbook/LiteLLM_batch_completion.ipynb new file mode 100644 index 000000000..a72fc3e87 --- /dev/null +++ b/cookbook/LiteLLM_batch_completion.ipynb @@ -0,0 +1,166 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# LiteLLM Batch Completions Example\n", + "\n", + "* This tutorial walks through using `batch_completion`\n", + "* Docs: https://docs.litellm.ai/docs/completion/batching" + ], + "metadata": { + "id": "MbLbs1tbISk-" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Ty6-ko_aDlPF" + }, + "outputs": [], + "source": [ + "!pip install litellm" + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Import Batch Completion" + ], + "metadata": { + "id": "KGhNJRUCIh1j" + } + }, + { + "cell_type": "code", + "source": [ + "import litellm\n", + "import os\n", + "from litellm import batch_completion\n", + "\n", + "# set your API_KEY\n", + "os.environ['ANTHROPIC_API_KEY'] = \"\"" + ], + "metadata": { + "id": "LOtI43snDrSK" + }, + "execution_count": 7, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Calling `litellm.batch_completion`\n", + "\n", + "In the batch_completion method, you provide a list of messages where each sub-list of messages is passed to litellm.completion(), allowing you to process multiple prompts efficiently in a single API call." + ], + "metadata": { + "id": "Xhv92NBaIpaw" + } + }, + { + "cell_type": "code", + "source": [ + "import litellm\n", + "import os\n", + "from litellm import batch_completion\n", + "\n", + "os.environ['ANTHROPIC_API_KEY'] = \"\"\n", + "\n", + "\n", + "responses = batch_completion(\n", + " model=\"claude-2\",\n", + " messages = [\n", + " [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"good morning? \"\n", + " }\n", + " ],\n", + " [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"what's the time? \"\n", + " }\n", + " ]\n", + " ]\n", + ")\n", + "responses" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "yY7GIRLsDywu", + "outputId": "009ea67f-95d5-462b-947f-b0d21e60c5bb" + }, + "execution_count": 11, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[ JSON: {\n", + " \"choices\": [\n", + " {\n", + " \"finish_reason\": \"stop\",\n", + " \"index\": 0,\n", + " \"message\": {\n", + " \"content\": \" Good morning!\",\n", + " \"role\": \"assistant\",\n", + " \"logprobs\": null\n", + " }\n", + " }\n", + " ],\n", + " \"created\": 1694030351.309254,\n", + " \"model\": \"claude-2\",\n", + " \"usage\": {\n", + " \"prompt_tokens\": 11,\n", + " \"completion_tokens\": 3,\n", + " \"total_tokens\": 14\n", + " }\n", + " },\n", + " JSON: {\n", + " \"choices\": [\n", + " {\n", + " \"finish_reason\": \"stop\",\n", + " \"index\": 0,\n", + " \"message\": {\n", + " \"content\": \" I'm an AI assistant created by Anthropic. I don't actually have a concept of the current time.\",\n", + " \"role\": \"assistant\",\n", + " \"logprobs\": null\n", + " }\n", + " }\n", + " ],\n", + " \"created\": 1694030352.1215081,\n", + " \"model\": \"claude-2\",\n", + " \"usage\": {\n", + " \"prompt_tokens\": 13,\n", + " \"completion_tokens\": 22,\n", + " \"total_tokens\": 35\n", + " }\n", + " }]" + ] + }, + "metadata": {}, + "execution_count": 11 + } + ] + } + ] +} \ No newline at end of file diff --git a/docs/my-website/docs/completion/batching.md b/docs/my-website/docs/completion/batching.md index 4b58a51b2..29c520efa 100644 --- a/docs/my-website/docs/completion/batching.md +++ b/docs/my-website/docs/completion/batching.md @@ -1,6 +1,6 @@ -# Batching Completion Calls - batch_completion +# Batching Completion() Calls -Batch Completion allows you to pass a batch of completion() requests to process multiple `messages` in a single API call. +In the batch_completion method, you provide a list of `messages` where each sub-list of messages is passed to `litellm.completion()`, allowing you to process multiple prompts efficiently in a single API call. ## Example Code ```python @@ -28,7 +28,4 @@ responses = batch_completion( ] ] ) -``` - - -In the batch_completion method, you provide a list of `messages` where each sub-list of messages is passed to `litellm.completion()`, allowing you to process multiple prompts efficiently in a single API call. \ No newline at end of file +``` \ No newline at end of file