From 7b1c1745472c27d06b2165550e21442a515677ac Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Wed, 6 Sep 2023 19:25:16 -0700 Subject: [PATCH] Add files via upload --- cookbook/Local_HF_Testing.ipynb | 444 ++++++++++++++++++++++++++++++++ 1 file changed, 444 insertions(+) create mode 100644 cookbook/Local_HF_Testing.ipynb diff --git a/cookbook/Local_HF_Testing.ipynb b/cookbook/Local_HF_Testing.ipynb new file mode 100644 index 000000000..da47d4aa4 --- /dev/null +++ b/cookbook/Local_HF_Testing.ipynb @@ -0,0 +1,444 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "machine_shape": "hm", + "gpuType": "V100" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Set up Environment" + ], + "metadata": { + "id": "vDOm5wfjdFLP" + } + }, + { + "cell_type": "code", + "source": [ + "!pip install --upgrade litellm" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Bx6mAA6MHiy_", + "outputId": "949ad348-5448-40f8-fdf5-acdf0b0ecc70" + }, + "execution_count": 1, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting litellm\n", + " Downloading litellm-0.1.555-py3-none-any.whl (100 kB)\n", + "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/100.4 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━\u001b[0m \u001b[32m92.2/100.4 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m100.4/100.4 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: importlib-metadata<7.0.0,>=6.8.0 in /usr/local/lib/python3.10/dist-packages (from litellm) (6.8.0)\n", + "Collecting openai<0.28.0,>=0.27.8 (from litellm)\n", + " Downloading openai-0.27.10-py3-none-any.whl (76 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.5/76.5 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting python-dotenv>=0.2.0 (from litellm)\n", + " Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n", + "Collecting tiktoken<0.5.0,>=0.4.0 (from litellm)\n", + " Downloading tiktoken-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m32.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7.0.0,>=6.8.0->litellm) (3.16.2)\n", + "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (2.31.0)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (4.66.1)\n", + "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm) (3.8.5)\n", + "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm) (2023.6.3)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (3.2.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (2.0.4)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm) (2023.7.22)\n", + "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (23.1.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (6.0.4)\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (4.0.3)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.9.2)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.4.0)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm) (1.3.1)\n", + "Installing collected packages: python-dotenv, tiktoken, openai, litellm\n", + "Successfully installed litellm-0.1.555 openai-0.27.10 python-dotenv-1.0.0 tiktoken-0.4.0\n" + ] + } + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "zIYv7JTyxSxR", + "outputId": "53890320-f9fa-4bf4-8362-0f17f52c6ed4" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Successfully installed fastapi-0.103.1 h11-0.14.0 huggingface-hub-0.16.4 ninja-1.11.1 pydantic-1.10.12 ray-2.6.3 safetensors-0.3.3 sentencepiece-0.1.99 starlette-0.27.0 tokenizers-0.13.3 transformers-4.33.1 uvicorn-0.23.2 vllm-0.1.4 xformers-0.0.21\n" + ] + } + ], + "source": [ + "!pip install vllm" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Load the Logs" + ], + "metadata": { + "id": "RMcoAni6WKEx" + } + }, + { + "cell_type": "code", + "source": [ + "import pandas as pd" + ], + "metadata": { + "id": "zchxB8c7WJe5" + }, + "execution_count": 4, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# path of the csv file\n", + "file_path = 'Model-prompts-example.csv'\n", + "\n", + "# load the csv file as a pandas DataFrame\n", + "data = pd.read_csv(file_path)\n", + "\n", + "data.head()" + ], + "metadata": { + "id": "aKcWr015WNPm", + "outputId": "6e226773-333f-46a2-9fc8-4f54f309d204", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 81 + } + }, + "execution_count": 6, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " Success Timestamp Input \\\n", + "0 True 1694041195 This is the templated query input \n", + "\n", + " Output RunId (Wandb Runid) \\\n", + "0 This is the query output from the model 8hlumwuk \n", + "\n", + " Model ID (or Name) \n", + "0 OpenAI/Turbo-3.5 " + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
SuccessTimestampInputOutputRunId (Wandb Runid)Model ID (or Name)
0True1694041195This is the templated query inputThis is the query output from the model8hlumwukOpenAI/Turbo-3.5
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + "
\n" + ] + }, + "metadata": {}, + "execution_count": 6 + } + ] + }, + { + "cell_type": "code", + "source": [ + "input_texts = data['Input'].values" + ], + "metadata": { + "id": "0DbL-kirWUyn" + }, + "execution_count": 7, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "messages = [[{\"role\": \"user\", \"content\": input_text}] for input_text in input_texts]" + ], + "metadata": { + "id": "cqpAvy8hWXyC" + }, + "execution_count": 8, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "# Running Inference" + ], + "metadata": { + "id": "SugCyom0Xy8U" + } + }, + { + "cell_type": "code", + "source": [ + "from litellm import batch_completion\n", + "model_name = \"facebook/opt-125m\"\n", + "provider = \"vllm\"\n", + "response_list = batch_completion(\n", + " model=model_name,\n", + " custom_llm_provider=provider, # can easily switch to huggingface, replicate, together ai, sagemaker, etc.\n", + " messages=messages,\n", + " temperature=0.2,\n", + " max_tokens=80,\n", + " )" + ], + "metadata": { + "id": "qpikx3uxHns3" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "response_list" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "QDPikHtwKJJ2", + "outputId": "06f47c44-e258-452a-f9db-232a5b6d2810" + }, + "execution_count": 10, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[ JSON: {\n", + " \"choices\": [\n", + " {\n", + " \"finish_reason\": \"stop\",\n", + " \"index\": 0,\n", + " \"message\": {\n", + " \"content\": \".\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is\",\n", + " \"role\": \"assistant\",\n", + " \"logprobs\": null\n", + " }\n", + " }\n", + " ],\n", + " \"created\": 1694053363.6139505,\n", + " \"model\": \"facebook/opt-125m\",\n", + " \"usage\": {\n", + " \"prompt_tokens\": 9,\n", + " \"completion_tokens\": 80,\n", + " \"total_tokens\": 89\n", + " }\n", + " }]" + ] + }, + "metadata": {}, + "execution_count": 10 + } + ] + }, + { + "cell_type": "code", + "source": [ + "response_values = [response['choices'][0]['message']['content'] for response in response_list]" + ], + "metadata": { + "id": "SYqTcCiJbQDF" + }, + "execution_count": 11, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "response_values" + ], + "metadata": { + "id": "wqs-Oy9FbiPo", + "outputId": "16a6a7b7-97c8-4b5b-eff8-09ea5eb5ad06", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "execution_count": 12, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "['.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is used to query the data.\\n\\nThe query input is the query input that is']" + ] + }, + "metadata": {}, + "execution_count": 12 + } + ] + }, + { + "cell_type": "code", + "source": [ + "data[f\"{model_name}_output\"] = response_values" + ], + "metadata": { + "id": "mElNbBehbkrz" + }, + "execution_count": 13, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "data.to_csv('model_responses.csv', index=False)" + ], + "metadata": { + "id": "F06NXssDc45k" + }, + "execution_count": 14, + "outputs": [] + } + ] +} \ No newline at end of file