diff --git a/cookbook/liteLLM_ChromaDB_Cache.ipynb b/cookbook/liteLLM_ChromaDB_Cache.ipynb new file mode 100644 index 000000000..2650a7263 --- /dev/null +++ b/cookbook/liteLLM_ChromaDB_Cache.ipynb @@ -0,0 +1,340 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "## This is a tutorial on how to build a Cache for GPT-4, chatGPT, Claude, Palm, Llama2\n", + "\n", + "In this notebook we:\n", + "- use chromaDB to define add_cache(), get_cache(). We cache responses from the LLM\n", + "- use liteLLM for calling `completion()` with GPT-4, chatGPT, Claude, llama2" + ], + "metadata": { + "id": "fqqYwS3jzN_t" + } + }, + { + "cell_type": "code", + "source": [ + "!pip install litellm\n", + "!pip install -Uq chromadb" + ], + "metadata": { + "id": "yQWPyKaEvl7c" + }, + "execution_count": 1, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Init ChromaDB collection\n" + ], + "metadata": { + "id": "oUVLNjt0pNUw" + } + }, + { + "cell_type": "code", + "source": [ + "import chromadb\n", + "# Global cache collection instance\n", + "cache_collection = None\n", + "\n", + "# Initialize the cache collection\n", + "def make_collection():\n", + " global cache_collection\n", + " client = chromadb.Client()\n", + " cache_collection = client.create_collection(\"llm_responses\")" + ], + "metadata": { + "id": "iyrAj4tjpMph" + }, + "execution_count": 10, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Add to Cache Function\n", + "We extract the user question and use chromaDB to embed it. For each question we store the model response as `metadata`\n", + "\n", + "`add_cache()` args\n", + "* `messages` - Expect this to be in the chatGPT messages format\n", + "* `model_response` - Response from LLM\n" + ], + "metadata": { + "id": "mr8ArGpWpZqi" + } + }, + { + "cell_type": "code", + "source": [ + "import uuid\n", + "\n", + "# Add a response to the cache\n", + "def add_cache(messages, model_response):\n", + " global cache_collection\n", + " if cache_collection is None:\n", + " make_collection()\n", + "\n", + " user_question = message_to_user_question(messages)\n", + "\n", + " # Add the user question and model response to the cache\n", + " cache_collection.add(\n", + " documents=[user_question],\n", + " metadatas=[{\"model_response\": str(model_response)}],\n", + " ids=[str(uuid.uuid4())]\n", + " )\n", + " return\n", + "\n", + "\n", + "# HELPER: Extract user's question from messages\n", + "def message_to_user_question(messages):\n", + " user_question = \"\"\n", + " for message in messages:\n", + " if message['role'] == 'user':\n", + " user_question += message[\"content\"]\n", + " return user_question" + ], + "metadata": { + "id": "9Yr9jrPspTl8" + }, + "execution_count": 11, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Get Cache Function\n", + "Given a user question, we check chromaDB for any embeddings with\n", + "`similarity > similarity_threshold`. Return the corresponding model_response if there is a match i.e `cache_hit`\n", + "\n", + "`get_cache()` args\n", + "* `messages` - Expect this to be in the chatGPT messages format\n", + "* `similarity_threshold` - Define a similarity_threshold on a scale of 0-1\n", + "0 -> everything is cache hit, 0.5 (50% similar), 1-> only return cache hits" + ], + "metadata": { + "id": "vpPjoHpNpxd0" + } + }, + { + "cell_type": "code", + "source": [ + "# Retrieve a response from the cache if similarity is above the threshold\n", + "def get_cache(messages, similarity_threshold):\n", + " try:\n", + " global cache_collection\n", + " if cache_collection is None:\n", + " make_collection()\n", + "\n", + " user_question = message_to_user_question(messages)\n", + "\n", + " # Query the cache for the user question\n", + " results = cache_collection.query(\n", + " query_texts=[user_question],\n", + " n_results=1\n", + " )\n", + "\n", + " if len(results['distances'][0]) == 0:\n", + " return None # Cache is empty\n", + "\n", + " distance = results['distances'][0][0]\n", + " sim = (1 - distance)\n", + "\n", + " if sim >= similarity_threshold:\n", + " return results['metadatas'][0][0][\"model_response\"] # Return cached response\n", + " else:\n", + " return None # No cache hit\n", + " except Exception as e:\n", + " print(\"Error in get cache\", e)\n", + " raise e\n" + ], + "metadata": { + "id": "SJaz-Mpnj7jd" + }, + "execution_count": 12, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Using liteLLM completion()\n", + "We use liteLLM completion to call our LLM APIs. LiteLLM allows the same Input/Output format for Azure OpenAI, chatGPT,\n", + "* Basic usage - `litellm.completion(model, messages)`.\n", + "\n", + "Use OpenAI, Claude, Anthropic, Replicate models. See supported models here: https://litellm.readthedocs.io/en/latest/supported/\n", + "\n" + ], + "metadata": { + "id": "8bM5GI9hqYPK" + } + }, + { + "cell_type": "code", + "source": [ + "import litellm, os, random\n", + "os.environ[\"OPENAI_API_KEY\"] = \"\" # @param\n", + "os.environ[\"REPLICATE_API_TOKEN\"] = \"\" #@param\n", + "\n", + "models = [\"gpt-4\", \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\"]\n", + "\n", + "def completion_with_cache(messages, similarity_threshold):\n", + " # check cache before calling model, return if there is a hit\n", + " cache_result = get_cache(messages, similarity_threshold)\n", + "\n", + " if cache_result != None:\n", + " return cache_result\n", + "\n", + " # randomly pick llama2, GPT-4\n", + " random_model_idx = random.randint(0, 1)\n", + " model = models[random_model_idx]\n", + " # use litellm to make completion request\n", + " print(f\"using model {model}\")\n", + " model_response = litellm.completion(model, messages)\n", + "\n", + " # add the user question + model response to cache\n", + " add_cache(messages, model_response)\n", + "\n", + " return model_response" + ], + "metadata": { + "id": "r3hW2whOkAEj" + }, + "execution_count": 13, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "## Testing + Running Cache" + ], + "metadata": { + "id": "tTkYOpo0rbJO" + } + }, + { + "cell_type": "code", + "source": [ + "import os\n", + "import time\n", + "import matplotlib.pyplot as plt\n", + "\n", + "\n", + "# List of example user messages\n", + "user_messages = [\n", + " \"Hello, what's the weather in San Francisco??\",\n", + " \"what's the weather in San Francisco??\"\n", + " \"Can you tell me about the latest news?\",\n", + " \"What's the capital of France?\",\n", + " \"How does photosynthesis work?\",\n", + " \"capital of france?\",\n", + " \"tell me a joke\",\n", + " \"tell me a joke right now\"\n", + " \"How do I bake a chocolate cake?\",\n", + " \"What are the benefits of exercise?\",\n", + " \"Tell me a joke!\",\n", + " # Add more questions here\n", + "]\n", + "\n", + "similarity_threshold = 0.5 # Adjust as needed\n", + "\n", + "### Testing / Measuring\n", + "cached_responses = 0\n", + "model_responses = 0\n", + "\n", + "for user_message in user_messages:\n", + " messages = [{\"content\": user_message, \"role\": \"user\"}]\n", + "\n", + " start = time.time()\n", + " response = completion_with_cache(messages=messages, similarity_threshold=similarity_threshold)\n", + " end = time.time()\n", + " response_time = end - start\n", + "\n", + " if response_time < 1: # Assuming cached responses come in less than 1s\n", + " cached_responses += 1\n", + " else:\n", + " model_responses += 1\n", + " print(f\"got response for {user_message}\")\n", + "\n", + "# Plotting\n", + "response_types = [\"Cached\", \"Model\"]\n", + "response_counts = [cached_responses, model_responses]\n", + "\n", + "fig, ax = plt.subplots()\n", + "ax.bar(response_types, response_counts)\n", + "ax.set_ylabel(\"Number of Responses\")\n", + "ax.set_title(\"Cached vs Model API Responses\")\n", + "plt.show()\n", + "\n", + "print(f\"Cached Responses: {cached_responses}\")\n", + "print(f\"Model Responses: {model_responses}\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 747 + }, + "id": "i650yqJfkokZ", + "outputId": "efd14d6f-500e-4e52-969f-974a2a2ac15a" + }, + "execution_count": 14, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "using model gpt-4\n", + "got response for Hello, what's the weather in San Francisco??\n", + "got response for what's the weather in San Francisco??Can you tell me about the latest news?\n", + "using model gpt-4\n", + "got response for What's the capital of France?\n", + "using model gpt-4\n", + "got response for How does photosynthesis work?\n", + "got response for capital of france?\n", + "using model replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\n", + "got response for tell me a joke\n", + "using model replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\n", + "got response for tell me a joke right nowHow do I bake a chocolate cake?\n", + "using model gpt-4\n", + "got response for What are the benefits of exercise?\n", + "got response for Tell me a joke!\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAGzCAYAAAABsTylAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA7S0lEQVR4nO3deVhU5f//8dfIMiKyqOGWCCqa4oJpWohrapZL9rFcW5C0LCnc0qQ+pbaIWpof07DNpdTcW6zUXDOX3HHFBZWyzNBkERdUOL8//DHfJjQZGpyTPB/XNdfVuc8993mfwYkX97nPjMUwDEMAAAAmVMzVBQAAAFwPQQUAAJgWQQUAAJgWQQUAAJgWQQUAAJgWQQUAAJgWQQUAAJgWQQUAAJgWQQUAAJgWQQX4/1q2bKk6derclGNZLBaNHDnyphzLjIKDg9W7d+8CPbeov3ZAUUNQgSkdOXJE/fr1U9WqVVW8eHH5+voqIiJC//vf/3ThwgVXl3dLWLt2rSwWiywWi2bNmnXNPhEREbJYLDctwBWGxMREWSwWFS9eXGlpadfs07JlS9trYbFYVLp0aTVq1EjTpk1TTk6OrV/v3r1VsmTJGx5z5MiRduN5eHgoODhYMTEx160BwLW5u7oA4K+++eYbde3aVVarVU888YTq1KmjS5cuaf369Ro6dKj27dunDz74wNVl3jKKFy+uOXPm6LHHHrNrT05O1saNG1W8eHEXVeYcs2bNUvny5ZWamqqFCxeqb9++1+xXqVIlxcXFSZJOnTqlTz75RH369NGhQ4c0ZsyYAh07Pj5eJUuW1Llz57Rq1Sq9++672rFjh9avX1/g8wGKGoIKTOXYsWPq0aOHgoKCtHr1alWoUMG2Lzo6WklJSfrmm29cWOGtp3379vrqq690+vRp3Xbbbbb2OXPmqFy5cqpevbpSU1NdWGHBGYahOXPmqFevXjp27Jhmz5593aDi5+dnF9b69eunO+64Q5MnT9brr78uDw8Ph4//yCOP2F7Tfv36qUePHpo3b562bNmixo0bF+ykgCKGSz8wlXHjxikzM1Mff/yxXUjJFRISogEDBti2p0+frnvvvVdly5aV1WpVaGio4uPjrzn20qVL1aJFC/n4+MjX11eNGjXSnDlz8vTbv3+/WrVqpRIlSuj222/XuHHj8vTJysrSiBEjFBISIqvVqsDAQA0bNkxZWVl5+g0aNEgBAQHy8fHRgw8+qF9++eWGr8Pvv/8ud3d3jRo1Ks++gwcPymKxaPLkyZKky5cva9SoUapevbqKFy+uMmXKqGnTplqxYsUNjyNJnTt3ltVq1YIFC+za58yZo27dusnNzS3Pc65cuaLXX39d1apVk9VqVXBwsF566aU8528Yht544w1VqlRJJUqUUKtWrbRv375r1pGWlqaBAwcqMDBQVqtVISEhGjt2rN2lF0dt2LBBycnJ6tGjh3r06KF169bl6/WXpBIlSuiee+7RuXPndOrUqQLX8GfNmjWTdPXS5p9t3rxZ999/v/z8/FSiRAm1aNFCGzZssOtz9uxZDRw4UMHBwbJarSpbtqzatm2rHTt22PrkrrPavn27mjRpIi8vL1WpUkVTp07NU0tKSor69OmjcuXKqXjx4goLC9PMmTPt+iQnJ8tisejtt9/WBx98YPt5N2rUSFu3brXre/LkSUVFRalSpUqyWq2qUKGCOnfurOTkZLt+S5cuVbNmzeTt7S0fHx916NAhz7+J/I6FooEZFZjKkiVLVLVqVTVp0iRf/ePj41W7dm09+OCDcnd315IlS9S/f3/l5OQoOjra1m/GjBl68sknVbt2bcXGxsrf3187d+7UsmXL1KtXL1u/1NRU3X///erSpYu6deumhQsX6sUXX1TdunX1wAMPSJJycnL04IMPav369Xr66adVq1Yt7dmzR++8844OHTqkL774wjZe3759NWvWLPXq1UtNmjTR6tWr1aFDhxueV7ly5dSiRQvNnz9fI0aMsNs3b948ubm5qWvXrpKuroeIi4tT37591bhxY2VkZGjbtm3asWOH2rZte8NjlShRQp07d9Znn32mZ599VpK0a9cu7du3Tx999JF2796d5zl9+/bVzJkz9cgjj2jIkCHavHmz4uLilJiYqM8//9zW79VXX9Ubb7yh9u3bq3379tqxY4fuu+8+Xbp0yW688+fPq0WLFvr111/Vr18/Va5cWRs3blRsbKx+++03TZw48YbncS2zZ89WtWrV1KhRI9WpU0clSpTQZ599pqFDh+br+UePHpWbm5v8/f0LdPy/yv1FW6pUKVvb6tWr9cADD6hhw4YaMWKEihUrZgvgP/zwg23m5ZlnntHChQv13HPPKTQ0VH/88YfWr1+vxMRENWjQwDZeamqq2rdvr27duqlnz56aP3++nn32WXl6eurJJ5+UJF24cEEtW7ZUUlKSnnvuOVWpUkULFixQ7969lZaWZvfHgHQ1tJ49e1b9+vWTxWLRuHHj1KVLFx09etQ20/Twww9r3759ev755xUcHKyUlBStWLFCP//8s4KDgyVJn376qSIjI9WuXTuNHTtW58+fV3x8vJo2baqdO3fa+uVnLBQhBmAS6enphiSjc+fO+X7O+fPn87S1a9fOqFq1qm07LS3N8PHxMe6++27jwoULdn1zcnJs/92iRQtDkvHJJ5/Y2rKysozy5csbDz/8sK3t008/NYoVK2b88MMPdmNNnTrVkGRs2LDBMAzDSEhIMCQZ/fv3t+vXq1cvQ5IxYsSIvz23999/35Bk7Nmzx649NDTUuPfee23bYWFhRocOHf52rGtZs2aNIclYsGCB8fXXXxsWi8X4+eefDcMwjKFDh9pewxYtWhi1a9e2PS/3vPr27Ws33gsvvGBIMlavXm0YhmGkpKQYnp6eRocOHexe55deesmQZERGRtraXn/9dcPb29s4dOiQ3ZjDhw833NzcbHUZhpGv184wDOPSpUtGmTJljJdfftnW1qtXLyMsLCxP3xYtWhg1a9Y0Tp06ZZw6dcpITEw0YmJiDElGp06dbP0iIyMNb2/vGx57xIgRhiTj4MGDxqlTp4zk5GRj2rRphpeXlxEQEGCcO3fOMIyr//6qV69utGvXzu41On/+vFGlShWjbdu2tjY/Pz8jOjr6b4+b+294/PjxtrasrCyjfv36RtmyZY1Lly4ZhmEYEydONCQZs2bNsnu9wsPDjZIlSxoZGRmGYRjGsWPHDElGmTJljDNnztj6fvnll4YkY8mSJYZhGEZqaqohyXjrrbeuW9vZs2cNf39/46mnnrJrP3nypOHn52drz89YKFq49APTyMjIkCT5+Pjk+zleXl62/05PT9fp06fVokULHT16VOnp6ZKkFStW6OzZsxo+fHiehaEWi8Vuu2TJknbrFDw9PdW4cWMdPXrU1rZgwQLVqlVLNWvW1OnTp22Pe++9V5K0Zs0aSdK3334rSYqJibE7xsCBA/N1bl26dJG7u7vmzZtna9u7d6/279+v7t2729r8/f21b98+HT58OF/jXst9992n0qVLa+7cuTIMQ3PnzlXPnj2v2Tf3vAYPHmzXPmTIEEmyrSFauXKlLl26pOeff97udb7W+S9YsEDNmjVTqVKl7F7TNm3aKDs7W+vWrXP4nJYuXao//vjD7jx69uxpmy36qwMHDiggIEABAQGqVauW3n33XXXo0EHTpk1z+Ni57rjjDgUEBCg4OFhPPvmkQkJCtHTpUpUoUUKSlJCQoMOHD6tXr176448/bOd97tw5tW7dWuvWrbNd+vL399fmzZt14sSJvz2mu7u7+vXrZ9v29PRUv379lJKSou3bt0u6+jMsX7683Wvj4eGhmJgYZWZm6vvvv7cbs3v37nazQLmXsHLfF15eXvL09NTatWuvu55pxYoVSktLU8+ePe1+xm5ubrr77rtt75v8jIWihUs/MA1fX19JV6/F59eGDRs0YsQIbdq0SefPn7fbl56eLj8/P9t6gPzcYlupUqU84aVUqVJ2lz8OHz6sxMREBQQEXHOMlJQUSdJPP/2kYsWKqVq1anb777jjjhufmKTbbrtNrVu31vz58/X6669LunrZx93dXV26dLH1e+2119S5c2fVqFFDderU0f3336/HH39c9erVy9dxpKu/pLp27ao5c+aocePGOn78uN0lsT/LPa+QkBC79vLly8vf318//fSTrZ8kVa9e3a5fQECA3S896eprunv37hu+po6YNWuWqlSpIqvVqqSkJElStWrVVKJECc2ePVujR4+26x8cHKwPP/zQditz9erVVbZsWYeP+2eLFi2Sr6+vTp06pUmTJunYsWN24To3XEZGRl53jPT0dJUqVUrjxo1TZGSkAgMD1bBhQ7Vv315PPPGEqlatate/YsWK8vb2tmurUaOGpKuXnu655x799NNPql69uooVs/9btVatWpL+72eXq3LlynbbuT+/3CBhtVo1duxYDRkyROXKldM999yjjh076oknnlD58uXtzjU30P9V7vs/P2OhaCGowDR8fX1VsWJF7d27N1/9jxw5otatW6tmzZqaMGGCAgMD5enpqW+//VbvvPNOgRZhXmvhqHR1UWiunJwc1a1bVxMmTLhm38DAQIePez09evRQVFSUEhISVL9+fc2fP1+tW7e2uzunefPmOnLkiL788kt99913+uijj/TOO+9o6tSp173D5Vp69eqlqVOnauTIkQoLC1NoaOjf9v9roPsncnJy1LZtWw0bNuya+3N/0eZXRkaGlixZoosXL+YJStLVNRdvvvmm3Tl4e3urTZs2jhV+A82bN7f9rDp16qS6devq0Ucf1fbt21WsWDHbv9G33npL9evXv+YYuZ/b0q1bNzVr1kyff/65vvvuO7311lsaO3asFi9ebFs/VVjy874YOHCgOnXqpC+++ELLly/XK6+8ori4OK1evVp33nmn7Vw//fTTawYOd3f3fI+FooWgAlPp2LGjPvjgA23atEnh4eF/23fJkiXKysrSV199ZfcXX+4Ucq7cGY29e/fmmQUoiGrVqmnXrl1q3br13/6yDgoKUk5Ojo4cOWI3i3Lw4MF8H+uhhx5Sv379bJd/Dh06pNjY2Dz9SpcuraioKEVFRSkzM1PNmzfXyJEjHQoqTZs2VeXKlbV27VqNHTv2hud1+PBh21/g0tU7ldLS0hQUFGTrJ139S/rPf/WfOnUqz5R+tWrVlJmZ6bSgsHjxYl28eFHx8fF2oU66+vr/97//1YYNG9S0aVOnHC8/SpYsqREjRigqKkrz589Xjx49bP82fX1983XuFSpUUP/+/dW/f3+lpKSoQYMGevPNN+2CyokTJ3Tu3Dm7WZVDhw5Jkm0halBQkHbv3q2cnBy7WZUDBw7Y9hdEtWrVNGTIEA0ZMkSHDx9W/fr1NX78eM2aNct2rmXLls3Xuf7dWChaWKMCUxk2bJi8vb3Vt29f/f7773n2HzlyRP/73/8k/d9feX/+qy49PV3Tp0+3e859990nHx8fxcXF6eLFi3b7/vzc/OrWrZt+/fVXffjhh3n2XbhwQefOnZMk2y+PSZMm2fVx5A4Wf39/tWvXTvPnz9fcuXPl6emphx56yK7PH3/8YbddsmRJhYSE5LlV+EYsFosmTZqkESNG6PHHH79uv/bt20vKex65M0y5dzW1adNGHh4eevfdd+1e52udf7du3bRp0yYtX748z760tDRduXLFoXOZNWuWqlatqmeeeUaPPPKI3eOFF15QyZIlNXv2bIfGdIZHH31UlSpVsgXBhg0bqlq1anr77beVmZmZp3/ubdHZ2dm2NVe5ypYtq4oVK+b5OV+5ckXvv/++bfvSpUt6//33FRAQoIYNG0q6+jM8efKk3fqnK1eu6N1331XJkiXVokULh87r/Pnzed5b1apVk4+Pj62+du3aydfXV6NHj9bly5eve675GQtFCzMqMJVq1appzpw56t69u2rVqmX3ybQbN2603UIpXQ0gnp6e6tSpk/r166fMzEx9+OGHKlu2rH777TfbmL6+vnrnnXfUt29fNWrUSL169VKpUqW0a9cunT9/Ps9nR9zI448/rvnz5+uZZ57RmjVrFBERoezsbB04cEDz58/X8uXLddddd6l+/frq2bOn3nvvPaWnp6tJkyZatWqVbb1EfnXv3l2PPfaY3nvvPbVr1y7PrbKhoaFq2bKlGjZsqNKlS2vbtm2221gd1blzZ3Xu3Plv+4SFhSkyMlIffPCB0tLS1KJFC23ZskUzZ87UQw89pFatWkm6uhblhRdeUFxcnDp27Kj27dtr586dWrp0aZ5ZjqFDh+qrr75Sx44d1bt3bzVs2FDnzp3Tnj17tHDhQiUnJ+d5zvWcOHFCa9asybOIOZfValW7du20YMECTZo0qUAf5FZQHh4eGjBggIYOHaply5bp/vvv10cffaQHHnhAtWvXVlRUlG6//Xb9+uuvWrNmjXx9fbVkyRKdPXtWlSpV0iOPPKKwsDCVLFlSK1eu1NatWzV+/Hi7Y1SsWFFjx45VcnKyatSooXnz5ikhIUEffPCB7Vyffvppvf/+++rdu7e2b9+u4OBgLVy4UBs2bNDEiRMdWtAuXZ2xad26tbp166bQ0FC5u7vr888/1++//64ePXpIuvo+jI+P1+OPP64GDRqoR48eCggI0M8//6xvvvlGERERmjx5cr7GQhHjyluOgOs5dOiQ8dRTTxnBwcGGp6en4ePjY0RERBjvvvuucfHiRVu/r776yqhXr55RvHhxIzg42Bg7dqwxbdo0Q5Jx7NgxuzG/+uoro0mTJoaXl5fh6+trNG7c2Pjss89s+/96G26uyMhIIygoyK7t0qVLxtixY43atWsbVqvVKFWqlNGwYUNj1KhRRnp6uq3fhQsXjJiYGKNMmTKGt7e30alTJ+P48eP5vsXWMAwjIyPD8PLyynM7aa433njDaNy4seHv7294eXkZNWvWNN58803brajX8+fbk//OtV6Xy5cvG6NGjTKqVKlieHh4GIGBgUZsbKzdz8YwDCM7O9sYNWqUUaFCBcPLy8to2bKlsXfvXiMoKMju9mTDuHr7amxsrBESEmJ4enoat912m9GkSRPj7bfftjuXG71248ePNyQZq1atum6fGTNmGJKML7/88rrneC2O3p586tSpPPvS09MNPz8/o0WLFra2nTt3Gl26dDHKlCljWK1WIygoyOjWrZvtHLKysoyhQ4caYWFhho+Pj+Ht7W2EhYUZ7733nt3Yueexbds2Izw83ChevLgRFBRkTJ48OU8dv//+uxEVFWXcdttthqenp1G3bl1j+vTpdn1yb0++1q3Cf/45nD592oiOjjZq1qxpeHt7G35+fsbdd99tzJ8/P8/z1qxZY7Rr187w8/MzihcvblSrVs3o3bu3sW3bNofHQtFgMYwCzH0DAEynZcuWOn36dL4XpAP/BqxRAQAApkVQAQAApkVQAQAApsUaFQAAYFrMqAAAANMiqAAAANP6V3/gW05Ojk6cOCEfHx+nfu8IAAAoPIZh6OzZs6pYsWKeL8f8q391UDlx4oRTvwAOAADcPMePH1elSpX+ts+/Oqjkfszz8ePHbV8RDgAAzC0jI0OBgYH5+rqGf3VQyb3c4+vrS1ABAOBfJj/LNlhMCwAATIugAgAATIugAgAATIugAgAATIugAgAATIugAgAATIugAgAATIugAgAATIugAgAATIugAgAATMvlQeXXX3/VY489pjJlysjLy0t169bVtm3bXF0WAAAwAZd+109qaqoiIiLUqlUrLV26VAEBATp8+LBKlSrlyrIAAIBJuDSojB07VoGBgZo+fbqtrUqVKi6sCAAAmIlLL/189dVXuuuuu9S1a1eVLVtWd955pz788MPr9s/KylJGRobdAwAA3LpcOqNy9OhRxcfHa/DgwXrppZe0detWxcTEyNPTU5GRkXn6x8XFadSoUS6oFMCtKnj4N64uATC15DEdXHp8i2EYhqsO7unpqbvuuksbN260tcXExGjr1q3atGlTnv5ZWVnKysqybWdkZCgwMFDp6eny9fW9KTUDuLUQVIC/VxhBJSMjQ35+fvn6/e3SSz8VKlRQaGioXVutWrX0888/X7O/1WqVr6+v3QMAANy6XBpUIiIidPDgQbu2Q4cOKSgoyEUVAQAAM3FpUBk0aJB+/PFHjR49WklJSZozZ44++OADRUdHu7IsAABgEi4NKo0aNdLnn3+uzz77THXq1NHrr7+uiRMn6tFHH3VlWQAAwCRcetePJHXs2FEdO3Z0dRkAAMCEXP4R+gAAANdDUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKZFUAEAAKbl0qAycuRIWSwWu0fNmjVdWRIAADARd1cXULt2ba1cudK27e7u8pIAAIBJuDwVuLu7q3z58q4uAwAAmJDL16gcPnxYFStWVNWqVfXoo4/q559/vm7frKwsZWRk2D0AAMCty6VB5e6779aMGTO0bNkyxcfH69ixY2rWrJnOnj17zf5xcXHy8/OzPQIDA29yxQAA4GayGIZhuLqIXGlpaQoKCtKECRPUp0+fPPuzsrKUlZVl287IyFBgYKDS09Pl6+t7M0sFcIsIHv6Nq0sATC15TAenj5mRkSE/P798/f52+RqVP/P391eNGjWUlJR0zf1Wq1VWq/UmVwUAAFzF5WtU/iwzM1NHjhxRhQoVXF0KAAAwAZcGlRdeeEHff/+9kpOTtXHjRv3nP/+Rm5ubevbs6cqyAACASbj00s8vv/yinj176o8//lBAQICaNm2qH3/8UQEBAa4sCwAAmIRLg8rcuXNdeXgAAGByplqjAgAA8GcEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoEFQAAYFoOB5Vly5Zp/fr1tu0pU6aofv366tWrl1JTU51aHAAAKNocDipDhw5VRkaGJGnPnj0aMmSI2rdvr2PHjmnw4MFOLxAAABRd7o4+4dixYwoNDZUkLVq0SB07dtTo0aO1Y8cOtW/f3ukFAgCAosvhGRVPT0+dP39ekrRy5Urdd999kqTSpUvbZloAAACcweEZlaZNm2rw4MGKiIjQli1bNG/ePEnSoUOHVKlSJacXCAAAii6HZ1QmT54sd3d3LVy4UPHx8br99tslSUuXLtX999/v9AIBAEDR5fCMSuXKlfX111/naX/nnXecUhAAAECuAn2OypEjR/Tf//5XPXv2VEpKiqSrMyr79u1zanEAAKBocziofP/996pbt642b96sxYsXKzMzU5K0a9cujRgxwukFAgCAosvhoDJ8+HC98cYbWrFihTw9PW3t9957r3788UenFgcAAIo2h4PKnj179J///CdPe9myZXX69GmnFAUAACAVIKj4+/vrt99+y9O+c+dO2x1AAAAAzuBwUOnRo4defPFFnTx5UhaLRTk5OdqwYYNeeOEFPfHEE4VRIwAAKKIcDiqjR49WzZo1FRgYqMzMTIWGhqp58+Zq0qSJ/vvf/xZGjQAAoIhy+HNUPD099eGHH+rVV1/Vnj17lJmZqTvvvFPVq1cvjPoAAEAR5nBQyRUYGKjAwEBlZ2drz549Sk1NValSpZxZGwAAKOIcvvQzcOBAffzxx5Kk7OxstWjRQg0aNFBgYKDWrl3r7PoAAEAR5nBQWbhwocLCwiRJS5Ys0dGjR3XgwAENGjRIL7/8stMLBAAARZfDQeX06dMqX768JOnbb79Vt27dVKNGDT355JPas2eP0wsEAABFl8NBpVy5ctq/f7+ys7O1bNkytW3bVpJ0/vx5ubm5Ob1AAABQdDkcVKKiotStWzfVqVNHFotFbdq0kSRt3rxZNWvWLHAhY8aMkcVi0cCBAws8BgAAuLU4fNfPyJEjVadOHR0/flxdu3aV1WqVJLm5uWn48OEFKmLr1q16//33Va9evQI9HwAA3JoKdHvyI488kqctMjKyQAVkZmbq0Ucf1Ycffqg33nijQGMAAIBbU4GCyqpVq7Rq1SqlpKQoJyfHbt+0adMcGis6OlodOnRQmzZtbhhUsrKylJWVZdvOyMhw6FgAAODfxeGgMmrUKL322mu66667VKFCBVkslgIffO7cudqxY4e2bt2ar/5xcXEaNWpUgY8HAAD+XRwOKlOnTtWMGTP0+OOP/6MDHz9+XAMGDNCKFStUvHjxfD0nNjZWgwcPtm1nZGQoMDDwH9UBAADMy+GgcunSJTVp0uQfH3j79u1KSUlRgwYNbG3Z2dlat26dJk+erKysrDy3O1utVtviXQAAcOtz+Pbkvn37as6cOf/4wK1bt9aePXuUkJBge9x111169NFHlZCQwGeyAAAAx2dULl68qA8++EArV65UvXr15OHhYbd/woQJ+RrHx8dHderUsWvz9vZWmTJl8rQDAICiyeGgsnv3btWvX1+StHfvXrt9/2RhLQAAwF85HFTWrFlTGHVIEt++DAAA7Di8RuXPfvnlF/3yyy/OqgUAAMCOw0ElJydHr732mvz8/BQUFKSgoCD5+/vr9ddfz/PhbwAAAP+Ew5d+Xn75ZX388ccaM2aMIiIiJEnr16/XyJEjdfHiRb355ptOLxIAABRNDgeVmTNn6qOPPtKDDz5oa6tXr55uv/129e/fn6ACAACcxuFLP2fOnFHNmjXztNesWVNnzpxxSlEAAABSAYJKWFiYJk+enKd98uTJCgsLc0pRAAAAUgEu/YwbN04dOnTQypUrFR4eLknatGmTjh8/rm+//dbpBQIAgKLL4RmVFi1a6NChQ/rPf/6jtLQ0paWlqUuXLjp48KCaNWtWGDUCAIAiyuEZFUmqWLEii2YBAEChK1BQSU1N1ccff6zExERJUmhoqKKiolS6dGmnFgcAAIo2hy/9rFu3TsHBwZo0aZJSU1OVmpqqSZMmqUqVKlq3bl1h1AgAAIooh2dUoqOj1b17d8XHx8vNzU2SlJ2drf79+ys6Olp79uxxepEAAKBocnhGJSkpSUOGDLGFFElyc3PT4MGDlZSU5NTiAABA0eZwUGnQoIFtbcqfJSYm8jkqAADAqRy+9BMTE6MBAwYoKSlJ99xzjyTpxx9/1JQpUzRmzBjt3r3b1rdevXrOqxQAABQ5FsMwDEeeUKzY30/CWCwWGYYhi8Wi7Ozsf1TcjWRkZMjPz0/p6eny9fUt1GMBuDUFD//G1SUAppY8poPTx3Tk97fDMyrHjh0rcGEAAACOcDioBAUFFUYdAAAAeTi8mHbmzJn65pv/myodNmyY/P391aRJE/30009OLQ4AABRtDgeV0aNHy8vLS9LVLyOcPHmyxo0bp9tuu02DBg1yeoEAAKDocvjSz/HjxxUSEiJJ+uKLL/TII4/o6aefVkREhFq2bOns+gAAQBHm8IxKyZIl9ccff0iSvvvuO7Vt21aSVLx4cV24cMG51QEAgCLN4RmVtm3bqm/fvrrzzjt16NAhtW/fXpK0b98+BQcHO7s+AABQhDk8ozJlyhSFh4fr1KlTWrRokcqUKSNJ2r59u3r27On0AgEAQNHl8IyKv7+/Jk+enKd91KhRTikIAAAgl8MzKpL0ww8/6LHHHlOTJk3066+/SpI+/fRTrV+/3qnFAQCAos3hoLJo0SK1a9dOXl5e2rFjh7KysiRJ6enpGj16tNMLBAAARZfDQeWNN97Q1KlT9eGHH8rDw8PWHhERoR07dji1OAAAULQ5HFQOHjyo5s2b52n38/NTWlqaM2oCAACQVICgUr58eSUlJeVpX79+vapWreqUogAAAKQCBJWnnnpKAwYM0ObNm2WxWHTixAnNnj1bL7zwgp599tnCqBEAABRRDt+ePHz4cOXk5Kh169Y6f/68mjdvLqvVqhdeeEHPP/98YdQIAACKKIeDisVi0csvv6yhQ4cqKSlJmZmZCg0NVcmSJXXhwgXbFxYCAAD8UwX6HBVJ8vT0VGhoqBo3biwPDw9NmDBBVapUcWZtAACgiMt3UMnKylJsbKzuuusuNWnSRF988YUkafr06apSpYreeecdDRo0qLDqBAAARVC+L/28+uqrev/999WmTRtt3LhRXbt2VVRUlH788UdNmDBBXbt2lZubW2HWCgAAiph8B5UFCxbok08+0YMPPqi9e/eqXr16unLlinbt2iWLxVKYNQIAgCIq35d+fvnlFzVs2FCSVKdOHVmtVg0aNIiQAgAACk2+g0p2drY8PT1t2+7u7ipZsmShFAUAACA5cOnHMAz17t1bVqtVknTx4kU988wz8vb2tuu3ePFi51YIAACKrHwHlcjISLvtxx57zOnFAAAA/Fm+g8r06dMLsw4AAIA8CvyBbwAAAIWNoAIAAEyLoAIAAEyLoAIAAEwrX0GlQYMGSk1NlSS99tprOn/+fKEWBQAAIOUzqCQmJurcuXOSpFGjRikzM9MpB4+Pj1e9evXk6+srX19fhYeHa+nSpU4ZGwAA/Pvl6/bk+vXrKyoqSk2bNpVhGHr77bev+6m0r776ar4PXqlSJY0ZM0bVq1eXYRiaOXOmOnfurJ07d6p27dr5HgcAANyaLIZhGDfqdPDgQY0YMUJHjhzRjh07FBoaKnf3vBnHYrFox44d/6ig0qVL66233lKfPn1u2DcjI0N+fn5KT0+Xr6/vPzougKIpePg3ri4BMLXkMR2cPqYjv7/zNaNyxx13aO7cuZKkYsWKadWqVSpbtuw/r/RPsrOztWDBAp07d07h4eHX7JOVlaWsrCzbdkZGhlNrAAAA5pLvT6bNlZOT49QC9uzZo/DwcF28eFElS5bU559/rtDQ0Gv2jYuL06hRo5x6/L/DX1rA9RXGX1kA8FcFuj35yJEjev7559WmTRu1adNGMTExOnLkSIEKuOOOO5SQkKDNmzfr2WefVWRkpPbv33/NvrGxsUpPT7c9jh8/XqBjAgCAfweHg8ry5csVGhqqLVu2qF69eqpXr542b96s2rVra8WKFQ4X4OnpqZCQEDVs2FBxcXEKCwvT//73v2v2tVqttjuEch8AAODW5fCln+HDh2vQoEEaM2ZMnvYXX3xRbdu2/UcF5eTk2K1DAQAARZfDQSUxMVHz58/P0/7kk09q4sSJDo0VGxurBx54QJUrV9bZs2c1Z84crV27VsuXL3e0LAAAcAtyOKgEBAQoISFB1atXt2tPSEhw+E6glJQUPfHEE/rtt9/k5+enevXqafny5f94VgYAANwaHA4qTz31lJ5++mkdPXpUTZo0kSRt2LBBY8eO1eDBgx0a6+OPP3b08AAAoAhxOKi88sor8vHx0fjx4xUbGytJqlixokaOHKmYmBinFwgAAIouh4OKxWLRoEGDNGjQIJ09e1aS5OPj4/TCAAAAHA4qf0ZAAQAAhalAH/gGAABwMxBUAACAaRFUAACAaTkUVC5fvqzWrVvr8OHDhVUPAACAjUNBxcPDQ7t37y6sWgAAAOw4fOnnscce44PaAADATeHw7clXrlzRtGnTtHLlSjVs2FDe3t52+ydMmOC04gAAQNHmcFDZu3evGjRoIEk6dOiQ3T6LxeKcqgAAAFSAoLJmzZrCqAMAACCPAt+enJSUpOXLl+vChQuSJMMwnFYUAACAVICg8scff6h169aqUaOG2rdvr99++02S1KdPHw0ZMsTpBQIAgKLL4aAyaNAgeXh46Oeff1aJEiVs7d27d9eyZcucWhwAACjaHF6j8t1332n58uWqVKmSXXv16tX1008/Oa0wAAAAh2dUzp07ZzeTkuvMmTOyWq1OKQoAAEAqQFBp1qyZPvnkE9u2xWJRTk6Oxo0bp1atWjm1OAAAULQ5fOln3Lhxat26tbZt26ZLly5p2LBh2rdvn86cOaMNGzYURo0AAKCIcnhGpU6dOjp06JCaNm2qzp0769y5c+rSpYt27typatWqFUaNAACgiHJ4RkWS/Pz89PLLLzu7FgAAADsFCiqpqan6+OOPlZiYKEkKDQ1VVFSUSpcu7dTiAABA0ebwpZ9169YpODhYkyZNUmpqqlJTUzVp0iRVqVJF69atK4waAQBAEeXwjEp0dLS6d++u+Ph4ubm5SZKys7PVv39/RUdHa8+ePU4vEgAAFE0Oz6gkJSVpyJAhtpAiSW5ubho8eLCSkpKcWhwAACjaHA4qDRo0sK1N+bPExESFhYU5pSgAAAApn5d+du/ebfvvmJgYDRgwQElJSbrnnnskST/++KOmTJmiMWPGFE6VAACgSMpXUKlfv74sFosMw7C1DRs2LE+/Xr16qXv37s6rDgAAFGn5CirHjh0r7DoAAADyyFdQCQoKKuw6AAAA8ijQB76dOHFC69evV0pKinJycuz2xcTEOKUwAAAAh4PKjBkz1K9fP3l6eqpMmTKyWCy2fRaLhaACAACcxuGg8sorr+jVV19VbGysihVz+O5mAACAfHM4aZw/f149evQgpAAAgELncNro06ePFixYUBi1AAAA2HH40k9cXJw6duyoZcuWqW7duvLw8LDbP2HCBKcVBwAAirYCBZXly5frjjvukKQ8i2kBAACcxeGgMn78eE2bNk29e/cuhHIAAAD+j8NrVKxWqyIiIgqjFgAAADsOB5UBAwbo3XffLYxaAAAA7Dh86WfLli1avXq1vv76a9WuXTvPYtrFixc7rTgAAFC0ORxU/P391aVLl8KoBQAAwI7DQWX69OmFUQcAAEAefLwsAAAwLYdnVKpUqfK3n5dy9OjRf1QQAABALoeDysCBA+22L1++rJ07d2rZsmUaOnSos+oCAABwPKgMGDDgmu1TpkzRtm3b/nFBAAAAuZy2RuWBBx7QokWLnDUcAACA84LKwoULVbp0aYeeExcXp0aNGsnHx0dly5bVQw89pIMHDzqrJAAA8C/n8KWfO++8024xrWEYOnnypE6dOqX33nvPobG+//57RUdHq1GjRrpy5Ypeeukl3Xfffdq/f7+8vb0dLQ0AANxiHA4qDz30kN12sWLFFBAQoJYtW6pmzZoOjbVs2TK77RkzZqhs2bLavn27mjdv7mhpAADgFuNwUBkxYkRh1CFJSk9Pl6TrXkLKyspSVlaWbTsjI6PQagEAAK5nmg98y8nJ0cCBAxUREaE6depcs09cXJz8/Pxsj8DAwJtcJQAAuJnyHVSKFSsmNze3v324uzs8QWMTHR2tvXv3au7cudftExsbq/T0dNvj+PHjBT4eAAAwv3wni88///y6+zZt2qRJkyYpJyenQEU899xz+vrrr7Vu3TpVqlTpuv2sVqusVmuBjgEAAP598h1UOnfunKft4MGDGj58uJYsWaJHH31Ur732mkMHNwxDzz//vD7//HOtXbtWVapUcej5AADg1lagNSonTpzQU089pbp16+rKlStKSEjQzJkzFRQU5NA40dHRmjVrlubMmSMfHx+dPHlSJ0+e1IULFwpSFgAAuMU4FFTS09P14osvKiQkRPv27dOqVau0ZMmS6y5+vZH4+Hilp6erZcuWqlChgu0xb968Ao0HAABuLfm+9DNu3DiNHTtW5cuX12effXbNS0GOMgzjH48BAABuXfkOKsOHD5eXl5dCQkI0c+ZMzZw585r9Fi9e7LTiAABA0ZbvoPLEE0/YfXQ+AABAYct3UJkxY0YhlgEAAJCXaT6ZFgAA4K8IKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLQIKgAAwLRcGlTWrVunTp06qWLFirJYLPriiy9cWQ4AADAZlwaVc+fOKSwsTFOmTHFlGQAAwKTcXXnwBx54QA888IArSwAAACbm0qDiqKysLGVlZdm2MzIyXFgNAAAobP+qxbRxcXHy8/OzPQIDA11dEgAAKET/qqASGxur9PR02+P48eOuLgkAABSif9WlH6vVKqvV6uoyAADATfKvmlEBAABFi0tnVDIzM5WUlGTbPnbsmBISElS6dGlVrlzZhZUBAAAzcGlQ2bZtm1q1amXbHjx4sCQpMjJSM2bMcFFVAADALFwaVFq2bCnDMFxZAgAAMDHWqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMiqAAAANMyRVCZMmWKgoODVbx4cd19993asmWLq0sCAAAm4PKgMm/ePA0ePFgjRozQjh07FBYWpnbt2iklJcXVpQEAABdzeVCZMGGCnnrqKUVFRSk0NFRTp05ViRIlNG3aNFeXBgAAXMzdlQe/dOmStm/frtjYWFtbsWLF1KZNG23atClP/6ysLGVlZdm209PTJUkZGRmFUl9O1vlCGRe4FRTW++5m430O/L3CeK/njmkYxg37ujSonD59WtnZ2SpXrpxde7ly5XTgwIE8/ePi4jRq1Kg87YGBgYVWI4Br85vo6goA3AyF+V4/e/as/Pz8/raPS4OKo2JjYzV48GDbdk5Ojs6cOaMyZcrIYrG4sDIUtoyMDAUGBur48ePy9fV1dTkACgHv86LDMAydPXtWFStWvGFflwaV2267TW5ubvr999/t2n///XeVL18+T3+r1Sqr1WrX5u/vX5glwmR8fX35Hxhwi+N9XjTcaCYll0sX03p6eqphw4ZatWqVrS0nJ0erVq1SeHi4CysDAABm4PJLP4MHD1ZkZKTuuusuNW7cWBMnTtS5c+cUFRXl6tIAAICLuTyodO/eXadOndKrr76qkydPqn79+lq2bFmeBbYo2qxWq0aMGJHn0h+AWwfvc1yLxcjPvUEAAAAu4PIPfAMAALgeggoAADAtggoAADAtggoAADAtggr+VVq2bKmBAwc6fdyRI0eqfv36Th8XwD+3du1aWSwWpaWl5fs5wcHBmjhxYqHVhJuHoAKnOnnypJ5//nlVrVpVVqtVgYGB6tSpk92H+gG4tfTu3VsWi0XPPPNMnn3R0dGyWCzq3bv3zS8MtwSCCpwmOTlZDRs21OrVq/XWW29pz549WrZsmVq1aqXo6GhXlwegEAUGBmru3Lm6cOGCre3ixYuaM2eOKleu7MLK8G9HUIHT9O/fXxaLRVu2bNHDDz+sGjVqqHbt2ho8eLB+/PFHSdKECRNUt25deXt7KzAwUP3791dmZqbdOBs2bFDLli1VokQJlSpVSu3atVNqaqptf05OjoYNG6bSpUurfPnyGjlypN3z09LS1LdvXwUEBMjX11f33nuvdu3aZddnzJgxKleunHx8fNSnTx9dvHixcF4UoIho0KCBAgMDtXjxYlvb4sWLVblyZd155522tqysLMXExKhs2bIqXry4mjZtqq1bt9qN9e2336pGjRry8vJSq1atlJycnOd469evV7NmzeTl5aXAwEDFxMTo3LlzhXZ+cB2CCpzizJkzWrZsmaKjo+Xt7Z1nf+6XRxYrVkyTJk3Svn37NHPmTK1evVrDhg2z9UtISFDr1q0VGhqqTZs2af369erUqZOys7NtfWbOnClvb29t3rxZ48aN02uvvaYVK1bY9nft2lUpKSlaunSptm/frgYNGqh169Y6c+aMJGn+/PkaOXKkRo8erW3btqlChQp67733CumVAYqOJ598UtOnT7dtT5s2Lc/XoQwbNkyLFi3SzJkztWPHDoWEhKhdu3a29+fx48fVpUsXderUSQkJCerbt6+GDx9uN8aRI0d0//336+GHH9bu3bs1b948rV+/Xs8991zhnyRuPgNwgs2bNxuSjMWLFzv0vAULFhhlypSxbffs2dOIiIi4bv8WLVoYTZs2tWtr1KiR8eKLLxqGYRg//PCD4evra1y8eNGuT7Vq1Yz333/fMAzDCA8PN/r372+3/+677zbCwsIcqh3AVZGRkUbnzp2NlJQUw2q1GsnJyUZycrJRvHhx49SpU0bnzp2NyMhIIzMz0/Dw8DBmz55te+6lS5eMihUrGuPGjTMMwzBiY2ON0NBQu/FffPFFQ5KRmppqGIZh9OnTx3j66aft+vzwww9GsWLFjAsXLhiGYRhBQUHGO++8U3gnjZvG5d/1g1uDkc9vYli5cqXi4uJ04MABZWRk6MqVK7p48aLOnz+vEiVKKCEhQV27dv3bMerVq2e3XaFCBaWkpEiSdu3apczMTJUpU8auz4ULF3TkyBFJUmJiYp5Ff+Hh4VqzZk2+zgHAtQUEBKhDhw6aMWOGDMNQhw4ddNttt9n2HzlyRJcvX1ZERIStzcPDQ40bN1ZiYqKkq+/Pu+++227c8PBwu+1du3Zp9+7dmj17tq3NMAzl5OTo2LFjqlWrVmGcHlyEoAKnqF69uiwWiw4cOHDdPsnJyerYsaOeffZZvfnmmypdurTWr1+vPn366NKlSypRooS8vLxueCwPDw+7bYvFopycHElSZmamKlSooLVr1+Z5Xu7lJwCF58knn7RdgpkyZUqhHCMzM1P9+vVTTExMnn0s3L31sEYFTlG6dGm1a9dOU6ZMueaCtrS0NG3fvl05OTkaP3687rnnHtWoUUMnTpyw61evXr1/dCtzgwYNdPLkSbm7uyskJMTukfuXXa1atbR582a75+Uu9gXwz9x///26dOmSLl++rHbt2tntq1atmjw9PbVhwwZb2+XLl7V161aFhoZKuvr+3LJli93z/vr+bNCggfbv35/nPR4SEiJPT89COjO4CkEFTjNlyhRlZ2ercePGWrRokQ4fPqzExERNmjRJ4eHhCgkJ0eXLl/Xuu+/q6NGj+vTTTzV16lS7MWJjY7V161b1799fu3fv1oEDBxQfH6/Tp0/nq4Y2bdooPDxcDz30kL777jslJydr48aNevnll7Vt2zZJ0oABAzRt2jRNnz5dhw4d0ogRI7Rv3z6nvx5AUeTm5qbExETt379fbm5udvu8vb317LPPaujQoVq2bJn279+vp556SufPn1efPn0kSc8884wOHz6soUOH6uDBg5ozZ45mzJhhN86LL76ojRs36rnnnlNCQoIOHz6sL7/8ksW0tyiCCpymatWq2rFjh1q1aqUhQ4aoTp06atu2rVatWqX4+HiFhYVpwoQJGjt2rOrUqaPZs2crLi7ObowaNWrou+++065du9S4cWOFh4fryy+/lLt7/q5SWiwWffvtt2revLmioqJUo0YN9ejRQz/99JPKlSsnSerevbteeeUVDRs2TA0bNtRPP/2kZ5991umvB1BU+fr6ytfX95r7xowZo4cffliPP/64GjRooKSkJC1fvlylSpWSdPXSzaJFi/TFF18oLCxMU6dO1ejRo+3GqFevnr7//nsdOnRIzZo105133qlXX31VFStWLPRzw81nMfK7ChIAAOAmY0YFAACYFkEFAACYFkEFAACYFkEFAACYFkEFAACYFkEFAACYFkEFAACYFkEFAACYFkEFAACYFkEFAACYFkEFAACY1v8D2kVLzIqfIssAAAAASUVORK5CYII=\n" + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Cached Responses: 3\n", + "Model Responses: 6\n" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/litellm/tests/test_cache.py b/litellm/tests/test_cache.py index 05004d1a6..d97d19142 100644 --- a/litellm/tests/test_cache.py +++ b/litellm/tests/test_cache.py @@ -1,44 +1,44 @@ -import sys, os -import traceback -from dotenv import load_dotenv -load_dotenv() -import os +# import sys, os +# import traceback +# from dotenv import load_dotenv +# load_dotenv() +# import os -sys.path.insert(0, os.path.abspath('../..')) # Adds the parent directory to the system path -import pytest -import litellm +# sys.path.insert(0, os.path.abspath('../..')) # Adds the parent directory to the system path +# import pytest +# import litellm -# set cache to True -litellm.cache = True -litellm.cache_similarity_threshold = 0.5 +# # set cache to True +# litellm.cache = True +# litellm.cache_similarity_threshold = 0.5 -user_message = "Hello, whats the weather in San Francisco??" -messages = [{ "content": user_message,"role": "user"}] +# user_message = "Hello, whats the weather in San Francisco??" +# messages = [{ "content": user_message,"role": "user"}] -def test_completion_with_cache_gpt4(): - try: - # in this test make the same call twice, measure the response time - # the 2nd response time should be less than half of the first, ensuring that the cache is working - import time - start = time.time() - print(litellm.cache) - response = litellm.completion(model="gpt-4", messages=messages) - end = time.time() - first_call_time = end-start - print(f"first call: {first_call_time}") +# def test_completion_with_cache_gpt4(): +# try: +# # in this test make the same call twice, measure the response time +# # the 2nd response time should be less than half of the first, ensuring that the cache is working +# import time +# start = time.time() +# print(litellm.cache) +# response = litellm.completion(model="gpt-4", messages=messages) +# end = time.time() +# first_call_time = end-start +# print(f"first call: {first_call_time}") - start = time.time() - response = litellm.completion(model="gpt-4", messages=messages) - end = time.time() - second_call_time = end-start - print(f"second call: {second_call_time}") +# start = time.time() +# response = litellm.completion(model="gpt-4", messages=messages) +# end = time.time() +# second_call_time = end-start +# print(f"second call: {second_call_time}") - if second_call_time > 1: - # the 2nd call should be less than 1s - pytest.fail(f"Cache is not working") - # Add any assertions here to check the response - print(response) - except Exception as e: - pytest.fail(f"Error occurred: {e}") +# if second_call_time > 1: +# # the 2nd call should be less than 1s +# pytest.fail(f"Cache is not working") +# # Add any assertions here to check the response +# print(response) +# except Exception as e: +# pytest.fail(f"Error occurred: {e}") -litellm.cache = False \ No newline at end of file +# litellm.cache = False \ No newline at end of file