forked from phoenix/litellm-mirror
348 lines
16 KiB
Text
Vendored
348 lines
16 KiB
Text
Vendored
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"id": "4FbDOmcj2VkM"
|
|
},
|
|
"source": [
|
|
"## Use LiteLLM with Langfuse\n",
|
|
"https://docs.litellm.ai/docs/observability/langfuse_integration"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"id": "21W8Woog26Ns"
|
|
},
|
|
"source": [
|
|
"## Install Dependencies"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"id": "xrjKLBxhxu2L"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"%pip install litellm lunary"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"id": "jHEu-TjZ29PJ"
|
|
},
|
|
"source": [
|
|
"## Set Env Variables"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {
|
|
"id": "QWd9rTysxsWO"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import litellm\n",
|
|
"from litellm import completion\n",
|
|
"import os\n",
|
|
"\n",
|
|
"# from https://app.lunary.ai/\n",
|
|
"os.environ[\"LUNARY_PUBLIC_KEY\"] = \"\"\n",
|
|
"\n",
|
|
"\n",
|
|
"# LLM provider keys\n",
|
|
"# You can use any of the litellm supported providers: https://docs.litellm.ai/docs/providers\n",
|
|
"os.environ['OPENAI_API_KEY'] = \"\"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"id": "NodQl0hp3Lma"
|
|
},
|
|
"source": [
|
|
"## Set Lunary as a callback for sending data\n",
|
|
"## OpenAI completion call"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {
|
|
"colab": {
|
|
"base_uri": "https://localhost:8080/"
|
|
},
|
|
"id": "vNAuwJY1yp_F",
|
|
"outputId": "c3a71e26-13f5-4379-fac9-409290ba79bb"
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))]ModelResponse(id='chatcmpl-8xIWykI0GiJSmYtXYuB8Z363kpIBm', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))], created=1709143276, model='gpt-3.5-turbo-0125', object='chat.completion', system_fingerprint='fp_86156a94a0', usage=Usage(completion_tokens=9, prompt_tokens=15, total_tokens=24))\n",
|
|
"\n",
|
|
"[Lunary] Add event: {\n",
|
|
" \"event\": \"start\",\n",
|
|
" \"type\": \"llm\",\n",
|
|
" \"name\": \"gpt-3.5-turbo\",\n",
|
|
" \"runId\": \"a363776a-bd07-4474-bce2-193067f01b2e\",\n",
|
|
" \"timestamp\": \"2024-02-28T18:01:15.188153+00:00\",\n",
|
|
" \"input\": {\n",
|
|
" \"role\": \"user\",\n",
|
|
" \"content\": \"Hi \\ud83d\\udc4b - i'm openai\"\n",
|
|
" },\n",
|
|
" \"extra\": {},\n",
|
|
" \"runtime\": \"litellm\",\n",
|
|
" \"metadata\": {}\n",
|
|
"}\n",
|
|
"\n",
|
|
"\n",
|
|
"[Lunary] Add event: {\n",
|
|
" \"event\": \"end\",\n",
|
|
" \"type\": \"llm\",\n",
|
|
" \"runId\": \"a363776a-bd07-4474-bce2-193067f01b2e\",\n",
|
|
" \"timestamp\": \"2024-02-28T18:01:16.846581+00:00\",\n",
|
|
" \"output\": {\n",
|
|
" \"role\": \"assistant\",\n",
|
|
" \"content\": \"Hello! How can I assist you today?\"\n",
|
|
" },\n",
|
|
" \"runtime\": \"litellm\",\n",
|
|
" \"tokensUsage\": {\n",
|
|
" \"completion\": 9,\n",
|
|
" \"prompt\": 15\n",
|
|
" }\n",
|
|
"}\n",
|
|
"\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"--- Logging error ---\n",
|
|
"Traceback (most recent call last):\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 537, in _make_request\n",
|
|
" response = conn.getresponse()\n",
|
|
" ^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connection.py\", line 466, in getresponse\n",
|
|
" httplib_response = super().getresponse()\n",
|
|
" ^^^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 1423, in getresponse\n",
|
|
" response.begin()\n",
|
|
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 331, in begin\n",
|
|
" version, status, reason = self._read_status()\n",
|
|
" ^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 292, in _read_status\n",
|
|
" line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n",
|
|
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/socket.py\", line 707, in readinto\n",
|
|
" return self._sock.recv_into(b)\n",
|
|
" ^^^^^^^^^^^^^^^^^^^^^^^\n",
|
|
"TimeoutError: timed out\n",
|
|
"\n",
|
|
"The above exception was the direct cause of the following exception:\n",
|
|
"\n",
|
|
"Traceback (most recent call last):\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/adapters.py\", line 486, in send\n",
|
|
" resp = conn.urlopen(\n",
|
|
" ^^^^^^^^^^^^^\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 847, in urlopen\n",
|
|
" retries = retries.increment(\n",
|
|
" ^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/util/retry.py\", line 470, in increment\n",
|
|
" raise reraise(type(error), error, _stacktrace)\n",
|
|
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/util/util.py\", line 39, in reraise\n",
|
|
" raise value\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 793, in urlopen\n",
|
|
" response = self._make_request(\n",
|
|
" ^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 539, in _make_request\n",
|
|
" self._raise_timeout(err=e, url=url, timeout_value=read_timeout)\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 370, in _raise_timeout\n",
|
|
" raise ReadTimeoutError(\n",
|
|
"urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\n",
|
|
"\n",
|
|
"During handling of the above exception, another exception occurred:\n",
|
|
"\n",
|
|
"Traceback (most recent call last):\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 59, in send_batch\n",
|
|
" response = requests.post(\n",
|
|
" ^^^^^^^^^^^^^^\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/api.py\", line 115, in post\n",
|
|
" return request(\"post\", url, data=data, json=json, **kwargs)\n",
|
|
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/api.py\", line 59, in request\n",
|
|
" return session.request(method=method, url=url, **kwargs)\n",
|
|
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/sessions.py\", line 589, in request\n",
|
|
" resp = self.send(prep, **send_kwargs)\n",
|
|
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/sessions.py\", line 703, in send\n",
|
|
" r = adapter.send(request, **kwargs)\n",
|
|
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/adapters.py\", line 532, in send\n",
|
|
" raise ReadTimeout(e, request=request)\n",
|
|
"requests.exceptions.ReadTimeout: HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\n",
|
|
"\n",
|
|
"During handling of the above exception, another exception occurred:\n",
|
|
"\n",
|
|
"Traceback (most recent call last):\n",
|
|
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 1160, in emit\n",
|
|
" msg = self.format(record)\n",
|
|
" ^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 999, in format\n",
|
|
" return fmt.format(record)\n",
|
|
" ^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 703, in format\n",
|
|
" record.message = record.getMessage()\n",
|
|
" ^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 392, in getMessage\n",
|
|
" msg = msg % self.args\n",
|
|
" ~~~~^~~~~~~~~~~\n",
|
|
"TypeError: not all arguments converted during string formatting\n",
|
|
"Call stack:\n",
|
|
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py\", line 1030, in _bootstrap\n",
|
|
" self._bootstrap_inner()\n",
|
|
" File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py\", line 1073, in _bootstrap_inner\n",
|
|
" self.run()\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 24, in run\n",
|
|
" self.send_batch()\n",
|
|
" File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 73, in send_batch\n",
|
|
" logging.error(\"[Lunary] Error sending events\", e)\n",
|
|
"Message: '[Lunary] Error sending events'\n",
|
|
"Arguments: (ReadTimeout(ReadTimeoutError(\"HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\")),)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# set langfuse as a callback, litellm will send the data to langfuse\n",
|
|
"litellm.success_callback = [\"lunary\"]\n",
|
|
"\n",
|
|
"# openai call\n",
|
|
"response = completion(\n",
|
|
" model=\"gpt-3.5-turbo\",\n",
|
|
" messages=[\n",
|
|
" {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n",
|
|
" ]\n",
|
|
")\n",
|
|
"\n",
|
|
"print(response)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Using LiteLLM with Lunary Templates\n",
|
|
"\n",
|
|
"You can use LiteLLM seamlessly with Lunary templates to manage your prompts and completions.\n",
|
|
"\n",
|
|
"Assuming you have created a template \"test-template\" with a variable \"question\", you can use it like this:"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {
|
|
"colab": {
|
|
"base_uri": "https://localhost:8080/"
|
|
},
|
|
"id": "2PMSLc_FziJL",
|
|
"outputId": "1c37605e-b406-4ffc-aafd-e1983489c6be"
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))]ModelResponse(id='chatcmpl-8xIXegwpudg4YKnLB6pmpFGXqTHcH', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))], created=1709143318, model='gpt-4-0125-preview', object='chat.completion', system_fingerprint='fp_c8aa5a06d6', usage=Usage(completion_tokens=9, prompt_tokens=21, total_tokens=30))\n",
|
|
"\n",
|
|
"[Lunary] Add event: {\n",
|
|
" \"event\": \"start\",\n",
|
|
" \"type\": \"llm\",\n",
|
|
" \"name\": \"gpt-4-turbo-preview\",\n",
|
|
" \"runId\": \"3a5b698d-cb55-4b3b-ab6d-04d2b99e40cb\",\n",
|
|
" \"timestamp\": \"2024-02-28T18:01:56.746249+00:00\",\n",
|
|
" \"input\": [\n",
|
|
" {\n",
|
|
" \"role\": \"system\",\n",
|
|
" \"content\": \"You are an helpful assistant.\"\n",
|
|
" },\n",
|
|
" {\n",
|
|
" \"role\": \"user\",\n",
|
|
" \"content\": \"Hi! Hello!\"\n",
|
|
" }\n",
|
|
" ],\n",
|
|
" \"extra\": {\n",
|
|
" \"temperature\": 1,\n",
|
|
" \"max_tokens\": 100\n",
|
|
" },\n",
|
|
" \"runtime\": \"litellm\",\n",
|
|
" \"metadata\": {}\n",
|
|
"}\n",
|
|
"\n",
|
|
"\n",
|
|
"[Lunary] Add event: {\n",
|
|
" \"event\": \"end\",\n",
|
|
" \"type\": \"llm\",\n",
|
|
" \"runId\": \"3a5b698d-cb55-4b3b-ab6d-04d2b99e40cb\",\n",
|
|
" \"timestamp\": \"2024-02-28T18:01:58.741244+00:00\",\n",
|
|
" \"output\": {\n",
|
|
" \"role\": \"assistant\",\n",
|
|
" \"content\": \"Hello! How can I assist you today?\"\n",
|
|
" },\n",
|
|
" \"runtime\": \"litellm\",\n",
|
|
" \"tokensUsage\": {\n",
|
|
" \"completion\": 9,\n",
|
|
" \"prompt\": 21\n",
|
|
" }\n",
|
|
"}\n",
|
|
"\n",
|
|
"\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"import lunary\n",
|
|
"from litellm import completion\n",
|
|
"\n",
|
|
"template = lunary.render_template(\"test-template\", {\"question\": \"Hello!\"})\n",
|
|
"\n",
|
|
"response = completion(**template)\n",
|
|
"\n",
|
|
"print(response)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"colab": {
|
|
"provenance": []
|
|
},
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.12.2"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 0
|
|
}
|