diff --git a/cookbook/logging_observability/LiteLLM_Lunary.ipynb b/cookbook/logging_observability/LiteLLM_Lunary.ipynb new file mode 100644 index 000000000..fbf9a365a --- /dev/null +++ b/cookbook/logging_observability/LiteLLM_Lunary.ipynb @@ -0,0 +1,348 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "4FbDOmcj2VkM" + }, + "source": [ + "## Use LiteLLM with Langfuse\n", + "https://docs.litellm.ai/docs/observability/langfuse_integration" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "21W8Woog26Ns" + }, + "source": [ + "## Install Dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xrjKLBxhxu2L" + }, + "outputs": [], + "source": [ + "%pip install litellm lunary" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jHEu-TjZ29PJ" + }, + "source": [ + "## Set Env Variables" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "QWd9rTysxsWO" + }, + "outputs": [], + "source": [ + "import litellm\n", + "from litellm import completion\n", + "import os\n", + "\n", + "# from https://app.lunary.ai/\n", + "os.environ[\"LUNARY_PUBLIC_KEY\"] = \"a9ea1ae5-0727-4aac-8cda-10d1d4a7dd88\"\n", + "\n", + "\n", + "# LLM provider keys\n", + "# You can use any of the litellm supported providers: https://docs.litellm.ai/docs/providers\n", + "os.environ['OPENAI_API_KEY'] = \"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\"\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NodQl0hp3Lma" + }, + "source": [ + "## Set Lunary as a callback for sending data\n", + "## OpenAI completion call" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vNAuwJY1yp_F", + "outputId": "c3a71e26-13f5-4379-fac9-409290ba79bb" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))]ModelResponse(id='chatcmpl-8xIWykI0GiJSmYtXYuB8Z363kpIBm', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))], created=1709143276, model='gpt-3.5-turbo-0125', object='chat.completion', system_fingerprint='fp_86156a94a0', usage=Usage(completion_tokens=9, prompt_tokens=15, total_tokens=24))\n", + "\n", + "[Lunary] Add event: {\n", + " \"event\": \"start\",\n", + " \"type\": \"llm\",\n", + " \"name\": \"gpt-3.5-turbo\",\n", + " \"runId\": \"a363776a-bd07-4474-bce2-193067f01b2e\",\n", + " \"timestamp\": \"2024-02-28T18:01:15.188153+00:00\",\n", + " \"input\": {\n", + " \"role\": \"user\",\n", + " \"content\": \"Hi \\ud83d\\udc4b - i'm openai\"\n", + " },\n", + " \"extra\": {},\n", + " \"runtime\": \"litellm\",\n", + " \"metadata\": {}\n", + "}\n", + "\n", + "\n", + "[Lunary] Add event: {\n", + " \"event\": \"end\",\n", + " \"type\": \"llm\",\n", + " \"runId\": \"a363776a-bd07-4474-bce2-193067f01b2e\",\n", + " \"timestamp\": \"2024-02-28T18:01:16.846581+00:00\",\n", + " \"output\": {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Hello! How can I assist you today?\"\n", + " },\n", + " \"runtime\": \"litellm\",\n", + " \"tokensUsage\": {\n", + " \"completion\": 9,\n", + " \"prompt\": 15\n", + " }\n", + "}\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "--- Logging error ---\n", + "Traceback (most recent call last):\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 537, in _make_request\n", + " response = conn.getresponse()\n", + " ^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connection.py\", line 466, in getresponse\n", + " httplib_response = super().getresponse()\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 1423, in getresponse\n", + " response.begin()\n", + " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 331, in begin\n", + " version, status, reason = self._read_status()\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/http/client.py\", line 292, in _read_status\n", + " line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/socket.py\", line 707, in readinto\n", + " return self._sock.recv_into(b)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^\n", + "TimeoutError: timed out\n", + "\n", + "The above exception was the direct cause of the following exception:\n", + "\n", + "Traceback (most recent call last):\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/adapters.py\", line 486, in send\n", + " resp = conn.urlopen(\n", + " ^^^^^^^^^^^^^\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 847, in urlopen\n", + " retries = retries.increment(\n", + " ^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/util/retry.py\", line 470, in increment\n", + " raise reraise(type(error), error, _stacktrace)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/util/util.py\", line 39, in reraise\n", + " raise value\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 793, in urlopen\n", + " response = self._make_request(\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 539, in _make_request\n", + " self._raise_timeout(err=e, url=url, timeout_value=read_timeout)\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 370, in _raise_timeout\n", + " raise ReadTimeoutError(\n", + "urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\n", + "\n", + "During handling of the above exception, another exception occurred:\n", + "\n", + "Traceback (most recent call last):\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 59, in send_batch\n", + " response = requests.post(\n", + " ^^^^^^^^^^^^^^\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/api.py\", line 115, in post\n", + " return request(\"post\", url, data=data, json=json, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/api.py\", line 59, in request\n", + " return session.request(method=method, url=url, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/sessions.py\", line 589, in request\n", + " resp = self.send(prep, **send_kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/sessions.py\", line 703, in send\n", + " r = adapter.send(request, **kwargs)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/requests/adapters.py\", line 532, in send\n", + " raise ReadTimeout(e, request=request)\n", + "requests.exceptions.ReadTimeout: HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\n", + "\n", + "During handling of the above exception, another exception occurred:\n", + "\n", + "Traceback (most recent call last):\n", + " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 1160, in emit\n", + " msg = self.format(record)\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 999, in format\n", + " return fmt.format(record)\n", + " ^^^^^^^^^^^^^^^^^^\n", + " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 703, in format\n", + " record.message = record.getMessage()\n", + " ^^^^^^^^^^^^^^^^^^^\n", + " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/logging/__init__.py\", line 392, in getMessage\n", + " msg = msg % self.args\n", + " ~~~~^~~~~~~~~~~\n", + "TypeError: not all arguments converted during string formatting\n", + "Call stack:\n", + " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py\", line 1030, in _bootstrap\n", + " self._bootstrap_inner()\n", + " File \"/opt/homebrew/Cellar/python@3.12/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/threading.py\", line 1073, in _bootstrap_inner\n", + " self.run()\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 24, in run\n", + " self.send_batch()\n", + " File \"/Users/vince/Library/Caches/pypoetry/virtualenvs/litellm-7WKnDWGw-py3.12/lib/python3.12/site-packages/lunary/consumer.py\", line 73, in send_batch\n", + " logging.error(\"[Lunary] Error sending events\", e)\n", + "Message: '[Lunary] Error sending events'\n", + "Arguments: (ReadTimeout(ReadTimeoutError(\"HTTPConnectionPool(host='localhost', port=3333): Read timed out. (read timeout=5)\")),)\n" + ] + } + ], + "source": [ + "# set langfuse as a callback, litellm will send the data to langfuse\n", + "litellm.success_callback = [\"lunary\"]\n", + "\n", + "# openai call\n", + "response = completion(\n", + " model=\"gpt-3.5-turbo\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"Hi 👋 - i'm openai\"}\n", + " ]\n", + ")\n", + "\n", + "print(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Using LiteLLM with Lunary Templates\n", + "\n", + "You can use LiteLLM seamlessly with Lunary templates to manage your prompts and completions.\n", + "\n", + "Assuming you have created a template \"test-template\" with a variable \"question\", you can use it like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "2PMSLc_FziJL", + "outputId": "1c37605e-b406-4ffc-aafd-e1983489c6be" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))]ModelResponse(id='chatcmpl-8xIXegwpudg4YKnLB6pmpFGXqTHcH', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Hello! How can I assist you today?', role='assistant'))], created=1709143318, model='gpt-4-0125-preview', object='chat.completion', system_fingerprint='fp_c8aa5a06d6', usage=Usage(completion_tokens=9, prompt_tokens=21, total_tokens=30))\n", + "\n", + "[Lunary] Add event: {\n", + " \"event\": \"start\",\n", + " \"type\": \"llm\",\n", + " \"name\": \"gpt-4-turbo-preview\",\n", + " \"runId\": \"3a5b698d-cb55-4b3b-ab6d-04d2b99e40cb\",\n", + " \"timestamp\": \"2024-02-28T18:01:56.746249+00:00\",\n", + " \"input\": [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are an helpful assistant.\"\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Hi! Hello!\"\n", + " }\n", + " ],\n", + " \"extra\": {\n", + " \"temperature\": 1,\n", + " \"max_tokens\": 100\n", + " },\n", + " \"runtime\": \"litellm\",\n", + " \"metadata\": {}\n", + "}\n", + "\n", + "\n", + "[Lunary] Add event: {\n", + " \"event\": \"end\",\n", + " \"type\": \"llm\",\n", + " \"runId\": \"3a5b698d-cb55-4b3b-ab6d-04d2b99e40cb\",\n", + " \"timestamp\": \"2024-02-28T18:01:58.741244+00:00\",\n", + " \"output\": {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Hello! How can I assist you today?\"\n", + " },\n", + " \"runtime\": \"litellm\",\n", + " \"tokensUsage\": {\n", + " \"completion\": 9,\n", + " \"prompt\": 21\n", + " }\n", + "}\n", + "\n", + "\n" + ] + } + ], + "source": [ + "import lunary\n", + "from litellm import completion\n", + "\n", + "template = lunary.render_template(\"test-template\", {\"question\": \"Hello!\"})\n", + "\n", + "response = completion(**template)\n", + "\n", + "print(response)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/litellm/integrations/lunary.py b/litellm/integrations/lunary.py index 2cd65fada..7fb0cca3c 100644 --- a/litellm/integrations/lunary.py +++ b/litellm/integrations/lunary.py @@ -98,9 +98,12 @@ class LunaryLogger: ) tags = litellm_params.pop("tags", None) or [] + template_id = litellm_params.pop("templateId", None) + + print(litellm_params) + print(template_id) if extra: - template_id = extra.pop("templateId", None) extra.pop("extra_body", None) extra.pop("user", None) @@ -119,8 +122,6 @@ class LunaryLogger: except: pass - - if response_obj: usage = ( parse_usage(response_obj["usage"]) @@ -130,7 +131,6 @@ class LunaryLogger: output = response_obj["choices"] if "choices" in response_obj else None - print(output) else: usage = None output = None @@ -140,8 +140,6 @@ class LunaryLogger: else: error_obj = None - - self.lunary_client.track_event( type, "start", diff --git a/litellm/main.py b/litellm/main.py index 1a6c8e178..8f18fec6e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -558,11 +558,12 @@ def completion( "output_cost_per_second", "hf_model_name", "model_info", + "templateId", "proxy_server_request", "preset_cache_key", "caching_groups", "ttl", - "cache", + "cache" ] default_params = openai_params + litellm_params non_default_params = { diff --git a/litellm/tests/test_lunary.py b/litellm/tests/test_lunary.py index 964843bfd..e719c5ce3 100644 --- a/litellm/tests/test_lunary.py +++ b/litellm/tests/test_lunary.py @@ -26,9 +26,21 @@ def test_lunary_logging(): print(e) -test_lunary_logging() +# test_lunary_logging() +def test_lunary_template(): + import lunary + + try: + template = lunary.render_template("test-template", {"question": "Hello!"}) + response = completion(**template) + print(response) + except Exception as e: + print(e) + +test_lunary_template() + def test_lunary_logging_with_metadata(): try: response = completion( @@ -68,4 +80,4 @@ def test_lunary_logging_with_streaming_and_metadata(): print(e) -test_lunary_logging_with_streaming_and_metadata() +#test_lunary_logging_with_streaming_and_metadata()