forked from phoenix/litellm-mirror
200 lines
No EOL
632 KiB
Text
200 lines
No EOL
632 KiB
Text
{
|
|
"nbformat": 4,
|
|
"nbformat_minor": 0,
|
|
"metadata": {
|
|
"colab": {
|
|
"provenance": []
|
|
},
|
|
"kernelspec": {
|
|
"name": "python3",
|
|
"display_name": "Python 3"
|
|
},
|
|
"language_info": {
|
|
"name": "python"
|
|
}
|
|
},
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [
|
|
"# Using LiteLLM with PromptLayer\n",
|
|
"Promptlayer allows you to track requests, responses and prompts\n",
|
|
"\n",
|
|
"LiteLLM allows you to use any litellm supported model and send data to promptlayer\n",
|
|
"\n",
|
|
"Getting started docs: https://docs.litellm.ai/docs/observability/promptlayer_integration"
|
|
],
|
|
"metadata": {
|
|
"id": "9AxeMfikUw2A"
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"source": [
|
|
"!pip install litellm"
|
|
],
|
|
"metadata": {
|
|
"id": "VwgSvAcVCiJX"
|
|
},
|
|
"execution_count": null,
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"source": [
|
|
"import litellm\n",
|
|
"from litellm import completion\n",
|
|
"import os\n",
|
|
"os.environ['OPENAI_API_KEY'] = \"\"\n",
|
|
"os.environ['REPLICATE_API_TOKEN'] = \"\"\n",
|
|
"os.environ['PROMPTLAYER_API_KEY'] = \"pl_4ea2bb00a4dca1b8a70cebf2e9e11564\"\n",
|
|
"\n",
|
|
"# Set Promptlayer as a success callback\n",
|
|
"litellm.success_callback =['promptlayer']\n",
|
|
"\n"
|
|
],
|
|
"metadata": {
|
|
"id": "r8QSgKbXFhpe"
|
|
},
|
|
"execution_count": 18,
|
|
"outputs": []
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [
|
|
"## Call OpenAI with LiteLLM x PromptLayer"
|
|
],
|
|
"metadata": {
|
|
"id": "gaklMZhxVFBv"
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"source": [
|
|
"\n",
|
|
"result = completion(model=\"gpt-3.5-turbo\", messages=[{\"role\": \"user\", \"content\": \"gm this is ishaan\"}])\n",
|
|
"print(result)"
|
|
],
|
|
"metadata": {
|
|
"colab": {
|
|
"base_uri": "https://localhost:8080/"
|
|
},
|
|
"id": "NOZL7MWiTFct",
|
|
"outputId": "039af693-c1d6-40ee-a081-0a494cf27c6a"
|
|
},
|
|
"execution_count": 11,
|
|
"outputs": [
|
|
{
|
|
"output_type": "stream",
|
|
"name": "stdout",
|
|
"text": [
|
|
"LiteLLM: self.optional_params: {}\n",
|
|
"LiteLLM: logging pre call for model: gpt-3.5-turbo\n",
|
|
"LiteLLM: model call details: {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'user', 'content': 'gm this is ishaan'}], 'optional_params': {}, 'litellm_params': {'return_async': False, 'api_key': None, 'force_timeout': 600, 'logger_fn': None, 'verbose': False, 'custom_llm_provider': None, 'custom_api_base': None, 'litellm_call_id': '6e509a5a-09df-4641-8c7b-9da6a8ff88de', 'model_alias_map': {}}, 'input': [{'role': 'user', 'content': 'gm this is ishaan'}], 'api_key': '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', 'additional_args': {'headers': None, 'api_base': None}}\n",
|
|
"Get help - https://discord.com/invite/wuPM9dRgDw\n",
|
|
"LiteLLM: Logging Details: logger_fn - None | callable(logger_fn) - False\n",
|
|
"LiteLLM: Logging Details: logger_fn - None | callable(logger_fn) - False\n",
|
|
"LiteLLM: reaches promptlayer for logging!\n",
|
|
"Get help - https://discord.com/invite/wuPM9dRgDw{\n",
|
|
" \"id\": \"chatcmpl-7rtC0HCsmZm7D0UONNjmt1BocKGgq\",\n",
|
|
" \"object\": \"chat.completion\",\n",
|
|
" \"created\": 1693077900,\n",
|
|
" \"model\": \"gpt-3.5-turbo-0613\",\n",
|
|
" \"choices\": [\n",
|
|
" {\n",
|
|
" \"index\": 0,\n",
|
|
" \"message\": {\n",
|
|
" \"role\": \"assistant\",\n",
|
|
" \"content\": \"Hello Ishaan! Good morning to you. How can I assist you today?\"\n",
|
|
" },\n",
|
|
" \"finish_reason\": \"stop\"\n",
|
|
" }\n",
|
|
" ],\n",
|
|
" \"usage\": {\n",
|
|
" \"prompt_tokens\": 13,\n",
|
|
" \"completion_tokens\": 17,\n",
|
|
" \"total_tokens\": 30\n",
|
|
" }\n",
|
|
"}\n",
|
|
"\n",
|
|
"LiteLLM: Prompt Layer Logging - Enters logging function for model {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'user', 'content': 'gm this is ishaan'}], 'litellm_call_id': '6e509a5a-09df-4641-8c7b-9da6a8ff88de'}\n",
|
|
"Get help - https://discord.com/invite/wuPM9dRgDw"
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [
|
|
"## Call Replicate-CodeLlama with LiteLLM x PromptLayer"
|
|
],
|
|
"metadata": {
|
|
"id": "Qt91CjpeVJ32"
|
|
}
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"source": [
|
|
"model=\"replicate/codellama-13b:1c914d844307b0588599b8393480a3ba917b660c7e9dfae681542b5325f228db\"\n",
|
|
"\n",
|
|
"result = completion(model=model, messages=[{\"role\": \"user\", \"content\": \"gm this is ishaan\"}])\n",
|
|
"print(result)"
|
|
],
|
|
"metadata": {
|
|
"id": "dTwhEKelDy_J",
|
|
"colab": {
|
|
"base_uri": "https://localhost:8080/"
|
|
},
|
|
"outputId": "751f7883-390f-47bd-9aa4-3b1523bd1af2"
|
|
},
|
|
"execution_count": 17,
|
|
"outputs": [
|
|
{
|
|
"output_type": "stream",
|
|
"name": "stdout",
|
|
"text": [
|
|
"LiteLLM: self.optional_params: {}\n",
|
|
"LiteLLM: replicate has already been imported.\n",
|
|
"LiteLLM: logging pre call for model: replicate/codellama-13b:1c914d844307b0588599b8393480a3ba917b660c7e9dfae681542b5325f228db\n",
|
|
"LiteLLM: model call details: {'model': 'replicate/codellama-13b:1c914d844307b0588599b8393480a3ba917b660c7e9dfae681542b5325f228db', 'messages': [{'role': 'user', 'content': 'gm this is ishaan'}], 'optional_params': {}, 'litellm_params': {'return_async': False, 'api_key': None, 'force_timeout': 600, 'logger_fn': None, 'verbose': False, 'custom_llm_provider': 'replicate', 'custom_api_base': None, 'litellm_call_id': '002aef0c-5cbb-4bed-8426-1ebea81c6a8a', 'model_alias_map': {}}, 'input': 'gm this is ishaan', 'api_key': 'r8_64iIxhdtFnCfAYXFTCn85yFztJkSRPO4XlW6n', 'additional_args': {'complete_input_dict': {'prompt': 'gm this is ishaan'}, 'max_tokens': inf}}\n",
|
|
"LiteLLM: Logging Details: logger_fn - None | callable(logger_fn) - False\n",
|
|
"Get help - https://discord.com/invite/wuPM9dRgDw\n",
|
|
"LiteLLM: Logging Details: logger_fn - None | callable(logger_fn) - False\n",
|
|
"Get help - https://discord.com/invite/wuPM9dRgDw\n",
|
|
"LiteLLM: reaches promptlayer for logging!{\n",
|
|
" \"choices\": [\n",
|
|
" {\n",
|
|
" \"finish_reason\": \"stop\",\n",
|
|
" \"index\": 0,\n",
|
|
" \"message\": {\n",
|
|
" \"content\": \"', 'Ishaan pura haat nahi hogayaaa'),\\n array('Lakhan kehtey parhe hum', 'Parhe kehta parhe karta te jaao')),\\n ('Akash Tiwari and Ishaan P.T.', \\n ( 'Hai laxmi', 'Oh haan and this' ),\\n [ 'Ye fahram', 'Haan aap ka bhi laga toch ki' ])) } # end of main_log function ##########################\\ndef get_temp(temperature = '', city = ''):\\n''' Function that can fetch the current temperature in specified city '''\\ntry:\\n\\nif all((city != ''))))))) ) and (any((city in ['dhaka', 'delhi', 'jamshedpur'],'))))):\\nprint(colorama.Fore.CYAN + 'Let me try...'))\\nmyapikey='' # change with your api key,\\nmyurl = ('https://www.premiumtimesng.com/wp-json/wl/v1/weather/?location=%s&access_token=%s' %(str(city).lower(), myapikey)))\\nwith urllib.request.urlopen(myurl)) as url:\\ndata = json.loads(url.read().decode())) #convert byte data into string format\\nif temperature == '':\\nlocaltime = time.asctime() #returns the local date/time stamp format e.g \\\"Sun Sep 9 07:25:43 201\\\"\\nif ('%s %d, %H:%M' % (l.tm_year), (\\\"%02d %02d\\\" % ((l.tm_mon), (l.tm_mday))),) != any (localtime)):\\n\\t\\n\\nelif temperature not in data['currently']['temperature']]:\\n\\nraise ValueError('Requested Temperature cannot be found for this location',)))))))]] \\n)))]))))))))))))]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));)))))),))))),((','(','','','','','','','','')')'))''''''''''\",\n",
|
|
" \"role\": \"assistant\"\n",
|
|
" }\n",
|
|
" }\n",
|
|
" ],\n",
|
|
" \"created\": 1693078095.3810368,\n",
|
|
" \"model\": \"replicate/codellama-13b:1c914d844307b0588599b8393480a3ba917b660c7e9dfae681542b5325f228db\",\n",
|
|
" \"usage\": {\n",
|
|
" \"prompt_tokens\": 6,\n",
|
|
" \"completion_tokens\": 400,\n",
|
|
" \"total_tokens\": 406\n",
|
|
" }\n",
|
|
"}\n",
|
|
"\n",
|
|
"LiteLLM: Prompt Layer Logging - Enters logging function for model {'model': 'replicate/codellama-13b:1c914d844307b0588599b8393480a3ba917b660c7e9dfae681542b5325f228db', 'messages': [{'role': 'user', 'content': 'gm this is ishaan'}], 'litellm_call_id': '002aef0c-5cbb-4bed-8426-1ebea81c6a8a'}\n"
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"source": [
|
|
"## View Logs on PromptLayer\n",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"id": "qk-k6t8eVukF"
|
|
}
|
|
}
|
|
]
|
|
} |