diff --git a/.circleci/config.yml b/.circleci/config.yml index 6014cb8f1..01ca0ee5e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -108,7 +108,7 @@ jobs: command: | pwd ls - python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not router and not assistants" + python -m pytest -vv tests/local_testing -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not router and not assistants" no_output_timeout: 120m # Store test results @@ -180,7 +180,7 @@ jobs: command: | pwd ls - python -m pytest litellm/tests/ -vv -k "router" -x -s -v --junitxml=test-results/junit.xml --durations=5 + python -m pytest tests/local_testing -vv -k "router" -x -s -v --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 120m # Store test results - store_test_results: @@ -210,7 +210,7 @@ jobs: command: | pwd ls - python -m pytest litellm/tests/ -vv -k "assistants" -x -s -v --junitxml=test-results/junit.xml --durations=5 + python -m pytest tests/local_testing/ -vv -k "assistants" -x -s -v --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 120m # Store test results - store_test_results: @@ -306,7 +306,7 @@ jobs: command: | pwd ls - python -m pytest -vv litellm/tests/test_python_38.py + python -m pytest -vv tests/local_testing/test_python_38.py build_and_test: machine: diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html deleted file mode 100644 index d8a0d885a..000000000 --- a/litellm/proxy/_experimental/out/404.html +++ /dev/null @@ -1 +0,0 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/model_hub.html b/litellm/proxy/_experimental/out/model_hub.html deleted file mode 100644 index 5ba73be91..000000000 --- a/litellm/proxy/_experimental/out/model_hub.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index 23e95e4c5..000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/tests/langfuse.log b/litellm/tests/langfuse.log deleted file mode 100644 index 6a59a8ff0..000000000 --- a/litellm/tests/langfuse.log +++ /dev/null @@ -1,173 +0,0 @@ -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -Creating trace id='8a5b5991-4a10-4dfe-ae1d-52a9195fcde1' timestamp=datetime.datetime(2024, 7, 11, 14, 57, 41, 996862, tzinfo=datetime.timezone.utc) name='litellm-acompletion' user_id='langfuse_latency_test_user' input={'messages': [{'role': 'user', 'content': 'This is a test'}]} output={'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None} session_id=None release=None version=None metadata=None tags=[] public=None -Creating trace id='dba8aaeb-d435-4c82-94f9-9615ace72313' timestamp=datetime.datetime(2024, 7, 11, 14, 57, 41, 998554, tzinfo=datetime.timezone.utc) name='litellm-acompletion' user_id='langfuse_latency_test_user' input={'messages': [{'role': 'user', 'content': 'This is a test'}]} output={'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None} session_id=None release=None version=None metadata=None tags=[] public=None -Creating generation trace_id='dba8aaeb-d435-4c82-94f9-9615ace72313' name='litellm-acompletion' start_time=datetime.datetime(2024, 7, 11, 7, 57, 41, 993400) metadata={'litellm_response_cost': None, 'cache_hit': False} input={'messages': [{'role': 'user', 'content': 'This is a test'}]} output={'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None} level= status_message=None parent_observation_id=None version=None id='time-07-57-41-993400_chatcmpl-296c9e6a-4142-431c-86ba-c68b550c8005' end_time=datetime.datetime(2024, 7, 11, 7, 57, 41, 995938) completion_start_time=datetime.datetime(2024, 7, 11, 7, 57, 41, 995938) model='chatgpt-v-2' model_parameters={'temperature': '0.7', 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'} usage=Usage(input=10, output=20, total=None, unit=, input_cost=None, output_cost=None, total_cost=None) prompt_name=None prompt_version=None... -item size 502 -item size 930 -Creating generation trace_id='8a5b5991-4a10-4dfe-ae1d-52a9195fcde1' name='litellm-acompletion' start_time=datetime.datetime(2024, 7, 11, 7, 57, 41, 992764) metadata={'litellm_response_cost': None, 'cache_hit': False} input={'messages': [{'role': 'user', 'content': 'This is a test'}]} output={'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None} level= status_message=None parent_observation_id=None version=None id='time-07-57-41-992764_chatcmpl-d449c7f4-47c5-4e46-a9f1-4cd2c1fcf257' end_time=datetime.datetime(2024, 7, 11, 7, 57, 41, 995293) completion_start_time=datetime.datetime(2024, 7, 11, 7, 57, 41, 995293) model='chatgpt-v-2' model_parameters={'temperature': '0.7', 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'} usage=Usage(input=10, output=20, total=None, unit=, input_cost=None, output_cost=None, total_cost=None) prompt_name=None prompt_version=None... -item size 502 -item size 930 -Creating trace id='d8d919d4-0c8d-47c5-b1a8-38d884379c74' timestamp=datetime.datetime(2024, 7, 11, 14, 57, 42, 766, tzinfo=datetime.timezone.utc) name='litellm-acompletion' user_id='langfuse_latency_test_user' input={'messages': [{'role': 'user', 'content': 'This is a test'}]} output={'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None} session_id=None release=None version=None metadata=None tags=[] public=None -Creating generation trace_id='d8d919d4-0c8d-47c5-b1a8-38d884379c74' name='litellm-acompletion' start_time=datetime.datetime(2024, 7, 11, 7, 57, 41, 994499) metadata={'litellm_response_cost': None, 'cache_hit': False} input={'messages': [{'role': 'user', 'content': 'This is a test'}]} output={'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None} level= status_message=None parent_observation_id=None version=None id='time-07-57-41-994499_chatcmpl-5b902172-1250-46e4-bcdd-fd071b951f8c' end_time=datetime.datetime(2024, 7, 11, 7, 57, 41, 996988) completion_start_time=datetime.datetime(2024, 7, 11, 7, 57, 41, 996988) model='chatgpt-v-2' model_parameters={'temperature': '0.7', 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'} usage=Usage(input=10, output=20, total=None, unit=, input_cost=None, output_cost=None, total_cost=None) prompt_name=None prompt_version=None... -Creating trace id='fac644ff-5f4c-4b45-a55d-a5c4f93ceeec' timestamp=datetime.datetime(2024, 7, 11, 14, 57, 42, 1164, tzinfo=datetime.timezone.utc) name='litellm-acompletion' user_id='langfuse_latency_test_user' input={'messages': [{'role': 'user', 'content': 'This is a test'}]} output={'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None} session_id=None release=None version=None metadata=None tags=[] public=None -item size 502 -item size 930 -Creating generation trace_id='fac644ff-5f4c-4b45-a55d-a5c4f93ceeec' name='litellm-acompletion' start_time=datetime.datetime(2024, 7, 11, 7, 57, 41, 993972) metadata={'litellm_response_cost': None, 'cache_hit': False} input={'messages': [{'role': 'user', 'content': 'This is a test'}]} output={'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None} level= status_message=None parent_observation_id=None version=None id='time-07-57-41-993972_chatcmpl-ecc3628b-922b-430d-85b5-068bc01b9cb9' end_time=datetime.datetime(2024, 7, 11, 7, 57, 41, 996217) completion_start_time=datetime.datetime(2024, 7, 11, 7, 57, 41, 996217) model='chatgpt-v-2' model_parameters={'temperature': '0.7', 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'} usage=Usage(input=10, output=20, total=None, unit=, input_cost=None, output_cost=None, total_cost=None) prompt_name=None prompt_version=None... -item size 502 -item size 930 -Creating trace id='979fb198-f54d-4e11-ba5c-dec4b5906322' timestamp=datetime.datetime(2024, 7, 11, 14, 57, 42, 2797, tzinfo=datetime.timezone.utc) name='litellm-acompletion' user_id='langfuse_latency_test_user' input={'messages': [{'role': 'user', 'content': 'This is a test'}]} output={'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None} session_id=None release=None version=None metadata=None tags=[] public=None -Creating generation trace_id='979fb198-f54d-4e11-ba5c-dec4b5906322' name='litellm-acompletion' start_time=datetime.datetime(2024, 7, 11, 7, 57, 41, 995013) metadata={'litellm_response_cost': None, 'cache_hit': False} input={'messages': [{'role': 'user', 'content': 'This is a test'}]} output={'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None} level= status_message=None parent_observation_id=None version=None id='time-07-57-41-995013_chatcmpl-41cfe580-e870-4af9-830d-53472ba99028' end_time=datetime.datetime(2024, 7, 11, 7, 57, 42, 2338) completion_start_time=datetime.datetime(2024, 7, 11, 7, 57, 42, 2338) model='chatgpt-v-2' model_parameters={'temperature': '0.7', 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'} usage=Usage(input=10, output=20, total=None, unit=, input_cost=None, output_cost=None, total_cost=None) prompt_name=None prompt_version=None... -item size 502 -item size 930 -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -~0 items in the Langfuse queue -uploading batch of 10 items -uploading data: {'batch': [{'id': 'b4eb919c-efde-48b5-9113-b089b46746c3', 'type': 'trace-create', 'body': {'id': 'dba8aaeb-d435-4c82-94f9-9615ace72313', 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 41, 998554, tzinfo=datetime.timezone.utc), 'name': 'litellm-acompletion', 'userId': 'langfuse_latency_test_user', 'input': {'messages': [{'role': 'user', 'content': 'This is a test'}]}, 'output': {'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'tags': []}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 41, 998725, tzinfo=datetime.timezone.utc)}, {'id': 'ef605d6e-effe-43eb-894b-57cdc6e28ae4', 'type': 'generation-create', 'body': {'traceId': 'dba8aaeb-d435-4c82-94f9-9615ace72313', 'name': 'litellm-acompletion', 'startTime': datetime.datetime(2024, 7, 11, 7, 57, 41, 993400), 'metadata': {'litellm_response_cost': None, 'cache_hit': False}, 'input': {'messages': [{'role': 'user', 'content': 'This is a test'}]}, 'output': {'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'level': , 'id': 'time-07-57-41-993400_chatcmpl-296c9e6a-4142-431c-86ba-c68b550c8005', 'endTime': datetime.datetime(2024, 7, 11, 7, 57, 41, 995938), 'completionStartTime': datetime.datetime(2024, 7, 11, 7, 57, 41, 995938), 'model': 'chatgpt-v-2', 'modelParameters': {'temperature': '0.7', 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'}, 'usage': {'input': 10, 'output': 20, 'unit': }}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 41, 999054, tzinfo=datetime.timezone.utc)}, {'id': '34ab3341-9b8f-45ef-ba32-31d6678c12e1', 'type': 'trace-create', 'body': {'id': '8a5b5991-4a10-4dfe-ae1d-52a9195fcde1', 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 41, 996862, tzinfo=datetime.timezone.utc), 'name': 'litellm-acompletion', 'userId': 'langfuse_latency_test_user', 'input': {'messages': [{'role': 'user', 'content': 'This is a test'}]}, 'output': {'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'tags': []}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 41, 999535, tzinfo=datetime.timezone.utc)}, {'id': '17e0038d-eb35-47f0-8501-2e9298e25ab3', 'type': 'generation-create', 'body': {'traceId': '8a5b5991-4a10-4dfe-ae1d-52a9195fcde1', 'name': 'litellm-acompletion', 'startTime': datetime.datetime(2024, 7, 11, 7, 57, 41, 992764), 'metadata': {'litellm_response_cost': None, 'cache_hit': False}, 'input': {'messages': [{'role': 'user', 'content': 'This is a test'}]}, 'output': {'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'level': , 'id': 'time-07-57-41-992764_chatcmpl-d449c7f4-47c5-4e46-a9f1-4cd2c1fcf257', 'endTime': datetime.datetime(2024, 7, 11, 7, 57, 41, 995293), 'completionStartTime': datetime.datetime(2024, 7, 11, 7, 57, 41, 995293), 'model': 'chatgpt-v-2', 'modelParameters': {'temperature': '0.7', 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'}, 'usage': {'input': 10, 'output': 20, 'unit': }}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 41, 999833, tzinfo=datetime.timezone.utc)}, {'id': '282dc858-0b89-44b2-9946-1b4580a3b99d', 'type': 'trace-create', 'body': {'id': 'd8d919d4-0c8d-47c5-b1a8-38d884379c74', 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 42, 766, tzinfo=datetime.timezone.utc), 'name': 'litellm-acompletion', 'userId': 'langfuse_latency_test_user', 'input': {'messages': [{'role': 'user', 'content': 'This is a test'}]}, 'output': {'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'tags': []}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 42, 890, tzinfo=datetime.timezone.utc)}, {'id': '8de8e06e-69a5-4636-a1be-7ce1179b7607', 'type': 'generation-create', 'body': {'traceId': 'd8d919d4-0c8d-47c5-b1a8-38d884379c74', 'name': 'litellm-acompletion', 'startTime': datetime.datetime(2024, 7, 11, 7, 57, 41, 994499), 'metadata': {'litellm_response_cost': None, 'cache_hit': False}, 'input': {'messages': [{'role': 'user', 'content': 'This is a test'}]}, 'output': {'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'level': , 'id': 'time-07-57-41-994499_chatcmpl-5b902172-1250-46e4-bcdd-fd071b951f8c', 'endTime': datetime.datetime(2024, 7, 11, 7, 57, 41, 996988), 'completionStartTime': datetime.datetime(2024, 7, 11, 7, 57, 41, 996988), 'model': 'chatgpt-v-2', 'modelParameters': {'temperature': '0.7', 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'}, 'usage': {'input': 10, 'output': 20, 'unit': }}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 42, 1121, tzinfo=datetime.timezone.utc)}, {'id': '576825dd-27ff-49b2-8af0-24a6ccd5f853', 'type': 'trace-create', 'body': {'id': 'fac644ff-5f4c-4b45-a55d-a5c4f93ceeec', 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 42, 1164, tzinfo=datetime.timezone.utc), 'name': 'litellm-acompletion', 'userId': 'langfuse_latency_test_user', 'input': {'messages': [{'role': 'user', 'content': 'This is a test'}]}, 'output': {'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'tags': []}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 42, 1564, tzinfo=datetime.timezone.utc)}, {'id': '4de8b9d4-96c2-492c-b273-f1d114a710da', 'type': 'generation-create', 'body': {'traceId': 'fac644ff-5f4c-4b45-a55d-a5c4f93ceeec', 'name': 'litellm-acompletion', 'startTime': datetime.datetime(2024, 7, 11, 7, 57, 41, 993972), 'metadata': {'litellm_response_cost': None, 'cache_hit': False}, 'input': {'messages': [{'role': 'user', 'content': 'This is a test'}]}, 'output': {'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'level': , 'id': 'time-07-57-41-993972_chatcmpl-ecc3628b-922b-430d-85b5-068bc01b9cb9', 'endTime': datetime.datetime(2024, 7, 11, 7, 57, 41, 996217), 'completionStartTime': datetime.datetime(2024, 7, 11, 7, 57, 41, 996217), 'model': 'chatgpt-v-2', 'modelParameters': {'temperature': '0.7', 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'}, 'usage': {'input': 10, 'output': 20, 'unit': }}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 42, 1799, tzinfo=datetime.timezone.utc)}, {'id': 'bdbc23a6-844e-415b-b33f-ede756955c74', 'type': 'trace-create', 'body': {'id': '979fb198-f54d-4e11-ba5c-dec4b5906322', 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 42, 2797, tzinfo=datetime.timezone.utc), 'name': 'litellm-acompletion', 'userId': 'langfuse_latency_test_user', 'input': {'messages': [{'role': 'user', 'content': 'This is a test'}]}, 'output': {'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'tags': []}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 42, 2921, tzinfo=datetime.timezone.utc)}, {'id': '8b74d44d-cd6d-469f-9077-efea4e82d81b', 'type': 'generation-create', 'body': {'traceId': '979fb198-f54d-4e11-ba5c-dec4b5906322', 'name': 'litellm-acompletion', 'startTime': datetime.datetime(2024, 7, 11, 7, 57, 41, 995013), 'metadata': {'litellm_response_cost': None, 'cache_hit': False}, 'input': {'messages': [{'role': 'user', 'content': 'This is a test'}]}, 'output': {'content': "It's simple to use and easy to get started", 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'level': , 'id': 'time-07-57-41-995013_chatcmpl-41cfe580-e870-4af9-830d-53472ba99028', 'endTime': datetime.datetime(2024, 7, 11, 7, 57, 42, 2338), 'completionStartTime': datetime.datetime(2024, 7, 11, 7, 57, 42, 2338), 'model': 'chatgpt-v-2', 'modelParameters': {'temperature': '0.7', 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'}, 'usage': {'input': 10, 'output': 20, 'unit': }}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 42, 3176, tzinfo=datetime.timezone.utc)}], 'metadata': {'batch_size': 10, 'sdk_integration': 'default', 'sdk_name': 'python', 'sdk_version': '2.39.1', 'public_key': 'pk-lf-b3db7e8e-c2f6-4fc7-825c-a541a8fbe003'}} -making request: {"batch": [{"id": "b4eb919c-efde-48b5-9113-b089b46746c3", "type": "trace-create", "body": {"id": "dba8aaeb-d435-4c82-94f9-9615ace72313", "timestamp": "2024-07-11T14:57:41.998554Z", "name": "litellm-acompletion", "userId": "langfuse_latency_test_user", "input": {"messages": [{"role": "user", "content": "This is a test"}]}, "output": {"content": "It's simple to use and easy to get started", "role": "assistant", "tool_calls": null, "function_call": null}, "tags": []}, "timestamp": "2024-07-11T14:57:41.998725Z"}, {"id": "ef605d6e-effe-43eb-894b-57cdc6e28ae4", "type": "generation-create", "body": {"traceId": "dba8aaeb-d435-4c82-94f9-9615ace72313", "name": "litellm-acompletion", "startTime": "2024-07-11T07:57:41.993400-07:00", "metadata": {"litellm_response_cost": null, "cache_hit": false}, "input": {"messages": [{"role": "user", "content": "This is a test"}]}, "output": {"content": "It's simple to use and easy to get started", "role": "assistant", "tool_calls": null, "function_call": null}, "level": "DEFAULT", "id": "time-07-57-41-993400_chatcmpl-296c9e6a-4142-431c-86ba-c68b550c8005", "endTime": "2024-07-11T07:57:41.995938-07:00", "completionStartTime": "2024-07-11T07:57:41.995938-07:00", "model": "chatgpt-v-2", "modelParameters": {"temperature": "0.7", "max_tokens": 5, "user": "langfuse_latency_test_user", "extra_body": "{}"}, "usage": {"input": 10, "output": 20, "unit": "TOKENS"}}, "timestamp": "2024-07-11T14:57:41.999054Z"}, {"id": "34ab3341-9b8f-45ef-ba32-31d6678c12e1", "type": "trace-create", "body": {"id": "8a5b5991-4a10-4dfe-ae1d-52a9195fcde1", "timestamp": "2024-07-11T14:57:41.996862Z", "name": "litellm-acompletion", "userId": "langfuse_latency_test_user", "input": {"messages": [{"role": "user", "content": "This is a test"}]}, "output": {"content": "It's simple to use and easy to get started", "role": "assistant", "tool_calls": null, "function_call": null}, "tags": []}, "timestamp": "2024-07-11T14:57:41.999535Z"}, {"id": "17e0038d-eb35-47f0-8501-2e9298e25ab3", "type": "generation-create", "body": {"traceId": "8a5b5991-4a10-4dfe-ae1d-52a9195fcde1", "name": "litellm-acompletion", "startTime": "2024-07-11T07:57:41.992764-07:00", "metadata": {"litellm_response_cost": null, "cache_hit": false}, "input": {"messages": [{"role": "user", "content": "This is a test"}]}, "output": {"content": "It's simple to use and easy to get started", "role": "assistant", "tool_calls": null, "function_call": null}, "level": "DEFAULT", "id": "time-07-57-41-992764_chatcmpl-d449c7f4-47c5-4e46-a9f1-4cd2c1fcf257", "endTime": "2024-07-11T07:57:41.995293-07:00", "completionStartTime": "2024-07-11T07:57:41.995293-07:00", "model": "chatgpt-v-2", "modelParameters": {"temperature": "0.7", "max_tokens": 5, "user": "langfuse_latency_test_user", "extra_body": "{}"}, "usage": {"input": 10, "output": 20, "unit": "TOKENS"}}, "timestamp": "2024-07-11T14:57:41.999833Z"}, {"id": "282dc858-0b89-44b2-9946-1b4580a3b99d", "type": "trace-create", "body": {"id": "d8d919d4-0c8d-47c5-b1a8-38d884379c74", "timestamp": "2024-07-11T14:57:42.000766Z", "name": "litellm-acompletion", "userId": "langfuse_latency_test_user", "input": {"messages": [{"role": "user", "content": "This is a test"}]}, "output": {"content": "It's simple to use and easy to get started", "role": "assistant", "tool_calls": null, "function_call": null}, "tags": []}, "timestamp": "2024-07-11T14:57:42.000890Z"}, {"id": "8de8e06e-69a5-4636-a1be-7ce1179b7607", "type": "generation-create", "body": {"traceId": "d8d919d4-0c8d-47c5-b1a8-38d884379c74", "name": "litellm-acompletion", "startTime": "2024-07-11T07:57:41.994499-07:00", "metadata": {"litellm_response_cost": null, "cache_hit": false}, "input": {"messages": [{"role": "user", "content": "This is a test"}]}, "output": {"content": "It's simple to use and easy to get started", "role": "assistant", "tool_calls": null, "function_call": null}, "level": "DEFAULT", "id": "time-07-57-41-994499_chatcmpl-5b902172-1250-46e4-bcdd-fd071b951f8c", "endTime": "2024-07-11T07:57:41.996988-07:00", "completionStartTime": "2024-07-11T07:57:41.996988-07:00", "model": "chatgpt-v-2", "modelParameters": {"temperature": "0.7", "max_tokens": 5, "user": "langfuse_latency_test_user", "extra_body": "{}"}, "usage": {"input": 10, "output": 20, "unit": "TOKENS"}}, "timestamp": "2024-07-11T14:57:42.001121Z"}, {"id": "576825dd-27ff-49b2-8af0-24a6ccd5f853", "type": "trace-create", "body": {"id": "fac644ff-5f4c-4b45-a55d-a5c4f93ceeec", "timestamp": "2024-07-11T14:57:42.001164Z", "name": "litellm-acompletion", "userId": "langfuse_latency_test_user", "input": {"messages": [{"role": "user", "content": "This is a test"}]}, "output": {"content": "It's simple to use and easy to get started", "role": "assistant", "tool_calls": null, "function_call": null}, "tags": []}, "timestamp": "2024-07-11T14:57:42.001564Z"}, {"id": "4de8b9d4-96c2-492c-b273-f1d114a710da", "type": "generation-create", "body": {"traceId": "fac644ff-5f4c-4b45-a55d-a5c4f93ceeec", "name": "litellm-acompletion", "startTime": "2024-07-11T07:57:41.993972-07:00", "metadata": {"litellm_response_cost": null, "cache_hit": false}, "input": {"messages": [{"role": "user", "content": "This is a test"}]}, "output": {"content": "It's simple to use and easy to get started", "role": "assistant", "tool_calls": null, "function_call": null}, "level": "DEFAULT", "id": "time-07-57-41-993972_chatcmpl-ecc3628b-922b-430d-85b5-068bc01b9cb9", "endTime": "2024-07-11T07:57:41.996217-07:00", "completionStartTime": "2024-07-11T07:57:41.996217-07:00", "model": "chatgpt-v-2", "modelParameters": {"temperature": "0.7", "max_tokens": 5, "user": "langfuse_latency_test_user", "extra_body": "{}"}, "usage": {"input": 10, "output": 20, "unit": "TOKENS"}}, "timestamp": "2024-07-11T14:57:42.001799Z"}, {"id": "bdbc23a6-844e-415b-b33f-ede756955c74", "type": "trace-create", "body": {"id": "979fb198-f54d-4e11-ba5c-dec4b5906322", "timestamp": "2024-07-11T14:57:42.002797Z", "name": "litellm-acompletion", "userId": "langfuse_latency_test_user", "input": {"messages": [{"role": "user", "content": "This is a test"}]}, "output": {"content": "It's simple to use and easy to get started", "role": "assistant", "tool_calls": null, "function_call": null}, "tags": []}, "timestamp": "2024-07-11T14:57:42.002921Z"}, {"id": "8b74d44d-cd6d-469f-9077-efea4e82d81b", "type": "generation-create", "body": {"traceId": "979fb198-f54d-4e11-ba5c-dec4b5906322", "name": "litellm-acompletion", "startTime": "2024-07-11T07:57:41.995013-07:00", "metadata": {"litellm_response_cost": null, "cache_hit": false}, "input": {"messages": [{"role": "user", "content": "This is a test"}]}, "output": {"content": "It's simple to use and easy to get started", "role": "assistant", "tool_calls": null, "function_call": null}, "level": "DEFAULT", "id": "time-07-57-41-995013_chatcmpl-41cfe580-e870-4af9-830d-53472ba99028", "endTime": "2024-07-11T07:57:42.002338-07:00", "completionStartTime": "2024-07-11T07:57:42.002338-07:00", "model": "chatgpt-v-2", "modelParameters": {"temperature": "0.7", "max_tokens": 5, "user": "langfuse_latency_test_user", "extra_body": "{}"}, "usage": {"input": 10, "output": 20, "unit": "TOKENS"}}, "timestamp": "2024-07-11T14:57:42.003176Z"}], "metadata": {"batch_size": 10, "sdk_integration": "default", "sdk_name": "python", "sdk_version": "2.39.1", "public_key": "pk-lf-b3db7e8e-c2f6-4fc7-825c-a541a8fbe003"}} to https://us.cloud.langfuse.com/api/public/ingestion -received response: {"errors":[],"successes":[{"id":"b4eb919c-efde-48b5-9113-b089b46746c3","status":201},{"id":"ef605d6e-effe-43eb-894b-57cdc6e28ae4","status":201},{"id":"34ab3341-9b8f-45ef-ba32-31d6678c12e1","status":201},{"id":"17e0038d-eb35-47f0-8501-2e9298e25ab3","status":201},{"id":"282dc858-0b89-44b2-9946-1b4580a3b99d","status":201},{"id":"8de8e06e-69a5-4636-a1be-7ce1179b7607","status":201},{"id":"576825dd-27ff-49b2-8af0-24a6ccd5f853","status":201},{"id":"4de8b9d4-96c2-492c-b273-f1d114a710da","status":201},{"id":"bdbc23a6-844e-415b-b33f-ede756955c74","status":201},{"id":"8b74d44d-cd6d-469f-9077-efea4e82d81b","status":201}]} -successfully uploaded batch of 10 items -~0 items in the Langfuse queue -consumer is running... -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -Creating trace id='litellm-test-41e731d8-eec7-484d-a123-3f5369227667' timestamp=datetime.datetime(2024, 7, 11, 14, 57, 43, 673224, tzinfo=datetime.timezone.utc) name='litellm-acompletion' user_id='langfuse_latency_test_user' input={'messages': [{'role': 'user', 'content': 'redacted-by-litellm'}]} output={'content': 'redacted-by-litellm', 'role': 'assistant', 'tool_calls': None, 'function_call': None} session_id=None release=None version=None metadata=None tags=[] public=None -Creating generation trace_id='litellm-test-41e731d8-eec7-484d-a123-3f5369227667' name='litellm-acompletion' start_time=datetime.datetime(2024, 7, 11, 7, 57, 43, 671900) metadata={'litellm_response_cost': 5.4999999999999995e-05, 'cache_hit': False} input={'messages': [{'role': 'user', 'content': 'redacted-by-litellm'}]} output={'content': 'redacted-by-litellm', 'role': 'assistant', 'tool_calls': None, 'function_call': None} level= status_message=None parent_observation_id=None version=None id='time-07-57-43-671900_chatcmpl-84840e13-5f25-494b-a771-43d6dc9f4825' end_time=datetime.datetime(2024, 7, 11, 7, 57, 43, 672638) completion_start_time=datetime.datetime(2024, 7, 11, 7, 57, 43, 672638) model='gpt-3.5-turbo' model_parameters={'temperature': '0.7', 'stream': False, 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'} usage=Usage(input=10, output=20, total=None, unit=, input_cost=None, output_cost=None, total_cost=5.4999999999999995e-05) prompt_name=None prompt_version=None... -item size 497 -item size 999 -flushing queue -successfully flushed about 0 items. -~0 items in the Langfuse queue -uploading batch of 2 items -uploading data: {'batch': [{'id': 'b37f0713-1089-44ec-99d2-12bd69d6b4ed', 'type': 'trace-create', 'body': {'id': 'litellm-test-41e731d8-eec7-484d-a123-3f5369227667', 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 43, 673224, tzinfo=datetime.timezone.utc), 'name': 'litellm-acompletion', 'userId': 'langfuse_latency_test_user', 'input': {'messages': [{'role': 'user', 'content': 'redacted-by-litellm'}]}, 'output': {'content': 'redacted-by-litellm', 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'tags': []}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 43, 673428, tzinfo=datetime.timezone.utc)}, {'id': '9e0c04ab-30c0-47ec-8507-9877c9bff1c7', 'type': 'generation-create', 'body': {'traceId': 'litellm-test-41e731d8-eec7-484d-a123-3f5369227667', 'name': 'litellm-acompletion', 'startTime': datetime.datetime(2024, 7, 11, 7, 57, 43, 671900), 'metadata': {'litellm_response_cost': 5.4999999999999995e-05, 'cache_hit': False}, 'input': {'messages': [{'role': 'user', 'content': 'redacted-by-litellm'}]}, 'output': {'content': 'redacted-by-litellm', 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'level': , 'id': 'time-07-57-43-671900_chatcmpl-84840e13-5f25-494b-a771-43d6dc9f4825', 'endTime': datetime.datetime(2024, 7, 11, 7, 57, 43, 672638), 'completionStartTime': datetime.datetime(2024, 7, 11, 7, 57, 43, 672638), 'model': 'gpt-3.5-turbo', 'modelParameters': {'temperature': '0.7', 'stream': False, 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'}, 'usage': {'input': 10, 'output': 20, 'unit': , 'totalCost': 5.4999999999999995e-05}}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 43, 673780, tzinfo=datetime.timezone.utc)}], 'metadata': {'batch_size': 2, 'sdk_integration': 'default', 'sdk_name': 'python', 'sdk_version': '2.39.1', 'public_key': 'pk-lf-b3db7e8e-c2f6-4fc7-825c-a541a8fbe003'}} -making request: {"batch": [{"id": "b37f0713-1089-44ec-99d2-12bd69d6b4ed", "type": "trace-create", "body": {"id": "litellm-test-41e731d8-eec7-484d-a123-3f5369227667", "timestamp": "2024-07-11T14:57:43.673224Z", "name": "litellm-acompletion", "userId": "langfuse_latency_test_user", "input": {"messages": [{"role": "user", "content": "redacted-by-litellm"}]}, "output": {"content": "redacted-by-litellm", "role": "assistant", "tool_calls": null, "function_call": null}, "tags": []}, "timestamp": "2024-07-11T14:57:43.673428Z"}, {"id": "9e0c04ab-30c0-47ec-8507-9877c9bff1c7", "type": "generation-create", "body": {"traceId": "litellm-test-41e731d8-eec7-484d-a123-3f5369227667", "name": "litellm-acompletion", "startTime": "2024-07-11T07:57:43.671900-07:00", "metadata": {"litellm_response_cost": 5.4999999999999995e-05, "cache_hit": false}, "input": {"messages": [{"role": "user", "content": "redacted-by-litellm"}]}, "output": {"content": "redacted-by-litellm", "role": "assistant", "tool_calls": null, "function_call": null}, "level": "DEFAULT", "id": "time-07-57-43-671900_chatcmpl-84840e13-5f25-494b-a771-43d6dc9f4825", "endTime": "2024-07-11T07:57:43.672638-07:00", "completionStartTime": "2024-07-11T07:57:43.672638-07:00", "model": "gpt-3.5-turbo", "modelParameters": {"temperature": "0.7", "stream": false, "max_tokens": 5, "user": "langfuse_latency_test_user", "extra_body": "{}"}, "usage": {"input": 10, "output": 20, "unit": "TOKENS", "totalCost": 5.4999999999999995e-05}}, "timestamp": "2024-07-11T14:57:43.673780Z"}], "metadata": {"batch_size": 2, "sdk_integration": "default", "sdk_name": "python", "sdk_version": "2.39.1", "public_key": "pk-lf-b3db7e8e-c2f6-4fc7-825c-a541a8fbe003"}} to https://us.cloud.langfuse.com/api/public/ingestion -~0 items in the Langfuse queue -received response: {"errors":[],"successes":[{"id":"b37f0713-1089-44ec-99d2-12bd69d6b4ed","status":201},{"id":"9e0c04ab-30c0-47ec-8507-9877c9bff1c7","status":201}]} -successfully uploaded batch of 2 items -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -Getting observations... None, None, None, None, litellm-test-41e731d8-eec7-484d-a123-3f5369227667, None, None, None, GENERATION -~0 items in the Langfuse queue -~0 items in the Langfuse queue -consumer is running... -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -flushing queue -successfully flushed about 0 items. -~0 items in the Langfuse queue -Creating trace id='litellm-test-e079602a-d96c-4178-a423-f45d4e8c2d21' timestamp=datetime.datetime(2024, 7, 11, 14, 57, 46, 266724, tzinfo=datetime.timezone.utc) name='litellm-acompletion' user_id='langfuse_latency_test_user' input={'messages': [{'role': 'user', 'content': 'redacted-by-litellm'}]} output={'content': 'redacted-by-litellm', 'role': 'assistant', 'tool_calls': None, 'function_call': None} session_id=None release=None version=None metadata=None tags=[] public=None -item size 497 -Creating generation trace_id='litellm-test-e079602a-d96c-4178-a423-f45d4e8c2d21' name='litellm-acompletion' start_time=datetime.datetime(2024, 7, 11, 7, 57, 46, 11092) metadata={'litellm_response_cost': 4.1e-05, 'cache_hit': False} input={'messages': [{'role': 'user', 'content': 'redacted-by-litellm'}]} output={'content': 'redacted-by-litellm', 'role': 'assistant', 'tool_calls': None, 'function_call': None} level= status_message=None parent_observation_id=None version=None id='time-07-57-46-011092_chatcmpl-bbd13417-0ddd-46d0-a4fc-ded148ff2ae9' end_time=datetime.datetime(2024, 7, 11, 7, 57, 46, 22279) completion_start_time=datetime.datetime(2024, 7, 11, 7, 57, 46, 13610) model='gpt-3.5-turbo' model_parameters={'temperature': '0.7', 'stream': True, 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'} usage=Usage(input=14, output=10, total=None, unit=, input_cost=None, output_cost=None, total_cost=4.1e-05) prompt_name=None prompt_version=None... -item size 968 -~0 items in the Langfuse queue -uploading batch of 2 items -uploading data: {'batch': [{'id': 'b05c61c3-d1c8-49fa-8bbe-7c9d1a9aba87', 'type': 'trace-create', 'body': {'id': 'litellm-test-e079602a-d96c-4178-a423-f45d4e8c2d21', 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 46, 266724, tzinfo=datetime.timezone.utc), 'name': 'litellm-acompletion', 'userId': 'langfuse_latency_test_user', 'input': {'messages': [{'role': 'user', 'content': 'redacted-by-litellm'}]}, 'output': {'content': 'redacted-by-litellm', 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'tags': []}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 46, 267376, tzinfo=datetime.timezone.utc)}, {'id': 'f1e3d54a-800e-40ba-8214-50fa8ba1a40e', 'type': 'generation-create', 'body': {'traceId': 'litellm-test-e079602a-d96c-4178-a423-f45d4e8c2d21', 'name': 'litellm-acompletion', 'startTime': datetime.datetime(2024, 7, 11, 7, 57, 46, 11092), 'metadata': {'litellm_response_cost': 4.1e-05, 'cache_hit': False}, 'input': {'messages': [{'role': 'user', 'content': 'redacted-by-litellm'}]}, 'output': {'content': 'redacted-by-litellm', 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'level': , 'id': 'time-07-57-46-011092_chatcmpl-bbd13417-0ddd-46d0-a4fc-ded148ff2ae9', 'endTime': datetime.datetime(2024, 7, 11, 7, 57, 46, 22279), 'completionStartTime': datetime.datetime(2024, 7, 11, 7, 57, 46, 13610), 'model': 'gpt-3.5-turbo', 'modelParameters': {'temperature': '0.7', 'stream': True, 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'}, 'usage': {'input': 14, 'output': 10, 'unit': , 'totalCost': 4.1e-05}}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 46, 268553, tzinfo=datetime.timezone.utc)}], 'metadata': {'batch_size': 2, 'sdk_integration': 'default', 'sdk_name': 'python', 'sdk_version': '2.39.1', 'public_key': 'pk-lf-b3db7e8e-c2f6-4fc7-825c-a541a8fbe003'}} -making request: {"batch": [{"id": "b05c61c3-d1c8-49fa-8bbe-7c9d1a9aba87", "type": "trace-create", "body": {"id": "litellm-test-e079602a-d96c-4178-a423-f45d4e8c2d21", "timestamp": "2024-07-11T14:57:46.266724Z", "name": "litellm-acompletion", "userId": "langfuse_latency_test_user", "input": {"messages": [{"role": "user", "content": "redacted-by-litellm"}]}, "output": {"content": "redacted-by-litellm", "role": "assistant", "tool_calls": null, "function_call": null}, "tags": []}, "timestamp": "2024-07-11T14:57:46.267376Z"}, {"id": "f1e3d54a-800e-40ba-8214-50fa8ba1a40e", "type": "generation-create", "body": {"traceId": "litellm-test-e079602a-d96c-4178-a423-f45d4e8c2d21", "name": "litellm-acompletion", "startTime": "2024-07-11T07:57:46.011092-07:00", "metadata": {"litellm_response_cost": 4.1e-05, "cache_hit": false}, "input": {"messages": [{"role": "user", "content": "redacted-by-litellm"}]}, "output": {"content": "redacted-by-litellm", "role": "assistant", "tool_calls": null, "function_call": null}, "level": "DEFAULT", "id": "time-07-57-46-011092_chatcmpl-bbd13417-0ddd-46d0-a4fc-ded148ff2ae9", "endTime": "2024-07-11T07:57:46.022279-07:00", "completionStartTime": "2024-07-11T07:57:46.013610-07:00", "model": "gpt-3.5-turbo", "modelParameters": {"temperature": "0.7", "stream": true, "max_tokens": 5, "user": "langfuse_latency_test_user", "extra_body": "{}"}, "usage": {"input": 14, "output": 10, "unit": "TOKENS", "totalCost": 4.1e-05}}, "timestamp": "2024-07-11T14:57:46.268553Z"}], "metadata": {"batch_size": 2, "sdk_integration": "default", "sdk_name": "python", "sdk_version": "2.39.1", "public_key": "pk-lf-b3db7e8e-c2f6-4fc7-825c-a541a8fbe003"}} to https://us.cloud.langfuse.com/api/public/ingestion -received response: {"errors":[],"successes":[{"id":"b05c61c3-d1c8-49fa-8bbe-7c9d1a9aba87","status":201},{"id":"f1e3d54a-800e-40ba-8214-50fa8ba1a40e","status":201}]} -successfully uploaded batch of 2 items -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -Getting observations... None, None, None, None, litellm-test-e079602a-d96c-4178-a423-f45d4e8c2d21, None, None, None, GENERATION -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -consumer is running... -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -Creating trace id='litellm-test-624f61d8-2604-406a-bd85-f7eb9f9878e4' timestamp=datetime.datetime(2024, 7, 11, 14, 57, 48, 311181, tzinfo=datetime.timezone.utc) name='litellm-acompletion' user_id='langfuse_latency_test_user' input='redacted-by-litellm' output='redacted-by-litellm' session_id=None release=None version=None metadata=None tags=[] public=None -Creating generation trace_id='litellm-test-624f61d8-2604-406a-bd85-f7eb9f9878e4' name='litellm-acompletion' start_time=datetime.datetime(2024, 7, 11, 7, 57, 48, 309974) metadata={'litellm_response_cost': 5.4999999999999995e-05, 'cache_hit': False} input='redacted-by-litellm' output='redacted-by-litellm' level= status_message=None parent_observation_id=None version=None id='time-07-57-48-309974_chatcmpl-9feff8c8-bddc-45fa-b3e7-3e83768589b4' end_time=datetime.datetime(2024, 7, 11, 7, 57, 48, 310635) completion_start_time=datetime.datetime(2024, 7, 11, 7, 57, 48, 310635) model='gpt-3.5-turbo' model_parameters={'temperature': '0.7', 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'} usage=Usage(input=10, output=20, total=None, unit=, input_cost=None, output_cost=None, total_cost=5.4999999999999995e-05) prompt_name=None prompt_version=None... -item size 375 -item size 860 -flushing queue -successfully flushed about 0 items. -~0 items in the Langfuse queue -uploading batch of 2 items -uploading data: {'batch': [{'id': '414f3e3a-35b4-4e24-a64a-e9303de94d0d', 'type': 'trace-create', 'body': {'id': 'litellm-test-624f61d8-2604-406a-bd85-f7eb9f9878e4', 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 48, 311181, tzinfo=datetime.timezone.utc), 'name': 'litellm-acompletion', 'userId': 'langfuse_latency_test_user', 'input': 'redacted-by-litellm', 'output': 'redacted-by-litellm', 'tags': []}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 48, 311330, tzinfo=datetime.timezone.utc)}, {'id': '890d05df-f7e0-4c1a-ad49-6b1704c36136', 'type': 'generation-create', 'body': {'traceId': 'litellm-test-624f61d8-2604-406a-bd85-f7eb9f9878e4', 'name': 'litellm-acompletion', 'startTime': datetime.datetime(2024, 7, 11, 7, 57, 48, 309974), 'metadata': {'litellm_response_cost': 5.4999999999999995e-05, 'cache_hit': False}, 'input': 'redacted-by-litellm', 'output': 'redacted-by-litellm', 'level': , 'id': 'time-07-57-48-309974_chatcmpl-9feff8c8-bddc-45fa-b3e7-3e83768589b4', 'endTime': datetime.datetime(2024, 7, 11, 7, 57, 48, 310635), 'completionStartTime': datetime.datetime(2024, 7, 11, 7, 57, 48, 310635), 'model': 'gpt-3.5-turbo', 'modelParameters': {'temperature': '0.7', 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'}, 'usage': {'input': 10, 'output': 20, 'unit': , 'totalCost': 5.4999999999999995e-05}}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 48, 311640, tzinfo=datetime.timezone.utc)}], 'metadata': {'batch_size': 2, 'sdk_integration': 'default', 'sdk_name': 'python', 'sdk_version': '2.39.1', 'public_key': 'pk-lf-b3db7e8e-c2f6-4fc7-825c-a541a8fbe003'}} -~0 items in the Langfuse queue -making request: {"batch": [{"id": "414f3e3a-35b4-4e24-a64a-e9303de94d0d", "type": "trace-create", "body": {"id": "litellm-test-624f61d8-2604-406a-bd85-f7eb9f9878e4", "timestamp": "2024-07-11T14:57:48.311181Z", "name": "litellm-acompletion", "userId": "langfuse_latency_test_user", "input": "redacted-by-litellm", "output": "redacted-by-litellm", "tags": []}, "timestamp": "2024-07-11T14:57:48.311330Z"}, {"id": "890d05df-f7e0-4c1a-ad49-6b1704c36136", "type": "generation-create", "body": {"traceId": "litellm-test-624f61d8-2604-406a-bd85-f7eb9f9878e4", "name": "litellm-acompletion", "startTime": "2024-07-11T07:57:48.309974-07:00", "metadata": {"litellm_response_cost": 5.4999999999999995e-05, "cache_hit": false}, "input": "redacted-by-litellm", "output": "redacted-by-litellm", "level": "DEFAULT", "id": "time-07-57-48-309974_chatcmpl-9feff8c8-bddc-45fa-b3e7-3e83768589b4", "endTime": "2024-07-11T07:57:48.310635-07:00", "completionStartTime": "2024-07-11T07:57:48.310635-07:00", "model": "gpt-3.5-turbo", "modelParameters": {"temperature": "0.7", "max_tokens": 5, "user": "langfuse_latency_test_user", "extra_body": "{}"}, "usage": {"input": 10, "output": 20, "unit": "TOKENS", "totalCost": 5.4999999999999995e-05}}, "timestamp": "2024-07-11T14:57:48.311640Z"}], "metadata": {"batch_size": 2, "sdk_integration": "default", "sdk_name": "python", "sdk_version": "2.39.1", "public_key": "pk-lf-b3db7e8e-c2f6-4fc7-825c-a541a8fbe003"}} to https://us.cloud.langfuse.com/api/public/ingestion -~0 items in the Langfuse queue -~0 items in the Langfuse queue -received response: {"errors":[],"successes":[{"id":"414f3e3a-35b4-4e24-a64a-e9303de94d0d","status":201},{"id":"890d05df-f7e0-4c1a-ad49-6b1704c36136","status":201}]} -successfully uploaded batch of 2 items -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -Getting trace litellm-test-624f61d8-2604-406a-bd85-f7eb9f9878e4 -~0 items in the Langfuse queue -~0 items in the Langfuse queue -Getting observations... None, None, None, None, litellm-test-624f61d8-2604-406a-bd85-f7eb9f9878e4, None, None, None, GENERATION -~0 items in the Langfuse queue -`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs. -Creating trace id='litellm-test-0020e225-3dc4-43d5-b75f-468a042e51be' timestamp=datetime.datetime(2024, 7, 11, 14, 57, 50, 588244, tzinfo=datetime.timezone.utc) name='litellm-acompletion' user_id='langfuse_latency_test_user' input={'messages': [{'role': 'user', 'content': 'This is a test'}]} output={'content': 'This is a test response', 'role': 'assistant', 'tool_calls': None, 'function_call': None} session_id=None release=None version=None metadata=None tags=[] public=None -Creating generation trace_id='litellm-test-0020e225-3dc4-43d5-b75f-468a042e51be' name='litellm-acompletion' start_time=datetime.datetime(2024, 7, 11, 7, 57, 50, 585058) metadata={'litellm_response_cost': 5.4999999999999995e-05, 'cache_hit': False} input={'messages': [{'role': 'user', 'content': 'This is a test'}]} output={'content': 'This is a test response', 'role': 'assistant', 'tool_calls': None, 'function_call': None} level= status_message=None parent_observation_id=None version=None id='time-07-57-50-585058_chatcmpl-05db6b0b-4847-4b87-a8ef-7d90bef7f81c' end_time=datetime.datetime(2024, 7, 11, 7, 57, 50, 587160) completion_start_time=datetime.datetime(2024, 7, 11, 7, 57, 50, 587160) model='gpt-3.5-turbo' model_parameters={'temperature': '0.7', 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'} usage=Usage(input=10, output=20, total=None, unit=, input_cost=None, output_cost=None, total_cost=5.4999999999999995e-05) prompt_name=None prompt_version=None... -item size 496 -item size 981 -flushing queue -successfully flushed about 0 items. -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -uploading batch of 2 items -uploading data: {'batch': [{'id': '27ef4895-21f0-4485-b9e6-083e1d64e5fa', 'type': 'trace-create', 'body': {'id': 'litellm-test-0020e225-3dc4-43d5-b75f-468a042e51be', 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 50, 588244, tzinfo=datetime.timezone.utc), 'name': 'litellm-acompletion', 'userId': 'langfuse_latency_test_user', 'input': {'messages': [{'role': 'user', 'content': 'This is a test'}]}, 'output': {'content': 'This is a test response', 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'tags': []}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 50, 589416, tzinfo=datetime.timezone.utc)}, {'id': '8f77e83d-d4b4-4890-b37a-bf04611f2ed6', 'type': 'generation-create', 'body': {'traceId': 'litellm-test-0020e225-3dc4-43d5-b75f-468a042e51be', 'name': 'litellm-acompletion', 'startTime': datetime.datetime(2024, 7, 11, 7, 57, 50, 585058), 'metadata': {'litellm_response_cost': 5.4999999999999995e-05, 'cache_hit': False}, 'input': {'messages': [{'role': 'user', 'content': 'This is a test'}]}, 'output': {'content': 'This is a test response', 'role': 'assistant', 'tool_calls': None, 'function_call': None}, 'level': , 'id': 'time-07-57-50-585058_chatcmpl-05db6b0b-4847-4b87-a8ef-7d90bef7f81c', 'endTime': datetime.datetime(2024, 7, 11, 7, 57, 50, 587160), 'completionStartTime': datetime.datetime(2024, 7, 11, 7, 57, 50, 587160), 'model': 'gpt-3.5-turbo', 'modelParameters': {'temperature': '0.7', 'max_tokens': 5, 'user': 'langfuse_latency_test_user', 'extra_body': '{}'}, 'usage': {'input': 10, 'output': 20, 'unit': , 'totalCost': 5.4999999999999995e-05}}, 'timestamp': datetime.datetime(2024, 7, 11, 14, 57, 50, 590731, tzinfo=datetime.timezone.utc)}], 'metadata': {'batch_size': 2, 'sdk_integration': 'default', 'sdk_name': 'python', 'sdk_version': '2.39.1', 'public_key': 'pk-lf-b3db7e8e-c2f6-4fc7-825c-a541a8fbe003'}} -making request: {"batch": [{"id": "27ef4895-21f0-4485-b9e6-083e1d64e5fa", "type": "trace-create", "body": {"id": "litellm-test-0020e225-3dc4-43d5-b75f-468a042e51be", "timestamp": "2024-07-11T14:57:50.588244Z", "name": "litellm-acompletion", "userId": "langfuse_latency_test_user", "input": {"messages": [{"role": "user", "content": "This is a test"}]}, "output": {"content": "This is a test response", "role": "assistant", "tool_calls": null, "function_call": null}, "tags": []}, "timestamp": "2024-07-11T14:57:50.589416Z"}, {"id": "8f77e83d-d4b4-4890-b37a-bf04611f2ed6", "type": "generation-create", "body": {"traceId": "litellm-test-0020e225-3dc4-43d5-b75f-468a042e51be", "name": "litellm-acompletion", "startTime": "2024-07-11T07:57:50.585058-07:00", "metadata": {"litellm_response_cost": 5.4999999999999995e-05, "cache_hit": false}, "input": {"messages": [{"role": "user", "content": "This is a test"}]}, "output": {"content": "This is a test response", "role": "assistant", "tool_calls": null, "function_call": null}, "level": "DEFAULT", "id": "time-07-57-50-585058_chatcmpl-05db6b0b-4847-4b87-a8ef-7d90bef7f81c", "endTime": "2024-07-11T07:57:50.587160-07:00", "completionStartTime": "2024-07-11T07:57:50.587160-07:00", "model": "gpt-3.5-turbo", "modelParameters": {"temperature": "0.7", "max_tokens": 5, "user": "langfuse_latency_test_user", "extra_body": "{}"}, "usage": {"input": 10, "output": 20, "unit": "TOKENS", "totalCost": 5.4999999999999995e-05}}, "timestamp": "2024-07-11T14:57:50.590731Z"}], "metadata": {"batch_size": 2, "sdk_integration": "default", "sdk_name": "python", "sdk_version": "2.39.1", "public_key": "pk-lf-b3db7e8e-c2f6-4fc7-825c-a541a8fbe003"}} to https://us.cloud.langfuse.com/api/public/ingestion -~0 items in the Langfuse queue -received response: {"errors":[],"successes":[{"id":"27ef4895-21f0-4485-b9e6-083e1d64e5fa","status":201},{"id":"8f77e83d-d4b4-4890-b37a-bf04611f2ed6","status":201}]} -successfully uploaded batch of 2 items -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -Getting trace litellm-test-0020e225-3dc4-43d5-b75f-468a042e51be -~0 items in the Langfuse queue -Getting observations... None, None, None, None, litellm-test-0020e225-3dc4-43d5-b75f-468a042e51be, None, None, None, GENERATION -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -joining 1 consumer threads -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -consumer thread 0 joined -joining 1 consumer threads -~0 items in the Langfuse queue -~0 items in the Langfuse queue -~0 items in the Langfuse queue -consumer thread 0 joined -joining 1 consumer threads -~0 items in the Langfuse queue -~0 items in the Langfuse queue -consumer thread 0 joined -joining 1 consumer threads -~0 items in the Langfuse queue -consumer thread 0 joined diff --git a/litellm/tests/litellm_uuid.txt b/litellm/tests/litellm_uuid.txt deleted file mode 100644 index e4252e46d..000000000 --- a/litellm/tests/litellm_uuid.txt +++ /dev/null @@ -1 +0,0 @@ -80888ede-4881-4876-ab3f-765d47282e66 \ No newline at end of file diff --git a/litellm/tests/.litellm_cache/cache.db b/tests/local_testing/.litellm_cache/cache.db similarity index 100% rename from litellm/tests/.litellm_cache/cache.db rename to tests/local_testing/.litellm_cache/cache.db diff --git a/litellm/tests/adroit-crow-413218-bc47f303efc9.json b/tests/local_testing/adroit-crow-413218-bc47f303efc9.json similarity index 100% rename from litellm/tests/adroit-crow-413218-bc47f303efc9.json rename to tests/local_testing/adroit-crow-413218-bc47f303efc9.json diff --git a/litellm/tests/azure_fine_tune.jsonl b/tests/local_testing/azure_fine_tune.jsonl similarity index 100% rename from litellm/tests/azure_fine_tune.jsonl rename to tests/local_testing/azure_fine_tune.jsonl diff --git a/litellm/tests/batch_job_results_furniture.jsonl b/tests/local_testing/batch_job_results_furniture.jsonl similarity index 100% rename from litellm/tests/batch_job_results_furniture.jsonl rename to tests/local_testing/batch_job_results_furniture.jsonl diff --git a/litellm/tests/conftest.py b/tests/local_testing/conftest.py similarity index 100% rename from litellm/tests/conftest.py rename to tests/local_testing/conftest.py diff --git a/litellm/tests/data_map.txt b/tests/local_testing/data_map.txt similarity index 100% rename from litellm/tests/data_map.txt rename to tests/local_testing/data_map.txt diff --git a/litellm/tests/eagle.wav b/tests/local_testing/eagle.wav similarity index 100% rename from litellm/tests/eagle.wav rename to tests/local_testing/eagle.wav diff --git a/litellm/tests/example_config_yaml/aliases_config.yaml b/tests/local_testing/example_config_yaml/aliases_config.yaml similarity index 100% rename from litellm/tests/example_config_yaml/aliases_config.yaml rename to tests/local_testing/example_config_yaml/aliases_config.yaml diff --git a/litellm/tests/example_config_yaml/azure_config.yaml b/tests/local_testing/example_config_yaml/azure_config.yaml similarity index 100% rename from litellm/tests/example_config_yaml/azure_config.yaml rename to tests/local_testing/example_config_yaml/azure_config.yaml diff --git a/litellm/tests/example_config_yaml/cache_no_params.yaml b/tests/local_testing/example_config_yaml/cache_no_params.yaml similarity index 100% rename from litellm/tests/example_config_yaml/cache_no_params.yaml rename to tests/local_testing/example_config_yaml/cache_no_params.yaml diff --git a/litellm/tests/example_config_yaml/cache_with_params.yaml b/tests/local_testing/example_config_yaml/cache_with_params.yaml similarity index 100% rename from litellm/tests/example_config_yaml/cache_with_params.yaml rename to tests/local_testing/example_config_yaml/cache_with_params.yaml diff --git a/litellm/tests/example_config_yaml/langfuse_config.yaml b/tests/local_testing/example_config_yaml/langfuse_config.yaml similarity index 100% rename from litellm/tests/example_config_yaml/langfuse_config.yaml rename to tests/local_testing/example_config_yaml/langfuse_config.yaml diff --git a/litellm/tests/example_config_yaml/load_balancer.yaml b/tests/local_testing/example_config_yaml/load_balancer.yaml similarity index 100% rename from litellm/tests/example_config_yaml/load_balancer.yaml rename to tests/local_testing/example_config_yaml/load_balancer.yaml diff --git a/litellm/tests/example_config_yaml/opentelemetry_config.yaml b/tests/local_testing/example_config_yaml/opentelemetry_config.yaml similarity index 100% rename from litellm/tests/example_config_yaml/opentelemetry_config.yaml rename to tests/local_testing/example_config_yaml/opentelemetry_config.yaml diff --git a/litellm/tests/example_config_yaml/simple_config.yaml b/tests/local_testing/example_config_yaml/simple_config.yaml similarity index 100% rename from litellm/tests/example_config_yaml/simple_config.yaml rename to tests/local_testing/example_config_yaml/simple_config.yaml diff --git a/litellm/tests/gettysburg.wav b/tests/local_testing/gettysburg.wav similarity index 100% rename from litellm/tests/gettysburg.wav rename to tests/local_testing/gettysburg.wav diff --git a/litellm/tests/large_text.py b/tests/local_testing/large_text.py similarity index 100% rename from litellm/tests/large_text.py rename to tests/local_testing/large_text.py diff --git a/litellm/tests/messages_with_counts.py b/tests/local_testing/messages_with_counts.py similarity index 90% rename from litellm/tests/messages_with_counts.py rename to tests/local_testing/messages_with_counts.py index d91138e3f..da27a9755 100644 --- a/litellm/tests/messages_with_counts.py +++ b/tests/local_testing/messages_with_counts.py @@ -3,7 +3,7 @@ system_message_short = { "role": "system", "content": "You are a bot.", }, - "count": 12 + "count": 12, } system_message = { @@ -11,7 +11,7 @@ system_message = { "role": "system", "content": "You are a helpful, pattern-following assistant that translates corporate jargon into plain English.", }, - "count": 25 + "count": 25, } system_message_long = { @@ -19,7 +19,7 @@ system_message_long = { "role": "system", "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.", }, - "count": 31 + "count": 31, } system_message_unicode = { @@ -27,7 +27,7 @@ system_message_unicode = { "role": "system", "content": "á", }, - "count": 8 + "count": 8, } system_message_with_name = { @@ -36,7 +36,7 @@ system_message_with_name = { "name": "example_user", "content": "New synergies will help drive top-line growth.", }, - "count": 20 + "count": 20, } user_message = { @@ -44,7 +44,7 @@ user_message = { "role": "user", "content": "Hello, how are you?", }, - "count": 13 + "count": 13, } user_message_unicode = { @@ -52,7 +52,7 @@ user_message_unicode = { "role": "user", "content": "á", }, - "count": 8 + "count": 8, } user_message_perf = { @@ -60,7 +60,7 @@ user_message_perf = { "role": "user", "content": "What happens in a performance review?", }, - "count": 14 + "count": 14, } assistant_message_perf = { @@ -68,7 +68,7 @@ assistant_message_perf = { "role": "assistant", "content": "During the performance review at Contoso Electronics, the supervisor will discuss the employee's performance over the past year and provide feedback on areas for improvement. They will also provide an opportunity for the employee to discuss their goals and objectives for the upcoming year. The review is a two-way dialogue between managers and employees, and employees will receive a written summary of their performance review which will include a rating of their performance, feedback, and goals and objectives for the upcoming year [employee_handbook-3.pdf].", }, - "count": 106 + "count": 106, } assistant_message_perf_short = { @@ -76,7 +76,7 @@ assistant_message_perf_short = { "role": "assistant", "content": "The supervisor will discuss the employee's performance and provide feedback on areas for improvement. They will also provide an opportunity for the employee to discuss their goals and objectives for the upcoming year. The review is a two-way dialogue between managers and employees, and employees will receive a written summary of their performance review which will include a rating of their performance, feedback, and goals for the upcoming year [employee_handbook-3.pdf].", }, - "count": 91 + "count": 91, } user_message_dresscode = { @@ -84,7 +84,7 @@ user_message_dresscode = { "role": "user", "content": "Is there a dress code?", }, - "count": 13 + "count": 13, } assistant_message_dresscode = { @@ -92,7 +92,7 @@ assistant_message_dresscode = { "role": "assistant", "content": "Yes, there is a dress code at Contoso Electronics. Look sharp! [employee_handbook-1.pdf]", }, - "count": 30 + "count": 30, } user_message_pm = { @@ -100,7 +100,7 @@ user_message_pm = { "role": "user", "content": "What does a Product Manager do?", }, - "count": 14 + "count": 14, } text_and_image_message = { @@ -117,11 +117,10 @@ text_and_image_message = { }, ], }, - "count": 266 + "count": 266, } - search_sources_toolchoice_auto = { "system_message": { "role": "system", @@ -217,7 +216,12 @@ integer_enum = { "function": { "name": "data_demonstration", "description": "This is the main function description", - "parameters": {"type": "object", "properties": {"integer_enum": {"type": "integer", "enum": [-1, 1]}}}, + "parameters": { + "type": "object", + "properties": { + "integer_enum": {"type": "integer", "enum": [-1, 1]} + }, + }, }, } ], @@ -237,7 +241,12 @@ integer_enum_tool_choice_name = { "function": { "name": "data_demonstration", "description": "This is the main function description", - "parameters": {"type": "object", "properties": {"integer_enum": {"type": "integer", "enum": [-1, 1]}}}, + "parameters": { + "type": "object", + "properties": { + "integer_enum": {"type": "integer", "enum": [-1, 1]} + }, + }, }, } ], @@ -280,7 +289,10 @@ no_parameters_tool_choice_name = { }, } ], - "tool_choice": {"type": "function", "function": {"name": "search_sources"}}, # 2 tokens for "search_sources" + "tool_choice": { + "type": "function", + "function": {"name": "search_sources"}, + }, # 2 tokens for "search_sources" "count": 51, } @@ -295,7 +307,10 @@ no_parameter_description_or_required = { "function": { "name": "search_sources", "description": "Retrieve sources from the Azure AI Search index", - "parameters": {"type": "object", "properties": {"search_query": {"type": "string"}}}, + "parameters": { + "type": "object", + "properties": {"search_query": {"type": "string"}}, + }, }, } ], @@ -424,7 +439,12 @@ inner_object_with_enum_only = { "object_1": { "type": "object", "description": "The object data type as a property", - "properties": {"string_2a": {"type": "string", "enum": ["Happy", "Sad"]}}, + "properties": { + "string_2a": { + "type": "string", + "enum": ["Happy", "Sad"], + } + }, } }, "required": ["object_1"], @@ -467,7 +487,10 @@ inner_object_with_enum = { "type": "object", "description": "The object data type as a property", "properties": { - "string_2a": {"type": "string", "enum": ["Happy", "Sad"]}, + "string_2a": { + "type": "string", + "enum": ["Happy", "Sad"], + }, "string_2b": { "type": "string", "description": "Description in a second object is lost", @@ -517,14 +540,20 @@ inner_object_and_string = { "type": "object", "description": "The object data type as a property", "properties": { - "string_2a": {"type": "string", "enum": ["Happy", "Sad"]}, + "string_2a": { + "type": "string", + "enum": ["Happy", "Sad"], + }, "string_2b": { "type": "string", "description": "Description in a second object is lost", }, }, }, - "string_1": {"type": "string", "description": "Not required gets a question mark"}, + "string_1": { + "type": "string", + "description": "Not required gets a question mark", + }, }, "required": ["object_1"], }, @@ -677,12 +706,10 @@ MESSAGES_TEXT = [ user_message_pm, assistant_message_perf, assistant_message_perf_short, - assistant_message_dresscode + assistant_message_dresscode, ] -MESSAGES_WITH_IMAGES = [ - text_and_image_message -] +MESSAGES_WITH_IMAGES = [text_and_image_message] MESSAGES_WITH_TOOLS = [ inner_object, @@ -704,4 +731,3 @@ MESSAGES_WITH_TOOLS = [ no_type, null, ] - diff --git a/litellm/tests/model_cost.json b/tests/local_testing/model_cost.json similarity index 100% rename from litellm/tests/model_cost.json rename to tests/local_testing/model_cost.json diff --git a/litellm/tests/openai_batch_completions.jsonl b/tests/local_testing/openai_batch_completions.jsonl similarity index 100% rename from litellm/tests/openai_batch_completions.jsonl rename to tests/local_testing/openai_batch_completions.jsonl diff --git a/litellm/tests/openai_batch_completions_router.jsonl b/tests/local_testing/openai_batch_completions_router.jsonl similarity index 100% rename from litellm/tests/openai_batch_completions_router.jsonl rename to tests/local_testing/openai_batch_completions_router.jsonl diff --git a/litellm/tests/speech_vertex.mp3 b/tests/local_testing/speech_vertex.mp3 similarity index 100% rename from litellm/tests/speech_vertex.mp3 rename to tests/local_testing/speech_vertex.mp3 diff --git a/litellm/tests/stream_chunk_testdata.py b/tests/local_testing/stream_chunk_testdata.py similarity index 100% rename from litellm/tests/stream_chunk_testdata.py rename to tests/local_testing/stream_chunk_testdata.py diff --git a/litellm/tests/test_acompletion.py b/tests/local_testing/test_acompletion.py similarity index 100% rename from litellm/tests/test_acompletion.py rename to tests/local_testing/test_acompletion.py diff --git a/litellm/tests/test_acooldowns_router.py b/tests/local_testing/test_acooldowns_router.py similarity index 100% rename from litellm/tests/test_acooldowns_router.py rename to tests/local_testing/test_acooldowns_router.py diff --git a/litellm/tests/test_add_function_to_prompt.py b/tests/local_testing/test_add_function_to_prompt.py similarity index 100% rename from litellm/tests/test_add_function_to_prompt.py rename to tests/local_testing/test_add_function_to_prompt.py diff --git a/litellm/tests/test_add_update_models.py b/tests/local_testing/test_add_update_models.py similarity index 100% rename from litellm/tests/test_add_update_models.py rename to tests/local_testing/test_add_update_models.py diff --git a/litellm/tests/test_alangfuse.py b/tests/local_testing/test_alangfuse.py similarity index 100% rename from litellm/tests/test_alangfuse.py rename to tests/local_testing/test_alangfuse.py diff --git a/litellm/tests/test_alerting.py b/tests/local_testing/test_alerting.py similarity index 100% rename from litellm/tests/test_alerting.py rename to tests/local_testing/test_alerting.py diff --git a/litellm/tests/test_amazing_s3_logs.py b/tests/local_testing/test_amazing_s3_logs.py similarity index 100% rename from litellm/tests/test_amazing_s3_logs.py rename to tests/local_testing/test_amazing_s3_logs.py diff --git a/litellm/tests/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py similarity index 100% rename from litellm/tests/test_amazing_vertex_completion.py rename to tests/local_testing/test_amazing_vertex_completion.py diff --git a/litellm/tests/test_anthropic_prompt_caching.py b/tests/local_testing/test_anthropic_prompt_caching.py similarity index 100% rename from litellm/tests/test_anthropic_prompt_caching.py rename to tests/local_testing/test_anthropic_prompt_caching.py diff --git a/litellm/tests/test_aproxy_startup.py b/tests/local_testing/test_aproxy_startup.py similarity index 100% rename from litellm/tests/test_aproxy_startup.py rename to tests/local_testing/test_aproxy_startup.py diff --git a/litellm/tests/test_arize_ai.py b/tests/local_testing/test_arize_ai.py similarity index 100% rename from litellm/tests/test_arize_ai.py rename to tests/local_testing/test_arize_ai.py diff --git a/litellm/tests/test_assistants.py b/tests/local_testing/test_assistants.py similarity index 100% rename from litellm/tests/test_assistants.py rename to tests/local_testing/test_assistants.py diff --git a/litellm/tests/test_async_fn.py b/tests/local_testing/test_async_fn.py similarity index 100% rename from litellm/tests/test_async_fn.py rename to tests/local_testing/test_async_fn.py diff --git a/litellm/tests/test_async_opentelemetry.py b/tests/local_testing/test_async_opentelemetry.py similarity index 100% rename from litellm/tests/test_async_opentelemetry.py rename to tests/local_testing/test_async_opentelemetry.py diff --git a/litellm/tests/test_audio_speech.py b/tests/local_testing/test_audio_speech.py similarity index 100% rename from litellm/tests/test_audio_speech.py rename to tests/local_testing/test_audio_speech.py diff --git a/litellm/tests/test_auth_checks.py b/tests/local_testing/test_auth_checks.py similarity index 100% rename from litellm/tests/test_auth_checks.py rename to tests/local_testing/test_auth_checks.py diff --git a/litellm/tests/test_azure_content_safety.py b/tests/local_testing/test_azure_content_safety.py similarity index 100% rename from litellm/tests/test_azure_content_safety.py rename to tests/local_testing/test_azure_content_safety.py diff --git a/litellm/tests/test_azure_openai.py b/tests/local_testing/test_azure_openai.py similarity index 100% rename from litellm/tests/test_azure_openai.py rename to tests/local_testing/test_azure_openai.py diff --git a/litellm/tests/test_azure_perf.py b/tests/local_testing/test_azure_perf.py similarity index 100% rename from litellm/tests/test_azure_perf.py rename to tests/local_testing/test_azure_perf.py diff --git a/litellm/tests/test_bad_params.py b/tests/local_testing/test_bad_params.py similarity index 100% rename from litellm/tests/test_bad_params.py rename to tests/local_testing/test_bad_params.py diff --git a/litellm/tests/test_banned_keyword_list.py b/tests/local_testing/test_banned_keyword_list.py similarity index 100% rename from litellm/tests/test_banned_keyword_list.py rename to tests/local_testing/test_banned_keyword_list.py diff --git a/litellm/tests/test_batch_completion_return_exceptions.py b/tests/local_testing/test_batch_completion_return_exceptions.py similarity index 98% rename from litellm/tests/test_batch_completion_return_exceptions.py rename to tests/local_testing/test_batch_completion_return_exceptions.py index 4e285771f..2d2ea8675 100644 --- a/litellm/tests/test_batch_completion_return_exceptions.py +++ b/tests/local_testing/test_batch_completion_return_exceptions.py @@ -1,4 +1,5 @@ """https://github.com/BerriAI/litellm/pull/3397/commits/a7ec1772b1457594d3af48cdcb0a382279b841c7#diff-44852387ceb00aade916d6b314dfd5d180499e54f35209ae9c07179febe08b4b.""" + """Test batch_completion's return_exceptions.""" import litellm @@ -14,4 +15,4 @@ def test_batch_completion_return_exceptions_true(): api_key="sk_xxx", # deliberately set invalid key ) - assert isinstance(res[0], litellm.exceptions.AuthenticationError) \ No newline at end of file + assert isinstance(res[0], litellm.exceptions.AuthenticationError) diff --git a/litellm/tests/test_batch_completions.py b/tests/local_testing/test_batch_completions.py similarity index 100% rename from litellm/tests/test_batch_completions.py rename to tests/local_testing/test_batch_completions.py diff --git a/litellm/tests/test_bedrock_completion.py b/tests/local_testing/test_bedrock_completion.py similarity index 100% rename from litellm/tests/test_bedrock_completion.py rename to tests/local_testing/test_bedrock_completion.py diff --git a/litellm/tests/test_blocked_user_list.py b/tests/local_testing/test_blocked_user_list.py similarity index 100% rename from litellm/tests/test_blocked_user_list.py rename to tests/local_testing/test_blocked_user_list.py diff --git a/litellm/tests/test_braintrust.py b/tests/local_testing/test_braintrust.py similarity index 100% rename from litellm/tests/test_braintrust.py rename to tests/local_testing/test_braintrust.py diff --git a/litellm/tests/test_budget_manager.py b/tests/local_testing/test_budget_manager.py similarity index 100% rename from litellm/tests/test_budget_manager.py rename to tests/local_testing/test_budget_manager.py diff --git a/litellm/tests/test_caching.py b/tests/local_testing/test_caching.py similarity index 100% rename from litellm/tests/test_caching.py rename to tests/local_testing/test_caching.py diff --git a/litellm/tests/test_caching_ssl.py b/tests/local_testing/test_caching_ssl.py similarity index 100% rename from litellm/tests/test_caching_ssl.py rename to tests/local_testing/test_caching_ssl.py diff --git a/litellm/tests/test_clarifai_completion.py b/tests/local_testing/test_clarifai_completion.py similarity index 100% rename from litellm/tests/test_clarifai_completion.py rename to tests/local_testing/test_clarifai_completion.py diff --git a/litellm/tests/test_class.py b/tests/local_testing/test_class.py similarity index 100% rename from litellm/tests/test_class.py rename to tests/local_testing/test_class.py diff --git a/litellm/tests/test_clickhouse_logger.py b/tests/local_testing/test_clickhouse_logger.py similarity index 100% rename from litellm/tests/test_clickhouse_logger.py rename to tests/local_testing/test_clickhouse_logger.py diff --git a/litellm/tests/test_cohere_completion.py b/tests/local_testing/test_cohere_completion.py similarity index 100% rename from litellm/tests/test_cohere_completion.py rename to tests/local_testing/test_cohere_completion.py diff --git a/litellm/tests/test_completion.py b/tests/local_testing/test_completion.py similarity index 100% rename from litellm/tests/test_completion.py rename to tests/local_testing/test_completion.py diff --git a/litellm/tests/test_completion_cost.py b/tests/local_testing/test_completion_cost.py similarity index 100% rename from litellm/tests/test_completion_cost.py rename to tests/local_testing/test_completion_cost.py diff --git a/litellm/tests/test_completion_with_retries.py b/tests/local_testing/test_completion_with_retries.py similarity index 100% rename from litellm/tests/test_completion_with_retries.py rename to tests/local_testing/test_completion_with_retries.py diff --git a/litellm/tests/test_config.py b/tests/local_testing/test_config.py similarity index 100% rename from litellm/tests/test_config.py rename to tests/local_testing/test_config.py diff --git a/litellm/tests/test_configs/custom_auth.py b/tests/local_testing/test_configs/custom_auth.py similarity index 100% rename from litellm/tests/test_configs/custom_auth.py rename to tests/local_testing/test_configs/custom_auth.py diff --git a/litellm/tests/test_configs/custom_callbacks.py b/tests/local_testing/test_configs/custom_callbacks.py similarity index 100% rename from litellm/tests/test_configs/custom_callbacks.py rename to tests/local_testing/test_configs/custom_callbacks.py diff --git a/litellm/tests/test_configs/test_bad_config.yaml b/tests/local_testing/test_configs/test_bad_config.yaml similarity index 100% rename from litellm/tests/test_configs/test_bad_config.yaml rename to tests/local_testing/test_configs/test_bad_config.yaml diff --git a/litellm/tests/test_configs/test_cloudflare_azure_with_cache_config.yaml b/tests/local_testing/test_configs/test_cloudflare_azure_with_cache_config.yaml similarity index 100% rename from litellm/tests/test_configs/test_cloudflare_azure_with_cache_config.yaml rename to tests/local_testing/test_configs/test_cloudflare_azure_with_cache_config.yaml diff --git a/litellm/tests/test_configs/test_config.yaml b/tests/local_testing/test_configs/test_config.yaml similarity index 100% rename from litellm/tests/test_configs/test_config.yaml rename to tests/local_testing/test_configs/test_config.yaml diff --git a/litellm/tests/test_configs/test_config_custom_auth.yaml b/tests/local_testing/test_configs/test_config_custom_auth.yaml similarity index 100% rename from litellm/tests/test_configs/test_config_custom_auth.yaml rename to tests/local_testing/test_configs/test_config_custom_auth.yaml diff --git a/litellm/tests/test_configs/test_config_no_auth.yaml b/tests/local_testing/test_configs/test_config_no_auth.yaml similarity index 100% rename from litellm/tests/test_configs/test_config_no_auth.yaml rename to tests/local_testing/test_configs/test_config_no_auth.yaml diff --git a/litellm/tests/test_configs/test_custom_logger.yaml b/tests/local_testing/test_configs/test_custom_logger.yaml similarity index 100% rename from litellm/tests/test_configs/test_custom_logger.yaml rename to tests/local_testing/test_configs/test_custom_logger.yaml diff --git a/litellm/tests/test_configs/test_guardrails_config.yaml b/tests/local_testing/test_configs/test_guardrails_config.yaml similarity index 100% rename from litellm/tests/test_configs/test_guardrails_config.yaml rename to tests/local_testing/test_configs/test_guardrails_config.yaml diff --git a/litellm/tests/test_cost_calc.py b/tests/local_testing/test_cost_calc.py similarity index 100% rename from litellm/tests/test_cost_calc.py rename to tests/local_testing/test_cost_calc.py diff --git a/litellm/tests/test_custom_api_logger.py b/tests/local_testing/test_custom_api_logger.py similarity index 100% rename from litellm/tests/test_custom_api_logger.py rename to tests/local_testing/test_custom_api_logger.py diff --git a/litellm/tests/test_custom_callback_input.py b/tests/local_testing/test_custom_callback_input.py similarity index 100% rename from litellm/tests/test_custom_callback_input.py rename to tests/local_testing/test_custom_callback_input.py diff --git a/litellm/tests/test_custom_callback_router.py b/tests/local_testing/test_custom_callback_router.py similarity index 100% rename from litellm/tests/test_custom_callback_router.py rename to tests/local_testing/test_custom_callback_router.py diff --git a/litellm/tests/test_custom_llm.py b/tests/local_testing/test_custom_llm.py similarity index 100% rename from litellm/tests/test_custom_llm.py rename to tests/local_testing/test_custom_llm.py diff --git a/litellm/tests/test_custom_logger.py b/tests/local_testing/test_custom_logger.py similarity index 100% rename from litellm/tests/test_custom_logger.py rename to tests/local_testing/test_custom_logger.py diff --git a/litellm/tests/test_datadog.py b/tests/local_testing/test_datadog.py similarity index 100% rename from litellm/tests/test_datadog.py rename to tests/local_testing/test_datadog.py diff --git a/litellm/tests/test_deployed_proxy_keygen.py b/tests/local_testing/test_deployed_proxy_keygen.py similarity index 100% rename from litellm/tests/test_deployed_proxy_keygen.py rename to tests/local_testing/test_deployed_proxy_keygen.py diff --git a/litellm/tests/test_dynamic_rate_limit_handler.py b/tests/local_testing/test_dynamic_rate_limit_handler.py similarity index 100% rename from litellm/tests/test_dynamic_rate_limit_handler.py rename to tests/local_testing/test_dynamic_rate_limit_handler.py diff --git a/litellm/tests/test_dynamodb_logs.py b/tests/local_testing/test_dynamodb_logs.py similarity index 100% rename from litellm/tests/test_dynamodb_logs.py rename to tests/local_testing/test_dynamodb_logs.py diff --git a/litellm/tests/test_embedding.py b/tests/local_testing/test_embedding.py similarity index 100% rename from litellm/tests/test_embedding.py rename to tests/local_testing/test_embedding.py diff --git a/litellm/tests/test_exceptions.py b/tests/local_testing/test_exceptions.py similarity index 100% rename from litellm/tests/test_exceptions.py rename to tests/local_testing/test_exceptions.py diff --git a/litellm/tests/test_file_types.py b/tests/local_testing/test_file_types.py similarity index 100% rename from litellm/tests/test_file_types.py rename to tests/local_testing/test_file_types.py diff --git a/litellm/tests/test_fine_tuning_api.py b/tests/local_testing/test_fine_tuning_api.py similarity index 100% rename from litellm/tests/test_fine_tuning_api.py rename to tests/local_testing/test_fine_tuning_api.py diff --git a/litellm/tests/test_function_call_parsing.py b/tests/local_testing/test_function_call_parsing.py similarity index 100% rename from litellm/tests/test_function_call_parsing.py rename to tests/local_testing/test_function_call_parsing.py diff --git a/litellm/tests/test_function_calling.py b/tests/local_testing/test_function_calling.py similarity index 100% rename from litellm/tests/test_function_calling.py rename to tests/local_testing/test_function_calling.py diff --git a/litellm/tests/test_function_setup.py b/tests/local_testing/test_function_setup.py similarity index 100% rename from litellm/tests/test_function_setup.py rename to tests/local_testing/test_function_setup.py diff --git a/litellm/tests/test_gcs_bucket.py b/tests/local_testing/test_gcs_bucket.py similarity index 100% rename from litellm/tests/test_gcs_bucket.py rename to tests/local_testing/test_gcs_bucket.py diff --git a/litellm/tests/test_get_llm_provider.py b/tests/local_testing/test_get_llm_provider.py similarity index 100% rename from litellm/tests/test_get_llm_provider.py rename to tests/local_testing/test_get_llm_provider.py diff --git a/litellm/tests/test_get_model_file.py b/tests/local_testing/test_get_model_file.py similarity index 100% rename from litellm/tests/test_get_model_file.py rename to tests/local_testing/test_get_model_file.py diff --git a/litellm/tests/test_get_model_info.py b/tests/local_testing/test_get_model_info.py similarity index 100% rename from litellm/tests/test_get_model_info.py rename to tests/local_testing/test_get_model_info.py diff --git a/litellm/tests/test_get_model_list.py b/tests/local_testing/test_get_model_list.py similarity index 100% rename from litellm/tests/test_get_model_list.py rename to tests/local_testing/test_get_model_list.py diff --git a/litellm/tests/test_get_optional_params_embeddings.py b/tests/local_testing/test_get_optional_params_embeddings.py similarity index 100% rename from litellm/tests/test_get_optional_params_embeddings.py rename to tests/local_testing/test_get_optional_params_embeddings.py diff --git a/litellm/tests/test_get_optional_params_functions_not_supported.py b/tests/local_testing/test_get_optional_params_functions_not_supported.py similarity index 100% rename from litellm/tests/test_get_optional_params_functions_not_supported.py rename to tests/local_testing/test_get_optional_params_functions_not_supported.py diff --git a/litellm/tests/test_get_secret.py b/tests/local_testing/test_get_secret.py similarity index 100% rename from litellm/tests/test_get_secret.py rename to tests/local_testing/test_get_secret.py diff --git a/litellm/tests/test_google_ai_studio_gemini.py b/tests/local_testing/test_google_ai_studio_gemini.py similarity index 100% rename from litellm/tests/test_google_ai_studio_gemini.py rename to tests/local_testing/test_google_ai_studio_gemini.py diff --git a/litellm/tests/test_guardrails_config.py b/tests/local_testing/test_guardrails_config.py similarity index 100% rename from litellm/tests/test_guardrails_config.py rename to tests/local_testing/test_guardrails_config.py diff --git a/litellm/tests/test_health_check.py b/tests/local_testing/test_health_check.py similarity index 100% rename from litellm/tests/test_health_check.py rename to tests/local_testing/test_health_check.py diff --git a/litellm/tests/test_helicone_integration.py b/tests/local_testing/test_helicone_integration.py similarity index 96% rename from litellm/tests/test_helicone_integration.py rename to tests/local_testing/test_helicone_integration.py index 8e1f0a94e..968a9aa5b 100644 --- a/litellm/tests/test_helicone_integration.py +++ b/tests/local_testing/test_helicone_integration.py @@ -16,10 +16,11 @@ from litellm import completion litellm.num_retries = 3 litellm.success_callback = ["helicone"] os.environ["HELICONE_DEBUG"] = "True" -os.environ['LITELLM_LOG'] = 'DEBUG' +os.environ["LITELLM_LOG"] = "DEBUG" import pytest + def pre_helicone_setup(): """ Set up the logging for the 'pre_helicone_setup' function. @@ -89,6 +90,7 @@ def create_async_task(**completion_kwargs): completion_args.update(completion_kwargs) return asyncio.create_task(litellm.acompletion(**completion_args)) + @pytest.mark.asyncio @pytest.mark.skipif( condition=not os.environ.get("OPENAI_API_KEY", False), @@ -96,13 +98,12 @@ def create_async_task(**completion_kwargs): ) async def test_helicone_logging_metadata(): import uuid + litellm.success_callback = ["helicone"] request_id = str(uuid.uuid4()) - trace_common_metadata = { - "Helicone-Property-Request-Id": request_id - } - + trace_common_metadata = {"Helicone-Property-Request-Id": request_id} + metadata = copy.deepcopy(trace_common_metadata) metadata["Helicone-Property-Conversation"] = "support_issue" metadata["Helicone-Auth"] = os.getenv("HELICONE_API_KEY") diff --git a/litellm/tests/test_hf_prompt_templates.py b/tests/local_testing/test_hf_prompt_templates.py similarity index 100% rename from litellm/tests/test_hf_prompt_templates.py rename to tests/local_testing/test_hf_prompt_templates.py diff --git a/litellm/tests/test_image_generation.py b/tests/local_testing/test_image_generation.py similarity index 100% rename from litellm/tests/test_image_generation.py rename to tests/local_testing/test_image_generation.py diff --git a/litellm/tests/test_img_resize.py b/tests/local_testing/test_img_resize.py similarity index 100% rename from litellm/tests/test_img_resize.py rename to tests/local_testing/test_img_resize.py diff --git a/litellm/tests/test_jwt.py b/tests/local_testing/test_jwt.py similarity index 100% rename from litellm/tests/test_jwt.py rename to tests/local_testing/test_jwt.py diff --git a/litellm/tests/test_key_generate_dynamodb.py b/tests/local_testing/test_key_generate_dynamodb.py similarity index 100% rename from litellm/tests/test_key_generate_dynamodb.py rename to tests/local_testing/test_key_generate_dynamodb.py diff --git a/litellm/tests/test_key_generate_prisma.py b/tests/local_testing/test_key_generate_prisma.py similarity index 100% rename from litellm/tests/test_key_generate_prisma.py rename to tests/local_testing/test_key_generate_prisma.py diff --git a/litellm/tests/test_lakera_ai_prompt_injection.py b/tests/local_testing/test_lakera_ai_prompt_injection.py similarity index 100% rename from litellm/tests/test_lakera_ai_prompt_injection.py rename to tests/local_testing/test_lakera_ai_prompt_injection.py diff --git a/litellm/tests/test_langchain_ChatLiteLLM.py b/tests/local_testing/test_langchain_ChatLiteLLM.py similarity index 100% rename from litellm/tests/test_langchain_ChatLiteLLM.py rename to tests/local_testing/test_langchain_ChatLiteLLM.py diff --git a/litellm/tests/test_langsmith.py b/tests/local_testing/test_langsmith.py similarity index 100% rename from litellm/tests/test_langsmith.py rename to tests/local_testing/test_langsmith.py diff --git a/litellm/tests/test_least_busy_routing.py b/tests/local_testing/test_least_busy_routing.py similarity index 100% rename from litellm/tests/test_least_busy_routing.py rename to tests/local_testing/test_least_busy_routing.py diff --git a/litellm/tests/test_litellm_max_budget.py b/tests/local_testing/test_litellm_max_budget.py similarity index 100% rename from litellm/tests/test_litellm_max_budget.py rename to tests/local_testing/test_litellm_max_budget.py diff --git a/litellm/tests/test_llm_guard.py b/tests/local_testing/test_llm_guard.py similarity index 100% rename from litellm/tests/test_llm_guard.py rename to tests/local_testing/test_llm_guard.py diff --git a/litellm/tests/test_load_test_router_s3.py b/tests/local_testing/test_load_test_router_s3.py similarity index 100% rename from litellm/tests/test_load_test_router_s3.py rename to tests/local_testing/test_load_test_router_s3.py diff --git a/litellm/tests/test_loadtest_router.py b/tests/local_testing/test_loadtest_router.py similarity index 100% rename from litellm/tests/test_loadtest_router.py rename to tests/local_testing/test_loadtest_router.py diff --git a/litellm/tests/test_logfire.py b/tests/local_testing/test_logfire.py similarity index 100% rename from litellm/tests/test_logfire.py rename to tests/local_testing/test_logfire.py diff --git a/litellm/tests/test_logging.py b/tests/local_testing/test_logging.py similarity index 100% rename from litellm/tests/test_logging.py rename to tests/local_testing/test_logging.py diff --git a/litellm/tests/test_longer_context_fallback.py b/tests/local_testing/test_longer_context_fallback.py similarity index 100% rename from litellm/tests/test_longer_context_fallback.py rename to tests/local_testing/test_longer_context_fallback.py diff --git a/litellm/tests/test_lowest_cost_routing.py b/tests/local_testing/test_lowest_cost_routing.py similarity index 100% rename from litellm/tests/test_lowest_cost_routing.py rename to tests/local_testing/test_lowest_cost_routing.py diff --git a/litellm/tests/test_lowest_latency_routing.py b/tests/local_testing/test_lowest_latency_routing.py similarity index 100% rename from litellm/tests/test_lowest_latency_routing.py rename to tests/local_testing/test_lowest_latency_routing.py diff --git a/litellm/tests/test_lunary.py b/tests/local_testing/test_lunary.py similarity index 100% rename from litellm/tests/test_lunary.py rename to tests/local_testing/test_lunary.py diff --git a/litellm/tests/test_max_tpm_rpm_limiter.py b/tests/local_testing/test_max_tpm_rpm_limiter.py similarity index 100% rename from litellm/tests/test_max_tpm_rpm_limiter.py rename to tests/local_testing/test_max_tpm_rpm_limiter.py diff --git a/litellm/tests/test_mem_usage.py b/tests/local_testing/test_mem_usage.py similarity index 100% rename from litellm/tests/test_mem_usage.py rename to tests/local_testing/test_mem_usage.py diff --git a/litellm/tests/test_mock_request.py b/tests/local_testing/test_mock_request.py similarity index 100% rename from litellm/tests/test_mock_request.py rename to tests/local_testing/test_mock_request.py diff --git a/litellm/tests/test_model_alias_map.py b/tests/local_testing/test_model_alias_map.py similarity index 100% rename from litellm/tests/test_model_alias_map.py rename to tests/local_testing/test_model_alias_map.py diff --git a/litellm/tests/test_model_max_token_adjust.py b/tests/local_testing/test_model_max_token_adjust.py similarity index 100% rename from litellm/tests/test_model_max_token_adjust.py rename to tests/local_testing/test_model_max_token_adjust.py diff --git a/litellm/tests/test_model_response_typing/server.py b/tests/local_testing/test_model_response_typing/server.py similarity index 100% rename from litellm/tests/test_model_response_typing/server.py rename to tests/local_testing/test_model_response_typing/server.py diff --git a/litellm/tests/test_model_response_typing/test.py b/tests/local_testing/test_model_response_typing/test.py similarity index 100% rename from litellm/tests/test_model_response_typing/test.py rename to tests/local_testing/test_model_response_typing/test.py diff --git a/litellm/tests/test_multiple_deployments.py b/tests/local_testing/test_multiple_deployments.py similarity index 100% rename from litellm/tests/test_multiple_deployments.py rename to tests/local_testing/test_multiple_deployments.py diff --git a/litellm/tests/test_ollama.py b/tests/local_testing/test_ollama.py similarity index 100% rename from litellm/tests/test_ollama.py rename to tests/local_testing/test_ollama.py diff --git a/litellm/tests/test_ollama_local.py b/tests/local_testing/test_ollama_local.py similarity index 100% rename from litellm/tests/test_ollama_local.py rename to tests/local_testing/test_ollama_local.py diff --git a/litellm/tests/test_ollama_local_chat.py b/tests/local_testing/test_ollama_local_chat.py similarity index 100% rename from litellm/tests/test_ollama_local_chat.py rename to tests/local_testing/test_ollama_local_chat.py diff --git a/litellm/tests/test_openai_batches_and_files.py b/tests/local_testing/test_openai_batches_and_files.py similarity index 100% rename from litellm/tests/test_openai_batches_and_files.py rename to tests/local_testing/test_openai_batches_and_files.py diff --git a/litellm/tests/test_openai_moderations_hook.py b/tests/local_testing/test_openai_moderations_hook.py similarity index 100% rename from litellm/tests/test_openai_moderations_hook.py rename to tests/local_testing/test_openai_moderations_hook.py diff --git a/litellm/tests/test_parallel_request_limiter.py b/tests/local_testing/test_parallel_request_limiter.py similarity index 100% rename from litellm/tests/test_parallel_request_limiter.py rename to tests/local_testing/test_parallel_request_limiter.py diff --git a/litellm/tests/test_pass_through_endpoints.py b/tests/local_testing/test_pass_through_endpoints.py similarity index 100% rename from litellm/tests/test_pass_through_endpoints.py rename to tests/local_testing/test_pass_through_endpoints.py diff --git a/litellm/tests/test_presidio_masking.py b/tests/local_testing/test_presidio_masking.py similarity index 100% rename from litellm/tests/test_presidio_masking.py rename to tests/local_testing/test_presidio_masking.py diff --git a/litellm/tests/test_profiling_router.py b/tests/local_testing/test_profiling_router.py similarity index 100% rename from litellm/tests/test_profiling_router.py rename to tests/local_testing/test_profiling_router.py diff --git a/litellm/tests/test_prometheus.py b/tests/local_testing/test_prometheus.py similarity index 100% rename from litellm/tests/test_prometheus.py rename to tests/local_testing/test_prometheus.py diff --git a/litellm/tests/test_prometheus_service.py b/tests/local_testing/test_prometheus_service.py similarity index 100% rename from litellm/tests/test_prometheus_service.py rename to tests/local_testing/test_prometheus_service.py diff --git a/litellm/tests/test_prompt_factory.py b/tests/local_testing/test_prompt_factory.py similarity index 100% rename from litellm/tests/test_prompt_factory.py rename to tests/local_testing/test_prompt_factory.py diff --git a/litellm/tests/test_prompt_injection_detection.py b/tests/local_testing/test_prompt_injection_detection.py similarity index 100% rename from litellm/tests/test_prompt_injection_detection.py rename to tests/local_testing/test_prompt_injection_detection.py diff --git a/litellm/tests/test_promptlayer_integration.py b/tests/local_testing/test_promptlayer_integration.py similarity index 99% rename from litellm/tests/test_promptlayer_integration.py rename to tests/local_testing/test_promptlayer_integration.py index 7935a69b6..f55765757 100644 --- a/litellm/tests/test_promptlayer_integration.py +++ b/tests/local_testing/test_promptlayer_integration.py @@ -100,6 +100,7 @@ def test_promptlayer_logging_with_metadata_tags(): except Exception as e: pytest.fail(f"Error occurred: {e}") + # def test_chat_openai(): # try: # response = completion(model="replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1", diff --git a/litellm/tests/test_provider_specific_config.py b/tests/local_testing/test_provider_specific_config.py similarity index 100% rename from litellm/tests/test_provider_specific_config.py rename to tests/local_testing/test_provider_specific_config.py diff --git a/litellm/tests/test_proxy_custom_auth.py b/tests/local_testing/test_proxy_custom_auth.py similarity index 100% rename from litellm/tests/test_proxy_custom_auth.py rename to tests/local_testing/test_proxy_custom_auth.py diff --git a/litellm/tests/test_proxy_custom_logger.py b/tests/local_testing/test_proxy_custom_logger.py similarity index 100% rename from litellm/tests/test_proxy_custom_logger.py rename to tests/local_testing/test_proxy_custom_logger.py diff --git a/litellm/tests/test_proxy_encrypt_decrypt.py b/tests/local_testing/test_proxy_encrypt_decrypt.py similarity index 100% rename from litellm/tests/test_proxy_encrypt_decrypt.py rename to tests/local_testing/test_proxy_encrypt_decrypt.py diff --git a/litellm/tests/test_proxy_exception_mapping.py b/tests/local_testing/test_proxy_exception_mapping.py similarity index 100% rename from litellm/tests/test_proxy_exception_mapping.py rename to tests/local_testing/test_proxy_exception_mapping.py diff --git a/litellm/tests/test_proxy_gunicorn.py b/tests/local_testing/test_proxy_gunicorn.py similarity index 100% rename from litellm/tests/test_proxy_gunicorn.py rename to tests/local_testing/test_proxy_gunicorn.py diff --git a/litellm/tests/test_proxy_pass_user_config.py b/tests/local_testing/test_proxy_pass_user_config.py similarity index 100% rename from litellm/tests/test_proxy_pass_user_config.py rename to tests/local_testing/test_proxy_pass_user_config.py diff --git a/litellm/tests/test_proxy_reject_logging.py b/tests/local_testing/test_proxy_reject_logging.py similarity index 100% rename from litellm/tests/test_proxy_reject_logging.py rename to tests/local_testing/test_proxy_reject_logging.py diff --git a/litellm/tests/test_proxy_routes.py b/tests/local_testing/test_proxy_routes.py similarity index 100% rename from litellm/tests/test_proxy_routes.py rename to tests/local_testing/test_proxy_routes.py diff --git a/litellm/tests/test_proxy_server.py b/tests/local_testing/test_proxy_server.py similarity index 100% rename from litellm/tests/test_proxy_server.py rename to tests/local_testing/test_proxy_server.py diff --git a/litellm/tests/test_proxy_server_caching.py b/tests/local_testing/test_proxy_server_caching.py similarity index 100% rename from litellm/tests/test_proxy_server_caching.py rename to tests/local_testing/test_proxy_server_caching.py diff --git a/litellm/tests/test_proxy_server_cost.py b/tests/local_testing/test_proxy_server_cost.py similarity index 100% rename from litellm/tests/test_proxy_server_cost.py rename to tests/local_testing/test_proxy_server_cost.py diff --git a/litellm/tests/test_proxy_server_keys.py b/tests/local_testing/test_proxy_server_keys.py similarity index 100% rename from litellm/tests/test_proxy_server_keys.py rename to tests/local_testing/test_proxy_server_keys.py diff --git a/litellm/tests/test_proxy_server_langfuse.py b/tests/local_testing/test_proxy_server_langfuse.py similarity index 100% rename from litellm/tests/test_proxy_server_langfuse.py rename to tests/local_testing/test_proxy_server_langfuse.py diff --git a/litellm/tests/test_proxy_server_spend.py b/tests/local_testing/test_proxy_server_spend.py similarity index 100% rename from litellm/tests/test_proxy_server_spend.py rename to tests/local_testing/test_proxy_server_spend.py diff --git a/litellm/tests/test_proxy_setting_guardrails.py b/tests/local_testing/test_proxy_setting_guardrails.py similarity index 100% rename from litellm/tests/test_proxy_setting_guardrails.py rename to tests/local_testing/test_proxy_setting_guardrails.py diff --git a/litellm/tests/test_proxy_token_counter.py b/tests/local_testing/test_proxy_token_counter.py similarity index 100% rename from litellm/tests/test_proxy_token_counter.py rename to tests/local_testing/test_proxy_token_counter.py diff --git a/litellm/tests/test_pydantic.py b/tests/local_testing/test_pydantic.py similarity index 100% rename from litellm/tests/test_pydantic.py rename to tests/local_testing/test_pydantic.py diff --git a/litellm/tests/test_pydantic_namespaces.py b/tests/local_testing/test_pydantic_namespaces.py similarity index 60% rename from litellm/tests/test_pydantic_namespaces.py rename to tests/local_testing/test_pydantic_namespaces.py index 8314216e1..61c5bd6b4 100644 --- a/litellm/tests/test_pydantic_namespaces.py +++ b/tests/local_testing/test_pydantic_namespaces.py @@ -1,10 +1,13 @@ import warnings import pytest + def test_namespace_conflict_warning(): with warnings.catch_warnings(record=True) as recorded_warnings: warnings.simplefilter("always") # Capture all warnings import litellm # Check that no warning with the specific message was raised - assert not any("conflict with protected namespace" in str(w.message) for w in recorded_warnings), "Test failed: 'conflict with protected namespace' warning was encountered!" + assert not any( + "conflict with protected namespace" in str(w.message) for w in recorded_warnings + ), "Test failed: 'conflict with protected namespace' warning was encountered!" diff --git a/litellm/tests/test_python_38.py b/tests/local_testing/test_python_38.py similarity index 100% rename from litellm/tests/test_python_38.py rename to tests/local_testing/test_python_38.py diff --git a/litellm/tests/test_register_model.py b/tests/local_testing/test_register_model.py similarity index 100% rename from litellm/tests/test_register_model.py rename to tests/local_testing/test_register_model.py diff --git a/litellm/tests/test_rerank.py b/tests/local_testing/test_rerank.py similarity index 100% rename from litellm/tests/test_rerank.py rename to tests/local_testing/test_rerank.py diff --git a/litellm/tests/test_router.py b/tests/local_testing/test_router.py similarity index 100% rename from litellm/tests/test_router.py rename to tests/local_testing/test_router.py diff --git a/litellm/tests/test_router_batch_completion.py b/tests/local_testing/test_router_batch_completion.py similarity index 100% rename from litellm/tests/test_router_batch_completion.py rename to tests/local_testing/test_router_batch_completion.py diff --git a/litellm/tests/test_router_caching.py b/tests/local_testing/test_router_caching.py similarity index 100% rename from litellm/tests/test_router_caching.py rename to tests/local_testing/test_router_caching.py diff --git a/litellm/tests/test_router_client_init.py b/tests/local_testing/test_router_client_init.py similarity index 100% rename from litellm/tests/test_router_client_init.py rename to tests/local_testing/test_router_client_init.py diff --git a/litellm/tests/test_router_cooldowns.py b/tests/local_testing/test_router_cooldowns.py similarity index 100% rename from litellm/tests/test_router_cooldowns.py rename to tests/local_testing/test_router_cooldowns.py diff --git a/litellm/tests/test_router_custom_routing.py b/tests/local_testing/test_router_custom_routing.py similarity index 100% rename from litellm/tests/test_router_custom_routing.py rename to tests/local_testing/test_router_custom_routing.py diff --git a/litellm/tests/test_router_debug_logs.py b/tests/local_testing/test_router_debug_logs.py similarity index 100% rename from litellm/tests/test_router_debug_logs.py rename to tests/local_testing/test_router_debug_logs.py diff --git a/litellm/tests/test_router_fallbacks.py b/tests/local_testing/test_router_fallbacks.py similarity index 100% rename from litellm/tests/test_router_fallbacks.py rename to tests/local_testing/test_router_fallbacks.py diff --git a/litellm/tests/test_router_get_deployments.py b/tests/local_testing/test_router_get_deployments.py similarity index 100% rename from litellm/tests/test_router_get_deployments.py rename to tests/local_testing/test_router_get_deployments.py diff --git a/litellm/tests/test_router_init.py b/tests/local_testing/test_router_init.py similarity index 100% rename from litellm/tests/test_router_init.py rename to tests/local_testing/test_router_init.py diff --git a/litellm/tests/test_router_max_parallel_requests.py b/tests/local_testing/test_router_max_parallel_requests.py similarity index 100% rename from litellm/tests/test_router_max_parallel_requests.py rename to tests/local_testing/test_router_max_parallel_requests.py diff --git a/litellm/tests/test_router_policy_violation.py b/tests/local_testing/test_router_policy_violation.py similarity index 100% rename from litellm/tests/test_router_policy_violation.py rename to tests/local_testing/test_router_policy_violation.py diff --git a/litellm/tests/test_router_retries.py b/tests/local_testing/test_router_retries.py similarity index 100% rename from litellm/tests/test_router_retries.py rename to tests/local_testing/test_router_retries.py diff --git a/litellm/tests/test_router_tag_routing.py b/tests/local_testing/test_router_tag_routing.py similarity index 100% rename from litellm/tests/test_router_tag_routing.py rename to tests/local_testing/test_router_tag_routing.py diff --git a/litellm/tests/test_router_timeout.py b/tests/local_testing/test_router_timeout.py similarity index 100% rename from litellm/tests/test_router_timeout.py rename to tests/local_testing/test_router_timeout.py diff --git a/litellm/tests/test_router_utils.py b/tests/local_testing/test_router_utils.py similarity index 100% rename from litellm/tests/test_router_utils.py rename to tests/local_testing/test_router_utils.py diff --git a/litellm/tests/test_router_with_fallbacks.py b/tests/local_testing/test_router_with_fallbacks.py similarity index 100% rename from litellm/tests/test_router_with_fallbacks.py rename to tests/local_testing/test_router_with_fallbacks.py diff --git a/litellm/tests/test_rules.py b/tests/local_testing/test_rules.py similarity index 100% rename from litellm/tests/test_rules.py rename to tests/local_testing/test_rules.py diff --git a/litellm/tests/test_sagemaker.py b/tests/local_testing/test_sagemaker.py similarity index 100% rename from litellm/tests/test_sagemaker.py rename to tests/local_testing/test_sagemaker.py diff --git a/litellm/tests/test_scheduler.py b/tests/local_testing/test_scheduler.py similarity index 100% rename from litellm/tests/test_scheduler.py rename to tests/local_testing/test_scheduler.py diff --git a/litellm/tests/test_secret_detect_hook.py b/tests/local_testing/test_secret_detect_hook.py similarity index 100% rename from litellm/tests/test_secret_detect_hook.py rename to tests/local_testing/test_secret_detect_hook.py diff --git a/litellm/tests/test_secret_manager.py b/tests/local_testing/test_secret_manager.py similarity index 100% rename from litellm/tests/test_secret_manager.py rename to tests/local_testing/test_secret_manager.py diff --git a/litellm/tests/test_simple_shuffle.py b/tests/local_testing/test_simple_shuffle.py similarity index 100% rename from litellm/tests/test_simple_shuffle.py rename to tests/local_testing/test_simple_shuffle.py diff --git a/litellm/tests/test_spend_calculate_endpoint.py b/tests/local_testing/test_spend_calculate_endpoint.py similarity index 100% rename from litellm/tests/test_spend_calculate_endpoint.py rename to tests/local_testing/test_spend_calculate_endpoint.py diff --git a/litellm/tests/test_spend_logs.py b/tests/local_testing/test_spend_logs.py similarity index 100% rename from litellm/tests/test_spend_logs.py rename to tests/local_testing/test_spend_logs.py diff --git a/litellm/tests/test_stream_chunk_builder.py b/tests/local_testing/test_stream_chunk_builder.py similarity index 100% rename from litellm/tests/test_stream_chunk_builder.py rename to tests/local_testing/test_stream_chunk_builder.py diff --git a/litellm/tests/test_streaming.py b/tests/local_testing/test_streaming.py similarity index 100% rename from litellm/tests/test_streaming.py rename to tests/local_testing/test_streaming.py diff --git a/litellm/tests/test_supabase_integration.py b/tests/local_testing/test_supabase_integration.py similarity index 100% rename from litellm/tests/test_supabase_integration.py rename to tests/local_testing/test_supabase_integration.py diff --git a/litellm/tests/test_team_config.py b/tests/local_testing/test_team_config.py similarity index 100% rename from litellm/tests/test_team_config.py rename to tests/local_testing/test_team_config.py diff --git a/litellm/tests/test_text_completion.py b/tests/local_testing/test_text_completion.py similarity index 100% rename from litellm/tests/test_text_completion.py rename to tests/local_testing/test_text_completion.py diff --git a/litellm/tests/test_timeout.py b/tests/local_testing/test_timeout.py similarity index 100% rename from litellm/tests/test_timeout.py rename to tests/local_testing/test_timeout.py diff --git a/litellm/tests/test_together_ai.py b/tests/local_testing/test_together_ai.py similarity index 100% rename from litellm/tests/test_together_ai.py rename to tests/local_testing/test_together_ai.py diff --git a/litellm/tests/test_token_counter.py b/tests/local_testing/test_token_counter.py similarity index 100% rename from litellm/tests/test_token_counter.py rename to tests/local_testing/test_token_counter.py diff --git a/litellm/tests/test_tpm_rpm_routing_v2.py b/tests/local_testing/test_tpm_rpm_routing_v2.py similarity index 100% rename from litellm/tests/test_tpm_rpm_routing_v2.py rename to tests/local_testing/test_tpm_rpm_routing_v2.py diff --git a/litellm/tests/test_traceloop.py b/tests/local_testing/test_traceloop.py similarity index 100% rename from litellm/tests/test_traceloop.py rename to tests/local_testing/test_traceloop.py diff --git a/litellm/tests/test_triton.py b/tests/local_testing/test_triton.py similarity index 86% rename from litellm/tests/test_triton.py rename to tests/local_testing/test_triton.py index cb1e1af28..122247c8a 100644 --- a/litellm/tests/test_triton.py +++ b/tests/local_testing/test_triton.py @@ -11,7 +11,9 @@ def test_split_embedding_by_shape_passes(): "data": [1, 2, 3, 4, 5, 6], } ] - split_output_data = triton.split_embedding_by_shape(data[0]["data"], data[0]["shape"]) + split_output_data = triton.split_embedding_by_shape( + data[0]["data"], data[0]["shape"] + ) assert split_output_data == [[1, 2, 3], [4, 5, 6]] except Exception as e: pytest.fail(f"An exception occured: {e}") diff --git a/litellm/tests/test_update_spend.py b/tests/local_testing/test_update_spend.py similarity index 100% rename from litellm/tests/test_update_spend.py rename to tests/local_testing/test_update_spend.py diff --git a/litellm/tests/test_user_api_key_auth.py b/tests/local_testing/test_user_api_key_auth.py similarity index 100% rename from litellm/tests/test_user_api_key_auth.py rename to tests/local_testing/test_user_api_key_auth.py diff --git a/litellm/tests/test_utils.py b/tests/local_testing/test_utils.py similarity index 100% rename from litellm/tests/test_utils.py rename to tests/local_testing/test_utils.py diff --git a/litellm/tests/test_validate_environment.py b/tests/local_testing/test_validate_environment.py similarity index 100% rename from litellm/tests/test_validate_environment.py rename to tests/local_testing/test_validate_environment.py diff --git a/litellm/tests/test_wandb.py b/tests/local_testing/test_wandb.py similarity index 100% rename from litellm/tests/test_wandb.py rename to tests/local_testing/test_wandb.py diff --git a/litellm/tests/test_whisper.py b/tests/local_testing/test_whisper.py similarity index 100% rename from litellm/tests/test_whisper.py rename to tests/local_testing/test_whisper.py diff --git a/litellm/tests/user_cost.json b/tests/local_testing/user_cost.json similarity index 100% rename from litellm/tests/user_cost.json rename to tests/local_testing/user_cost.json diff --git a/litellm/tests/vertex_ai.jsonl b/tests/local_testing/vertex_ai.jsonl similarity index 100% rename from litellm/tests/vertex_ai.jsonl rename to tests/local_testing/vertex_ai.jsonl diff --git a/litellm/tests/vertex_key.json b/tests/local_testing/vertex_key.json similarity index 100% rename from litellm/tests/vertex_key.json rename to tests/local_testing/vertex_key.json