From 6d000742c66322bc97eeb954bffc8cc480c415db Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 20 Nov 2024 16:52:07 -0800 Subject: [PATCH 1/6] fix - don't block proxy startup if not a premium user --- litellm/proxy/common_utils/callback_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index c3c19f551..40f66e90b 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -242,7 +242,9 @@ def initialize_callbacks_on_proxy( # noqa: PLR0915 if "prometheus" in value: if premium_user is not True: - raise Exception(CommonProxyErrors.not_premium_user.value) + verbose_proxy_logger.warning( + f"Prometheus metrics are only available for premium users. {CommonProxyErrors.not_premium_user.value}" + ) from litellm.proxy.proxy_server import app verbose_proxy_logger.debug("Starting Prometheus Metrics on /metrics") From e1247533176590d789c93827690b8aad11355638 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 20 Nov 2024 17:10:32 -0800 Subject: [PATCH 2/6] test_litellm_proxy_server_config_with_prometheus --- .../test_configs/test_enterprise_config.yaml | 26 ++++++++ tests/local_testing/test_python_38.py | 62 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 tests/local_testing/test_configs/test_enterprise_config.yaml diff --git a/tests/local_testing/test_configs/test_enterprise_config.yaml b/tests/local_testing/test_configs/test_enterprise_config.yaml new file mode 100644 index 000000000..def055fdc --- /dev/null +++ b/tests/local_testing/test_configs/test_enterprise_config.yaml @@ -0,0 +1,26 @@ +model_list: + - model_name: Azure OpenAI GPT-4 Canada + litellm_params: + model: azure/chatgpt-v-2 + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: "2023-07-01-preview" + model_info: + mode: chat + input_cost_per_token: 0.0002 + id: gm + - model_name: azure-embedding-model + litellm_params: + model: azure/azure-embedding-model + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: "2023-07-01-preview" + model_info: + mode: embedding + input_cost_per_token: 0.002 + id: hello + +litellm_settings: + drop_params: True + set_verbose: True + callbacks: ["prometheus", "gcs_bucket"] \ No newline at end of file diff --git a/tests/local_testing/test_python_38.py b/tests/local_testing/test_python_38.py index 5fa48f096..027ee2d32 100644 --- a/tests/local_testing/test_python_38.py +++ b/tests/local_testing/test_python_38.py @@ -96,3 +96,65 @@ def test_litellm_proxy_server_config_no_general_settings(): # Additional assertions can be added here assert True + + +def _put_sso_on(): + os.environ["GOOGLE_CLIENT_ID"] = "xxxx" + os.environ["GOOGLE_CLIENT_SECRET"] = "xxxx" + + +def test_litellm_proxy_server_config_with_prometheus(): + # Install the litellm[proxy] package + # Start the server + server_process = None + try: + _put_sso_on() + _old_license = os.environ.pop("LITELLM_LICENSE", None) + subprocess.run(["pip", "install", "-U", "litellm[proxy]"]) + subprocess.run(["pip", "install", "-U", "litellm[extra_proxy]"]) + filepath = os.path.dirname(os.path.abspath(__file__)) + config_fp = f"{filepath}/test_configs/test_enterprise_config.yaml" + server_process = subprocess.Popen( + [ + "python3", + "-m", + "litellm.proxy.proxy_cli", + "--config", + config_fp, + ] + ) + + # Allow some time for the server to start + time.sleep(60) # Adjust the sleep time if necessary + + # Send a request to the /health/liveliness endpoint + response = requests.get("http://localhost:4000/health/liveliness") + print("response= ", response.json()) + + # Check if the response is successful + assert response.status_code == 200 + assert response.json() == "I'm alive!" + + # Test /chat/completions + response = requests.post( + "http://localhost:4000/chat/completions", + headers={"Authorization": "Bearer 1234567890"}, + json={ + "model": "test_openai_models", + "messages": [{"role": "user", "content": "Hello, how are you?"}], + }, + ) + print("response= ", response.json()) + assert response.status_code == 200 + + except ImportError: + pytest.fail("Failed to import litellm.proxy_server") + except requests.ConnectionError: + pytest.fail("Failed to connect to the server") + finally: + # Shut down the server + if _old_license: + os.environ["LITELLM_LICENSE"] = _old_license + if server_process: + server_process.terminate() + server_process.wait() From 048c137f865fd64a4f24629e5569dab85e621064 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 20 Nov 2024 17:20:59 -0800 Subject: [PATCH 3/6] add test for proxy startup --- .circleci/config.yml | 41 +++++++++++++++++ .../enterprise_config.yaml | 17 +++++++ .../test_basic_proxy_startup.py | 44 +++++++++++++++++++ .../test_configs/test_enterprise_config.yaml | 26 ----------- 4 files changed, 102 insertions(+), 26 deletions(-) create mode 100644 litellm/proxy/example_config_yaml/enterprise_config.yaml create mode 100644 tests/basic_proxy_startup_tests/test_basic_proxy_startup.py delete mode 100644 tests/local_testing/test_configs/test_enterprise_config.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a6327bb3..e7005b5d1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1039,6 +1039,47 @@ jobs: ls python -m pytest -vv tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 120m + # Clean up first container + - run: + name: Stop and remove first container + command: | + docker stop my-app + docker rm my-app + + # Second Docker Container Run with Different Config + - run: + name: Run Second Docker container + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e LITELLM_LICENSE="bad-license" \ + --name my-app-3 \ + -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \ + my-app:latest \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug + + - run: + name: Start outputting logs for second container + command: docker logs -f my-app-2 + background: true + + - run: + name: Wait for second app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + + - run: + name: Run second round of tests + command: | + python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5 + no_output_timeout: 120m # Store test results - store_test_results: diff --git a/litellm/proxy/example_config_yaml/enterprise_config.yaml b/litellm/proxy/example_config_yaml/enterprise_config.yaml new file mode 100644 index 000000000..337e85177 --- /dev/null +++ b/litellm/proxy/example_config_yaml/enterprise_config.yaml @@ -0,0 +1,17 @@ +model_list: + - model_name: gpt-4 + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + tags: ["teamA"] + model_info: + id: "team-a-model" + +litellm_settings: + cache: true + callbacks: ["prometheus"] + +router_settings: + enable_tag_filtering: True # 👈 Key Change + diff --git a/tests/basic_proxy_startup_tests/test_basic_proxy_startup.py b/tests/basic_proxy_startup_tests/test_basic_proxy_startup.py new file mode 100644 index 000000000..e49a01413 --- /dev/null +++ b/tests/basic_proxy_startup_tests/test_basic_proxy_startup.py @@ -0,0 +1,44 @@ +import pytest +import aiohttp +from typing import Optional + + +@pytest.mark.asyncio +async def test_health_and_chat_completion(): + """ + Test health endpoints and chat completion: + 1. Check /health/readiness + 2. Check /health/liveness + 3. Make a chat completion call + """ + async with aiohttp.ClientSession() as session: + # Test readiness endpoint + async with session.get("http://0.0.0.0:4000/health/readiness") as response: + assert response.status == 200 + readiness_response = await response.json() + assert readiness_response["status"] == "OK" + + # Test liveness endpoint + async with session.get("http://0.0.0.0:4000/health/liveness") as response: + assert response.status == 200 + liveness_response = await response.json() + assert liveness_response["status"] == "OK" + + # Make a chat completion call + url = "http://0.0.0.0:4000/chat/completions" + headers = { + "Authorization": "Bearer sk-1234", + "Content-Type": "application/json", + } + data = { + "model": "gpt-4", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"}, + ], + } + + async with session.post(url, headers=headers, json=data) as response: + assert response.status == 200 + completion_response = await response.json() + assert "choices" in completion_response diff --git a/tests/local_testing/test_configs/test_enterprise_config.yaml b/tests/local_testing/test_configs/test_enterprise_config.yaml deleted file mode 100644 index def055fdc..000000000 --- a/tests/local_testing/test_configs/test_enterprise_config.yaml +++ /dev/null @@ -1,26 +0,0 @@ -model_list: - - model_name: Azure OpenAI GPT-4 Canada - litellm_params: - model: azure/chatgpt-v-2 - api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY - api_version: "2023-07-01-preview" - model_info: - mode: chat - input_cost_per_token: 0.0002 - id: gm - - model_name: azure-embedding-model - litellm_params: - model: azure/azure-embedding-model - api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY - api_version: "2023-07-01-preview" - model_info: - mode: embedding - input_cost_per_token: 0.002 - id: hello - -litellm_settings: - drop_params: True - set_verbose: True - callbacks: ["prometheus", "gcs_bucket"] \ No newline at end of file From 9e040e12fc0baa7d16e2252672186773a889c4bf Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 20 Nov 2024 17:22:41 -0800 Subject: [PATCH 4/6] fix remove unused test --- tests/local_testing/test_python_38.py | 62 --------------------------- 1 file changed, 62 deletions(-) diff --git a/tests/local_testing/test_python_38.py b/tests/local_testing/test_python_38.py index 027ee2d32..5fa48f096 100644 --- a/tests/local_testing/test_python_38.py +++ b/tests/local_testing/test_python_38.py @@ -96,65 +96,3 @@ def test_litellm_proxy_server_config_no_general_settings(): # Additional assertions can be added here assert True - - -def _put_sso_on(): - os.environ["GOOGLE_CLIENT_ID"] = "xxxx" - os.environ["GOOGLE_CLIENT_SECRET"] = "xxxx" - - -def test_litellm_proxy_server_config_with_prometheus(): - # Install the litellm[proxy] package - # Start the server - server_process = None - try: - _put_sso_on() - _old_license = os.environ.pop("LITELLM_LICENSE", None) - subprocess.run(["pip", "install", "-U", "litellm[proxy]"]) - subprocess.run(["pip", "install", "-U", "litellm[extra_proxy]"]) - filepath = os.path.dirname(os.path.abspath(__file__)) - config_fp = f"{filepath}/test_configs/test_enterprise_config.yaml" - server_process = subprocess.Popen( - [ - "python3", - "-m", - "litellm.proxy.proxy_cli", - "--config", - config_fp, - ] - ) - - # Allow some time for the server to start - time.sleep(60) # Adjust the sleep time if necessary - - # Send a request to the /health/liveliness endpoint - response = requests.get("http://localhost:4000/health/liveliness") - print("response= ", response.json()) - - # Check if the response is successful - assert response.status_code == 200 - assert response.json() == "I'm alive!" - - # Test /chat/completions - response = requests.post( - "http://localhost:4000/chat/completions", - headers={"Authorization": "Bearer 1234567890"}, - json={ - "model": "test_openai_models", - "messages": [{"role": "user", "content": "Hello, how are you?"}], - }, - ) - print("response= ", response.json()) - assert response.status_code == 200 - - except ImportError: - pytest.fail("Failed to import litellm.proxy_server") - except requests.ConnectionError: - pytest.fail("Failed to connect to the server") - finally: - # Shut down the server - if _old_license: - os.environ["LITELLM_LICENSE"] = _old_license - if server_process: - server_process.terminate() - server_process.wait() From 8334cdc3cfaf54128550ae5bea96b261f8aa5eea Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 20 Nov 2024 17:26:42 -0800 Subject: [PATCH 5/6] fix startup test --- tests/basic_proxy_startup_tests/test_basic_proxy_startup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/basic_proxy_startup_tests/test_basic_proxy_startup.py b/tests/basic_proxy_startup_tests/test_basic_proxy_startup.py index e49a01413..a7ce3854d 100644 --- a/tests/basic_proxy_startup_tests/test_basic_proxy_startup.py +++ b/tests/basic_proxy_startup_tests/test_basic_proxy_startup.py @@ -16,13 +16,13 @@ async def test_health_and_chat_completion(): async with session.get("http://0.0.0.0:4000/health/readiness") as response: assert response.status == 200 readiness_response = await response.json() - assert readiness_response["status"] == "OK" + assert readiness_response["status"] == "connected" # Test liveness endpoint async with session.get("http://0.0.0.0:4000/health/liveness") as response: assert response.status == 200 liveness_response = await response.json() - assert liveness_response["status"] == "OK" + print("liveness_response", liveness_response) # Make a chat completion call url = "http://0.0.0.0:4000/chat/completions" From 8df5dc092074472827df753652eea8af93c43d8e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 20 Nov 2024 17:29:31 -0800 Subject: [PATCH 6/6] add comment on bad-license --- .circleci/config.yml | 1 + .../basic_proxy_startup_tests/test_basic_proxy_startup.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e7005b5d1..3c8bdbfc1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1047,6 +1047,7 @@ jobs: docker rm my-app # Second Docker Container Run with Different Config + # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license - run: name: Run Second Docker container command: | diff --git a/tests/basic_proxy_startup_tests/test_basic_proxy_startup.py b/tests/basic_proxy_startup_tests/test_basic_proxy_startup.py index a7ce3854d..db09e38ea 100644 --- a/tests/basic_proxy_startup_tests/test_basic_proxy_startup.py +++ b/tests/basic_proxy_startup_tests/test_basic_proxy_startup.py @@ -1,3 +1,10 @@ +""" +This test ensures that the proxy starts and serves requests even with a bad license. + + +in ci/cd config.yml, we set the license to "bad-license" +""" + import pytest import aiohttp from typing import Optional