build(proxy_cli.py): make running gunicorn an optional cli arg

when running proxy locally, running with uvicorn is much better for debugging

.circleci/config.yml

@@ -156,6 +156,7 @@ jobs:
               --config /app/config.yaml \
               --port 4000 \
               --num_workers 8 \
+              --run_gunicorn \ 
               --debug
       - run:
           name: Install curl and dockerize

Dockerfile

@@ -52,4 +52,4 @@ RUN chmod +x entrypoint.sh
 EXPOSE 4000/tcp
 
 ENTRYPOINT ["litellm"]
-CMD ["--port", "4000", "--config", "./proxy_server_config.yaml", "--detailed_debug"]
+CMD ["--port", "4000", "--config", "./proxy_server_config.yaml", "--detailed_debug", "--run_gunicorn"]

Dockerfile.database

@@ -56,4 +56,4 @@ EXPOSE 4000/tcp
 # # Set your entrypoint and command
 
 ENTRYPOINT ["litellm"]
-CMD ["--port", "4000"]
+CMD ["--port", "4000", "--run_gunicorn"]

litellm/proxy/proxy_cli.py

@@ -157,6 +157,12 @@ def is_port_in_use(port):
     type=int,
     help="Number of requests to hit async endpoint with",
 )
+@click.option(
+    "--run_gunicorn",
+    default=False,
+    is_flag=True,
+    help="Starts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)",
+)
 @click.option("--local", is_flag=True, default=False, help="for local debugging")
 def run_server(
     host,
@@ -186,6 +192,7 @@ def run_server(
     use_queue,
     health,
     version,
+    run_gunicorn,
 ):
     global feature_telemetry
     args = locals()
@@ -439,9 +446,9 @@ def run_server(
             port = random.randint(1024, 49152)
         from litellm.proxy.proxy_server import app
 
-        if os.name == "nt":
+        if run_gunicorn == False:
             uvicorn.run(app, host=host, port=port)  # run uvicorn
-        else:
+        elif run_gunicorn == True:
             import gunicorn.app.base
 
             # Gunicorn Application Class

litellm/proxy/proxy_server.py

@@ -245,8 +245,6 @@ async def user_api_key_auth(
             response = await user_custom_auth(request=request, api_key=api_key)
             return UserAPIKeyAuth.model_validate(response)
         ### LITELLM-DEFINED AUTH FUNCTION ###
-        if isinstance(api_key, str):
-            assert api_key.startswith("sk-")  # prevent token hashes from being used
         if master_key is None:
             if isinstance(api_key, str):
                 return UserAPIKeyAuth(api_key=api_key)
@@ -283,6 +281,10 @@ async def user_api_key_auth(
         if is_master_key_valid:
             return UserAPIKeyAuth(api_key=master_key)
 
+        if isinstance(
+            api_key, str
+        ):  # if generated token, make sure it starts with sk-.
+            assert api_key.startswith("sk-")  # prevent token hashes from being used
         if route.startswith("/config/") and not is_master_key_valid:
             raise Exception(f"Only admin can modify config")