From 0d0f46a826c42f52db56bfdc4e0dbf6913652671 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Sat, 28 Sep 2024 15:03:50 -0700
Subject: [PATCH] [Feat Proxy] Allow using hypercorn for http v2  (#5950)

* use run_hypercorn

* add docs on using hypercorn
---
 docs/my-website/docs/proxy/deploy.md | 22 +++++++++++++++++++-
 litellm/proxy/proxy_cli.py           | 30 +++++++++++++++++++++++++++-
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md
index 1f8d3ec2c..ed1ffbddf 100644
--- a/docs/my-website/docs/proxy/deploy.md
+++ b/docs/my-website/docs/proxy/deploy.md
@@ -684,7 +684,27 @@ docker run ghcr.io/berriai/litellm:main-latest \
 
 Provide an ssl certificate when starting litellm proxy server 
 
-### 3. Providing LiteLLM config.yaml file as a s3, GCS Bucket Object/url
+### 3. Using Http/2 with Hypercorn
+
+Use this if you want to run the proxy with hypercorn to support http/2
+
+**Usage**
+Pass the `--run_hypercorn` flag when starting the proxy
+
+```shell
+docker run \
+    -v $(pwd)/proxy_config.yaml:/app/config.yaml \
+    -p 4000:4000 \
+    -e LITELLM_LOG="DEBUG"\
+    -e SERVER_ROOT_PATH="/api/v1"\
+    -e DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname> \
+    -e LITELLM_MASTER_KEY="sk-1234"\
+    ghcr.io/berriai/litellm:main-latest \
+    --config /app/config.yaml
+    --run_hypercorn
+```
+
+### 4. Providing LiteLLM config.yaml file as a s3, GCS Bucket Object/url
 
 Use this if you cannot mount a config file on your deployment service (example - AWS Fargate, Railway etc)
 
diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py
index 9003b885e..e5abf6e03 100644
--- a/litellm/proxy/proxy_cli.py
+++ b/litellm/proxy/proxy_cli.py
@@ -195,6 +195,12 @@ def is_port_in_use(port):
     is_flag=True,
     help="Starts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)",
 )
+@click.option(
+    "--run_hypercorn",
+    default=False,
+    is_flag=True,
+    help="Starts proxy via hypercorn, instead of uvicorn (supports HTTP/2)",
+)
 @click.option(
     "--ssl_keyfile_path",
     default=None,
@@ -240,6 +246,7 @@ def run_server(
     health,
     version,
     run_gunicorn,
+    run_hypercorn,
     ssl_keyfile_path,
     ssl_certfile_path,
 ):
@@ -668,7 +675,7 @@ def run_server(
         import litellm
         from litellm.proxy.proxy_server import app
 
-        if run_gunicorn == False:
+        if run_gunicorn == False and run_hypercorn == False:
             if ssl_certfile_path is not None and ssl_keyfile_path is not None:
                 print(  # noqa
                     f"\033[1;32mLiteLLM Proxy: Using SSL with certfile: {ssl_certfile_path} and keyfile: {ssl_keyfile_path}\033[0m\n"  # noqa
@@ -771,6 +778,27 @@ def run_server(
             StandaloneApplication(
                 app=app, options=gunicorn_options
             ).run()  # Run gunicorn
+        elif run_hypercorn == True:
+            import asyncio
+
+            from hypercorn.asyncio import serve
+            from hypercorn.config import Config
+
+            print(  # noqa
+                f"\033[1;32mLiteLLM Proxy: Starting server on {host}:{port} using Hypercorn\033[0m\n"  # noqa
+            )  # noqa
+            config = Config()
+            config.bind = [f"{host}:{port}"]
+
+            if ssl_certfile_path is not None and ssl_keyfile_path is not None:
+                print(  # noqa
+                    f"\033[1;32mLiteLLM Proxy: Using SSL with certfile: {ssl_certfile_path} and keyfile: {ssl_keyfile_path}\033[0m\n"  # noqa
+                )
+                config.certfile = ssl_certfile_path
+                config.keyfile = ssl_keyfile_path
+
+            # hypercorn serve raises a type warning when passing a fast api app - even though fast API is a valid type
+            asyncio.run(serve(app, config))  # type: ignore
 
 
 if __name__ == "__main__":