phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 10:44:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(fix) use gunicorn to start proxt

Browse source
This commit is contained in:
ishaan-jaff 2024-01-10 17:09:03 +05:30
parent c7fe33202d
commit 5136d5980f
2 changed files with 17 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

25
litellm/proxy/proxy_cli.py
View file

@ -366,37 +366,46 @@ def run_server(
use_queue=use_queue,
)
try:
import gunicorn
import uvicorn
except:
raise ImportError(
"Uvicorn needs to be imported. Run - `pip install uvicorn`"
)
if port == 8000 and is_port_in_use(port):
port = random.randint(1024, 49152)
# uvicorn.run(
# "litellm.proxy.proxy_server:app", host=host, port=port, workers=num_workers
# )
from gunicorn.app.base import BaseApplication
import gunicorn.app.base
class StandaloneApplication(BaseApplication):
class StandaloneApplication(gunicorn.app.base.BaseApplication):
def __init__(self, app, options=None):
self.options = options or {}
self.application = app
super().__init__()
def load_config(self):
for key, value in self.options.items():
self.cfg.set(key, value)
config = {
key: value
for key, value in self.options.items()
if key in self.cfg.settings and value is not None
}
for key, value in config.items():
self.cfg.set(key.lower(), value)
def load(self):
return self.application
num_workers = 4 # Set the desired number of Gunicorn workers
host = "0.0.0.0"
gunicorn_options = {
"bind": f"{host}:{port}",
"workers": num_workers,
"worker_class": "uvicorn.workers.UvicornWorker",
"preload": True, # Add the preload flag
}
from litellm.proxy.proxy_server import app
StandaloneApplication(app, gunicorn_options).run()
StandaloneApplication(app=app, options=gunicorn_options).run()
if __name__ == "__main__":