phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(feat) add comments on starting with gunicorn

Browse source
This commit is contained in:
ishaan-jaff 2024-01-10 17:50:51 +05:30
parent 67dc9adc71
commit 2b9174c3d7
1 changed files with 7 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

11
litellm/proxy/proxy_cli.py
View file

@ -375,13 +375,15 @@ def run_server(
if port == 8000 and is_port_in_use(port): if port == 8000 and is_port_in_use(port):
port = random.randint(1024, 49152) port = random.randint(1024, 49152)
# Gunicorn Application Class
class StandaloneApplication(gunicorn.app.base.BaseApplication): class StandaloneApplication(gunicorn.app.base.BaseApplication):
def __init__(self, app, options=None): def __init__(self, app, options=None):
self.options = options or {} self.options = options or {} # gunicorn options
self.application = app self.application = app # FastAPI app
super().__init__() super().__init__()
def load_config(self): def load_config(self):
# note: This Loads the gunicorn config - has nothing to do with LiteLLM Proxy config
config = { config = {
key: value key: value
for key, value in self.options.items() for key, value in self.options.items()
@ -391,17 +393,18 @@ def run_server(
self.cfg.set(key.lower(), value) self.cfg.set(key.lower(), value)
def load(self): def load(self):
# gunicorn app function
return self.application return self.application
gunicorn_options = { gunicorn_options = {
"bind": f"{host}:{port}", "bind": f"{host}:{port}",
"workers": num_workers, "workers": num_workers, # default is 1
"worker_class": "uvicorn.workers.UvicornWorker", "worker_class": "uvicorn.workers.UvicornWorker",
"preload": True, # Add the preload flag "preload": True, # Add the preload flag
} }
from litellm.proxy.proxy_server import app from litellm.proxy.proxy_server import app
StandaloneApplication(app=app, options=gunicorn_options).run() StandaloneApplication(app=app, options=gunicorn_options).run() # Run gunicorn
if __name__ == "__main__": if __name__ == "__main__":