phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 10:44:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: support async redis caching

Browse source
This commit is contained in:
Krrish Dholakia 2024-01-12 21:46:41 +05:30
parent 817a3d29b7
commit 007870390d
6 changed files with 357 additions and 122 deletions
Download patch file Download diff file Expand all files Collapse all files

58
litellm/proxy/proxy_cli.py
View file

@ -346,7 +346,7 @@ def run_server(
import gunicorn.app.base
except:
raise ImportError(
"Uvicorn, gunicorn needs to be imported. Run - `pip 'litellm[proxy]'`"
"uvicorn, gunicorn needs to be imported. Run - `pip install 'litellm[proxy]'`"
)
if config is not None:
@ -427,36 +427,40 @@ def run_server(
f"\033[1;34mSee all Router/Swagger docs on http://0.0.0.0:{port} \033[0m\n"
) # noqa
# Gunicorn Application Class
class StandaloneApplication(gunicorn.app.base.BaseApplication):
def __init__(self, app, options=None):
self.options = options or {} # gunicorn options
self.application = app # FastAPI app
super().__init__()
uvicorn.run(
"litellm.proxy.proxy_server:app", host=host, port=port, workers=num_workers
)
def load_config(self):
# note: This Loads the gunicorn config - has nothing to do with LiteLLM Proxy config
config = {
key: value
for key, value in self.options.items()
if key in self.cfg.settings and value is not None
}
for key, value in config.items():
self.cfg.set(key.lower(), value)
# # Gunicorn Application Class
# class StandaloneApplication(gunicorn.app.base.BaseApplication):
# def __init__(self, app, options=None):
# self.options = options or {} # gunicorn options
# self.application = app # FastAPI app
# super().__init__()
def load(self):
# gunicorn app function
return self.application
# def load_config(self):
# # note: This Loads the gunicorn config - has nothing to do with LiteLLM Proxy config
# config = {
# key: value
# for key, value in self.options.items()
# if key in self.cfg.settings and value is not None
# }
# for key, value in config.items():
# self.cfg.set(key.lower(), value)
gunicorn_options = {
"bind": f"{host}:{port}",
"workers": num_workers, # default is 1
"worker_class": "uvicorn.workers.UvicornWorker",
"preload": True, # Add the preload flag
}
from litellm.proxy.proxy_server import app
# def load(self):
# # gunicorn app function
# return self.application
StandaloneApplication(app=app, options=gunicorn_options).run() # Run gunicorn
# gunicorn_options = {
# "bind": f"{host}:{port}",
# "workers": num_workers, # default is 1
# "worker_class": "uvicorn.workers.UvicornWorker",
# "preload": True, # Add the preload flag
# }
# from litellm.proxy.proxy_server import app
# StandaloneApplication(app=app, options=gunicorn_options).run() # Run gunicorn
if __name__ == "__main__":