phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

build(litellm_server/utils.py): add support for general settings + num retries as a module variable

Browse source
This commit is contained in:
Krrish Dholakia 2023-11-02 20:56:33 -07:00
parent 3f1b4c0759
commit e3a1c58dd9
7 changed files with 52 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

18
litellm_server/utils.py
View file

@ -43,16 +43,17 @@ def set_callbacks():
## CACHING
### REDIS
if len(os.getenv("REDIS_HOST", "")) > 0 and len(os.getenv("REDIS_PORT", "")) > 0 and len(os.getenv("REDIS_PASSWORD", "")) > 0:
from litellm.caching import Cache
litellm.cache = Cache(type="redis", host=os.getenv("REDIS_HOST"), port=os.getenv("REDIS_PORT"), password=os.getenv("REDIS_PASSWORD"))
print("\033[92mLiteLLM: Switched on Redis caching\033[0m")
# if len(os.getenv("REDIS_HOST", "")) > 0 and len(os.getenv("REDIS_PORT", "")) > 0 and len(os.getenv("REDIS_PASSWORD", "")) > 0:
# print(f"redis host: {os.getenv('REDIS_HOST')}; redis port: {os.getenv('REDIS_PORT')}; password: {os.getenv('REDIS_PASSWORD')}")
# from litellm.caching import Cache
# litellm.cache = Cache(type="redis", host=os.getenv("REDIS_HOST"), port=os.getenv("REDIS_PORT"), password=os.getenv("REDIS_PASSWORD"))
# print("\033[92mLiteLLM: Switched on Redis caching\033[0m")
def load_router_config(router: Optional[litellm.Router], config_file_path: Optional[str]='/app/config.yaml'):
config = {}
server_settings = {}
try:
if os.path.exists(config_file_path):
with open(config_file_path, 'r') as file:
@ -62,6 +63,11 @@ def load_router_config(router: Optional[litellm.Router], config_file_path: Optio
except:
pass
## SERVER SETTINGS (e.g. default completion model = 'ollama/mistral')
server_settings = config.get("server_settings", None)
if server_settings:
server_settings = server_settings
## LITELLM MODULE SETTINGS (e.g. litellm.drop_params=True,..)
litellm_settings = config.get('litellm_settings', None)
if litellm_settings:
@ -79,4 +85,4 @@ def load_router_config(router: Optional[litellm.Router], config_file_path: Optio
for key, value in environment_variables.items():
os.environ[key] = value
return router, model_list
return router, model_list, server_settings