phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(fix) proxy cleanup

Browse source
This commit is contained in:
ishaan-jaff 2023-11-09 11:31:31 -08:00
parent 333268c8b7
commit 64a50d6b98
1 changed files with 14 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

27
litellm/proxy/proxy_server.py
View file

@ -79,7 +79,8 @@ def generate_feedback_box():
"\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m" "\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m"
) )
print() print()
print("\033[1;34mDocs: https://docs.litellm.ai/docs/proxy_server\033[0m") print("\033[1;34mDocs: https://docs.litellm.ai/docs/simple_proxy\033[0m\n")
print(f"\033[32mLiteLLM: Test your local endpoint with: \"litellm --test\" [In a new terminal tab]\033[0m\n")
print() print()
import litellm import litellm
@ -207,12 +208,8 @@ def load_router_config(router: Optional[litellm.Router], config_file_path: str):
pass pass
except: except:
pass pass
## SERVER SETTINGS (e.g. default completion model = 'ollama/mistral') print_verbose(f"Configs passed in, loaded config YAML\n{config}")
_server_settings = config.get("server_settings", None)
if _server_settings:
server_settings = _server_settings
## LITELLM MODULE SETTINGS (e.g. litellm.drop_params=True,..) ## LITELLM MODULE SETTINGS (e.g. litellm.drop_params=True,..)
litellm_settings = config.get('litellm_settings', None) litellm_settings = config.get('litellm_settings', None)
if litellm_settings: if litellm_settings:
@ -221,9 +218,13 @@ def load_router_config(router: Optional[litellm.Router], config_file_path: str):
## MODEL LIST ## MODEL LIST
model_list = config.get('model_list', None) model_list = config.get('model_list', None)
if model_list: if model_list:
router = litellm.Router(model_list=model_list) router = litellm.Router(model_list=model_list)
print(f"\033[32mLiteLLM: Proxy initialized with Config, Set models:\033[0m")
for model in model_list:
print(f"\033[32m {model.get('model_name', '')}\033[0m")
print()
## ENVIRONMENT VARIABLES ## ENVIRONMENT VARIABLES
environment_variables = config.get('environment_variables', None) environment_variables = config.get('environment_variables', None)
if environment_variables: if environment_variables:
@ -392,7 +393,7 @@ def initialize(
if max_budget: # litellm-specific param if max_budget: # litellm-specific param
litellm.max_budget = max_budget litellm.max_budget = max_budget
dynamic_config["general"]["max_budget"] = max_budget dynamic_config["general"]["max_budget"] = max_budget
if debug: # litellm-specific param if debug==True: # litellm-specific param
litellm.set_verbose = True litellm.set_verbose = True
if experimental: if experimental:
pass pass
@ -415,7 +416,7 @@ def data_generator(response):
def litellm_completion(*args, **kwargs): def litellm_completion(*args, **kwargs):
global user_temperature, user_request_timeout, user_max_tokens, user_api_base global user_temperature, user_request_timeout, user_max_tokens, user_api_base
call_type = kwargs.pop("call_type") call_type = kwargs.pop("call_type")
# override with user settings # override with user settings, these are params passed via cli
if user_temperature: if user_temperature:
kwargs["temperature"] = user_temperature kwargs["temperature"] = user_temperature
if user_request_timeout: if user_request_timeout:
@ -427,7 +428,7 @@ def litellm_completion(*args, **kwargs):
## CHECK CONFIG ## ## CHECK CONFIG ##
if llm_model_list and kwargs["model"] in [m["model_name"] for m in llm_model_list]: if llm_model_list and kwargs["model"] in [m["model_name"] for m in llm_model_list]:
for m in llm_model_list: for m in llm_model_list:
if kwargs["model"] == m["model_name"]: if kwargs["model"] == m["model_name"]: # if user has specified a config, this will use the config
for key, value in m["litellm_params"].items(): for key, value in m["litellm_params"].items():
kwargs[key] = value kwargs[key] = value
break break
@ -445,7 +446,7 @@ def startup_event():
import json import json
worker_config = json.loads(os.getenv("WORKER_CONFIG")) worker_config = json.loads(os.getenv("WORKER_CONFIG"))
initialize(**worker_config) initialize(**worker_config)
print(f"\033[32mWorker Initialized\033[0m\n") # print(f"\033[32mWorker Initialized\033[0m\n")
#### API ENDPOINTS #### #### API ENDPOINTS ####
@router.get("/v1/models") @router.get("/v1/models")