forked from phoenix/litellm-mirror
fix(lowest_tpm_rpm_routing.py): fix parallel rate limit check (#6577)
* fix(lowest_tpm_rpm_routing.py): fix parallel rate limit check * fix(lowest_tpm_rpm_v2.py): return headers in correct format * test: update test * build(deps): bump cookie and express in /docs/my-website (#6566) Bumps [cookie](https://github.com/jshttp/cookie) and [express](https://github.com/expressjs/express). These dependencies needed to be updated together. Updates `cookie` from 0.6.0 to 0.7.1 - [Release notes](https://github.com/jshttp/cookie/releases) - [Commits](https://github.com/jshttp/cookie/compare/v0.6.0...v0.7.1) Updates `express` from 4.20.0 to 4.21.1 - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/4.21.1/History.md) - [Commits](https://github.com/expressjs/express/compare/4.20.0...4.21.1) --- updated-dependencies: - dependency-name: cookie dependency-type: indirect - dependency-name: express dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * docs(virtual_keys.md): update Dockerfile reference (#6554) Signed-off-by: Emmanuel Ferdman <emmanuelferdman@gmail.com> * (proxy fix) - call connect on prisma client when running setup (#6534) * critical fix - call connect on prisma client when running setup * fix test_proxy_server_prisma_setup * fix test_proxy_server_prisma_setup * Add 3.5 haiku (#6588) * feat: add claude-3-5-haiku-20241022 entries * feat: add claude-3-5-haiku-20241022 and vertex_ai/claude-3-5-haiku@20241022 models * add missing entries, remove vision * remove image token costs * Litellm perf improvements 3 (#6573) * perf: move writing key to cache, to background task * perf(litellm_pre_call_utils.py): add otel tracing for pre-call utils adds 200ms on calls with pgdb connected * fix(litellm_pre_call_utils.py'): rename call_type to actual call used * perf(proxy_server.py): remove db logic from _get_config_from_file was causing db calls to occur on every llm request, if team_id was set on key * fix(auth_checks.py): add check for reducing db calls if user/team id does not exist in db reduces latency/call by ~100ms * fix(proxy_server.py): minor fix on existing_settings not incl alerting * fix(exception_mapping_utils.py): map databricks exception string * fix(auth_checks.py): fix auth check logic * test: correctly mark flaky test * fix(utils.py): handle auth token error for tokenizers.from_pretrained * build: fix map * build: fix map * build: fix json for model map * test: remove eol model * fix(proxy_server.py): fix db config loading logic * fix(proxy_server.py): fix order of config / db updates, to ensure fields not overwritten * test: skip test if required env var is missing * test: fix test --------- Signed-off-by: dependabot[bot] <support@github.com> Signed-off-by: Emmanuel Ferdman <emmanuelferdman@gmail.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Emmanuel Ferdman <emmanuelferdman@gmail.com> Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com> Co-authored-by: paul-gauthier <69695708+paul-gauthier@users.noreply.github.com>
This commit is contained in:
Krish Dholakia
GitHub
parent
f3071161ad
commit
695f48a8f1
7 changed files with 148 additions and 64 deletions
|
|
@ -757,12 +757,6 @@ async def _PROXY_track_cost_callback(
|
|||
verbose_proxy_logger.debug("INSIDE _PROXY_track_cost_callback")
|
||||
global prisma_client
|
||||
try:
|
||||
# check if it has collected an entire stream response
|
||||
verbose_proxy_logger.debug(
|
||||
"Proxy: In track_cost_callback for: kwargs=%s and completion_response: %s",
|
||||
kwargs,
|
||||
completion_response,
|
||||
)
|
||||
verbose_proxy_logger.debug(
|
||||
f"kwargs stream: {kwargs.get('stream', None)} + complete streaming response: {kwargs.get('complete_streaming_response', None)}"
|
||||
)
|
||||
|
|
@ -1359,7 +1353,7 @@ class ProxyConfig:
|
|||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
self.config: Dict[str, Any] = {}
|
||||
|
||||
def is_yaml(self, config_file_path: str) -> bool:
|
||||
if not os.path.isfile(config_file_path):
|
||||
|
|
@ -1465,7 +1459,7 @@ class ProxyConfig:
|
|||
"""
|
||||
|
||||
# load existing config
|
||||
config = await self.get_config()
|
||||
config = self.config
|
||||
|
||||
## LITELLM MODULE SETTINGS (e.g. litellm.drop_params=True,..)
|
||||
litellm_settings = config.get("litellm_settings", {})
|
||||
|
|
@ -1518,7 +1512,9 @@ class ProxyConfig:
|
|||
dict: config
|
||||
|
||||
"""
|
||||
global prisma_client, store_model_in_db
|
||||
# Load existing config
|
||||
|
||||
if os.environ.get("LITELLM_CONFIG_BUCKET_NAME") is not None:
|
||||
bucket_name = os.environ.get("LITELLM_CONFIG_BUCKET_NAME")
|
||||
object_key = os.environ.get("LITELLM_CONFIG_BUCKET_OBJECT_KEY")
|
||||
|
|
@ -1540,12 +1536,21 @@ class ProxyConfig:
|
|||
else:
|
||||
# default to file
|
||||
config = await self._get_config_from_file(config_file_path=config_file_path)
|
||||
## UPDATE CONFIG WITH DB
|
||||
if prisma_client is not None:
|
||||
config = await self._update_config_from_db(
|
||||
config=config,
|
||||
prisma_client=prisma_client,
|
||||
store_model_in_db=store_model_in_db,
|
||||
)
|
||||
|
||||
## PRINT YAML FOR CONFIRMING IT WORKS
|
||||
printed_yaml = copy.deepcopy(config)
|
||||
printed_yaml.pop("environment_variables", None)
|
||||
|
||||
config = self._check_for_os_environ_vars(config=config)
|
||||
|
||||
self.config = config
|
||||
return config
|
||||
|
||||
async def load_config( # noqa: PLR0915
|
||||
|
|
@ -2357,6 +2362,55 @@ class ProxyConfig:
|
|||
pass_through_endpoints=general_settings["pass_through_endpoints"]
|
||||
)
|
||||
|
||||
async def _update_config_from_db(
|
||||
self,
|
||||
prisma_client: PrismaClient,
|
||||
config: dict,
|
||||
store_model_in_db: Optional[bool],
|
||||
):
|
||||
|
||||
if store_model_in_db is not True:
|
||||
verbose_proxy_logger.info(
|
||||
"'store_model_in_db' is not True, skipping db updates"
|
||||
)
|
||||
return config
|
||||
|
||||
_tasks = []
|
||||
keys = [
|
||||
"general_settings",
|
||||
"router_settings",
|
||||
"litellm_settings",
|
||||
"environment_variables",
|
||||
]
|
||||
for k in keys:
|
||||
response = prisma_client.get_generic_data(
|
||||
key="param_name", value=k, table_name="config"
|
||||
)
|
||||
_tasks.append(response)
|
||||
|
||||
responses = await asyncio.gather(*_tasks)
|
||||
for response in responses:
|
||||
if response is not None:
|
||||
param_name = getattr(response, "param_name", None)
|
||||
verbose_proxy_logger.info(f"loading {param_name} settings from db")
|
||||
if param_name == "litellm_settings":
|
||||
verbose_proxy_logger.info(
|
||||
f"litellm_settings: {response.param_value}"
|
||||
)
|
||||
param_value = getattr(response, "param_value", None)
|
||||
if param_name is not None and param_value is not None:
|
||||
# check if param_name is already in the config
|
||||
if param_name in config:
|
||||
if isinstance(config[param_name], dict):
|
||||
config[param_name].update(param_value)
|
||||
else:
|
||||
config[param_name] = param_value
|
||||
else:
|
||||
# if it's not in the config - then add it
|
||||
config[param_name] = param_value
|
||||
|
||||
return config
|
||||
|
||||
async def add_deployment(
|
||||
self,
|
||||
prisma_client: PrismaClient,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue