From efe6d375e9adec23cc24f1934a41fc8e7e6fe298 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 18:40:03 -0700 Subject: [PATCH 01/22] add new SpendUpdateQueue --- litellm/proxy/db/spend_update_queue.py | 54 ++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 litellm/proxy/db/spend_update_queue.py diff --git a/litellm/proxy/db/spend_update_queue.py b/litellm/proxy/db/spend_update_queue.py new file mode 100644 index 0000000000..778361be84 --- /dev/null +++ b/litellm/proxy/db/spend_update_queue.py @@ -0,0 +1,54 @@ +import asyncio +from typing import TYPE_CHECKING, Any, Dict, List + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient +else: + PrismaClient = Any + + +class SpendUpdateQueue: + """ + Handles buffering database `UPDATE` transactions in Redis before committing them to the database + + This is to prevent deadlocks and improve reliability + """ + + def __init__( + self, + ): + self.update_queue: asyncio.Queue[Dict[str, Any]] = asyncio.Queue() + + async def add_update(self, update: Dict[str, Any]) -> None: + """Enqueue an update. Each update might be a dict like {'entity_type': 'user', 'entity_id': '123', 'amount': 1.2}.""" + await self.update_queue.put(update) + + async def flush_all_updates_from_in_memory_queue(self) -> List[Dict[str, Any]]: + """Get all updates from the queue.""" + updates: List[Dict[str, Any]] = [] + while not self.update_queue.empty(): + updates.append(await self.update_queue.get()) + return updates + + async def flush_and_get_all_aggregated_updates_by_entity_type( + self, + ) -> Dict[str, Any]: + """Flush all updates from the queue and return all updates aggregated by entity type.""" + updates = await self.flush_all_updates_from_in_memory_queue() + return self.aggregate_updates_by_entity_type(updates) + + def aggregate_updates_by_entity_type( + self, updates: List[Dict[str, Any]] + ) -> Dict[str, Any]: + """Aggregate updates by entity type.""" + aggregated_updates = {} + for update in updates: + entity_type = update["entity_type"] + entity_id = update["entity_id"] + amount = update["amount"] + if entity_type not in aggregated_updates: + aggregated_updates[entity_type] = {} + if entity_id not in aggregated_updates[entity_type]: + aggregated_updates[entity_type][entity_id] = 0 + aggregated_updates[entity_type][entity_id] += amount + return aggregated_updates From bcd49204f6153629baa5a26cfd62d1d3ddba6294 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 18:40:52 -0700 Subject: [PATCH 02/22] use spend_update_queue for RedisUpdateBuffer --- litellm/proxy/db/redis_update_buffer.py | 42 ++++++++++--------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/db/redis_update_buffer.py b/litellm/proxy/db/redis_update_buffer.py index f98fc9300f..7370b0ed08 100644 --- a/litellm/proxy/db/redis_update_buffer.py +++ b/litellm/proxy/db/redis_update_buffer.py @@ -12,6 +12,7 @@ from litellm.caching import RedisCache from litellm.constants import MAX_REDIS_BUFFER_DEQUEUE_COUNT, REDIS_UPDATE_BUFFER_KEY from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import DBSpendUpdateTransactions +from litellm.proxy.db.spend_update_queue import SpendUpdateQueue from litellm.secret_managers.main import str_to_bool if TYPE_CHECKING: @@ -32,6 +33,7 @@ class RedisUpdateBuffer: redis_cache: Optional[RedisCache] = None, ): self.redis_cache = redis_cache + self.spend_update_queue = SpendUpdateQueue() @staticmethod def _should_commit_spend_updates_to_redis() -> bool: @@ -54,7 +56,6 @@ class RedisUpdateBuffer: async def store_in_memory_spend_updates_in_redis( self, - prisma_client: PrismaClient, ): """ Stores the in-memory spend updates to Redis @@ -78,13 +79,21 @@ class RedisUpdateBuffer: "redis_cache is None, skipping store_in_memory_spend_updates_in_redis" ) return - db_spend_update_transactions: DBSpendUpdateTransactions = DBSpendUpdateTransactions( - user_list_transactions=prisma_client.user_list_transactions, - end_user_list_transactions=prisma_client.end_user_list_transactions, - key_list_transactions=prisma_client.key_list_transactions, - team_list_transactions=prisma_client.team_list_transactions, - team_member_list_transactions=prisma_client.team_member_list_transactions, - org_list_transactions=prisma_client.org_list_transactions, + + aggregated_updates = ( + await self.spend_update_queue.flush_and_get_all_aggregated_updates_by_entity_type() + ) + verbose_proxy_logger.debug("ALL AGGREGATED UPDATES: ", aggregated_updates) + + db_spend_update_transactions: DBSpendUpdateTransactions = ( + DBSpendUpdateTransactions( + user_list_transactions=aggregated_updates.get("user", {}), + end_user_list_transactions=aggregated_updates.get("end_user", {}), + key_list_transactions=aggregated_updates.get("key", {}), + team_list_transactions=aggregated_updates.get("team", {}), + team_member_list_transactions=aggregated_updates.get("team_member", {}), + org_list_transactions=aggregated_updates.get("org", {}), + ) ) # only store in redis if there are any updates to commit @@ -100,9 +109,6 @@ class RedisUpdateBuffer: values=list_of_transactions, ) - # clear the in-memory spend updates - RedisUpdateBuffer._clear_all_in_memory_spend_updates(prisma_client) - @staticmethod def _number_of_transactions_to_store_in_redis( db_spend_update_transactions: DBSpendUpdateTransactions, @@ -116,20 +122,6 @@ class RedisUpdateBuffer: num_transactions += len(v) return num_transactions - @staticmethod - def _clear_all_in_memory_spend_updates( - prisma_client: PrismaClient, - ): - """ - Clears all in-memory spend updates - """ - prisma_client.user_list_transactions = {} - prisma_client.end_user_list_transactions = {} - prisma_client.key_list_transactions = {} - prisma_client.team_list_transactions = {} - prisma_client.team_member_list_transactions = {} - prisma_client.org_list_transactions = {} - @staticmethod def _remove_prefix_from_keys(data: Dict[str, Any], prefix: str) -> Dict[str, Any]: """ From 3e16a51ca6986095b89c0a3b967a8f28bdcc348b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 19:01:00 -0700 Subject: [PATCH 03/22] fix update_database helper on db_spend_update_writer --- litellm/proxy/db/db_spend_update_writer.py | 181 ++++++------------ litellm/proxy/db/redis_update_buffer.py | 8 +- litellm/proxy/db/spend_update_queue.py | 4 + .../proxy/hooks/proxy_track_cost_callback.py | 6 +- 4 files changed, 73 insertions(+), 126 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index f46b03b57a..6b5719acbb 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -25,6 +25,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.db.pod_lock_manager import PodLockManager from litellm.proxy.db.redis_update_buffer import RedisUpdateBuffer +from litellm.proxy.db.spend_update_queue import SpendUpdateQueue if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient, ProxyLogging @@ -48,10 +49,11 @@ class DBSpendUpdateWriter: self.redis_cache = redis_cache self.redis_update_buffer = RedisUpdateBuffer(redis_cache=self.redis_cache) self.pod_lock_manager = PodLockManager(cronjob_id=DB_SPEND_UPDATE_JOB_NAME) + self.spend_update_queue = SpendUpdateQueue() - @staticmethod async def update_database( # LiteLLM management object fields + self, token: Optional[str], user_id: Optional[str], end_user_id: Optional[str], @@ -84,7 +86,7 @@ class DBSpendUpdateWriter: hashed_token = token asyncio.create_task( - DBSpendUpdateWriter._update_user_db( + self._update_user_db( response_cost=response_cost, user_id=user_id, prisma_client=prisma_client, @@ -94,14 +96,14 @@ class DBSpendUpdateWriter: ) ) asyncio.create_task( - DBSpendUpdateWriter._update_key_db( + self._update_key_db( response_cost=response_cost, hashed_token=hashed_token, prisma_client=prisma_client, ) ) asyncio.create_task( - DBSpendUpdateWriter._update_team_db( + self._update_team_db( response_cost=response_cost, team_id=team_id, user_id=user_id, @@ -109,7 +111,7 @@ class DBSpendUpdateWriter: ) ) asyncio.create_task( - DBSpendUpdateWriter._update_org_db( + self._update_org_db( response_cost=response_cost, org_id=org_id, prisma_client=prisma_client, @@ -135,56 +137,8 @@ class DBSpendUpdateWriter: f"Error updating Prisma database: {traceback.format_exc()}" ) - @staticmethod - async def _update_transaction_list( - response_cost: Optional[float], - entity_id: Optional[str], - transaction_list: dict, - entity_type: Litellm_EntityType, - debug_msg: Optional[str] = None, - prisma_client: Optional[PrismaClient] = None, - ) -> bool: - """ - Common helper method to update a transaction list for an entity - - Args: - response_cost: The cost to add - entity_id: The ID of the entity to update - transaction_list: The transaction list dictionary to update - entity_type: The type of entity (from EntityType enum) - debug_msg: Optional custom debug message - - Returns: - bool: True if update happened, False otherwise - """ - try: - if debug_msg: - verbose_proxy_logger.debug(debug_msg) - else: - verbose_proxy_logger.debug( - f"adding spend to {entity_type.value} db. Response cost: {response_cost}. {entity_type.value}_id: {entity_id}." - ) - if prisma_client is None: - return False - - if entity_id is None: - verbose_proxy_logger.debug( - f"track_cost_callback: {entity_type.value}_id is None. Not tracking spend for {entity_type.value}" - ) - return False - transaction_list[entity_id] = response_cost + transaction_list.get( - entity_id, 0 - ) - return True - - except Exception as e: - verbose_proxy_logger.info( - f"Update {entity_type.value.capitalize()} DB failed to execute - {str(e)}\n{traceback.format_exc()}" - ) - raise e - - @staticmethod async def _update_key_db( + self, response_cost: Optional[float], hashed_token: Optional[str], prisma_client: Optional[PrismaClient], @@ -193,13 +147,12 @@ class DBSpendUpdateWriter: if hashed_token is None or prisma_client is None: return - await DBSpendUpdateWriter._update_transaction_list( - response_cost=response_cost, - entity_id=hashed_token, - transaction_list=prisma_client.key_list_transactions, - entity_type=Litellm_EntityType.KEY, - debug_msg=f"adding spend to key db. Response cost: {response_cost}. Token: {hashed_token}.", - prisma_client=prisma_client, + await self.spend_update_queue.add_update( + update={ + "entity_type": Litellm_EntityType.KEY.value, + "entity_id": hashed_token, + "amount": response_cost, + } ) except Exception as e: verbose_proxy_logger.exception( @@ -207,8 +160,8 @@ class DBSpendUpdateWriter: ) raise e - @staticmethod async def _update_user_db( + self, response_cost: Optional[float], user_id: Optional[str], prisma_client: Optional[PrismaClient], @@ -234,21 +187,21 @@ class DBSpendUpdateWriter: for _id in user_ids: if _id is not None: - await DBSpendUpdateWriter._update_transaction_list( - response_cost=response_cost, - entity_id=_id, - transaction_list=prisma_client.user_list_transactions, - entity_type=Litellm_EntityType.USER, - prisma_client=prisma_client, + await self.spend_update_queue.add_update( + update={ + "entity_type": Litellm_EntityType.USER.value, + "entity_id": _id, + "amount": response_cost, + } ) if end_user_id is not None: - await DBSpendUpdateWriter._update_transaction_list( - response_cost=response_cost, - entity_id=end_user_id, - transaction_list=prisma_client.end_user_list_transactions, - entity_type=Litellm_EntityType.END_USER, - prisma_client=prisma_client, + await self.spend_update_queue.add_update( + update={ + "entity_type": Litellm_EntityType.END_USER.value, + "entity_id": end_user_id, + "amount": response_cost, + } ) except Exception as e: verbose_proxy_logger.info( @@ -256,8 +209,8 @@ class DBSpendUpdateWriter: + f"Update User DB call failed to execute {str(e)}\n{traceback.format_exc()}" ) - @staticmethod async def _update_team_db( + self, response_cost: Optional[float], team_id: Optional[str], user_id: Optional[str], @@ -270,12 +223,12 @@ class DBSpendUpdateWriter: ) return - await DBSpendUpdateWriter._update_transaction_list( - response_cost=response_cost, - entity_id=team_id, - transaction_list=prisma_client.team_list_transactions, - entity_type=Litellm_EntityType.TEAM, - prisma_client=prisma_client, + await self.spend_update_queue.add_update( + update={ + "entity_type": Litellm_EntityType.TEAM.value, + "entity_id": team_id, + "amount": response_cost, + } ) try: @@ -283,12 +236,12 @@ class DBSpendUpdateWriter: if user_id is not None: # key is "team_id::::user_id::" team_member_key = f"team_id::{team_id}::user_id::{user_id}" - await DBSpendUpdateWriter._update_transaction_list( - response_cost=response_cost, - entity_id=team_member_key, - transaction_list=prisma_client.team_member_list_transactions, - entity_type=Litellm_EntityType.TEAM_MEMBER, - prisma_client=prisma_client, + await self.spend_update_queue.add_update( + update={ + "entity_type": Litellm_EntityType.TEAM_MEMBER.value, + "entity_id": team_member_key, + "amount": response_cost, + } ) except Exception: pass @@ -298,8 +251,8 @@ class DBSpendUpdateWriter: ) raise e - @staticmethod async def _update_org_db( + self, response_cost: Optional[float], org_id: Optional[str], prisma_client: Optional[PrismaClient], @@ -311,12 +264,12 @@ class DBSpendUpdateWriter: ) return - await DBSpendUpdateWriter._update_transaction_list( - response_cost=response_cost, - entity_id=org_id, - transaction_list=prisma_client.org_list_transactions, - entity_type=Litellm_EntityType.ORGANIZATION, - prisma_client=prisma_client, + await self.spend_update_queue.add_update( + update={ + "entity_type": Litellm_EntityType.ORGANIZATION.value, + "entity_id": org_id, + "amount": response_cost, + } ) except Exception as e: verbose_proxy_logger.info( @@ -435,7 +388,7 @@ class DBSpendUpdateWriter: - Only 1 pod will commit to db at a time (based on if it can acquire the lock over writing to DB) """ await self.redis_update_buffer.store_in_memory_spend_updates_in_redis( - prisma_client=prisma_client, + spend_update_queue=self.spend_update_queue, ) # Only commit from redis to db if this pod is the leader @@ -447,7 +400,7 @@ class DBSpendUpdateWriter: await self.redis_update_buffer.get_all_update_transactions_from_redis_buffer() ) if db_spend_update_transactions is not None: - await DBSpendUpdateWriter._commit_spend_updates_to_db( + await self._commit_spend_updates_to_db( prisma_client=prisma_client, n_retry_times=n_retry_times, proxy_logging_obj=proxy_logging_obj, @@ -471,23 +424,26 @@ class DBSpendUpdateWriter: Note: This flow causes Deadlocks in production (1K RPS+). Use self._commit_spend_updates_to_db_with_redis() instead if you expect 1K+ RPS. """ - db_spend_update_transactions = DBSpendUpdateTransactions( - user_list_transactions=prisma_client.user_list_transactions, - end_user_list_transactions=prisma_client.end_user_list_transactions, - key_list_transactions=prisma_client.key_list_transactions, - team_list_transactions=prisma_client.team_list_transactions, - team_member_list_transactions=prisma_client.team_member_list_transactions, - org_list_transactions=prisma_client.org_list_transactions, + aggregated_updates = ( + await self.spend_update_queue.flush_and_get_all_aggregated_updates_by_entity_type() ) - await DBSpendUpdateWriter._commit_spend_updates_to_db( + db_spend_update_transactions = DBSpendUpdateTransactions( + user_list_transactions=aggregated_updates.get("user", {}), + end_user_list_transactions=aggregated_updates.get("end_user", {}), + key_list_transactions=aggregated_updates.get("key", {}), + team_list_transactions=aggregated_updates.get("team", {}), + team_member_list_transactions=aggregated_updates.get("team_member", {}), + org_list_transactions=aggregated_updates.get("organization", {}), + ) + await self._commit_spend_updates_to_db( prisma_client=prisma_client, n_retry_times=n_retry_times, proxy_logging_obj=proxy_logging_obj, db_spend_update_transactions=db_spend_update_transactions, ) - @staticmethod async def _commit_spend_updates_to_db( # noqa: PLR0915 + self, prisma_client: PrismaClient, n_retry_times: int, proxy_logging_obj: ProxyLogging, @@ -526,9 +482,6 @@ class DBSpendUpdateWriter: where={"user_id": user_id}, data={"spend": {"increment": response_cost}}, ) - prisma_client.user_list_transactions = ( - {} - ) # Clear the remaining transactions after processing all batches in the loop. break except DB_CONNECTION_ERROR_TYPES as e: if ( @@ -583,9 +536,6 @@ class DBSpendUpdateWriter: where={"token": token}, data={"spend": {"increment": response_cost}}, ) - prisma_client.key_list_transactions = ( - {} - ) # Clear the remaining transactions after processing all batches in the loop. break except DB_CONNECTION_ERROR_TYPES as e: if ( @@ -632,9 +582,6 @@ class DBSpendUpdateWriter: where={"team_id": team_id}, data={"spend": {"increment": response_cost}}, ) - prisma_client.team_list_transactions = ( - {} - ) # Clear the remaining transactions after processing all batches in the loop. break except DB_CONNECTION_ERROR_TYPES as e: if ( @@ -684,9 +631,6 @@ class DBSpendUpdateWriter: where={"team_id": team_id, "user_id": user_id}, data={"spend": {"increment": response_cost}}, ) - prisma_client.team_member_list_transactions = ( - {} - ) # Clear the remaining transactions after processing all batches in the loop. break except DB_CONNECTION_ERROR_TYPES as e: if ( @@ -725,9 +669,6 @@ class DBSpendUpdateWriter: where={"organization_id": org_id}, data={"spend": {"increment": response_cost}}, ) - prisma_client.org_list_transactions = ( - {} - ) # Clear the remaining transactions after processing all batches in the loop. break except DB_CONNECTION_ERROR_TYPES as e: if ( diff --git a/litellm/proxy/db/redis_update_buffer.py b/litellm/proxy/db/redis_update_buffer.py index 7370b0ed08..0dfaa72a16 100644 --- a/litellm/proxy/db/redis_update_buffer.py +++ b/litellm/proxy/db/redis_update_buffer.py @@ -33,7 +33,6 @@ class RedisUpdateBuffer: redis_cache: Optional[RedisCache] = None, ): self.redis_cache = redis_cache - self.spend_update_queue = SpendUpdateQueue() @staticmethod def _should_commit_spend_updates_to_redis() -> bool: @@ -56,6 +55,7 @@ class RedisUpdateBuffer: async def store_in_memory_spend_updates_in_redis( self, + spend_update_queue: SpendUpdateQueue, ): """ Stores the in-memory spend updates to Redis @@ -81,9 +81,9 @@ class RedisUpdateBuffer: return aggregated_updates = ( - await self.spend_update_queue.flush_and_get_all_aggregated_updates_by_entity_type() + await spend_update_queue.flush_and_get_all_aggregated_updates_by_entity_type() ) - verbose_proxy_logger.debug("ALL AGGREGATED UPDATES: ", aggregated_updates) + verbose_proxy_logger.debug("ALL AGGREGATED UPDATES: %s", aggregated_updates) db_spend_update_transactions: DBSpendUpdateTransactions = ( DBSpendUpdateTransactions( @@ -92,7 +92,7 @@ class RedisUpdateBuffer: key_list_transactions=aggregated_updates.get("key", {}), team_list_transactions=aggregated_updates.get("team", {}), team_member_list_transactions=aggregated_updates.get("team_member", {}), - org_list_transactions=aggregated_updates.get("org", {}), + org_list_transactions=aggregated_updates.get("organization", {}), ) ) diff --git a/litellm/proxy/db/spend_update_queue.py b/litellm/proxy/db/spend_update_queue.py index 778361be84..2462e6a547 100644 --- a/litellm/proxy/db/spend_update_queue.py +++ b/litellm/proxy/db/spend_update_queue.py @@ -1,6 +1,8 @@ import asyncio from typing import TYPE_CHECKING, Any, Dict, List +from litellm._logging import verbose_proxy_logger + if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient else: @@ -21,6 +23,7 @@ class SpendUpdateQueue: async def add_update(self, update: Dict[str, Any]) -> None: """Enqueue an update. Each update might be a dict like {'entity_type': 'user', 'entity_id': '123', 'amount': 1.2}.""" + verbose_proxy_logger.debug("Adding update to queue: %s", update) await self.update_queue.put(update) async def flush_all_updates_from_in_memory_queue(self) -> List[Dict[str, Any]]: @@ -35,6 +38,7 @@ class SpendUpdateQueue: ) -> Dict[str, Any]: """Flush all updates from the queue and return all updates aggregated by entity type.""" updates = await self.flush_all_updates_from_in_memory_queue() + verbose_proxy_logger.debug("Aggregating updates by entity type: %s", updates) return self.aggregate_updates_by_entity_type(updates) def aggregate_updates_by_entity_type( diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 39c1eeace9..dc0c27eb3e 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -37,6 +37,8 @@ class _ProxyDBLogger(CustomLogger): if _ProxyDBLogger._should_track_errors_in_db() is False: return + from litellm.proxy.proxy_server import proxy_logging_obj + _metadata = dict( StandardLoggingUserAPIKeyMetadata( user_api_key_hash=user_api_key_dict.api_key, @@ -66,7 +68,7 @@ class _ProxyDBLogger(CustomLogger): request_data.get("proxy_server_request") or {} ) request_data["litellm_params"]["metadata"] = existing_metadata - await DBSpendUpdateWriter.update_database( + await proxy_logging_obj.db_spend_update_writer.update_database( token=user_api_key_dict.api_key, response_cost=0.0, user_id=user_api_key_dict.user_id, @@ -136,7 +138,7 @@ class _ProxyDBLogger(CustomLogger): end_user_id=end_user_id, ): ## UPDATE DATABASE - await DBSpendUpdateWriter.update_database( + await proxy_logging_obj.db_spend_update_writer.update_database( token=user_api_key, response_cost=response_cost, user_id=user_id, From 95e674d1e98729e8c010046cda707dc9da645c50 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 19:01:17 -0700 Subject: [PATCH 04/22] test spend accuracy --- litellm/proxy/proxy_config.yaml | 24 +++++++-------- .../local_test_spend_accuracy_tests.py | 30 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index c0107d7ea6..fe8d73d26a 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,15 +1,15 @@ model_list: - - model_name: gpt-4o + - model_name: fake-openai-endpoint litellm_params: - model: openai/gpt-4o - api_key: sk-xxxxxxx + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ -mcp_servers: - { - "zapier_mcp": { - "url": "https://actions.zapier.com/mcp/sk-akxxxxx/sse" - }, - "fetch": { - "url": "http://localhost:8000/sse" - } - } +general_settings: + use_redis_transaction_buffer: true + +litellm_settings: + cache: True + cache_params: + type: redis + supported_call_types: [] \ No newline at end of file diff --git a/tests/otel_tests/local_test_spend_accuracy_tests.py b/tests/otel_tests/local_test_spend_accuracy_tests.py index 6d756219c7..bbb12c58a1 100644 --- a/tests/otel_tests/local_test_spend_accuracy_tests.py +++ b/tests/otel_tests/local_test_spend_accuracy_tests.py @@ -52,7 +52,7 @@ Additional Test Scenarios: async def create_organization(session, organization_alias: str): """Helper function to create a new organization""" - url = "http://0.0.0.0:4002/organization/new" + url = "http://0.0.0.0:4000/organization/new" headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"} data = {"organization_alias": organization_alias} async with session.post(url, headers=headers, json=data) as response: @@ -61,7 +61,7 @@ async def create_organization(session, organization_alias: str): async def create_team(session, org_id: str): """Helper function to create a new team under an organization""" - url = "http://0.0.0.0:4002/team/new" + url = "http://0.0.0.0:4000/team/new" headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"} data = {"organization_id": org_id, "team_alias": f"test-team-{uuid.uuid4()}"} async with session.post(url, headers=headers, json=data) as response: @@ -70,7 +70,7 @@ async def create_team(session, org_id: str): async def create_user(session, org_id: str): """Helper function to create a new user""" - url = "http://0.0.0.0:4002/user/new" + url = "http://0.0.0.0:4000/user/new" headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"} data = {"user_name": f"test-user-{uuid.uuid4()}"} async with session.post(url, headers=headers, json=data) as response: @@ -79,7 +79,7 @@ async def create_user(session, org_id: str): async def generate_key(session, user_id: str, team_id: str): """Helper function to generate a key for a specific user and team""" - url = "http://0.0.0.0:4002/key/generate" + url = "http://0.0.0.0:4000/key/generate" headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"} data = {"user_id": user_id, "team_id": team_id} async with session.post(url, headers=headers, json=data) as response: @@ -91,7 +91,7 @@ async def chat_completion(session, key: str): from openai import AsyncOpenAI import uuid - client = AsyncOpenAI(api_key=key, base_url="http://0.0.0.0:4002/v1") + client = AsyncOpenAI(api_key=key, base_url="http://0.0.0.0:4000/v1") response = await client.chat.completions.create( model="fake-openai-endpoint", @@ -102,7 +102,7 @@ async def chat_completion(session, key: str): async def get_spend_info(session, entity_type: str, entity_id: str): """Helper function to get spend information for an entity""" - url = f"http://0.0.0.0:4002/{entity_type}/info" + url = f"http://0.0.0.0:4000/{entity_type}/info" headers = {"Authorization": "Bearer sk-1234", "Content-Type": "application/json"} if entity_type == "key": data = {"key": entity_id} @@ -266,14 +266,14 @@ async def test_long_term_spend_accuracy_with_bursts(): abs(key_info["info"]["spend"] - expected_spend) < TOLERANCE ), f"Key spend {key_info['info']['spend']} does not match expected {expected_spend}" - assert ( - abs(user_info["user_info"]["spend"] - expected_spend) < TOLERANCE - ), f"User spend {user_info['user_info']['spend']} does not match expected {expected_spend}" + # assert ( + # abs(user_info["user_info"]["spend"] - expected_spend) < TOLERANCE + # ), f"User spend {user_info['user_info']['spend']} does not match expected {expected_spend}" - assert ( - abs(team_info["team_info"]["spend"] - expected_spend) < TOLERANCE - ), f"Team spend {team_info['team_info']['spend']} does not match expected {expected_spend}" + # assert ( + # abs(team_info["team_info"]["spend"] - expected_spend) < TOLERANCE + # ), f"Team spend {team_info['team_info']['spend']} does not match expected {expected_spend}" - assert ( - abs(org_info["spend"] - expected_spend) < TOLERANCE - ), f"Organization spend {org_info['spend']} does not match expected {expected_spend}" + # assert ( + # abs(org_info["spend"] - expected_spend) < TOLERANCE + # ), f"Organization spend {org_info['spend']} does not match expected {expected_spend}" From 73d6af55723975711fca28f5d64d4ed9cd9e8dc9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 19:03:02 -0700 Subject: [PATCH 05/22] fix docstring --- litellm/proxy/db/spend_update_queue.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/proxy/db/spend_update_queue.py b/litellm/proxy/db/spend_update_queue.py index 2462e6a547..2a1e336877 100644 --- a/litellm/proxy/db/spend_update_queue.py +++ b/litellm/proxy/db/spend_update_queue.py @@ -11,9 +11,7 @@ else: class SpendUpdateQueue: """ - Handles buffering database `UPDATE` transactions in Redis before committing them to the database - - This is to prevent deadlocks and improve reliability + In memory buffer for spend updates that should be committed to the database """ def __init__( From 49595121703279ce924a0f88efb02124a0c20a4b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 19:05:58 -0700 Subject: [PATCH 06/22] fix update_end_user_spend --- litellm/proxy/db/db_spend_update_writer.py | 1 + litellm/proxy/utils.py | 17 +++++------------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 6b5719acbb..5899ab3416 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -514,6 +514,7 @@ class DBSpendUpdateWriter: n_retry_times=n_retry_times, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj, + end_user_list_transactions=end_user_list_transactions, ) ### UPDATE KEY TABLE ### key_list_transactions = db_spend_update_transactions["key_list_transactions"] diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 900b26f3f1..76591d9fca 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1100,12 +1100,6 @@ def jsonify_object(data: dict) -> dict: class PrismaClient: - user_list_transactions: dict = {} - end_user_list_transactions: dict = {} - key_list_transactions: dict = {} - team_list_transactions: dict = {} - team_member_list_transactions: dict = {} # key is ["team_id" + "user_id"] - org_list_transactions: dict = {} spend_log_transactions: List = [] daily_user_spend_transactions: Dict[str, DailyUserSpendTransaction] = {} @@ -2422,7 +2416,10 @@ def _hash_token_if_needed(token: str) -> str: class ProxyUpdateSpend: @staticmethod async def update_end_user_spend( - n_retry_times: int, prisma_client: PrismaClient, proxy_logging_obj: ProxyLogging + n_retry_times: int, + prisma_client: PrismaClient, + proxy_logging_obj: ProxyLogging, + end_user_list_transactions: Dict[str, float], ): for i in range(n_retry_times + 1): start_time = time.time() @@ -2434,7 +2431,7 @@ class ProxyUpdateSpend: for ( end_user_id, response_cost, - ) in prisma_client.end_user_list_transactions.items(): + ) in end_user_list_transactions.items(): if litellm.max_end_user_budget is not None: pass batcher.litellm_endusertable.upsert( @@ -2461,10 +2458,6 @@ class ProxyUpdateSpend: _raise_failed_update_spend_exception( e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj ) - finally: - prisma_client.end_user_list_transactions = ( - {} - ) # reset the end user list transactions - prevent bad data from causing issues @staticmethod async def update_spend_logs( From a753fc9d9f8c6c833b8afd34f4e934f94957c5fe Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 19:17:13 -0700 Subject: [PATCH 07/22] test_long_term_spend_accuracy_with_bursts --- .../local_test_spend_accuracy_tests.py | 18 +++++++++--------- tests/proxy_unit_tests/test_update_spend.py | 6 ------ 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/tests/otel_tests/local_test_spend_accuracy_tests.py b/tests/otel_tests/local_test_spend_accuracy_tests.py index bbb12c58a1..d60bc985a6 100644 --- a/tests/otel_tests/local_test_spend_accuracy_tests.py +++ b/tests/otel_tests/local_test_spend_accuracy_tests.py @@ -266,14 +266,14 @@ async def test_long_term_spend_accuracy_with_bursts(): abs(key_info["info"]["spend"] - expected_spend) < TOLERANCE ), f"Key spend {key_info['info']['spend']} does not match expected {expected_spend}" - # assert ( - # abs(user_info["user_info"]["spend"] - expected_spend) < TOLERANCE - # ), f"User spend {user_info['user_info']['spend']} does not match expected {expected_spend}" + assert ( + abs(user_info["user_info"]["spend"] - expected_spend) < TOLERANCE + ), f"User spend {user_info['user_info']['spend']} does not match expected {expected_spend}" - # assert ( - # abs(team_info["team_info"]["spend"] - expected_spend) < TOLERANCE - # ), f"Team spend {team_info['team_info']['spend']} does not match expected {expected_spend}" + assert ( + abs(team_info["team_info"]["spend"] - expected_spend) < TOLERANCE + ), f"Team spend {team_info['team_info']['spend']} does not match expected {expected_spend}" - # assert ( - # abs(org_info["spend"] - expected_spend) < TOLERANCE - # ), f"Organization spend {org_info['spend']} does not match expected {expected_spend}" + assert ( + abs(org_info["spend"] - expected_spend) < TOLERANCE + ), f"Organization spend {org_info['spend']} does not match expected {expected_spend}" diff --git a/tests/proxy_unit_tests/test_update_spend.py b/tests/proxy_unit_tests/test_update_spend.py index 641768a7d2..1fb2479792 100644 --- a/tests/proxy_unit_tests/test_update_spend.py +++ b/tests/proxy_unit_tests/test_update_spend.py @@ -27,12 +27,6 @@ class MockPrismaClient: # Initialize transaction lists self.spend_log_transactions = [] - self.user_list_transactons = {} - self.end_user_list_transactons = {} - self.key_list_transactons = {} - self.team_list_transactons = {} - self.team_member_list_transactons = {} - self.org_list_transactons = {} self.daily_user_spend_transactions = {} def jsonify_object(self, obj): From 71e772dd4a5a6547c4ca97374d1ba3898e4660e9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 19:28:17 -0700 Subject: [PATCH 08/22] use typed data structure for queue --- litellm/proxy/_types.py | 6 ++ litellm/proxy/db/db_spend_update_writer.py | 73 +++++++++----------- litellm/proxy/db/redis_update_buffer.py | 17 ++--- litellm/proxy/db/spend_update_queue.py | 78 ++++++++++++++++------ 4 files changed, 101 insertions(+), 73 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7f13717e29..16d302aa9a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2749,3 +2749,9 @@ class DBSpendUpdateTransactions(TypedDict): team_list_transactions: Optional[Dict[str, float]] team_member_list_transactions: Optional[Dict[str, float]] org_list_transactions: Optional[Dict[str, float]] + + +class SpendUpdateQueueItem(TypedDict, total=False): + entity_type: Litellm_EntityType + entity_id: str + response_cost: Optional[float] diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 5899ab3416..5bf255feae 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -22,6 +22,7 @@ from litellm.proxy._types import ( Litellm_EntityType, LiteLLM_UserTable, SpendLogsPayload, + SpendUpdateQueueItem, ) from litellm.proxy.db.pod_lock_manager import PodLockManager from litellm.proxy.db.redis_update_buffer import RedisUpdateBuffer @@ -148,11 +149,11 @@ class DBSpendUpdateWriter: return await self.spend_update_queue.add_update( - update={ - "entity_type": Litellm_EntityType.KEY.value, - "entity_id": hashed_token, - "amount": response_cost, - } + update=SpendUpdateQueueItem( + entity_type=Litellm_EntityType.KEY, + entity_id=hashed_token, + response_cost=response_cost, + ) ) except Exception as e: verbose_proxy_logger.exception( @@ -188,20 +189,20 @@ class DBSpendUpdateWriter: for _id in user_ids: if _id is not None: await self.spend_update_queue.add_update( - update={ - "entity_type": Litellm_EntityType.USER.value, - "entity_id": _id, - "amount": response_cost, - } + update=SpendUpdateQueueItem( + entity_type=Litellm_EntityType.USER, + entity_id=_id, + response_cost=response_cost, + ) ) if end_user_id is not None: await self.spend_update_queue.add_update( - update={ - "entity_type": Litellm_EntityType.END_USER.value, - "entity_id": end_user_id, - "amount": response_cost, - } + update=SpendUpdateQueueItem( + entity_type=Litellm_EntityType.END_USER, + entity_id=end_user_id, + response_cost=response_cost, + ) ) except Exception as e: verbose_proxy_logger.info( @@ -224,11 +225,11 @@ class DBSpendUpdateWriter: return await self.spend_update_queue.add_update( - update={ - "entity_type": Litellm_EntityType.TEAM.value, - "entity_id": team_id, - "amount": response_cost, - } + update=SpendUpdateQueueItem( + entity_type=Litellm_EntityType.TEAM, + entity_id=team_id, + response_cost=response_cost, + ) ) try: @@ -237,11 +238,11 @@ class DBSpendUpdateWriter: # key is "team_id::::user_id::" team_member_key = f"team_id::{team_id}::user_id::{user_id}" await self.spend_update_queue.add_update( - update={ - "entity_type": Litellm_EntityType.TEAM_MEMBER.value, - "entity_id": team_member_key, - "amount": response_cost, - } + update=SpendUpdateQueueItem( + entity_type=Litellm_EntityType.TEAM_MEMBER, + entity_id=team_member_key, + response_cost=response_cost, + ) ) except Exception: pass @@ -265,11 +266,11 @@ class DBSpendUpdateWriter: return await self.spend_update_queue.add_update( - update={ - "entity_type": Litellm_EntityType.ORGANIZATION.value, - "entity_id": org_id, - "amount": response_cost, - } + update=SpendUpdateQueueItem( + entity_type=Litellm_EntityType.ORGANIZATION, + entity_id=org_id, + response_cost=response_cost, + ) ) except Exception as e: verbose_proxy_logger.info( @@ -424,16 +425,8 @@ class DBSpendUpdateWriter: Note: This flow causes Deadlocks in production (1K RPS+). Use self._commit_spend_updates_to_db_with_redis() instead if you expect 1K+ RPS. """ - aggregated_updates = ( - await self.spend_update_queue.flush_and_get_all_aggregated_updates_by_entity_type() - ) - db_spend_update_transactions = DBSpendUpdateTransactions( - user_list_transactions=aggregated_updates.get("user", {}), - end_user_list_transactions=aggregated_updates.get("end_user", {}), - key_list_transactions=aggregated_updates.get("key", {}), - team_list_transactions=aggregated_updates.get("team", {}), - team_member_list_transactions=aggregated_updates.get("team_member", {}), - org_list_transactions=aggregated_updates.get("organization", {}), + db_spend_update_transactions = ( + await self.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions() ) await self._commit_spend_updates_to_db( prisma_client=prisma_client, diff --git a/litellm/proxy/db/redis_update_buffer.py b/litellm/proxy/db/redis_update_buffer.py index 0dfaa72a16..1a3fd3d42d 100644 --- a/litellm/proxy/db/redis_update_buffer.py +++ b/litellm/proxy/db/redis_update_buffer.py @@ -80,20 +80,11 @@ class RedisUpdateBuffer: ) return - aggregated_updates = ( - await spend_update_queue.flush_and_get_all_aggregated_updates_by_entity_type() + db_spend_update_transactions = ( + await spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions() ) - verbose_proxy_logger.debug("ALL AGGREGATED UPDATES: %s", aggregated_updates) - - db_spend_update_transactions: DBSpendUpdateTransactions = ( - DBSpendUpdateTransactions( - user_list_transactions=aggregated_updates.get("user", {}), - end_user_list_transactions=aggregated_updates.get("end_user", {}), - key_list_transactions=aggregated_updates.get("key", {}), - team_list_transactions=aggregated_updates.get("team", {}), - team_member_list_transactions=aggregated_updates.get("team_member", {}), - org_list_transactions=aggregated_updates.get("organization", {}), - ) + verbose_proxy_logger.debug( + "ALL DB SPEND UPDATE TRANSACTIONS: %s", db_spend_update_transactions ) # only store in redis if there are any updates to commit diff --git a/litellm/proxy/db/spend_update_queue.py b/litellm/proxy/db/spend_update_queue.py index 2a1e336877..2d9792ac85 100644 --- a/litellm/proxy/db/spend_update_queue.py +++ b/litellm/proxy/db/spend_update_queue.py @@ -2,6 +2,11 @@ import asyncio from typing import TYPE_CHECKING, Any, Dict, List from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import ( + DBSpendUpdateTransactions, + Litellm_EntityType, + SpendUpdateQueueItem, +) if TYPE_CHECKING: from litellm.proxy.utils import PrismaClient @@ -17,40 +22,73 @@ class SpendUpdateQueue: def __init__( self, ): - self.update_queue: asyncio.Queue[Dict[str, Any]] = asyncio.Queue() + self.update_queue: asyncio.Queue[SpendUpdateQueueItem] = asyncio.Queue() - async def add_update(self, update: Dict[str, Any]) -> None: + async def add_update(self, update: SpendUpdateQueueItem) -> None: """Enqueue an update. Each update might be a dict like {'entity_type': 'user', 'entity_id': '123', 'amount': 1.2}.""" verbose_proxy_logger.debug("Adding update to queue: %s", update) await self.update_queue.put(update) - async def flush_all_updates_from_in_memory_queue(self) -> List[Dict[str, Any]]: + async def flush_all_updates_from_in_memory_queue( + self, + ) -> List[SpendUpdateQueueItem]: """Get all updates from the queue.""" - updates: List[Dict[str, Any]] = [] + updates: List[SpendUpdateQueueItem] = [] while not self.update_queue.empty(): updates.append(await self.update_queue.get()) return updates - async def flush_and_get_all_aggregated_updates_by_entity_type( + async def flush_and_get_aggregated_db_spend_update_transactions( self, - ) -> Dict[str, Any]: + ) -> DBSpendUpdateTransactions: """Flush all updates from the queue and return all updates aggregated by entity type.""" updates = await self.flush_all_updates_from_in_memory_queue() verbose_proxy_logger.debug("Aggregating updates by entity type: %s", updates) - return self.aggregate_updates_by_entity_type(updates) + return self.get_aggregated_db_spend_update_transactions(updates) - def aggregate_updates_by_entity_type( - self, updates: List[Dict[str, Any]] - ) -> Dict[str, Any]: + def get_aggregated_db_spend_update_transactions( + self, updates: List[SpendUpdateQueueItem] + ) -> DBSpendUpdateTransactions: """Aggregate updates by entity type.""" - aggregated_updates = {} + # Initialize all transaction lists as empty dicts + db_spend_update_transactions = DBSpendUpdateTransactions( + user_list_transactions={}, + end_user_list_transactions={}, + key_list_transactions={}, + team_list_transactions={}, + team_member_list_transactions={}, + org_list_transactions={}, + ) + + # Map entity types to their corresponding transaction dictionary keys + entity_type_to_dict_key = { + Litellm_EntityType.USER: "user_list_transactions", + Litellm_EntityType.END_USER: "end_user_list_transactions", + Litellm_EntityType.KEY: "key_list_transactions", + Litellm_EntityType.TEAM: "team_list_transactions", + Litellm_EntityType.TEAM_MEMBER: "team_member_list_transactions", + Litellm_EntityType.ORGANIZATION: "org_list_transactions", + } + for update in updates: - entity_type = update["entity_type"] - entity_id = update["entity_id"] - amount = update["amount"] - if entity_type not in aggregated_updates: - aggregated_updates[entity_type] = {} - if entity_id not in aggregated_updates[entity_type]: - aggregated_updates[entity_type][entity_id] = 0 - aggregated_updates[entity_type][entity_id] += amount - return aggregated_updates + entity_type = update.get("entity_type") + entity_id = update.get("entity_id") + response_cost = update.get("response_cost") + + if entity_type is None or entity_id is None or response_cost is None: + raise ValueError("Invalid update: %s", update) + + dict_key = entity_type_to_dict_key.get(entity_type) + if dict_key is None: + continue # Skip unknown entity types + + transactions_dict = db_spend_update_transactions[dict_key] + if transactions_dict is None: + transactions_dict = {} + db_spend_update_transactions[dict_key] = transactions_dict + + transactions_dict[entity_id] = ( + transactions_dict.get(entity_id, 0) + response_cost + ) + + return db_spend_update_transactions From 811f488ca39f83bb56b02bac05992feebb52aae7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 19:31:53 -0700 Subject: [PATCH 09/22] get_aggregated_db_spend_update_transactions --- litellm/proxy/db/spend_update_queue.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/db/spend_update_queue.py b/litellm/proxy/db/spend_update_queue.py index 2d9792ac85..d502c2cd84 100644 --- a/litellm/proxy/db/spend_update_queue.py +++ b/litellm/proxy/db/spend_update_queue.py @@ -87,8 +87,9 @@ class SpendUpdateQueue: transactions_dict = {} db_spend_update_transactions[dict_key] = transactions_dict - transactions_dict[entity_id] = ( - transactions_dict.get(entity_id, 0) + response_cost - ) + if entity_id not in transactions_dict: + transactions_dict[entity_id] = 0 + + transactions_dict[entity_id] += response_cost return db_spend_update_transactions From aa8261af89e04f75da92ab4628d73028364eee90 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 19:33:10 -0700 Subject: [PATCH 10/22] test fixes --- litellm/proxy/hooks/proxy_track_cost_callback.py | 1 - tests/proxy_unit_tests/test_unit_test_proxy_hooks.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index dc0c27eb3e..9ae1eb4c34 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -13,7 +13,6 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_checks import log_db_metrics -from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter from litellm.proxy.utils import ProxyUpdateSpend from litellm.types.utils import ( StandardLoggingPayload, diff --git a/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py b/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py index 129be6d754..46863889d2 100644 --- a/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py +++ b/tests/proxy_unit_tests/test_unit_test_proxy_hooks.py @@ -24,9 +24,10 @@ async def test_disable_spend_logs(): "litellm.proxy.proxy_server.prisma_client", mock_prisma_client ): from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter + db_spend_update_writer = DBSpendUpdateWriter() # Call update_database with disable_spend_logs=True - await DBSpendUpdateWriter.update_database( + await db_spend_update_writer.update_database( token="fake-token", response_cost=0.1, user_id="user123", From 271b8b95bc8d12cbd378591c35d6136624164b17 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 19:35:07 -0700 Subject: [PATCH 11/22] test spend accuracy --- .../test_spend_accuracy_tests.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{otel_tests/local_test_spend_accuracy_tests.py => spend_tracking_tests/test_spend_accuracy_tests.py} (100%) diff --git a/tests/otel_tests/local_test_spend_accuracy_tests.py b/tests/spend_tracking_tests/test_spend_accuracy_tests.py similarity index 100% rename from tests/otel_tests/local_test_spend_accuracy_tests.py rename to tests/spend_tracking_tests/test_spend_accuracy_tests.py From 6d4a6a84029acd7b2b2365b4e18bfff846c1a91f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 19:42:00 -0700 Subject: [PATCH 12/22] add spend tracking config.yaml --- .circleci/config.yml | 99 ++++++++++++++++++- .../spend_tracking_config.yaml | 15 +++ 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 litellm/proxy/example_config_yaml/spend_tracking_config.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index e1488a9083..d2a6aafef7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1450,7 +1450,7 @@ jobs: command: | pwd ls - python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests + python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/spend_tracking_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/llm_responses_api_testing --ignore=tests/mcp_tests --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests no_output_timeout: 120m # Store test results @@ -1743,6 +1743,96 @@ jobs: # Store test results - store_test_results: path: test-results + proxy_spend_accuracy_tests: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + - run: + name: Install Python 3.9 + command: | + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda init bash + source ~/.bashrc + conda create -n myenv python=3.9 -y + conda activate myenv + python --version + - run: + name: Install Dependencies + command: | + pip install "pytest==7.3.1" + pip install "pytest-asyncio==0.21.1" + pip install aiohttp + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + - run: + name: Build Docker image + command: docker build -t my-app:latest -f ./docker/Dockerfile.database . + - run: + name: Run Docker container + # intentionally give bad redis credentials here + # the OTEL test - should get this as a trace + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=$PROXY_DATABASE_URL \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ + -e USE_DDTRACE=True \ + -e DD_API_KEY=$DD_API_KEY \ + -e DD_SITE=$DD_SITE \ + -e AWS_REGION_NAME=$AWS_REGION_NAME \ + --name my-app \ + -v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \ + my-app:latest \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Install curl and dockerize + command: | + sudo apt-get update + sudo apt-get install -y curl + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Run tests + command: | + pwd + ls + python -m pytest -vv tests/spend_tracking_tests -x --junitxml=test-results/junit.xml --durations=5 + no_output_timeout: + 120m + # Clean up first container + - run: + name: Stop and remove first container + command: | + docker stop my-app + docker rm my-app proxy_multi_instance_tests: machine: @@ -2553,6 +2643,12 @@ workflows: only: - main - /litellm_.*/ + - proxy_spend_accuracy_tests: + filters: + branches: + only: + - main + - /litellm_.*/ - proxy_multi_instance_tests: filters: branches: @@ -2714,6 +2810,7 @@ workflows: - installing_litellm_on_python - installing_litellm_on_python_3_13 - proxy_logging_guardrails_model_info_tests + - proxy_spend_accuracy_tests - proxy_multi_instance_tests - proxy_store_model_in_db_tests - proxy_build_from_pip_tests diff --git a/litellm/proxy/example_config_yaml/spend_tracking_config.yaml b/litellm/proxy/example_config_yaml/spend_tracking_config.yaml new file mode 100644 index 0000000000..fe8d73d26a --- /dev/null +++ b/litellm/proxy/example_config_yaml/spend_tracking_config.yaml @@ -0,0 +1,15 @@ +model_list: + - model_name: fake-openai-endpoint + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + +general_settings: + use_redis_transaction_buffer: true + +litellm_settings: + cache: True + cache_params: + type: redis + supported_call_types: [] \ No newline at end of file From 5fa5c1154eeb24d07e88ec8b034c61e5695af752 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 19:42:20 -0700 Subject: [PATCH 13/22] add SpendUpdateQueue --- litellm/proxy/db/spend_update_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/db/spend_update_queue.py b/litellm/proxy/db/spend_update_queue.py index d502c2cd84..e695f42841 100644 --- a/litellm/proxy/db/spend_update_queue.py +++ b/litellm/proxy/db/spend_update_queue.py @@ -1,5 +1,5 @@ import asyncio -from typing import TYPE_CHECKING, Any, Dict, List +from typing import TYPE_CHECKING, Any, List from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ( From 923ac2303beedafea7a5f16e088eda586e8ff098 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 20:55:13 -0700 Subject: [PATCH 14/22] test_end_user_transactions_reset --- tests/proxy_unit_tests/test_proxy_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index d613118fc8..31c9711b0e 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -1509,13 +1509,16 @@ from litellm.proxy.utils import ProxyUpdateSpend async def test_end_user_transactions_reset(): # Setup mock_client = MagicMock() - mock_client.end_user_list_transactions = {"1": 10.0} # Bad log + end_user_list_transactions = {"1": 10.0} # Bad log mock_client.db.tx = AsyncMock(side_effect=Exception("DB Error")) # Call function - should raise error with pytest.raises(Exception): await ProxyUpdateSpend.update_end_user_spend( - n_retry_times=0, prisma_client=mock_client, proxy_logging_obj=MagicMock() + n_retry_times=0, + prisma_client=mock_client, + proxy_logging_obj=MagicMock(), + end_user_list_transactions=end_user_list_transactions, ) # Verify cleanup happened From 9951b356da8ad3e274902507d0ee1b62244871be Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 21:09:29 -0700 Subject: [PATCH 15/22] test_long_term_spend_accuracy_with_bursts --- tests/spend_tracking_tests/test_spend_accuracy_tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/spend_tracking_tests/test_spend_accuracy_tests.py b/tests/spend_tracking_tests/test_spend_accuracy_tests.py index d60bc985a6..93228c2d06 100644 --- a/tests/spend_tracking_tests/test_spend_accuracy_tests.py +++ b/tests/spend_tracking_tests/test_spend_accuracy_tests.py @@ -156,7 +156,7 @@ async def test_basic_spend_accuracy(): response = await chat_completion(session, key) print("response: ", response) - # wait 10 seconds for spend to be updated + # wait 15 seconds for spend to be updated await asyncio.sleep(15) # Get spend information for each entity @@ -235,7 +235,7 @@ async def test_long_term_spend_accuracy_with_bursts(): print(f"Burst 1 - Request {i+1}/{BURST_1_REQUESTS} completed") # Wait for spend to be updated - await asyncio.sleep(8) + await asyncio.sleep(15) # Check intermediate spend intermediate_key_info = await get_spend_info(session, "key", key) @@ -248,7 +248,7 @@ async def test_long_term_spend_accuracy_with_bursts(): print(f"Burst 2 - Request {i+1}/{BURST_2_REQUESTS} completed") # Wait for spend to be updated - await asyncio.sleep(8) + await asyncio.sleep(15) # Get final spend information for each entity key_info = await get_spend_info(session, "key", key) From f7ddc583f0371c95c2582d924ac6c1b0745b4c35 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 21:15:21 -0700 Subject: [PATCH 16/22] linting fix get_aggregated_db_spend_update_transactions --- litellm/proxy/db/spend_update_queue.py | 31 ++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/db/spend_update_queue.py b/litellm/proxy/db/spend_update_queue.py index e695f42841..29118073ef 100644 --- a/litellm/proxy/db/spend_update_queue.py +++ b/litellm/proxy/db/spend_update_queue.py @@ -1,5 +1,5 @@ import asyncio -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any, Dict, List from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ( @@ -82,7 +82,34 @@ class SpendUpdateQueue: if dict_key is None: continue # Skip unknown entity types - transactions_dict = db_spend_update_transactions[dict_key] + # Type-safe access using if/elif statements + if dict_key == "user_list_transactions": + transactions_dict = db_spend_update_transactions[ + "user_list_transactions" + ] + elif dict_key == "end_user_list_transactions": + transactions_dict = db_spend_update_transactions[ + "end_user_list_transactions" + ] + elif dict_key == "key_list_transactions": + transactions_dict = db_spend_update_transactions[ + "key_list_transactions" + ] + elif dict_key == "team_list_transactions": + transactions_dict = db_spend_update_transactions[ + "team_list_transactions" + ] + elif dict_key == "team_member_list_transactions": + transactions_dict = db_spend_update_transactions[ + "team_member_list_transactions" + ] + elif dict_key == "org_list_transactions": + transactions_dict = db_spend_update_transactions[ + "org_list_transactions" + ] + else: + continue + if transactions_dict is None: transactions_dict = {} db_spend_update_transactions[dict_key] = transactions_dict From 115946d402450e800d848636a6aa6159479dc724 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 21:25:24 -0700 Subject: [PATCH 17/22] unit testing for SpendUpdateQueue --- litellm/proxy/db/spend_update_queue.py | 20 ++- .../proxy/db/test_spend_update_queue.py | 152 ++++++++++++++++++ 2 files changed, 166 insertions(+), 6 deletions(-) create mode 100644 tests/litellm/proxy/db/test_spend_update_queue.py diff --git a/litellm/proxy/db/spend_update_queue.py b/litellm/proxy/db/spend_update_queue.py index 29118073ef..fbd96c6c2d 100644 --- a/litellm/proxy/db/spend_update_queue.py +++ b/litellm/proxy/db/spend_update_queue.py @@ -1,5 +1,5 @@ import asyncio -from typing import TYPE_CHECKING, Any, Dict, List +from typing import TYPE_CHECKING, Any, List from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ( @@ -72,14 +72,22 @@ class SpendUpdateQueue: for update in updates: entity_type = update.get("entity_type") - entity_id = update.get("entity_id") - response_cost = update.get("response_cost") + entity_id = update.get("entity_id") or "" + response_cost = update.get("response_cost") or 0 - if entity_type is None or entity_id is None or response_cost is None: - raise ValueError("Invalid update: %s", update) + if entity_type is None: + verbose_proxy_logger.debug( + "Skipping update spend for update: %s, because entity_type is None", + update, + ) + continue dict_key = entity_type_to_dict_key.get(entity_type) if dict_key is None: + verbose_proxy_logger.debug( + "Skipping update spend for update: %s, because entity_type is not in entity_type_to_dict_key", + update, + ) continue # Skip unknown entity types # Type-safe access using if/elif statements @@ -117,6 +125,6 @@ class SpendUpdateQueue: if entity_id not in transactions_dict: transactions_dict[entity_id] = 0 - transactions_dict[entity_id] += response_cost + transactions_dict[entity_id] += response_cost or 0 return db_spend_update_transactions diff --git a/tests/litellm/proxy/db/test_spend_update_queue.py b/tests/litellm/proxy/db/test_spend_update_queue.py new file mode 100644 index 0000000000..89d494a070 --- /dev/null +++ b/tests/litellm/proxy/db/test_spend_update_queue.py @@ -0,0 +1,152 @@ +import asyncio +import json +import os +import sys + +import pytest +from fastapi.testclient import TestClient + +from litellm.proxy._types import Litellm_EntityType, SpendUpdateQueueItem +from litellm.proxy.db.spend_update_queue import SpendUpdateQueue + +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path + + +@pytest.fixture +def spend_queue(): + return SpendUpdateQueue() + + +@pytest.mark.asyncio +async def test_add_update(spend_queue): + # Test adding a single update + update: SpendUpdateQueueItem = { + "entity_type": Litellm_EntityType.USER, + "entity_id": "user123", + "response_cost": 0.5, + } + await spend_queue.add_update(update) + + # Verify update was added by checking queue size + assert spend_queue.update_queue.qsize() == 1 + + +@pytest.mark.asyncio +async def test_missing_response_cost(spend_queue): + # Test with missing response_cost - should default to 0 + update: SpendUpdateQueueItem = { + "entity_type": Litellm_EntityType.USER, + "entity_id": "user123", + } + + await spend_queue.add_update(update) + aggregated = ( + await spend_queue.flush_and_get_aggregated_db_spend_update_transactions() + ) + + # Should have created entry with 0 cost + assert aggregated["user_list_transactions"]["user123"] == 0 + + +@pytest.mark.asyncio +async def test_missing_entity_id(spend_queue): + # Test with missing entity_id - should default to empty string + update: SpendUpdateQueueItem = { + "entity_type": Litellm_EntityType.USER, + "response_cost": 1.0, + } + + await spend_queue.add_update(update) + aggregated = ( + await spend_queue.flush_and_get_aggregated_db_spend_update_transactions() + ) + + # Should use empty string as key + assert aggregated["user_list_transactions"][""] == 1.0 + + +@pytest.mark.asyncio +async def test_none_values(spend_queue): + # Test with None values + update: SpendUpdateQueueItem = { + "entity_type": Litellm_EntityType.USER, + "entity_id": None, # type: ignore + "response_cost": None, + } + + await spend_queue.add_update(update) + aggregated = ( + await spend_queue.flush_and_get_aggregated_db_spend_update_transactions() + ) + + # Should handle None values gracefully + assert aggregated["user_list_transactions"][""] == 0 + + +@pytest.mark.asyncio +async def test_multiple_updates_with_missing_fields(spend_queue): + # Test multiple updates with various missing fields + updates: list[SpendUpdateQueueItem] = [ + { + "entity_type": Litellm_EntityType.USER, + "entity_id": "user123", + "response_cost": 0.5, + }, + { + "entity_type": Litellm_EntityType.USER, + "entity_id": "user123", # missing response_cost + }, + { + "entity_type": Litellm_EntityType.USER, # missing entity_id + "response_cost": 1.5, + }, + ] + + for update in updates: + await spend_queue.add_update(update) + + aggregated = ( + await spend_queue.flush_and_get_aggregated_db_spend_update_transactions() + ) + + # Verify aggregation + assert ( + aggregated["user_list_transactions"]["user123"] == 0.5 + ) # only the first update with valid cost + assert ( + aggregated["user_list_transactions"][""] == 1.5 + ) # update with missing entity_id + + +@pytest.mark.asyncio +async def test_unknown_entity_type(spend_queue): + # Test with unknown entity type + update: SpendUpdateQueueItem = { + "entity_type": "UNKNOWN_TYPE", # type: ignore + "entity_id": "123", + "response_cost": 0.5, + } + + await spend_queue.add_update(update) + aggregated = ( + await spend_queue.flush_and_get_aggregated_db_spend_update_transactions() + ) + + # Should ignore unknown entity type + assert all(len(transactions) == 0 for transactions in aggregated.values()) + + +@pytest.mark.asyncio +async def test_missing_entity_type(spend_queue): + # Test with missing entity type + update: SpendUpdateQueueItem = {"entity_id": "123", "response_cost": 0.5} + + await spend_queue.add_update(update) + aggregated = ( + await spend_queue.flush_and_get_aggregated_db_spend_update_transactions() + ) + + # Should ignore updates without entity type + assert all(len(transactions) == 0 for transactions in aggregated.values()) From d0a7e44a6ee232c8d2a75c5096304b61105c449f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 31 Mar 2025 21:33:05 -0700 Subject: [PATCH 18/22] fix linting --- litellm/proxy/db/spend_update_queue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/db/spend_update_queue.py b/litellm/proxy/db/spend_update_queue.py index fbd96c6c2d..28e05246fa 100644 --- a/litellm/proxy/db/spend_update_queue.py +++ b/litellm/proxy/db/spend_update_queue.py @@ -120,7 +120,9 @@ class SpendUpdateQueue: if transactions_dict is None: transactions_dict = {} - db_spend_update_transactions[dict_key] = transactions_dict + + # type ignore: dict_key is guaranteed to be one of "one of ("user_list_transactions", "end_user_list_transactions", "key_list_transactions", "team_list_transactions", "team_member_list_transactions", "org_list_transactions")" + db_spend_update_transactions[dict_key] = transactions_dict # type: ignore if entity_id not in transactions_dict: transactions_dict[entity_id] = 0 From 7a2442d6c052dd1f82a56d70087f8400ea58d2e2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 1 Apr 2025 07:12:29 -0700 Subject: [PATCH 19/22] test_batch_update_spend --- tests/local_testing/test_update_spend.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/local_testing/test_update_spend.py b/tests/local_testing/test_update_spend.py index fffa3062d7..cc2c94af27 100644 --- a/tests/local_testing/test_update_spend.py +++ b/tests/local_testing/test_update_spend.py @@ -62,6 +62,8 @@ from litellm.proxy._types import ( KeyRequest, NewUserRequest, UpdateKeyRequest, + SpendUpdateQueueItem, + Litellm_EntityType, ) proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache()) @@ -93,7 +95,13 @@ def prisma_client(): @pytest.mark.asyncio async def test_batch_update_spend(prisma_client): - prisma_client.user_list_transactions["test-litellm-user-5"] = 23 + await proxy_logging_obj.db_spend_update_writer.spend_update_queue.add_update( + SpendUpdateQueueItem( + entity_type=Litellm_EntityType.USER, + entity_id="test-litellm-user-5", + response_cost=23, + ) + ) setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client) setattr(litellm.proxy.proxy_server, "master_key", "sk-1234") await litellm.proxy.proxy_server.prisma_client.connect() From 55763ae276ba1413ce70d7e76057ca9e73e76d10 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 1 Apr 2025 07:13:25 -0700 Subject: [PATCH 20/22] test_end_user_transactions_reset --- tests/proxy_unit_tests/test_proxy_utils.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 31c9711b0e..a8c0ebe2ff 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -1521,12 +1521,6 @@ async def test_end_user_transactions_reset(): end_user_list_transactions=end_user_list_transactions, ) - # Verify cleanup happened - assert ( - mock_client.end_user_list_transactions == {} - ), "Transactions list should be empty after error" - - @pytest.mark.asyncio async def test_spend_logs_cleanup_after_error(): # Setup test data From 40a792472b909297102f3cbc9aacc61fdddf1e4d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 1 Apr 2025 11:27:03 -0700 Subject: [PATCH 21/22] build(enterprise.md): add why enterprise to docs --- docs/my-website/docs/enterprise.md | 4 ++ docs/my-website/img/enterprise_vs_oss.png | Bin 0 -> 62493 bytes ...odel_prices_and_context_window_backup.json | 36 ++++++++++++++++++ 3 files changed, 40 insertions(+) create mode 100644 docs/my-website/img/enterprise_vs_oss.png diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md index 5aeeb710ff..706ca33714 100644 --- a/docs/my-website/docs/enterprise.md +++ b/docs/my-website/docs/enterprise.md @@ -1,3 +1,5 @@ +import Image from '@theme/IdealImage'; + # Enterprise For companies that need SSO, user management and professional support for LiteLLM Proxy @@ -7,6 +9,8 @@ Get free 7-day trial key [here](https://www.litellm.ai/#trial) Includes all enterprise features. + + [**Procurement available via AWS / Azure Marketplace**](./data_security.md#legalcompliance-faqs) diff --git a/docs/my-website/img/enterprise_vs_oss.png b/docs/my-website/img/enterprise_vs_oss.png new file mode 100644 index 0000000000000000000000000000000000000000..f2b58fbc14a82d18077ba777b9e97d6a136bf8e2 GIT binary patch literal 62493 zcmd?R_g53&_wP?H(gH|_gd#=hAU$*tK@co7K{_JRMIcD%EfAV?q$Aisx^$5eLX{xW zrG};;flws$JG|cIe(oReU3cB}{XtzOGjk?$&faJ5{dn$)Gr6lrcZKr`5fKrc{%xK6 zL_}m^L`1}RDhl8e7F=%*@DH{3ZA)JwA{yq)Kg2|@^Voq8iGAeZL*}As zq)9|nlT34JPfkR%SF5k1`4C3@C!aF?YR_Q<%^oICfI{c$5b?c-s02%C${}Lit5TF# zr6aDeOAEv^g2;P_;u(jCSy))4C{cL=3u=dZ!(SVI=j7)Y)zxkL4(Cnh<0>m}{xJ2L z3o6&YZND~@Lu1~?^9cXU7XPRb`}sadi;6GmUw@VL#j;dQirNU|-gK2w+=>$FXSjhX zoD-e*`~VZ@{`Wz^;X2yb?s)Ui?dcuEZ4fzkm7wH?_up4+J?Uk@5cbK!l7QNy>lkLMKOn`1XIt zD;au4bnU-axwrQJ;&3iWCK0D6+h6~__Gp;y8p@IF{!@o3ID!x=^wWfv3PaEPt(Vu2 zl?svy8LXKvwkLafH%Ks{riFs~nZi~%%NLu<$#SOhu?@yHg`2Yuy!i_uC*r4s<>bu2 zr+XL(tA$;-!I_}=Mc&|g+d1&@&JSbt>5~W=3HG)8fO*4tKJ)z3v%>}dEy=H^4u@-U zz1+Sl!Cvi@vjLUOYXjE8QJ|935rRnma|lGRS0zP$Z3$Y1WFN@iR4WLX`8>U||E%Ti z-iNgNf`uU#=0_v!_o!Ya3d#N`xN$(sjJ=z$Jk7wCw(B|961~;%vvO1^P1u-TIRC|c zi>JlWf8auM3bWqYWw8{xD*GeavT44?>r;-&bPMpkM5E)A{ol+J7LJJ~lhD6Yj`ma4 z_Di7`C$<*P`vtpmTncg~{nGKBl#qJJ;o0Axm&x8FI?zE@c=58W#JH4y_#ti6MQ%~xtqqV_%b+j z|GVB2n#+sUs||l^Eu04GWYy@&qa1+aX0?Um$B2(H>KELeEx+GNg`C1Rks;#SqE&6R z2S3Kmti(-;1U@mS5jrY`8Q{ii1wo#hcXB;jToo6?NbmQ^0%ky$g_crlIl;Ad15@XL zY5qKEqrx4$|2^=rdF8UeZs?G7Z`5}CiNui7dfs0SkI=J&$yn#C#|@*+38L(fnv|=d zXp46r`!C9?+Mg{;9UGaR9!%QG-z|a4$)O3r8tLBtdf~ZT7&8>Op3i0zdggcHdGPDI zfP39H`C^Ihz=Ymo5&TVp){fVtWMw=>NH^cz#UdqVbn@BMZ44Jf2}`jC>pRtXYQZf$ z>y$qEeiQu_Njun?N zv$Z(yTTkGSeRiho#pPvw9{Rg1>xokGu@*Opy28Tlad%Qk4KP=+ospDQR#J<_3wRBu z>Y>P>Un9l#RbT$rc5bH5nTv>z?ie}Z2=OYmh({PiXX*X#n@`aCI9 zuCaKy5VZZph-#(14H``GcHd=&(Wn2%K~Nei>6}0^RHh9W<*(g3H^p7rB3#R+#Gt}f z5oIe#E1e%@fRp|Z;we=98Yid*^T>|2V7|2tA%plOK*%q>$ch3>Ki;mHe5am!o zO0CnK=5g{&z6vu9xKVH?FTET%&CvFHKjlUgrFvD<-4I}2M`|4P%CpL{Y+7y&+-mO= zsL~p`h5V94`Kpm7^hZ0DM!%+r^NSy5>R&;JlHizFNJ!J;WIuXR*YkP1NII=n|1HBX z`)b2NO#NGO_Eh?fEa=yMvO2{k_8~TTY^0fax^PMYa!aSYrxQWf|-!k&?K` zat_i5l}j^#$547;Gupm4tm0uR(XPOHry@n(V`j|E_0q_^0i%hr`SLQS2XtWk1rjjp zvjnVrTEMRl!~#qBwM#*uJMF!`x-Zg+$xlzrZ33M$_t+)9lO51{_@*oDUrlUXtDG|} z(G~h1X%3a~DCTbwxp)h{<)!=NOR+WPwU7yNPJt_YyTwlVnv;**#83E^_2OBG!Hb@w z15r8SX}YYIC^O#EUfQ%fLZibf+ii}nJETo`G?#g8*L(f=gH4I6;2V!yUPwQ%D}vI( zL{vg$i!>f~K+ zKE^F4t9z1~bVS5WmLMgbF)UbleHlOZmm1W2QBvrrf71bls6Id5bV;%e+WKr?)5BWV zmL_REo|h;;(`R@geWW|p)%CQOC*Yq48F`-rvMT%cJK_CWF*G6E+eO<6q06iz(RjO( zejiz0*>YoB?UiwBZ((n5pk9>I5SEXKw=mo2XkI}`e9pm9QLq?Oqxft2K2CdKzsAw% zu&jRn=B=w*4|V&mXjDLVI{wCD03*rDA*;iZ(|Vd&*ptgTu7vjv=y4Mg_F%VV6dwi5 z`MESxy87n15g1qCAr&0f9-qg}a`u`EsMQ&SzzthJ8ynk{a7cCgF0zw6ebx3kIaRww z;$2+pc1*JR`PA-wTcGcrb8jBIf*^+3Ot(q+UZs)zG>Rg|XEsOL5j~ZB@rT9Pqvdqh z9x2`+MO^ZbLTRPgwA@U-iP-vmpMXhYwxaB!>-UoA&#KM?3Xh}MD7q@FgHN^^*C=jP zgt2dYGRl=-n)Sudz03E1osaw=FHL=_xxuQsPkDTU)L^^3?T4h&;a1~90}C6tp8n7; zOY-pd&awQm44IL-Fl*X-u8oLV*;`sv9CFXn6DONu zT9(>pW6!_2j7>NBEO!Cs?D`v@zvJ`nU7^=Zg5{Bo_wZ}CO4m%q#bw1~He`bJ9Uf_l z(yuZ$0etctpxix++ocjm3DNqvWH~cW+0GQIzQj-1zWLEcDxJxk1_) zg3mjNY6<(W`y^v7&l;7lIH}|ZB4GmSVrlpuCwReL@=g)ZyWSkAq zy%1R#crm(zDB|XImEcHTjQ%OHVdxtMhiIJzyOCccBOl8vpSem&m7f0=AUt$r-8$$U zxMN*-uE6k;LaH6x6R5}S6Gi=KULIOR&CO1kM6v7hO>WW&OI%8KRYbtqQEG5e0|?oV zObnIalo62mKn82{;M@36K&JKr@{YAw6_%R20=Tbuqo(14*;11TH>xc#!Xd<`8=L zT=Mt_5T|+Zvv#I21}G*s$qDCqYoT8vgTBkxLm~uRQI32SpFgk-FU^xVuPJYkhsQ2X z?=zzfth>)t6&21ReScS#Vqi!arK8fs6BZJN<+i$EWl;g5?*Yeux7;Fbg!<;4$cDTq zm#7SwRF-5$7Noy=eo|>OTd1qWDiwFIy#6dfB&shqHIbN%mLFy(1&V_>RmdMP$S|mx zvCMXLeY%xhzvtM`0A7E4)`wRjWxMJP&a5nz4O{9&B`fYM->tR{@wN=qjy8^H>Pm*H zFScu&dMvUEw9P)ur0nIUhIs9Ao&+;@uL0q&Jj88yy>s9SB?IX|(6n>bG|~HZN`;Qd ztFNLo$viL`z2-ISv&v+j#KuB%G$mbk7oEeHte*D}NYg?(iKb7A)+348MIY@O@AeO{ z@zayy&d*L_f!zlUu9Oh!`56WrPc(FcDHwDQ*-7u|h(I!^?x&D<8sa!v*9T|XN^FH7 zf2iQJB#)a6ar6Q}1ibX%# z8YR4UfHNj1gHjFb;V4+t-|BBdH)cfUMMwvk11JUF@~yMdEXTeQ(0gr?`nLs$T8(y) zy+UIuNQW_1EW=rJ4gJ<#^)d!&3de-y$q{ z;pyDLjwF|#!2ugLre4lb#&&<1d$h!s2@(sJZg*j12v2vb!D_<7VQES5(KT+Hu{Yg! z-4W?4LCvd~b$bj`p`pmb+tW>yzKM!W^2<78w>MH6%IK%!olB+hB%7qo}+@mt^aV#>rYP&WSR`&gxKd#_yNng6uBcsG!ce4}7|@51H4WQgz!aMaaX zb(=wOoz{^`i&0QtF56b0Qvv=JW%iRl5wKK`SLHWuGl*8XZ|OYdLrLL`y0Q`N8lgU$ z5E4}ZA#A1&#J-z`b?H|wLTumaSRYe2EYtWw>4N$0uh8_I+L3i?Cqvt=kBQmFi0e&T zTNi~XOx?jXB6=@M8K8%=zO1Cef%IAo!TyJH0gagVdaQ=}xoTm3F{UqzU4=C(_>IJJ zs)yc4ERJ;3*IA^)6$q908tWl`7{xTa;9=Ly2OxLxr!%|P3l6F{uoBDY3u?gIG)VJX zq1+Ngbe2m?#lzUe2pdH4isf{t?2)b1;U`tSt^D3#Qc&d`SucvvslCWeQA(Y`fW7-k zC+k$A?{V#^=*8Z^D9D@VIuEr}fj6UL1< zaxyrNiAZ%QbfHc)A01Nus3eAO*M=AM6(PczCS{8F9)thZGkc(mV&wbfEY{dXbu5^q zr%D!NI{eZCfFx^G*AJ~nbxTbRtCV|-Z&^TI%Wn6ZG;XWXwWBmE2v56jSLerZM_5+v z7!&cPI61lP3nQ+NdDY%JYFx?#i?c-pj0?^)N^E6(;$fTy30t2oK{qnmHS>S45%OwA zPcOF2*7qjmjVPvpX!dnhg@mf9o_HVTb*8Vv)CW;+s%CQa;w8+E zE$rwFIHmPzu^PuFGn%+Krl%FwW_x=^-x^wav80S9fbemc8Z6&0c|PJ<@`RH*-rFrJ zGzyB8OY32z^wry{ei8386WtwkLbQPhuf*s6D60mv$#QjZX(;(>_z!jqscgHVCADoR zH+$)hcAs_%W#P&0a~pYLDgrTOXbN7Nv)4T?lsjPBnazrNF+4h_j^PQ~l-8S&02x1y ze_MFUm>ht12wT>S?H6Wvs32XIWPP=mp8a<9@%@FJYGKgLKSoZ{m^Y~A7 z+5pR+RcN+C?Yr0S$qK3(WqJ3h>I%_hl#`v^Kv8(RI&5drAoI>;ihk*p>yNghzXMUaw0qAI!v`Jz!atSL3g;=x!Rk8Uc17mhvybA?eQl@Fv)GE+cL9 zZ>zidESnU9578&|e!nA}XQsYkkb8+KsEak686PSEqOO((IacymuY7>|V9;g3$Pi}j zeb>Et`HYL0WsrZ=7%7)yJfja{mi)4Rzg(fc5!Zs|=bdoo7WT|QK0#=wr|ky5CvJS{ z@~59Y^W~~aVpAGjB4ad2-{27u3576p7FM(f?!LnMOfi$SVp#FiVj*B5_-~l~PG@A~ zSB?aypxc^%3Tf2(lg{FJbSQD(XoS*nQDT zfqZvTKOJFQX8?(>z3NmnP%gS&hG(3>P-!4UEClR6b!c&8l!kjgc0IN+h>KBFb6`r2 zmc+nja;1qJ!g>{SxNp1&7o&CrM|C@$mcsQnRicxopC%_bo9J?ZoRk~-*1&4FcJbD< z>rycl@izzaqeanvVD*6j8f5Roqqke$s%Bd0aT=pzlqH!oiin#DN1zA((@{FH^_Vn^eTV3W9Jon&d(Sr&PNR&Nu!(){TQmJ4C6{Qr+!i;KI@} zbh$G~Elf!3H^g2oA+8A{6#s>Z(|SVHXmQE~p1~=zFUObodM*$*s2*mo?RJn2EWK?= zk6s)s3v1Qq+pmu3yCB4$Xje?ZM+<-HL3b>r9dt3`$kc<_j^5_|>$vIm;FE+SxwnmD z9FBpM;FkLhGnCEUxqE zypn_rZFWik{>tcjVlwP46LX9Y<1*6q$J=RCnh&!h^0+q1f4~(^wt{3=wn*i@Dn-92 zJCold!(eVAhRZa~-z=wDR`n7^E0DP*{X~n`Dy?aLO}iHT{#t3WpLHta1{$956^@tu z1T`(=={_ip{Y(G{{jv9ShN~m48!YS*t)Ws-*jyjJi<3&*`b}N7$LTdh&xsRmUjN#f zSXO*l;n44=(ExAue7kk@-NB)NwhtXSqDu+}ofizYaa;-RkVe=GCWc>K%btx4X9(OC z@1iMtOC}(BM@DrP0hdF>c9>atnEqCL@)mKm{Yq+sj7gt=k(^I{M^iG$j4Q}&1s7E`?!!F#H`bwN1h*aFa+s;|YlHT@JJY-Sh zd|D#?=B-3d=k|5A6yjvzp0KoV)!%E6l$=dgR9aA-Z=-=ik7Sv=T36T7(RyK{%QYHy z=sLT(6urpUWAt`mGP!{orY#1lHbAi~Al5N^g~O+V?c>KgZN9FLQzq09zU>D}&Tc;& zJOqnlrI{qH-($DKmV?#U5(akeE4K>j;_-g|}Ijddd-whJgJ=1$|gi{}HMIu8mQE+4OKSkz4RRO~I7RV+pl zC3~3A?!VJ!P#yDcv|H?rWOw5l8vo$9t1U*Jv^U(HE#qz}EmFF5*4ZDK-QwU(byyj4 zc5K;UxRFCC?gO9VPp6b6^%(fYPd~}eDgD&G`Y!L&2(DwZwlTJ@hixtCqBtT=`QOR3 z4-pSc4a(C2XZ|eqJ{Lrq9;?(FL6OTY~9FUHaJUsH{})|{}o3g))OlB zZwpOFvROse3F4L-=;*WyB}ytfjg#dFbW`e-ss)Jnpj)CIuKu28b~&}H>dk-bpo2GB z{8r7>VsT%$&=w`IhgiG8!@3!M1MwmIGfuF~q_wXR`bD-ryk_lNI=M5XZrPoId9-)S zJ)FDM>=nJFEuF@Q-HsMF=SFfYnw-trE|RSws8Gs2f{vT|c`T??J8IFleOWJ$&8&S2 zn>t188mQBOB}V(_OZV8Ybw_$HU13~Xh!2l>cQ2oq{wy|w#esRpMCXM)sQXGBrO1lr z;78EbV8`p14nP0GxDHhhXMgy6+tHYTN3-`*3mBZ-vufh~7p+Hmsc!D5;~f5Qhi|eA zt!J;-=0>0N`G$&xve!H6>R1h5EckzRO<_RYp!m$?)m8(>X;FQ@EduNUu1qvfSNdLY zl#>v(t# zYo)tb6CJnWNJSOnou_O%NkBk4 z^BUKyknm}ilyk|(`tW0EShwKY-WM*UAT^iMTeD)gZ?KfPzhUjS7S?;@5D&$mb-aDq zKg3_cR%@$&T(XN&O~vqC$6fi3znL;p;upAL7KNc(+PoPS4Z?HaSJ?Vpw;_R z(oL^2v=JT4E_$Pa8^3rB0=2GoAP!yh9op<_+!_*29{-Y5D@`EYZ86h5c_5K=Pa+id z%5&E2h)nf``kw;Tpc(B9WvuR*w8nnG255=GKABkGT2g|9Z$nDT4%9Uezs-bh^`bFW zkuBO6pPnvdi zTXW5^&e~goD}eW{J+%(4=thRWo<%bOi`;^;h3@LBcse!~m8c;unx>!RPG`Sr5%?v$ zJox>Mo|Wb8G;b38Vf40wnK3GSMV5?LtnX-{@k4L0Se9bhKV}zMP1J>rWZ|L^J-zMl z_caAW7m)YO1fKqS+x%^32VI(_Nu(ZMaRX&{W#}U%1JU7^#HHfi$e6`A!sc^&tK7Fe zDgETmMTSS#dX}e9T(qr}I(CI`c=Dbek25I?g`{=&$ArlYBt4Jvi)vGG!=QeZa)t^j zUOPbuZH3uRpy}EdXrIGosY6vaK#2gwV@Wl*UgBVh8#b`FgzUW|Q#I@442-9GWx=Qg#Z5_A`~U%w58;t(y1#NnJ3! zH;KnX7eU;{022{7eD}#R`s08t7gC7g;mQ|zv3UT6rV9a^7?u;wf)Fz_nR{If3gEy_ zNrm8Dg5|N<;wHZceV^i%@r-7?7V~SeYW%?s32Ljf!>$VY7wHQrUy8aWZcZ%gtrr7JG3uK#1c+Gt-tEpK*k)sXh*e5d%je)?x}pd&;im9dX}ZO zjZB?xcl=jOmou%gx_Hxf2(ETVMR=0I<&m9oNPQev_e$Q^X*22h=by9N%ypj^rDd8m zyg1v_BKgXarf2)<c56bHmABUf*WZRhjUO!O=f#d zzcN!SQV3kZksD%ay5NS~3Q*>ct>1Yv_u`J|{ z0w_pg^0N^5L(`nJ-hb{5KJJmbvG(TUi`_5AH4DWn!%ofbe@mBLmUM1%RuDa$1VQna zMK)jaA#>?hJ&7N^7TW#W8ls6VnLFZsWLUj>LO7D$^NyiZ+`Z=jj+^mGOuqbTRFlM{ zpqh&`lGEnv`7ZL4XFgC9u}?FSf2CA*N$5ER6OuuJ?3DK8C=!m`Yt_qTXuXoF_OuuY zThtG{kseS^sTr|zBpR2ab&&E!zssLeC#xS+Fe$h^Sw|^C1-dLiLSV|v)EFTn%_ImI zyjqcPBYWvj`~bK%>Hn!0=$`}5`8WFi|CL_g|F#-pOhwjZcxbau3&Zl+xZ&^ZlhcLJ z3%Z`SfKotfg@3ZHQLNGN66ncvSzC@lk_&&P%YLtRp&M2jDu< z&1_cD@j6gE_3C|Vn8PGw^!nx4tF**@Hq@%oPTn(= zoA;$sLA%`SQx~UO#VgCU+ezi8N4;(vV;}r4s=l5pp91XN{`_d2VNkp=Z?16NI?o(f z&?Q%0WggsN6^vw1t8CjH*xBi3__-30q_kGuNmY2Uhfx=TNj+8iU7DtqEAKI&Rdflr zyL@_i*jO!{7)2r50G=~9^P%Zn1GFBEFA2Zj6?R?nM+Z$cSTNtIH5}g~&s&-lNX$35OV443S2&nrmT_)phaG&r^*xKGB03>Qv*?ciHjK^=PU!3}? zoPXDYIHPI%Z(2Z=_l9IW+)H`{fM-+||HM=<0;x`G0}fnyRgGf069s4&6l9w;qZXs3 zI*YLB@!=ythT|E!zWnIHfK;dLvkzrmS~TKLt7zcd_7???3&O^T&t3Z2nb(>;mp0pK+*`kpXsb`1Qh4CD&ZRXh39lhA&FP{YJwn!A~9Txl4-z zrj^e0oLH0?p5grZDiqLa(n$s29wi9#^i10bv7yXnnQ`vZDW*4gw+Wq}k1x)%k6n6H0w z9KPJ`7w!g@{xcGk%%B+aU6rkybx!?)FVj^3F^ev{ihsp;Y*jt_Ps?Vd{hLJjT1P-x z^77J!Ka1Cc1dmx)fuL~;?WHwzW1afj96%NFXNG-8GvzB}%%M{BtrK8ZoJ95UCqfqBgdTD~#yB`AOy213HXxH^nl@#9v1A&OvN zVaa>i4MnJMy4H5Q02ZWKZ8{4uya742h9FTF*^_T5Rr}PlZU*(JZwd>us91I#Qc!6O{Tj0LH@=gOM0b>_FL;7ON*bZLXqQ6TJo~y@nV+Fgf=0Zpx-5U+&A64# z8+=Gop#WzQ%SG%F9FwP|;YS>XOOu$6KR}fLLztYrc6M^8;1jFK8QBle_m88ziQ!9t z1j*s;n#CtL;EtB!gbS#CdbWD3-twDCK^(Ji44wQ`2gUWX8)SSWS7oM}mtwevDvoLg zC1j<&K~v9 zVaKR|Y9_DZ!LR@ZMO+fxPg?lBopx;R2#gMg{&pFAx1QgAOa0APeKIC3#79?=Wr2e5 z1Y`CKyP9s_OnhoLKM}^Faifejsdss4TK9;(4uyHqF_*iphACAn;63Q-;&!}R5C{l& z^5fL|UoJJzv}h4MGq3SJ1UAN>s)6lT`ibmMv6qE0A?Guv-PoJ0YSquW$;Lmi!A1^o zb3ZAR4|rCxPMCGrL}@5E3=wrX^bg*Co_I7!MhfUKta*i`V8jD`>$DN@)a}kv{lXx= zD^iTx0QnajOn8%|JRtPjlXHugicf6z`Ji~im36|GKGDN2OhJ&h`Y7+WLCLSP6wm&Q zWspfz+>2qkIqWr2Vfl2S{Y!JRxXE`lhKDQp`oTt@>9lf?0xNMunEvp*!(>31Bw0GH z+ADn^)Fgc*#G`X{v~2dz6EqRaVVh$1wer+4mV?;$GB5X^mW$4i<0JgHzrTt7tSU`= zJIitZfsTNgxb$^BUXtCuU{{yQo*M~G`e3)%Hoj{$&*Sx5t$bc#e20~KllXRF3+wEM zaKLXGb?|4BxC(H~JWUEvC{Fe^Zq(2cb6Wfs^cu@}IrKC~;Qr79qX|#JqisX#Sz^I)JiTQDeR;NVs zcCYt87Wx8*wnA|X%d}{9i$C6~zjqB+Ex&1*Gl|s&p0|A71a?}lLg7Mj8+AYoQ)mgo zi>+G5wq|d~U%&JUSDbm~GGq4vO^CZWg-`-maq96i5Of~!zb_%PQq>(FJA2|_2RGsv zpgS%ue(~XS#FRX2{1%Mbh()NwY#!hqggB)|EbXIN3_I*baZiR3WTo8(Z2Tl?PlxS; z9*%0cH!Vic`~l<57DK844bkN6MD;U(g5}nFhoXw>HO9$8pD-n7cf^B#Hy`wIv;4k$ zalDz=Sws^Z{$`s~i)Sa`_3e~T>fRFNrRLT?(R3TPDH*BV#3~#_1P*!+EV^JOu0;jb zh&RSE1Rd_~aauO9jc`W`%6sc$st3PVb@r_`6l3-{o;oB(9k5K?w2Q+VNQ^;t~b1FuP0Bs1bEFv7g8SJ(ih&d`0-QNcH7%~DZ$a!P<37iC3Q^aB z;t=&%@>oLMM|r^_oWr3Dn)@;v`fzW81Vlo~;4id^PonjtA6j8<%srmh9gJqTt6uG- z@eR-Jwu^br%X_WZ5TQ*aLnT|0z9ipGM9~?i%&z?6eg0rkTqkq6osA2b#WgB9QsIRj z>dh+w+wq!GyzFu%jCfxQ({8o`xHO?F{#t_w37H^;Hx)hAbx%Ij1?}IySjxy4xc`s$ z8{APs8_1UHPUaukQ8W>nsS|+A)&0pj!n_G=f1s_zQN-V7x0)7JTiJw-H>;|s9F<4# z&Bs_H5#5>f6;Ju=7q7(htwnx!%K307BHEEWiiVBUA&O^q7Ni?P$MvKOGSVmk0JA|^ zPvo#Lh2RbP9>j#DvU^1fZF<5HqF43CU0|3-LYZHF^feD`-}z3Jl-o zkR{0~TmT@JA&1vll;y4kb8z&5*i)&>C0)yA3dT6{cy`>fO1;dJ_ZvB}Qmfu{m}{3& z`Iv|sF@l{hL-k*G&Hio&cj zEMU~ZBq}8m6xFoy7K0#@SX-=2i|P*^g>%-qNwRHIL8L8$d4-x-=t{yObL1k(NhO9p z^n0nX@_><4<{t<>F6ljUMbI_PV%`u&6DDc&T&)AUVQPTR)hIEt%4=ieK>YpSw=9Zw zR2%xJQL96<9xwk!eAZy_llLfZN?G@3DrMZR9vAi*Wj)QII*>6Kit!sAtK&1P3zj&C z13IZ3@WYW%Hj+tZUt;`q|8HGilAk{)toUP=A+biN=L>!%lahJ(U?U+!C-CTQYZxGg za4_Q>AW&J8X_a=W=D0}GJcPPgx-W4=f>6F;J;GvsbI<$dW-LS&AF+P4fG;+8el86s zzaGI>`CY6-QLSNj-OCj7ex*((N%&H;f9a4&5gFjVCmdMmd)eHwffbUo8EDEIYL>>` z!Xt*+If}9(c%m=TxRBM~%Cyf3g>H-?ipc?^tFpL6xjS%mHND#oa*1}t3+ZL8Y0i)Pv8q8txV#L~@~H(f&w zN0ega(Yt-yk-1l~oUC#cWC9=URJB$1LXBRZ2{e=)=u-T_$;U|DI$57NNDNyWqdbT` z;f*|k9V*R??-^P3NM{2-{}e)J55KDZ<}8054QN%}C0TEH&pG76{5Mx}hI|v~q^elopfXzk4lxxXtQT#E~s_Qa7y^lo_OMhlCdu4q?QmYeO^pX5( z0r#$1y{<7KZpQvfIZReN?lapY6e_B_|Kzmo=1o44Arg3E)ZSU7I&pVbb~m{-#mZ26 z$8jh7=I_6n+dzuxIqC8B?C)B9wA9i8cHK$TR_6VSU08#u=4ifb8O~SpC9UC)V*~ez z?Oymh3DngpQ3{m%>fKx@(2$Vh=5__+IJ4foaf5nm)y%6+c=Nlt%LkfA{5eT@7VUh_ zO#vZ${;f1uHbZN&-Eb2>F`epje7_0_YoLDoT?;Pz-<^S|cFk)gYR@}6JR0Zxb9Kiw z6?u6olg*iHl(j741G=!j{$Kb-q<`}wK<_!qMZ^qdOJYx7dAbLZB1CMyT6`%Tf1rj> zxDp=3hwUj(oF%2XHejR{EX?GfHX;eC|E1u5gkyb5mV+)4vyuKp-A8(WJ=&GG`#iRM zPkN{FMmx!bTa&om^(fH}K!K}vN4*Sp1}>`(N(3yBpUxSM@>0alvvcuL@L^L9OUWfA zzO|e5v|c0z$(pHox*CjZ5~e#A#BSkf4J2M%3Z9>^pf0$D-#P%54_UKTyaOka+XCzZ z#@s^{T2E;GvkdP|)CmZ!8_oys{%-Q_(#htHm2=%P5u;)QyQdISss!+n2ntvKdEXO8 z@xTSeT%mv!Z{ZOlxGxMV`5bxYYg*WAEL+7p2BQ|E$8X~Ld9TlZj?b58p8|RW&<-RI zUG9JDpt={W$rPz%-CEg-YZC{@3g|E(B}RN2OTH{KRN>}YbNe)v zX~FoDcjpvzoaxl;)MLz8k_q)ONCa6RZzj`@LU{xj&J`fL)1vp3Z!M*fjyQSq%4{*V zt$J1UVDsnXaprb~d#d{GOQ6JI@dY*6M8$mgZt8yQ-#$6l+mZhEx|u<}!LNuGN_7j1 zB+bUFsu!`=$MnIBa7r1qcXL1|LFEpR`8>)IJO>uE^5rA1Yx3KikXN&>=q53hk>JG$ zqG_RDVQK=igf0qcTFWRq&6uf3R130@?(|!B{JP*WPi26)XA(ZoX(_=mfnmyy4{VMf zzDQ6F+f#%f`TwaJU3a756{^{q`Aidh^7pg3yLd-Y#N*uThnW#&M5I}lVQa%O`cY&j zJt>`GN{w}A7isy`)4w@G0*fYKvpU{yj?=Etx zNm<;TFgud#&mls;-B|{&fd|>K_=l8c*x7@4|Bl+2pts@4>Q?D>yS+t;nzn6&w^FT1 zP1R#~qm-0Ll}4SVlz`0tr(l%wEfC3WEID1PY0uAmS#~GU=QC8{iJAf=j1yv12A<(s z7Y^p1T8IWA{>RvPDr@hAJ~ap#!3cI^VQKyl5EpUuI5`&w?N zue;#EVkE;%Sk~WumO5dyFjrHdeT4B)OpRmjBZz?dA2Bl)GMrP2vD&X*sx)^~!}FjpH&ZisMMFm*{0Lgkbl3POCbFt7Sk`6k_?a3Tqmhne4Fc8 zyL&IWUs=d&jF^P<@cE$F_8nSfjz5e#980v{6Y{kcV~_XqwVL#1aPFUp4?80$M0v4- zuZ&URI-F5#I->6!dy^XSB}gaDZG+{Qceu($+S#bo5yAM6QhquMK5k-S4T_;aacJNU zk;CHLTO%&!d2daD;EmVN`-gCKtIy9&SM741rv^A^NbTrV^BeJomflhMZVV)wPVa!Y zt*?8=t%@f}5ldC4gT9ir4$4HeKe&$J3%Otn1;G(>Y`r&Eaykex>ty^REB$F(O2a;A zUMLdw_7$r`_GI!xVSXp6kSvxKQxuLHT2 ztshR`T>PX*1{6oD(4?A@ zwVyMNzr&*`v`8|qTcoUzh;q-q2Ngcmn=OYfO1q}YABxl$Fg~k}d2S!|0*Ypg)IgDo zJq5J$O$G;dzy!zxFN&xNiu>y0r_5(f`#&Yy($`Wj>+A3OcRj9rfm7(gLggeV>WJwh&V{fh^Tjgm($;~C@DD20u& zpD}IH`b7-B$xS%mkq#LEocPG16zKPA_!Z3p!th-rt~YnUVx?(WKUv)G#dst!_x=K^ zxtt- zV@ATlG&tK?p2@tgY*iKv$cZO4dq@83AUtOamLUj9T1XDhTqt}2y``DwUlS=_+53rJ znq(8ABWFgo{K>j*m4+Sh()t=jj_3EQS)%5TkJiCsKW#&3e^C}>S~>H*Wu(fYn(W2zT?4k2C=us@8S;wa?l3a9t2R9#8al%%$59HA za}Lsv_2Ld`;mp2EZzrW)t?Jh~MUGTQ8;q*1AKt(v6LpCm1)3g_u&s`c8mXuQ^j)vE z?3Kq)LVXZlgxyh^zt}KMrW2S9s+O6_aFL*VIEPB@tz8w}QVfUsK3+Z?@Z%ocP@}gH zzZ6O%LMLl9jDkGZeS_Pw&BlXkgNJ=H&G{y20N+;@k=YL{l7ADs0d7m8436w;%fnGf z1_|2|C?2K@?$#^F`_BQ-`H0^S*-^%`ia)ZT%@{YgRYmMZg=M)XBCoeO%>Re`y}v>p zE+H`5@V)Yxo5ww{$WH0MM?hHrpN+syoczEN`qydyZ#D`4--dqwdE}t~Pdlzftj>WJ z7fZEVCEvGSp1uA!I^+ip6cb;V9wn?7hJJ)f{`|6xajU&syZW{A$9vq_hB5w16rjlalI}8d@{BGN?%&Qro6RR zU!MjC2+$JHg-l%tKlLHt0m|04H_Y`~&W#ngY+W@#A6*7g(R2#4Ab{qX+w=TM()CL)lqtL+`Q&*2_djyP zn_(@h7fw9Rkx=ovAsPS2e5b%~vi~zx;}6+wn18uxX*Euvc2>-j>TwM8KF0v%iIHJ^ z3yT)wxqTAAFM$W66NjBp1`2DsFrf4Evc(*{mhDKNTfe6?Aqnhe#uiRX(cN*v`B!J^ zU1gDdGS9|Dm|tf95vXV)x(R^Sk$1+0)!sdyr+Lo=GcEXEY)*&+Wc6ic{bR$$HeynW zC_tPgmZmp<#BJ}m*b#n-3F$8zls-~Lc|X56JB+mrJ(Ig6Mtp3Sgfjc4YigX39*r*4C3Caz{U+B9RM}W1 z+6ma|K7M2rT;rj3$BlbeE9 ztnYttPrm?mp*nvREJEP&g#-gvn#2BiEde4G)l=x0q$C90?Gq3}+1=Zlv<;mmSWc`SiP~Np@o+N>=Q()aJp_q+n1#Tr| z3wrQM>VX5D*cCkURvE*iF(5yf&joM~GC-dt=@LK-8*irGB0xeWa=Qaey3q4=B71rOIdO0BW)*?keVa)di|Mv#}@Q)g2@p z_XI;$k(1p(t@#_acP~cn2N0H-zQHDEkxJu=BI&O{fPQhYOr!dY$>odl!>_=SSd9`3 z``5zgUn=PW#)YVz?LS~PtA6APG-&=B`|!ZRwJgJM1P07x7XaJ6*7=TyEsZ&XH#)*wGe3o4_u%hOuUwk6J>U$} z_~0l+Bnu-p6Uj?uWv$0!(pX+mYhdBi!_o~jLO=6#4AY4IOfNb7=J7Ce&#>+mNtEP3 zzT_ChFOpsR(QH=GD`Bz5YVVKw-U8gpFP>U>Hm`OWtDij-ttkRFt%v`4e}n;jR5)Ok z>d=F+2TvGx|IH4ed4RwrV_19N6-ak~&3rCzruXr@2s!K83YfLeR6L3(8>ZlLH|h6? zKiLBKD?W2tPTD(sMmoG$F1{A~1HWIzu}@I% z6t@U-cndXs8u3PJtfFi4IFsZ#hV|$5yO1R+pZJ!g*RTDJfTC%OhJ@We8)YJG;S~Qf zUML7c>6W#EKc*liA|@$kB0~ZR^}B-JsB<8tZO~UP88~Ed_zt{wL7Pu1M=$JlS5%>e zk_D2N-kvX#eU;b7jwO-8Phxo0b*ewwb7Je(X@uTpmmT^%Nd0RpCn?r5~K?aVj( zu2%!a{#^3=vv>6bG zXeEPH-BXdEbk_B^l}qOq6TFqY?Vz^l<+;l$Jmik#mwL<3ysS0Lrbgi+k#^m|uwK7` zy}rtORH6tLEBBh{;DW2_P4qtRZNG*HwJ$zm(T0P`uIa>IMmSdo0hx&X_6no)YU_4; z-36lp=n-rQtfvMviPVE)xCeSZuS7p?+K5;$2pT36oS{=tneCm*Pmq7NMcqfMUlV)I7Oew*?o?y*q`l_ zWI)@t-U=yS1WzMTIB|fW4)gBspFJ*vw(`~In$<;g-m;>sVvoFw`Nca_>AWv?Px+Yw zi=H(;eI6e5l~GB=(NFBvl}_>ZQ>yNtqo;81H90}kYI=-i=84IZ&q2+mI5_^k=b*?|Kjc~ADV36_+gsS8%PPrMk9@M zGg?AGg;7dKNk~Wv(lA0okQM|n2}h4mVGNN*q$H$4Ndb`*pL2fi`+k1Ef8crXyzrem z*ma%faU7pI3Y5x@9H!H^dJpHt(dyi7moOcxfl_b?xtv>)~A|oH+U_+ocDC69KeP>u<#up5)Jbz8wup z|B&qu75n{d=KZ^%PF~c z(7I%fM7NS1qKVqPZO|kjD8%oj#Pl+Ca98+?iwX(dimR-QM9siHSi%>`l{CxM*eWdJ zC2IVd<4;#Rvv*&vl*{$Ss3)*!41KA$Pi^#ArH$+K66VnFKuPot^d<`^uco5Y30$o{ zTFGMF&^|)YD1)$UF}Av^Xt(UT$;2SdWX(vvA|~;h;y)1A zcHYxOWr0kh^Nj!K@y^5WDp8)qp5cvr4VT_rA^u2Cm8PeCzh<84?n29Wugg{bvU*nOH4wCgZM^9h7m7NRJn5sUcpZrFbTiB+7RhVb21Y+ z)j!bQPlfi?%F~~kNr%cn4TH@DY||GuzuQI5wU+HuyTe}}T5N+~wRV(>hk9ySxmUc! z^TWlDL9|-5$jDbzhJ3@z%w4OAe?BlikmFr}9o+-XrI06gvPY6wJygAsw z6=ti~xgeflP($33U9}Cc(gCmo0SS+K<$05Al6WFAeRz{nA;d)Xd6_HaO3E3E_&r3MH1Xr#qX2|6lDreG&Z|INYUvgqo+01*OM`yMIEg3ikGCY_Dh2^ZPICj^Wlh z5)H)1q?gEHk&nwme{HafW9LZx{gX7Wg@9VumrqvC7)vK=>*$g3T0_*p`m@jPOM@y* zpzgEO>bBc?0G;2fIxf{evw-;y8SiD1#fN!jm)h;JK?;K;MKT;;dD!|a z+xnu@_iLgf&M! z;+jMXKug&dkkp1Z`gTR_=oXV6Gzzn;p`m$A(?~<(iHfH$LmV2F^)_hxYo0VOCp;b@ zp5D+B$rbja{ZSR~m?SC`^&3HE(x39Mzt2S6Kmk1r_cdh+7w zDgqrRs*~XTw(>wiF%??$6PD=%fQd>O4=xhTF6XY_9xa;<1<%w}=+m{>sXo}+?Ywz? zUn~9gfM#9E*G3>ks#x`-^+5EbKmqKQkT(=#x*XMHP!72A693hKF>g`|TFG_9q!<;h z;AYn>DARA22@oPmJntx4th($oo;p;Tr;1OZ*aLs0vOcObbIKPP-X|mzh<%-tE zK4>C}>+A!07rC$qpM>UxBoXr<<&NXTd%;!;>cYQ>EVQ#|<`Zfj&gP^4TxBM^_nZ^u zu?Zrj6bGMqCoph(dvQ_5JeL@fFAD!<#T4_BTeKV+8SKmDl}=Vo*DHSv#SH?zOyxFa zrUgWzX*cl}bTRVK3w2S$V(2W2P%}xEt36^rI7tO>WM!}}_tuvq!@Lo1%wIJkd)LSL zdP|97K{uDw*~tYSoetqEAF6RSWQurl&9M-R>KvN~vsjMtDLb!}+=AlAOE4I5dz=@} zmz)<;v)u={ZxXuI-A1kRXzEslR3cc-Gn^RM{Lw0%7HROh3+K=m`$V#E^ zJf5aWoPbMr-OhyBEI@cY<&SUM8)bb1zH+aX2$Y_ z>y%AA>W2DKVuO}G?+;liG_?DptKWK%D1JJiFUI+v;=;R!tpg`CT_Hz@wxCc^8@t?Q zv0^-=rfa98a{QQN+S)-}uAnlM#CkK#cTK|tAR|ISY?P9|f`PP5LE7v$ zz5whlZy}C})$;U_N25^cDs?UwlMOwBF7U7{VR)BkFN-r-`)Ml`!!t zF9)N}T_i10>}>5Aj^hP{U&emi0fSv`6(Ki6ArTdztqRn$KW z`Kd8|Uv(QhET!w1VibNNHTd+sZN=W>51?Y6!L;6fJym@tKJfm}=9nclwOFj1&`0tL zvL6ZK7>AC&x?Ex+QgRC(6*z)qXrkJQFgJQFKLsw*l0tno({iO8;_IHfM!!2rAvc@+ z72apn4NOoA&4%$;#uFNp50JqK+xx-GTGwe@+{LeTLX(M77K4|S(p;Oi1y=qD#*zIg z_JvRN3wB$`{b_w&bK=YTVizHtbS}DU#^6MSqiOfscoQU7B|BU1Ps*Y}&YRb3Vm4>M zIb8DOCWbK*_{odI7zwJqJ%bNRS3f;vt&izefv?>@ z{&@0$`(x9bLJYidWoHy_R2r^B341_U_hxRXh1t!n{c_$N|NQOOurXxCef}VVQW3Dq zyaf|LwSydCVT`;?)*#;bjB4ivNDXR`^-zJ@C(tysYm?zD3pIi^EYfx5{O3GZ>W}Uh z1>Pt*cufU0_F?zOGLSG#VR#D75>Vz2JX|(Ce0x{j!msHVXDN!&$=AOFQby_J68dSr zW+ktOVuSAPBQ6Eh{{yIEU)r62Omyywr=dI2&__em0<%RJv@hmSb^vt5L-Ppj-0`af zg!p-_SmV2!wtcyEyDAHK@e5Mg)c?LH2`4x5*rY(FZPDj_^6LL(s1 zV@F! zkX*?lnZ2v{SXBp>SpWVcI#HYYL4u3t@ zYr>xB4s-_J;vm-wT7>NtM_YpK^orP&GhR*f(ucK9JbD_kw<1u7h$rai^2DsWGf(a< zJ|73_=JGDK9vnRawbF(-HR-76=&WWUzO2Zuj2_kCUr)4m|u3+?(S?+6WO zH{l0*$}sHDVZ%+^z~1!#QZwmtYGx^l5|3bdoG`#!v8e&vv=(jOMILD8$aC{z``v0r z-*N`BL;zySbNYQmG;pw#l3MT^4TM{gqDe;`j$-Pm#uU&*XespN!ZfHxdJE`p`$Uyg zTKRwH(`owW2WmOF6_|+?_&dc~vNHytB0?NqsaX|Jc zYbgWCv_OHUqlG0}*j5?jbS29#nSRK;qV~m6wpvWY`KbwX@1?|7gFS#P4>nz;Y<~+I z!pxP3!${n<)J@*Q|Fk+L3*&j#&cW!$703#~EI!i{)7B-WeTD3a!hI+b;f1s=>sQW3 z)@{Btds@NpP~gB{3XGfJ&B>$OKrR_IfIhFi9({NxCNWsENuQ~me4TDOR_IbToRF0D>^D&0 z7LYTPk~?+*b@HuoqH+8F=uwdAGN#{s&)0MN?EyU@o#l60+ZbrqQ*!@WG{hPva0A>* zp1D*gi1t$Ay`updV1;Q(a~- zDeqz=$w22=M&&WW8{PXvGsY-|w4!9HuH+@ovr>=uV7A|ZV=rVv2|*t|(dPR`b?GUxAP;et2Nm zFA+T=)y0`vErfIS1j&@u_cU$aKsX_ScqOINA05WuepC)A2dv#9v1^bXr9cVJACH0l zs21sWP!WFcrHhQ?2Do(WSHe;q_?K8@MjmMOU6_ooaj1?vT(nRoV7n7wWbA)N>ru`( z@(1t*4lM2?R8=Un#82{|&poVbX)+)PCbS~*BSROoq~F$5-n{$}2SeX`<9!UB=v?dWx;#&7MT=m2pwx|z+w0BqmXnb{kV z-I89}#~+OdIhtbae zDp%O+>RqX$v3+>HCrLrq)Z?Gp2c;{nVTs&*U}K(BrRRQFR@kl1L&p>-uwc{{0uaMd z6N3B4vrUKUfK$|c0VQel1IFm+&?)#6qK1m_p?9QR=mVF^Z4&bRy8wts`OF_$Z^b;L zVL?JDs)@9VU=k1U(FHF$-})Hu`5~(KG|OK16Aei%rOD~l8QeEH!_?q(9!T>%G&R{Z z2_I4u=OjTwk>`(At=HZ|v6SQXaUJ0=IY_QAKDT@c!5_JMJ#6U6w;>_k%=*Qp(N|tB z2m4=1f|#{r)e~t~FwI_8?tc_FSUm|JyBr=3_Xmjf)GS(T1C^cf78%Aq?KfdZgNTOX zv&4#~Pkt>!#MMfIU{QO*5)n-P@$X?``W>XsEgac2oM^=nhBsv>l4^Js%`nekkz5EL zgA|0Z{w+=JBGSOzcK#251eI+;KIF<}9j;H}s8M^dSRpMHf!R^om!y;NN~LL^?(cY8 zO(*-k80Ai<&$a}pTJENroYMzdnw|%;*G_(IYL%2Fs=D(qdTnEWd|P;Qz<&IUQh@cJ z-{=!Yc-^iI3}x=r>zPY@6M|noSM6q_ysHP=sVNV51Xc(X(OL+ERNO_%>I##6Zr`F3 zg?D1i+=BOIPC)_M@1oZcILRj0-S;2Pd&B~>llTU}5?I_q3%nYrJSyugK;U1@sF@rw zFE^AQD|o=Oy=8Ik&ZATs>-_!8Ug=^DsoSlcCO1deH4DDba~%i`=uI0mx9$uaK{S3! zIhu4jYuP?ENvsBjHCE>ZUd3rw&;v1mKgCTy9lq;Cz~PtNS*|vOsXbMlu}fP-U!h-~ zYuN$4#&}CW;lH;|en|y6T%ZB_*}f#onLN{1&57}B^yhaMMC^6r@R;N#QI))l2O&$gLWDhFt73N5222d&wr+m{1p}Lp?(X#fGN^K*uu|%t&)yul8 z9I6$fXhwh=&Z#8=VSDa;nt%a8BDduQ02q3JIaqz*Nt}6uk?S5WeJP+X!axF*NF!e& zpGh z8f@y^99q<*h^roS2T0xBn*Jg%heaY#GlaCET`GlWQtxYMRI7qIov0kzpu_tbnT>7= z+&Gs1OzqF<-lf%=sR`hz@^t5wo{z5sS%=o??j%=g*O{kEwgIVHqGZ|y-qe;RO9b+h z0IFrqI__iV*iu&0yH1L=D$oL1xKud(rD^&!eY7SL#S+32E~#}mt^tx%R$du@AyAi> zYwcWAT%Fp{64TUo5h~~Bs&*AluF|sxDaVd_656$s-(H~P>9~|e$OIiV5kC>M^EL6# zP?ozWRe}xXZ&gOeU$L?w>cl(?`g8WX1K9m85E2d-7_xenSk_rZ!+hj%=Ygf9TcJv+ zbq^<&aKPb>ZLbI(ZfMZ6RRkEAQE>29)2eifIzj_FNKr1BmY{77sKB=~92pQZCy+xO z_;}kpPDlfO`#rdJK2cQ1)MeT?;od5CHC}#^+QgczkIMBC8{@}@8yBg3skni`9|ITX zCaSeF^quL=>CfAbS@shB^^5(3`AhoHYSYEXPegwo?_XLM3N(9qhOsTB1qxiZ=bpCm zpqZ(Cmv8n-7h-_c_iop@_fR~#a5cGkML~iLo^Ys%T|kWa76aR!ODf_+dPLO@*I4A9 zikN7|!y%KeF1{+ z=y|AYQh8JDyJNn^U@*Y>IBBtumqB7Q_4SjTVla_d9iw93#SeMP43040FwA9x8wtyF zV`c=ziC{xD;%jpxM?~Mna3+b})F-W`_!Oza-~JjxBeE4^nA!0MTMV0pu*%1B_f!My z%BB#37Z7#DGT^jIO!Xi)A@-1q?!J#9e*KCNnQ3L=+omS57b+Y-^77Dr=ZVbFJEkY0 z!BK6aCAS)Mrx4<5M|h_BWy>4g8=>+1X%QR5lcW*Cvq-#IO;9oaGfnx4k~l$5 z5d>*VYrcDBNXaVQkT1BoH1-PvC=~1a*nzZr>TovsF=PA*I3Gcx#UOY4Ra5hW^Rhj= z`0}cSOaG?TzvRmA6J_QWa=675=oi(?BJP0Q@llO7$sf9IG+Di`m(Qy^oM$HV;ECE*&3EJFW>&$)<}`+8*$1F_4^dWGtNl?m+YJt-zLSHH%|R!kTwN z{QtRy5tn0h9f(KLUsioH`|RFDqL)7bdy$uYU+tjc;*8^~%Z0)tmBM*1kX31eV$X}H zn*?U%+M7K=z>3S;r7dp)xh5~ECg@YoZCES%-!kqhtw9Js1LILSAtiOLS%bT<8c4879F*bESB z9iHukzP1!n630R|>9?%)DVKn*q(P3D4%?}(+K^*Oo z|F|$vwmu%X6LN-ETv;ZPwxOo1mfv4iZrPYpgw=-rhU-0jTF#Fa?EWpPC8m zs&{ev_wkH@yF!TWp9L3aikM_|Fnir&lf8Wn<&x)Yq-_Pf*;=5x-3T^6?G`*Uo>c56 zgo0Rl9s_?VPWVNk<59{?E%rirK8#m4w|t5amH+_B`XNAtUEIq7LcBpBIdZM?M=kT& z4ej!q6aKyoMa?@4-4h>{W7Pm~McPz}Ay zCgWy`3jE^SA9ov=z-d`;1Us0(R>7=Hbwy<-wiAM*e&O7kP4HPe5Y|6rMF9PR0|3um zI&0cUXktMmxm)%>{YYn`7gx*b?;a=nM!%E=6<^0D^g7TK|Zm06FQdubdA7 zcTho#pKG6_=)KT0ROkM=Pcdr|^cr*^ts%atlNi|t#)!UQH<0v=kv=&p>`D$@y3}Kw zCpYcWVWb9LrxQuv1?}V$t>ZGoKJJPXG-(}x*z*n9s)F4ERu6av@w}lxT_k2aU<1Ga zqYXc)z7xbBbs+Xxx!4szu(nbe+{#RObiI(mJFw93ozN6v3kHjw zXEjFWX3# z0)Yj8d6U($<-EbaK4XmLVY_tPpwJ1>b}s9$aT)9e3zAM2i67;8Usp z{+Vg@6Z)jN8H>^kgU7OKJkig-2b5D56a$(=z9Jm4m|x7L6u$RcPdgGA{z7n6x4vcr zmvGcFM|GUc*(?2n*u^vk0*;(0cS8ajI0N`B7pnkJhvPvt8^_mP8c^V^z~HC@#aE%V z-vInWs~Y-Kk{CUqgLhguy>4N}&r1|?26k=q57oL}VY;i|u@?$15$h0l%2-kL;;@_H z`=ev>N3@!2pp~Z~DCDU03NLw1Nc#V20lx5x$J58_K=VTdE|vznuHO|!B5R&oNq#r& zKCOWFr#^D(+0sR!SZF}h{hhxT{CRexq%@%DW@09QOiap95+LGoOx`n3(hRFUY*R{L zQ-^REeT9kDY|1S=!;9#V?I3x=tGM#*ugL~OW*#zcBM@QLaHxp^Ge+({P1RKhH9$qf zp2?^mf8(t!)x>E0oCscgp!Lq@PTHRsei{if!Qu7KFVdDNg)(iJn>50#FoC0 zpVLMxu+8(tYb^Slf_wyO0tPcGLIShT&^kK(4O|FvzJe^b6m&zgPI8@6r7YE>!z8Nl zjDba6_XzdG|^R`!N zVpi;>`?V+I7zGZEQmJ2(rtm>w?P{zAPiS3aYB=h%lapw=W z zrt}YKB;}3^R9#`k+aBMJh?FUwKa0C=h9Ds!A=`4Dy7T@mxygs)&Q2Of3B*U}5Xj8S zN(=lkp;O2@P6_?fWmkvsLkf|wYQx>l7Hx(y#h0~zv;eWKF&r_FN`stMx6-|me7Z!s zbrWJc?Fxij{n+M^vuYV@a03}fKBdJ|U&g7v+B;O*z0ig9!zSbu55lmRd?ZRMjF44w zDS*%;fzv~i`xlbsExnjb=UR2q_TCz!z!8KDB?ud0hXMM|M6yw=0*SeeGj{)`Qwtv zO&>+vKH6)JxR)(fWH*hUp5!y&PHcpYwRs!a7QUiL_%$)R9@Ec6HdVUTDN7(ZN?XZs z^ret&O^6L@2@6Hd*w0NT+{H}emL=&NkKlJRA^@S6o^b2^W8Ng!bE`1TM73xR&gIhI zGogugJw+iac|L%y$eSu|$pkdGljgsQ01|m2fkh*xcxjx#o+y6UdvLhI19LP!FFwux<_eMv zzC=JA<^g6<8>n^IzR*LecTYvQqNmDf^2;|C03T&1I@KSG<CE<&lGuzdWWMrdhzv}2g;$kAg4>x zPb#TxaVhTom&6K`{%3OaK!v9l+$GfXupRj{Ig$(-51v@eG@L`C<6|BFRlDTUE-J-6 zqs?aB+KS=tv;6#`;5wbM*`xE~p?I5&+S+rP8F|xAM@XW1kYXalv`Eq3)k%qGw#9K) zX#JWAx;e$4H5Et*0(vUAJ+baTu9f=arQOY za36WLrFVO$EMdEizyRzBS;uR)6$i24B1X%6dFu_PhdQ+|aCn{7^u8g`4dcW|uX?O6 zgPD%*KTVHH5%ttlCZoK^RhU@92A8Jmus_ccWL*JL3n00r{!j%?joO1w#rfI8@MGVB zneo!WMKXy2opX-6X|j+pVX}x1%T(pXxu7YIcj?d1!$g;}1w_l5f*ckg=d-cTi$z5( zGlC%9r(P3dh0+d*+C#SkYjz{YQF0lqqA`cG#I(I|N4O$L!c%^?$Oc4xhYrLs*J&+` zN2s>UHq<`2lIaV`!M_ajQ{4#fZB&Yae?anan=-MBhNEIedwKGzErA|^QB zK{%>|$X$UQGzqY3%I;8J<`<_PI}hPOwf(qE8WlyQjw4%z9v~)zkci;MPS<;Wk*v`G zB0Cwf4bkFgr(V(8ltgo|ElWXN4ZnJZRzQnSa~udv^s$OByc7=!q zd}3fYV5Fpd?_vpTl|U(?+5T-E=nrj%Jn?)B>}HK9>Q0-nd%t-!9&pGH*zRKmEv!A> zylr@<$=)I;q3zrwOVq_8`+6e3RHXkXPS z{O9=YW02+ZA@ZO7hh~0diPB#LHouS+1_xDmDqDBD3w(YL`no=)o<`W}osX6Z`Ym^= zmwWh8=PtTn-LfqNn|2E;FYJlABF>p#xz1HS&;+E9CPEbd+n^|3nv)Z&CH za`@(-{sumhg1L@?YXIO+v-&gVqW$C>QYWVFgvF-H_CQ?b<^|5?*Y^a}do9wAvTLE# zuU@=Py}oJ-US`36hF!=GH-WQkQlGbu6=B+O_wnGdAbi@VbhG~nS|NzDn9AjF5C4T} zdt!Xx!Uh?z-6Cafog@9pD3`2{0ad%h)6GjuOtp6rOEu=*mnSck2@i+ zJQjAPqgJ<@g(3pL7A#!?0*$lv@4x=tUxHz?-ve}^3wy5LQqj3PgTrdICswpeMa#RAe`wn1h zVx7TBwvD&dDoJ`WKY#;luuBgOMHxmkwdR0xf|ekSLfy*H@hj3}CnV;@5k^r$5TmtY z>^SQt5JznL0HtL-VFn$c0RQ>bZ)24O`)NQZf|v%NlLI)<6}!y@)5P&8K#J}xu#uR0 z?GSPZ!5FLJ&&?XSR#WuPBPf0M$uB4|k{PnVFs5qs+$pzA+RB-vWx_Ez4(N!ae{Oua zG(tX20I(XuGTt9!n&*oExDq@8##S7-OI=De?IIzIr3E%g=Wa_TpjI|7t2Td|lW6>b zgyYUX*Gs!F4?2~~Y<}wRgsdcsn~-FPVLkc5uA1SIkl{OgfHiA8y8C3rS@j>ZSUa4T z<_RMqyizKPG&vSvTbC}mcfMW8(7QlI*yRPbNAlwUfX6LhQpg4pz#b2=f%#h5h|m9^ zv6%?A0tBvf0Oz|T^$loNXutw2n$BqQ&je@F=K0x0@N}i3l5761o&y`Ks0j(;LM+EU zbL}z4*Ny=~XDH3`2~29<*;?rQ5lye=8(>fkExC8FC14`!H7TAH0ZO^~AufD5v%!RX~8R?&ToVMppuih??r7VW@7QSv?> zo}A7rRqL;ti~sxvc9z_AsPl(tK4G%Mrmg4?f3j5FGZw#RG6?LAP)cR0TrjQ{zm8Qf zE>@eV(vy5izvKl#iXwh)Y*=O8@a1{%hRMO)0LZiwx*BqzG{?1rR>`2~0`ue7K1F(D zf=$MbVz!QNcxO0id&bX?99}uCu3j2;58ne^Kx8j(R^m(ATf0{b$5bI@eZDOxQn6Pj zRn&BkKyqk?@8-g#$qc2}uZtE=uBwA&jmgg}nSO(wQQNIPQG+8{UB)kOasEhUvo}-~yQ< zdpsd&l>b{vtz&o@B&ZCab&NKGCf#570P~M)&lw>R671*d-A$>7NGuQ`tU2!=&O7AT z1y=775=*PVIrT3lDL&<`xVttYOt8-B)XO^SvHfWKJc_W1blr{9pI@NtdMmCvh@t{A zb+36KD!NqNcLHHCARzv8yvQHDu2R2-Xw8-6RU|W~MwjFA&l8 zi0YdbVSaqC*g$CC@0H~g`>BG&Gl42!?qlLH4YDUyuZ!SYmS4}N6Sl`RDY>rQeg7|c z*ZjPK{7LX`3Ou?#(S}`ezN4QS@p8msr~7F#Ieo-pQtZ$|^<}x4yPq$Q&pXBv;rW!l z58(F0id>v``cv5F=d|dh9HQ!~>*mh@a3=qWwwy-bt-(7mC;RAk29zZRFOEN8lj0#n zGX~NHn7ms!O=l1oeU4k6HW zGosO_L}ddP0G`zs5Ac%GH|eiaLSPqYK300YZWgdSb|%cTwc*s|(h;XI#Up zH{NA(EnU6ReSdjhTMhbfZ`1N0g2{{;2p90N54s%z6fOEoPvw=1`Nt*tUeZR2i{!bs z3%J+-8f*YmbL zeIjC9-mq_SrPh@Ka+V$E}#iDCa()AuWVGf{tSP z7T*0Nu5j{UUp&jAJCMh195P{T+1xU8rZVpsCZJ75;{^fS5ub;)I` zT}Sz?id8w?^AcD@8Nz zScFOh*0QNSsY~plHqpWvVLjWkU!`6%vh>O40?Ygw<>qWFA~5*U-=J7dbXMmOc|d=yRqugLdA$Advc&M=h!D-7y&{=hX4|Y=2ExV z6DY}dk0matg!`AdHS0>RV2QGHukULs-+Ny=h`7cw_2%CW;2a2N`$h=o3<{dj$7#bF z8AMG8nZ~BnScQ5VwaBf7fnXx*SQdd3BWswYH+G&P+4m|JkQii2TI#66e-adH>^ohZ zf_RpSm*#H@Jj7e**I}#?r_F?w7@|=R?ZcyekRfujVg5{(w$~hZjA>b8`Fm3N^}uAX zkvksOCF19%wcB^=aWg=bd7$4bKrE4fJwEC|InHT4xHwTqi}gay{)@2T=NtFm(omzF zvBNNU{;!mE7%@qE8mp=0Pph?z#~rr0T5Ud#L8@tuOISm$G2Jke3XV4sySlUjlLZZ( zGZ-~;_yoAhFGcWDNMP z4CR?JG`ylAr%FM2-3Lkly=Z$tNPT$}Na zW=_#tYD;{tn-yf^?9PqFc#MJ=RPS+zjM$;h@ZctAt9NkQ>c^_%=7AMaZhB{WvG_;A z?w^*A_{Zp?2^LW2F}*S!F2|_2!^}6#Aej0BZ{q!USwm63BX3WLP~G)#N9D)AUBj2* zho9sXR!FSFDg-)n^Vj~5L$!a_iTbpJBW180PkR2`*Mq4`>}IZ;w6ODTS{%ie5}MDQ zk5tuZ*=%%4L2Srh&V#ivb>~pnKxM+4kyXSt%xO6FTU&3?)UKtP1!rNxGYTYImC*>D zw?-EEf!JC~TN@UFD7;o*`J0e~=xqm_CiPs^&`wPLeB3_ePLM+UKI0ZhQ?0uA29-JI zQU~fzfe4FK7&ITsGT|#`HG6ucklk^}o?V9%9(`%CDz2VsdB3?&*LLNW#@4OhU79$j z?71UTZK@^n1Ssh6W3*+(fm{I}6tO-45tj8&y_v9XB+U@XL08N)8|Sr>cxefj zY4hZ;%0^uG7L&Z=e9S4d^?XS@Q5%7rv-(2VhwUO9y-YKvf)DA|oeWpGkWT42u$u45 zM^5251RY_|h|-wCw!&f(v2r)OV-Yt{$3wrsXiG-he4aYED%SShW+=j`w z3h~S|%KAqEWjPIo>`+&9a~C5;qd?D`vJB!3lQ{uw1unjB)1zIzs}_3RNA0c~`Lb?9 zPY6rla}7DO&9Dy$%rA>d9-A&2RrX-cOlS$jls~i5XHHHICa7mYE|MmhukQiSe**c*oH)+LfYxvpLXWK;# zK^Mm0W;cuFu&OBvHj}Ua^56Ye1@6C(olF9Sge>MJALvB(gjm1j4aGhysl4)w_b-RH z#YO;7=g~s@p~a8ZOT^9#X~B$I8q0v#>l3|qEAiG zA;@WtHK?W_)p+&QbD8$LS}ErppGi6h=o+`upf+yAVB^-ANKsmd8i=Kwz{dg*NrIM5vaLeK=T7Kc6lEWO!s*pCqhVMtWWrEghz}*P z{1pJw8`o{pxqI#c+AF`4s+2w80h?w4Mub(|5iNqshkwOFqV)CHUkL^^E#6s!8I}{A zR|^4 zHu@Q;EYAG`F}TT}Vihbt|93FgVrl}3Nm+w(F%JFJfE=6PV*{9_^Kd*f2~!x^InhXr zl0)hDF+uTk`wWdatLZYIRbPiV1Nm&o24nm)a&awE{IDQvH$oGn>T zd`k`3#=;XT151G#hDC1&V{X152s++>Xu&$ziX;liLJEy6#J0Uzho^}aX zE-8AM1?B)ETFFOp^j0ygTV4{jT6BZkeX7J?{Ah7{g1>jr`C*!Aw0`t51@5M&!L^)2 zOA+=ZrOF434atkchexLm6k>@O8(+QH9#4<|v$<(dalRs@)!Xjwg2KU56L-oA`aPtdC#* zIo|rODAO`VYMV*2f5=Z_lST2&JI3oI#+;1Hj@vr1Fk{OcnEwT#kmzGg!--NM%tMsZ z1KB=55})smt9)dLuZfhgvQ*jn_``n{*a+?42pYKAxw{goS}ThtKwHm`Csp ztV8{s*{{*GA|_y2`~N0=vk0K+(3y{>j|%$YQY3~??AJ14x5YtBQVzA|yb{$3f6a~7 ztq*v@O0z&B$Zcl2SVC`UM!RG30!q1erHJ*8_*?-e#z4n-#v?~vEGp; zd1s{`etIAM=V|qC?-AyIm|1NDY42(5(EEspsq5!x7Qz3m7?3jr+T=^EV9Ia+G+?#e zCFOsGUE}t2Gu2Ldy^91bmUz?B!`7e&VENlL=%MPz1aSfVKx~7f5-hZjW+V9^);MiG zmgzD1aK=ZNccGsU=hA|;&}r2YX88qnUz-L4O!Ja*9ZAF;03^%5z99R?L7EQWJz#g)c2)2AIkWpE^|$ws3plRXLYAaLkh;GQM93g_TH0W#b4e^DC2<>QRakF??_ZxnFMu@?)P#_X zMD=|ssSo@`-PBAq1f%%^e~oXv)_si8ZtHI_GUquNsbo5fvMvPvWd7~fB{obgED;{f zmJe(-`TP8$_gSpn5VdC|cffV6aSy-G6jt zQy0%b6~%#Dnq@#~y&|*#`W^_@O(h3xv&o78-z5 z2y}aS&icw|-1d_Q)=VTr@%WaAwRn7&`mJ8o zU~<%5kXq%G&H0nzSf*u@u(#vUB!y)c!H`wahY9V_d3b-N)Um2rXeIdF>dNJoq0ozb zo5&}rfEvj=;@){RF&m#xWGJrfxf?)0mVkocVitAW83psnN#5oqH{kXrL|C~VsTAM! z&)7U&+P3$qzZd?`Q!341VhA$~A2R$(AYL+WEX3LKM|ILwOZpj05(d5>Ae{-70(T^@ zULaT*ne1Cn3D!I25K&(YX2R|11D7Uw7PgA%OmP;qX^vd%o}%~-Y{b!#6pHf5b$W0n zbXedLL>pjRQlH#@CHWVKzi(}7boiZ7h(0+1E^s#z;sD%IAZQ_F97Ml9dc+zKImId2 zZ4L0%4$vBM6ou^?v}_aBKLQaRYq?-s{}LFxYZZ7$jtFuQ9-ZoJ3tO!ww0}&!;oeGr z2Vuyt5O;TeT7>z@BLH!JI=})XdmhCg81Gv1hdu4%5lSR=RF?0H1vUJjK@$I&{HiLT zA}nsGbu_zx*N+xN(%c<05cEy}N~u<>RnuAJ4NS~v#;WSgkE;ZJV9Zne44NzuFXLWL zm;<$+EQ=hIEtk^6Vm~s3tm?2UetmttylBB*x?X{^RnZ!!rHHoL}e9-65wbl3Vf<|IDmDwt{&g9!5@4jJ^p;7DD{A0(iemf)ZRz-DGWa! z9NzX1A9o+jxe?lZ|EkcX_wFeB9j?2Lm&lsJoUpQd|BzTRJPR}RXx?2A$~A>g>=q~e z4q)?UJ{@>58bAR4EE`$fyi6m6e70yYE$gDA4MM^b2>HTLv69|Xd%6Ll1FjRym6yOe z1gZ*jE+kz}GuV#IcR+EKlj)w%GVU0ssX`(^D^h_$3Q@cXip>Qti^AeB-J+ut5OP)j z;ptbp5pkP-jIDydq`HCnpSvm=-Mj*E5OAPefA*2okv&_q zFo%|HfQ-dA%O8A^gO^F5} zD>kN&HqR>9rrgwRNl!zr3(wZQ5{XXcrO&8}Xgn&opBw=Hj^z;Os;)bZIfq&w$OzQM@5ur$=*&M`-W&gNTlq z(5zYdUCpN#2!*hPe9KP4)Yoc!B#JfIbj!k>2_A=vh&h;j|i#ry!t zS*)IKO<5(W{%RdjD9InLHPLKY@i|cj^qIts3vEi2$rP;IN_O2 zEV?ka`(DdfD4@C}VL>Wq4B0xJRV$Yi9dw$36;c2xr8uv)Zn2c-gua`_8ud{yB`j4) zo5S%@hxRLir#*TQpXvdXe@WRT=Jt}Z;uZ8>J`gQb+d3|2D}gyAm`yDVyG&e8v}N!G z<_o4XJ0Ep~Ix1UIILMF`q2_Ce_9PdQ6@Z^XE*!qKUj0fcm#)i4zj}0`!~obKVU%2YB>S9<>6d3kTR1tL{(_ zK&=Z1_!J(OMVMch+yp9tK?wO#w2+>XkqXS~A=nbChD$yvd#q)LpxwDV1xpGW{?!`h zXBdjTY98eMR{M{GW4~gZqC4!a61_~{t%AERJ+v9$b8qUtCCygs9J#9I{+aUVoe#ZN zbUnwc!=1+OpI_MAOx~$^k?88qck=mu$j=utFR#XmlZoEfa&u+nh?=Dzo?~165nlCR z)`2wws(T31a@?HOqPN+)+&PYUXa6W1N{_?ri)S^fo^GI6%~uNt_c|(v7v5QJ1S}=K zTlHdD{2jA>k%Uod4vKaYtnxQh!dI~2T>`_N7W-d#uZ+U&%mA+QbLDB7=H&9XGx8du zznq5~-{Bm$AtgoEmLI7&3mDgx8zyJLq+xn7U0#$0Ljq4Tf``=~|4JIQZNQr}^ZIgD z6MHZVc%SxDOR#9Qm0g&RNhB-pHWK(7x+woO_^EN3uW;LDPy6-gZbqg$KwrKc{tYE- z)5&uDf7ts9uc)@LZ(@`oBqda8KtVcHxLzV)qdt-F@@&RWhrbIv|{KhN*^g)wjEDh!{oPnLkHwQ4ChPWkIsBc-LH zF-3oW^PyD?T?}7O9r)3 z#+b4m#nod>im8V|7ay6(#EYwEucEJeWr!o9K>Tx(Y*520qIX0K;NkiW-ORh%&OAw` zqtE6lL3J*c7lmy7A*cA~Ar7_2l<~S|muUG&wolUi7jMpQn3?!T-_QeYIXidRF{4F~ zwXKQRVHS=VnGUxyUgA^=Qn0K>xfp$o$kz@f?uAX1m|Cn*9SS&a{oMDKrBe>4P#j4t zd#J)g@sI}?A%rV=B;~&Rs~zRuOb80yBDz_Z4K9N`p4F67%}`;u1N`axH67<|mZZlN zp1JY{o6MI{5~#EGXCE+rg)03i+JXgfRf?MhJYGG)rxa+;@S@Z!UU}C%_Q%_`zg?7E zvhs_)tG3Ty;uj?1XHmv8p3^rdUo^*fm7UK2pe!eavESGX5qYwKFT!kr-x~W#2&Els zQr3^Np#K4dF?_St0oPUZoX;C`TeRj+mWWqNFOF^KA2vG@q+{QYEoQm@q4~9$(kUkM z7sQ^foNjTES?h)4CCyGghT|wAwON=m;Mj`Kl8`HpJMZ)1#^Dc{9d;Wnbp4~R4xR_C zaJ5{1E$KBqlJ+8D@=*Mfbej)h_k;w7*y>~^BAn!{FlEq9qe%+w@RP-klVVEayv87N z===UUydEIA)(@sh5D~z7p?1owL%-7EqL0R!rwdNFTiBkz?Fr z0@g{LbM6+N>$Lsfr2C%hC)wirzBYc`isIR}fPA`$zC_;(E25ArU||IIP!taqLRo$a zFT_EfU^3J-@Q(AF+akrc;%7Bu!0{vHfhIDDq7mqjE<6MEm? z){=RW&bAi$AL$49za_9L@_((P+!u`FhuxetdUoUHzT$+Ay?GUN4Sq2l@vU;!Rx4&f z%nPt*@*Qv>9}=4f;WBb(8D|c|lC8@v5Y4z`jCzKkd0u{hkIJ4!`%%v;zc40FHQLq$ zYZBjC*6$p2;g|;jjjC%@y*yS?HQ_uwqFicfGe zICRxADl`-t&j(QzRIT1CjwCnUD^S>nEh`9&7<3kf1>NggN+9KQ`ygNXc{cTVLtQ@( zYCxAq=b~1M%Cjl^t@aHyxh?-{)V%#K;_so`fu}*p?uFLZE`TdowL-g?P+7ppG&7rW zEpf?6Y1+%?+$YR{y-TnFaC*QfT!DA+CIWd!_d!a z)tUX*O?kzN=@o_9bO%FCv;N&PA~)4s0&YD zO2uIF-njkeXUELNBemk2c}IDABY9_H?u)KI5gXPLbF~lqTm{ec3t66KxO|>J-Fpk# zrXic8}>&5vzPd626a7v-2Lz(zvFaVZO2$?c{-uN&Lp-%V$_X z@jxF(a91DR+(6~t?SbW5B+#Nr;N9N8I*o4aym>-e{DfG)HhBDlsVQsV*Zw41_ zQP^*0`?^MvbV~LlyMUa_)PVkoh@@Xn8+@rdn6D!*zs1pNI$jRHPIDkL4Kn2fjqHm9 zo#b&DrJpZfO<&XGiI4myF(-LZOSBs$(0_m7Law4h4{ukcPKJIo`<-B_{0H}io3tm& zu#;NOLz4J#7P1z)pAkKOp0)GTocm|6`Mj(p#GEaA9>LXh!uv}+you8=KVy}TgMxhJ zJcmX!RQN>k#k1kY>Hl?F!y@x#;v%RWV+!z!MoR7^zG&l+21X9PCX1>RokX43?C+5D;9DyjJWQpcE=GIaL+4_tW;5AP#5_OyBfW3R&+Zg~@E zk+Y{g%$p?Av}~k4%;;}ovAkMbQ*a#5VXrkcmh8A9@eXyBq08PYP-)>TX~|V;&tlqQ z;jjTvtuMWI^tZJqdnvZ$HDF+*>#1q(T4Hkkva~YxbjXF%&hwWQ*`9nli4KeItA>5S zmL$%N!y)I(A%^O@A|q1vsn`~xE(0yN<+95wkJXMjEzy17%X{(}h00=g@%~q`02fA{ z@`Xv#hv765Ox&Sb8H%IWsA{%Ld`OhORlyfZ;|QG1eg(#dv6ULZ*pDSeGZ~r$MWzZW6NhJt)UJ;h-pDmF-r8Jok(w_g`-_usrLuv+#Q6P^*HU^Q zOsniS=+qTO(xYjFr(ciT-F`Rshla^T>YT)!vs?b2OIA+f8d^H_3Tmw!O{3F4Z@f5fY#{Ow zts6~*YNO9p$7la(@`_CHq|2d_%)mb(Ysbcb-Gc1^9?8vPl0t#0_5 z3^h8`VTNb@*ykHa6fkng`p9p05i!TAPmMac{p4vN7dzuwX{R?^ zD$cJY;+q}hhT{rjzDe?OZ=RY?SGZ#)Np-mWFs5wJ^VfS9N$SJzDTz@ELVF%C*ENXw zc9D9XRc06Yc-4b4i5R2e3xoGLMniYk38IyK;{G2T{~&BBaQy3*Eefq2y(bQFTk-2a z2Z;Z+y+lYZe;RGsJO_x=~xFyq1#aXz} zcF`e>?*Rp_nHfXRDve}7OPJECCY5C=4n}8hYJlDT(9;Ttkr@deXv1nBCDfiri-@hekT;y}9%HaC`TTw2vZVJ-`Je<=2mJ zm(}^7em$DTQui9}ld4imhBhx_QBcsn?TFe?xD$7l?OS(*=t5;0 zD$ylp{>*2jCDfN6@67TIPv-pvz--L!?8IIb6q80Kn1NM5-$bBPO zOzrW-=`Q+=uzMAv`62v2A_N{bTkY8g#vVMIY6|vTC;Jg1DKne@9^2`^r%9w>3k8voW|&P#9#khkg$~Mb34c zrT4sUq+R5TwpVV}l;xB#9+&ZY(AVRQTe(Z-RIH0kHO5>e<7Q7senL3a-oRg;`fL50 zGjlb6R4j5~ac|ES9~NSSe?Q`8Vo>?%@*bzLu|0+Bm7S}rKL_b+d+pS&xkn2kl=EWo zUmEf}kqW_mXkl|?(jI4n#@y;Ps;S1}9xILok&rFy5f2?4ZspyVM)3cNiy9dM_)wn`MwxJ-~8tTF`nTblU0L8JUr21>_yH zCyCTSPk- zHi|D^2$;O)&&DDNmqEIC9Co}1ITn~g^f}^nbq>si6@(JvrQx*G8O+cghb%ce6`K@MwkS=$NR$e{nN)8)k zau_N;TJWL{Bz|a33kAQdAka{BdiYCRs5;e9XncoJTH%zoCh?t|qLAW)e&9k7Tc(56 z3TF~{GaVaNULe4xSwbEYWE5)kU|kdOH0y5gis>Dx8sr5V)6APYtWv<**xw2CwCjt3 z)*;);D>;mVyn1Eob+^u)Jp(oL-sxpDHouaeDQI%;@c>G`NiW;{{gq3{D?1ze)Ibl& zm}TuWe3EODeDa-rEz$PUd8Yod=2}IXW?3HM!MxVs?5_%eAKrTX>ilXSyST;0%;C-2 z%o%3tcwLPWKL~lUo@C^yOYrj4Sh19T^SDx8y|YzG)ab35Am!zuK0DlxuKeDbu@wI zBgaspb4<-;e9L8P&bI#7B!%I^2HioI^C23HSUw43%+7_d z5(vT7CAig`j*hM8lpDzZ=XXpN?i5zV3|6a{I|pyg$2JG8*gAPvPWq&c>%99sC%m2d zILG$FC3Emqox!W7^zEvbC6kjL#qld|op!5F&JoU{w{T0SwH^&oW(mX#s*}Aa=$+M4 z|HVBJefBNCIjb)sJb9r(uSzZh6&3v;J?BM4p+NJ?P3Z^s9pw*%$z2FunrhR=o2!q+ z^IZD<`w63E$jXEkO+K&p3Tk)W$e1qyiZ{($H~;&$p2OcFhr=?1mqG&td16R41icJ+ z|0FEA4scM$KcUet1^l>!t~xvmAi5pC=ql&JVm)RK{_uiD#z;0^^-etB1lR^*h29HV zEhns8Y9R44p_$UY{3v%+W+3i(!Km>bHP-F+yHaFW_tsOEDbNE)FA400>h;V`J)!Al z^S3)~VoNFnEGxoox6wj8ema-XQ3NtZnew4Nr{B36DIQ83UdRX*Qdq{X7$kkuK*q^UXo~{>JV0ge{QH;K zXG0iI|9w`lO7Q;w)sJigbF*wmG9?RukY!hFB59@r3<5=_RyuB-Jg3#35;bg=jvZ35dKp@^u?OMut z1malpzzp$Gl-T)3-I#ADiF__FGijb%1A?OxC!k}<HuyWg48bFEhlv4_Baxoe)6d$DT4qeWCzlA)q4`{=D}ZOw*nN&YUuoqjV^^ z_v@Sae`|!X7-NeG{?5SXY>oP>y4DQ;Ay9Fk*5D3+G!&7La0N6Rhm>$hVAi_yXC+4b z3yB=nkbPvm2Pv=%uM|pXPflZ=V>ogwNpH7khE-xi&sd-?lYbNG^cI{d8g*f@j~7?ASw%rj7irbGtu^cApD8GXi9qV7r)$UQjZ{WIS2{-d0nncsz1 z(MN>4E+3&Ua8~Ki!;l#vA!l$&@V(>l2)M(cGR=Rg$tywTWv>8#hjm!|c*-;$VGI}m zo*rN<{6c=AxE;iGV9F#s2)O@g9@opsi>G%EyLpC1 z&;yph^7*?XE_n^PIN#(5jqB`3kYf_?2iO^{ZG8!2N`44zPfgA|k9ZH#Y`B?8&=hJq zkGgKZ%!Lepq6C&h1N1YhI_;`~M}er8kNtHw$P@Tkq*c~;oWEKRMlEAt1cnTFQq2Rj zD=7Eq{C2B%_n>Exd&uI9q24$%?mVfVxCiE$)b~g379poRo{OY@z_`wRedTatXJYUgv z@Ve_D&22f%5@_Cg$CdrekRqPIrb*>DNQ9i1Q`bMZ0_&lFCdn*8=dMAV2q^g)P#7$% zUpX81+xvLI(lHeRPXz~mc8aV0v+vcm;U>t3dWO$Y`C%y!dak!jpBdhiU`7f9KF6qn zyU)G7{Lb9(PRE=~xlzd*E?!)K-Wmg=&ac`rW5DBp5afq8JL512O~0rg?&(xWgRnEigGt|mT(mGVtU zj}sf;eUohYMej!^FHDJ=&rfU6<@!iyt^RWp6Vk|^dceMhuP{n`9moR|pmFD2KyE&K ztf^<;!4M04?-7v!}mHM6BG#r*sEy3cemZ<7-On`ZMp^)ZE*a%QPOU~j^y_1w;S z-}nyV7bMC|LZK@Bu*aw!hmwjEB``YC5Px;+p?~y@Z3Blsux)C=rK9>HW7Y?lJRg0T zA7*0#hvYRVvFO?7TQxb5Kau0qAy78v4Ylh}O@Obg1or^7a7Rp*UfP*(vLDC-Ce%c6`Wg1!ch1arz~o~Sh_arE zU+znn(oO>j(ao6}Nt^Ao%-!<``@mehVHE7x866Lf!!k^tQeskUuM66Gt5{G#L7O3? zCBQXYa4IMJ-xkEb^UwxErdPtK?ZX$7Q?k>vaCq?LNEn@IB+qDh_*Mdt*NB9`VUz?H z({>%V(0Y#uG#HSX=j$ghPW%3&SLk8&1@cI=Gf4eHStZp9zd}0nv7lXtxDaaQnOk8J zEbVl5Hjow9W+-EH@3>&4!t4#3lB$V5V-m^d>gZ1Ua$m^K>^cpDEjmrm6YeFzB&FaJ z{UJ6pYz_YFOR`nr!p;Jf%?z^yWIf|BW7iBhcIjEFza6F`?5Rive1uKl`#6LkHbw!y zNJb2w!HUSg0)YQ5%MrB%R2b8qzj%kFj2S~PO&LuMmxpuHGl??sdxFynZ3=ua)Cmpq zS+kado@GWeRBnJ{fKbrHHhI~xOE|{QQJP9Nw9S2$`U)lf_j|lxAZPIkU?3=_`8f}Q zXWZiM1$3Q*lQ@RzA1g$$D&V$L;>TfSQ{vMfkF!hQL#+<%!z;yXh3`z-t%GB2>edpR zPhy;Tc^BVQaqhtR;^N+z@V3%y$KGv;dnU$9!9tcsm1!=Fq`Ml_7qI>necsh zJY&>I!6X+NgXA@L4FNebeJVnr^*8P<__`Z%!daOUI1Jtm z&L5_G8|PDBeE+S^#8b@1>h2b~g;>YPo(}1$yT&bOU7b@$s6BE5DSW6xCTIC=c~o0z`VKpL~A&G^CY_jH*VVVmLFU?|M8SjK%))Q zd%y-9N7Hm#{BBp$4@Z4ngb2^c#X!L{t?^WkAs>D{wx>eH%W-@x{bLg)mcR58^mLHAEsr(A(r*amt7}U}_AbY=Ix^cQjE2 z=QO?|h0=}jHXOQC=;c@FNO+-zL9)YL86qs#wVx!>ll0xLPQz8PdnU~(xBfGgkQ1!N zJ%*Uc)^cdr(M1a5Cmxf>n+q6sQE9!^G_amx8|Dd7PVE`Od!Ns4QmYG7=IPu9A$5J| zb?(53?+Gg|f7IpG6r|Tw2D-@liL1*E?Pom7mHip8xM}nHSl2(B+_A-P0m80zjA%S& zTXMr$~G|P%SaRl1~!N1`Nvs!EYks3_q~=m4Fkm^IO{D4gOHz+_jf>~bKh_5 z^{#N!_@Y z?SX|3o4sug`S&k~q=PNukU1gcejuAIROEie6+ClUfN^z^&zQPA|;FXt3g^-UJHoyxgH1hCI^hcth}L(j%@`s(`DTHesap0-znmmi1^aK%zW4 zW@OfAp3v0tBegV0o271hG4#j*6mFV@zVMIdN85?O=DxuRyoOrUgJ+Bzk?wHMX z7iM|F7maJ?T#6cO*FOjX))P*HsN17;%;n>?ftS)Z2y5QZjgs9Q1}@>7KodV4|LDoA zoiw4u7eH#gX&9pt@4u}`pT@^l(- zxY7b1u?B=okA)l0K;rrs&_+#icSaql2FcMY_I`GyI?;Sgfobk zL^B2N+7EL79R?cQG+>Q8Kz%w0aFMgd1)%hDk7g+SD>s<{JBnk}#x_CMrNdnauo;5YCnRvqP@)YpG-$U+?mqHUW-?%R$>a5)&k2*Pq zW~B4mc;DtEnN4xJeuJD9$3Uf;E=^IWW0JiFrs9lg;Mf!V3E2raX13{9n&gv_b2e5_ zxm*s35oh22Ezx~#=gK)DYj+3=_%+>UXmrHKU#YQl9xCqWiKa$gtsjYB3&|4?+eg0v z@1>V3{}5vq?L{SU6j+}m!>ecl(-`47KofQs^jDpJ1NNw)n(`Q%f2~{aEzz6KHKa)6 z+s56XR?x-0dl%uhF;nx7_OXncJrF8CcJARzV7yPyo#h`CPT9^a6;eA!>DRNCXqb)- zGXU8-$V;R|@D@Gi9VX%R$D6wd7^2sx2}TJtm%0u8G6HS-MIaRciA`K+2B=YKCr4nb z7(mZ<=JxOn^1qzdTCX#1!Igyznjp!t2qnSc1$4EnHL2#Dw1w@h#AOhM*plE9raJ}^ zYU|QWGnPwrVY+W)jJ0R8BA%syB8{nrIZnor7WV^W1<*)WLA@*Sh|rOcM3G0tx{oDq zK)L}QwCudt_(E33?%U57&p#_LCFPEa5g>AG3iM1w15@6nc&{eYOPJasuu7@AdxVgt z`L(zH+ko3J^6za}Jd%?aKCDWxuH-x}o6xVBp7AM~DVTp#^0=UMv9M-ntM$wIk%h~` zD?#z`?TrCP-NrS~BcYVJ=y-AHrQ?6givj(4Ux8Jy$M3&}IX)Hh!@rMSe82OrH;=vl z|3CVF=1!t0ZW$BO0^mL0>AO_H}p4~MLBsu&IHU{}Q z^r`$G;1Bu*ROJG63qbJ^59KreTUT3Ra!Rcf}BgS6>_Oke%d-~EM zL-yQUqacMbec0y6zT2nse7TW?^T)M_R%$c*&_9@PG;n(7bjB74&o8zZVN=N65cM2*d`C0=0n-F5MI%`c_HZ zdU64Y{*1|G zpUz<@U+CJT#rDlx1pxe;1ECL9WI1bN% zQkMfv3>e7w_w!)#J7SO&D}lplGvn{2xtTb0s!-4K5JaQBLA|u@oo%Btd_PVx7W7^z z{J=cs7RAIt+6mh>KsFK`HoCS2&F|S%szIJ(haQ$Vr8Xg=&!D&7`*S+>L!pmga?_jE z1yu?(;K^Bn3=TyILKp)pSX`U@nX5z|a_9*;lX~h%o4J%e7CTlZxKI(-@DyC}4R!!L zUTy*%%m5}v+Mi_?=v<9S+r7M4_)Gg%=NOZnlI)x`edNFgG>wiH(jlp91-M- zJeDti1GEi7IQn+w;B5z_6y?LZN{I558yHyZuHmok)$NbEQ&WO{CjFj}cf!D_)CE$7aU0MithWzxG0pT00AE3dMxW~OF8%^gWHmGFsuKcuum>QqPX=8n z4r>wIoGO+Q^{G76I?PeZPIm1!xp*;tpkFGbfiAVj3DCu?K4U3ug8v1wnOX>IMinv_ zeyvTgNL;@7LdC10wY)DZdH2E<{5sUh9N4u!0g$iFQO}%&_9kn*z~cn-l*^%J$R2qU z$lK`Nru^HdyPwfusC|^^B3e?YFbVwaba#__tYAl(Tmo#o&gqmvAA+*9{FJ$3cqR!k zp9VhKO#<=YXuT)u5vnCpp|Ew*aCe(iWmPozMidGjV8K{j18k@?)*Z4sXr;TL$T5+~ z=iHX)O1K*NqZs|8b>=e!ODvk4B=`6EFk8V(i@Ln4dV^GhGtdy`LX8HyW?D3W zF)Kk+w6+4p(@&RRnEb`HfLp<1szFQ~bviK?@C17!_{TW^ZbwSKYW8xjkoxjnlfP4( zHVX7oRd`u=7ncCc){L%UQ{8PpAPfI7!JO`9V(W{=9xO_-EVD*vtRny+QX}8Ft07A+ zq9s!ciXLUVp0m<%ANHOjsTnoVYGClQx_ihs6TOuaZIni^z=&aD2+7&>nr@?6?f0-J zS#6%*bKC(9+1q5`LO%*{iwtaiEa_-T=+6F3eSsqpNS&|6B6S37!$40v;S8WjJBr}S z31BnBme{r$;!|UO>@ER;*>@wz4!L9t9L*h)J zTxQpbRunz@DHxfB7kpHug((w{U$!EUpP6+l0zvOMv@iQXB4q(2owexTEJ+inu~DpL26MbtXdX8OSZ$?f}3`p6}9xzX&U!6To8VL10$e3@EL^^WpIa{`&zt`%FQM) zdO#K4iH=_kWyLGhmk?;RyBq99f^i^>sY-wj`~J1f1t}e@P*^|;S+Y8EOZKGOF|kUA zUwi{5>g(kp@0Y>b)0>4AFQE=2c}Oh5gXu(#Lyl;ScBGU(q6Oiw)U>j0ZM)sLgww(< ztjWuSFVu<2PdJ*t?Sd!FR3p43|7e(lW7kX_c@Wlqoq zwq&eXRMG`-hg#{o0QjTIBBOB(7Oqf)ai}kGpK)G1&2J~Juk$!o#IL-ernuA3`_Sbl zuJ)9Cbc~=c{1GxEcvAIy=ZDN?-W}$SbR>SLQ8+?TL_UbH4R%ov1Z=3YxB2syxdKlV zo65hOx;15TN3#(WEN0dB6->B9pierjiww1_L+?temCSTgvgKN7OsRez8-UFob*8O^ zUl_jAw^!tOTM@clP{djWRk;O`&#svNfOEKH30C3s23I4y-N?e^A$c4A#q*FvyTBm_ z#nF4{?M|8)_=DOg4?Z`JajD`&R!XKBvtEiUJdFpHFK2DjjDW>uH4EWilR9Za!TVb8 z%c+r9*Ja2tH3M2Go`G4 z&qzWAyN4kE2B}6aDeUmY3D<-yhB<#7fU2rXZFQEeYPVFB;JjvO0D?q}xblZw<-G;m zmO?caX{vU#I;%ZXRC@40A#{>k&RD+g@ZPz_+&yAG8;NLa+5BTt{%X$Sy2EM8FXB{ z-BOEB?pV%^dR`PK$>&N)PD9Nmoo+%%Ep+93K9<26Po<7ULO^`?2zcxF9sYxFDd5Hs zo*=@{F0`mPue%ee<+_!$lZe872#hWcsTs=Y2Bf8tWM$@mk-bPO1|~eF9g}?0j^0{N za#Xmnvb!yktL5zzjLgxnV}uO-fRTl6g-I~6pVj^8G{7Rf`y#o{ksOr;6jrb9(L_Bj*>ShGwcSJq4440vla5wZ7I$s>D&l8A*0-vjM(s_^mW zBA(YVj`FJCt9rE5VLbA;(aIV`%>U+d<62t?BPWlfK%_$8S$Gz=h%!zh;SmAfzz5+` zX-6@VUG)7`G;4Q)A&Z z^L@f6X|31=l1)Si8H^H8*M>1^$uP)n&H+zocVQR?i3`7!Go^+iR2Ys+Kw){b4syDx z8d9xoD?9yC_#B^5JX>3R>9x*+ews}UAtbYQc}39F4ss0>?=^=9b#T^(L zPE@FY6mUJ?{PLW!`UW8n$)AzVS)o8a7?x9c1Q{1)McF`u^csQmSGZ|u?Q-}REP5&0 z>G#j@60lov_V$m!@J4G7Lqq?Tv@EV4EDIx^5{e%s1U%*qm`or)H8w(XpyjqPriKUs z;vTT$XNz$(2zQIjGK1>t&r%`u0o6+)q|_DfV6?N{O`?^QHTfvccY!J}T=f<`FuXo< zmm*|JVLFB4i&!tX?+3f81!-_SHLleg@a)(Z@u#+JQ`YWU{S_#`!-(>8CdQ)iy*!8) zHUqst7_bS{5qlD7HBcl|3yyDHZt60(MlbVwvo?{e?4UrGsHDug?Kc44l_ciP=tE6q?#$9_Jqjd&%`7+*{u)UgPlfU?2lQKjJ4=nkOq`b1_(zsv*L5B-6xj5~d&k zJC=QAr`aEVVH>q(9-=X}!*kX7LOU@NNNA`*kn`#oJNmeo-!0>9oJXIy>Z{c=udX!JM2 z=8kKrRdtD#4c!m1!JT&{HvS_4rbd5}4+Z_px2%H;>9HRBi_yl|^fZdYq}2n|mcD_i zQlqd}`-x5a*9vMAU$`OCD|Ip0*bM4}2UjAt%QST<$`o#6S8GQThn3Z$Y0Av5?3s?~D+k334xAJIoH6?rstaevnK z>-;)UBOQc;Ak4ZEcUc{>@6T1}e|J~KbGgcknSHV7V=eksj`Jg{B9H!`cqy-RXw_HL zRebaa&f6g+@^5zenFD7qT-R!+!>R&sj8F{+0zY2XZ>tcb_VR|9s=wvA&lODY;#TLS zWFV*d$1aDc=&~G}@5J(0((7F&y$hf=HmGBxfy@!WC`6m!3I8PoUF3EXy#M)N;}75M zNcT5X%N<@~>W)b!zK_hyy*JRfspg|JkHc1QQoxN3BRBi~<3d~W5lgOx5$e59$=}C<2MVr8DVF`wG zO8@mC)L36Q9l>6=Cj3qbq}B-X8=aCm-sDLf8A*3*7%=Ij>S+8&N=V+PLMW@JstHOz zAVM?B%Br)w0?rJ|y&!)At1N1bst+ev52A_`6qR$qvot zr1XWd&%X+N?hZyl|A?RY5EZLZ{)CaqCQyS6O?H&e=+_TR{hf+DFz?I+TMw)LA|^o}}4z`%u} zHd`uRB}44Y;|HT}k6p)A?D&Cp(4O{Ua_xfSM8q zrOhQky>h{P_aWE@^v}>mRN;0_Wq=c68APtxn_=m@x_tyNrl(LVPgvt|v=t9!`TUGO zZCrfy+7p_*PJ@{ldZqFJg2uY3Oe6n*sn?h1oOSo3c>5kgtEOMrO?dUn@QQA}66HFovI%4J6Z@1!4Xf-5q7VAuvpSEokqwJ@XSZi}E+IY!stgq6yUht`DpahsF)%cG9;e8wkPO&N zXsyexjI2j5l^$vAX+uUmniv{cw}!zmxf0+OtB9wSx@^m$0_?X+!d7h7zujjYt*v$i z(lY*ar0IH@iue_O6hNCchkdUdOSPB2g>5?nhoyntTJTMI5P*KAMqU`WoV~ZTm%#v_9ZcAtP$V{UqGSqwtPRv8zR+>0;7dbY#gVc zccIyq>lvG0Ir{10t?u*FLqsTNP>-nO?YE}2h~chbuPA^|obgPzO!&A29Y=$BFiH7| zu^(nuVaA3KFqaHbbexq_((T%DEzdXe=ys~0Aq(Nx0w0KAHU<&lLITHUGTw;AC97>G zTe#@c%qe^J*L*dY{+?ImSVw97V z9YIv3?S`|!FJ{v#)pg$8+yk^n=!5RBJh;|;lRrR_{}T5P@O!HM$!bi3>|fo2gK1_TO=IwU2Gs1QvX5Tcdy4pg5!V-pepVwqxeRRG)4!y9Y%Ms= zuUr7^KO;oJp+prSYt$#m!r9?E^nmQ$m>3kqdT6SLT{3W^vwr#fi;VzNX%Juo2^uC+ zdmaddXIS2gj;c1@pzT)#Qv&xp#|Q(;i-Ec~SE>Ebe556PKm0fotJpx|d%2pJaBZ_r zuwbi2%vO1(+mPVRQp@?@sr_Id5&J!sDOqyMnb9qBF$C=eST(5OsI6UI^#fqgH9X0P z3Y(;W0_ozPC4xl7+!rO}h%w+9h7|8OZOjxkX*x)=PIWPAYUC_+p3bJa*IM)+Dow|qOvu=Y0Kn*J9#Ua@e_|G0INA=*F%#f?`4w8)JkKu#W-fsz|>im2-r{qYZdMha%Y zOy2>cvl;RRt#gz%Pez&eTM6CSL4)CAGQ>EX{U``#7LMkzge*1&bH;)ISw+hfQZImT z27}F!gVP{N1LR3YE9`BI@(5JG0*8yKXZs@to?i)t{(=dT?mW@)BcWVsx;9L5ei(Jv z$os7urR2v^AI!i=X$ZZ`T^Y2N^e%*O4?NUge*%kOR+VHOHB@)eo?Xt*d!mi}9rZ-Z zh}-bux=4#x=|3oY%sA|bcXV4kAJVW-1AvGQpcR%ln14Jr&PaNnU>)}T4S4R0vvLDR zq^Kn@_!`)Knuk0tB-BLSMhX^B^#BLkGE$5R(P?LJas4#rLqyPW8Fie`QBSzNke4dY zB!apue@AjyVKao2W?pKOV64xsBL9SG?2X~36IXdF7RL>VoqL#saaW}LBk5iw+V&!l znfhlTuF@u82UF^h)-|hVA6Y3nEFVwMBf;zo9__%)(say`-P^_*w+uVm@JUue;D%0Z zCA-+?NWi!zWO}cQ8?!O^gcrdv$#v0vwpOnV@B`gs8RXGdc4K_hhc)+ioz0YG9JJdD zExy;Zh^IUlrtOx~al`kAeC=z2Wld9J;+kmhwSD@E+bOl_s2A(~Iv0k>W%KrjVkf}t zf8!Yj?e?G7NoIqJl!`=1?aU9%J_SR>0|kwMj&zD^LK?LJ=sJVLKXm?M zW&c>>%*Iw6$D7HFs{t8)sGhUu!kEPUBVoA#Qj3pPBKg)IFIA~a984a*nU1sbvx`70 z;t+VDT(K~Z9f76T079n%yVF|0#XkUoW?b2E^QX9kK04*Q@BF>~{rAPU{4ZIica2|t zW8e>!U~GoJgvOzN!0KNoq)8hGMiaVANYk^*xT- zj@@4diL-ZiW~)OGMe19!<+8P)J1udGG+ptg9;O_|*3k2^FHv7O_%Lbj-Dw@ZJqv&< zG|!=cjcjcX4_HM})`@mS9{Q>GhM38DcPJ+r2zD1X6`hCA1G|Zmp30CQjYE|1B0{ge+5univ)h5EuHAeH^GhjM7~1tPIk#>MI0Sp zC2-#FzX6htLIPY9_vVd8zE8o?)MYsx(jkw76@IW}p{1F_Q3V1^Wm{@pl9|({CG1v@Sh>t0D*I7aQyXHPQ}{)Rt?^SHOzrSh=HoKOO;U;v)%#J&7aIlBe$ z$g4HsnY{C048Iey$tPKa`EFTTAFVSoV>3B|?c_f6H>-Z^a>hP}G6+iUERT2a$lOpN zP+?s@=$UE%vQ&sgvp>)9*78es1#Kx#bQ;(6M@JFzI zxRo?0&UySw<~BO2!fwx;re_5n+e{lw151}YHClQxsh*{tg)^s@Yk$@3ZC{OV{m){; z=oGg)>k*ce=DEbZ95UQ1pfK8yD002Ih3DkI`F<=Zx1-IA+K*6Z*q)u>LH0`bbxlY# zHH}JPINW6;_bat;(w8`yab>6G4Q^F6NmSr*&bNNUD9N5^W8B7cW7@7Pxl3x_k{Dg` zZ^{4>tmMlLb++fs7JA=D*iI~!ckA&~=GW4Er>tA~quJkvc%8DRd`bj0t-_)?{V)+& zeiY*hPQ5zOXeDwO4VS@l)W;9u`}|CU|k}jA_b^>%j{6#a}o@DJ@yVkrd9gOu*?rj z>`q<_py~Q~r&ihRZ-rqhU2tKSbEiAQeuS~v4r341u3}G$u^_>C4li5qj@>{-wA(Ox zy0)~{nbL-;=Ag8>h}5Jssr4dmSLU?CL0#Tnl>7BZ=NE$V7@gI%%>r`s{aq)lyZt;p z+z+I%e9T+4qpkVYA7-P6FG9KF>rBIYy4{&Bv%KNPr_^l0TMt;~NlQCu)mvX`!{P*s1qtv)DVNcD9vha9u!ct?;etz^a-$z7nV~apD+_x2_ zvO{^HgTLEa7j75gNFc>KztTuqd+aV&SOrb|XOGnHVh?ZK~P$?j{ z8IBx1cRkPf{t@5%x9dKfi*xRC^S*e$*88t=q|sUEc?==$$ss}R8bw-;^7rppI3OWO zP0t9R)yQP8iump7LEu;Ms`$X+izR#DDfj4xau?Qe&Oy|}IikxVIH>)g!#?}{#55lZ z7e06^=pH9Q50VfpvAZ&a0`m-CEP}gi&tVXWc2Do~q(uIZ42I6kUu(di&w}N${^&oV zj09>y7$a3yXAufU(`W7SZdqF$kPBfI`R^WLSUj$O{i*iIzk%A=;$L^hMHh8KlL)S8 zjwy2N|kZ#CJq}7!)&* zkv!`FA-&8qNn7W8(|>6MBFN5TOT#3l4~ZWrK=lx!9eaBzd7TTo^r%vLK)sx8bq&aZ z6L?`-!a`{44C8Q)awni4+C=jSt2Y@*R15GaLuW#B91O-myw*sbQ5igPn;3S|O?XtU zFNPRhsOyy>suv;G$0%}_NLF3>oN7B-dIaCZzFE9yH*^yBZL>-Hvtg)4ZgxoOPIfV@ zKoZY#>h76qVHq2Iu6d?S(4^~v+JZ#LX3=J<{j95r5b-<}HE`3@tkb3=@0_;_j3 zkU`$F2B~?5v_D6bbU2Y8D)9_}X*y|h=_F-7IliI|jZ@UCQa2B~?3uUQ<`#ynnv5Rj zH(wdwZqWeq9_p5qlV#BYm)?K>`CJ=Yi`sCFR7+tpTLRvDH|BY@l(&4kVq%ZAM&0_? z)ubJJMnJlJnTO;>Mg`)?1$oTWX-MB|#U`r+*o*q#h*|rGlFjy@?$!#?lBCSMH+Pt1 zeP=EwY>s*QpfAeEv0S`m>}IC$4W#6-K-swpDdpu3WzNj6OYR=09aN-7HH8ZOeU1C2 z`p>rXaQp9e%y@s3nf_Q$3J_PMxtWSMAiTV??y9IlR3lQ=!A)H>WT+35IW|qN!KnGqdqGyFAh+nE5OeR zbU4lB%|I)K(eEV|4qIHSIxq~9o~vkHe1DZs8h`n@=9)2|30-;%!W&@I$L+FlElxhs)eC=HzR`z*6DKC+l*swDrjRRtapRW5MSjEzkq}S5BypDYi zI4Qn9-4O~~VHde!`LdUZ%oVNHS>j(UtM%>?#UE2t&AzO6vB^#?*OutUnKT?6YdRq( zfp32|t32F&spCkZ8X+xMOr5~Vj5yurAC3klZCR=a-v*0W)rUe%EQEOvDemQ=F zyAx10k&0cqn6_j~F3YrH933iAqvC}!UHC+KDzRp`i0^L&eaw?0zN)McDnU7_Wz#RU zJ<^8ec>UG|-yFDeSk8{{ko%)=b1I5Fz$sdlftHQVtW9tU^*%0#x18j98Mg_v=ts4W zjV|hL^$vL%_UYiK5-8y%_3Znb0rn;jRt1TPhr7wPtUwE5gsTqDepj;p^P<|hmI{G4w2VxA>C z0Yfkm-=L38rC{Qq)VwnkR~}@nugek~;s&0(8Yd+36)>ggpig6-ICE(j{@~C`*JYV$ z6u0qJi1dxuj1Qnd@``(DMBV4sO%euiY^brT2^3XvY-Tm@ZuYtAnZ{u-+7xY;|KCnL z9O%S1Xs5)Odd>DicR<_J{LW>)LfR5BzNCJ6d<%4hIaf^z=}ogPxL5yoD-H zuxh5CmV#ihn+W{N8=#RTY&Q>lj(;^C!m^$1ey1RGZ4j5k_4X| zzIj+WAX-w~GFcNM!dA0X%XcuKy>gQ8n5kL$D3L$y$ZS|JoC~}L63DT$Q9GeZs;-_@ z^h@^~r75?Q0PP)M;%q+3>*S2!8{~HPzQle6ZIts*Z;yZp2S{m?(At`&NWTizqLUD6 z+!+&EtW%$oPx2n*$g;`w-Z#CM-p2MVcg;p9lkr9hD_$%4$3BzIA~fq1bj=0f=hNA0 zFe&_?xW*MuL;u&dfe@-BO)Bbm23eC_9T4qbC+4$Fj8jg&d&!7i$JYq+iP10?);jhCHQVc(4aBTaUx zkGt!&Pi>E!@J%xx&(}#rrK8$ocSl>{9+lARra%d(_;@Z(HhQ;ivNQhoM}bm4WmbQY z*ss4fq~aNpGhSks*Fgui++)7GX=f&uay_MVMlyT2qU3-Y^cuwNB=v4O)jz=K+`tMf z5{B(BWm?@AkOOev;d|-AA=rT#W4^K(25B}}1I75*ttaOknMv-?{Yd$FgQ4l|s?DerC^%aS-s}(4C zOkgjiUB9^{J47>Fz|o|aG?^ae-KKJyckj%qqI%HG+6-Fu$85SmZrS-L@E@y~;vCAg)lS$iKKWL@L2gT-Xc{&NsXrH&c09b%!%*i1TV)wBvmtDuCr3%k#mCvIq%Yap_??*sJ!6I z#~j_JbLX-D%!FU&Kg!j4mndv6bT8Q*4AnByWByjwOFJ>so z4&U|{mVpjn7db!w@SqfOYG>lxZnf0Z@sZ-=H!?W|K#q^Zhv5~bKGq@K9e*Gfz<4#> zu7RV6o&8~ftPFq9p2?>EJ&VrJM;Q4q8P zqCIL((*GXTpufrz&9h3SEeUA?1%dVP3W*6E=aGn#!A)ixx><=3m!&`UaxOh2TU`fs z2zLK3II_8DR#>k+F%5}JpEK0Jod600Mk>r==m-iyIafH=o4iM1K zA^D73$+fz}T3a%38Op6vzU1|Tm?_aF@IJk9H=vM(1Q28<516-Mt$sY}4$>Hsv^*KZrA{eM?Ho>|SmQJh8Tuhd9# z#}8RlD<#6JqgdL^tK_Wc(~JFMCkQDz*LpgS;VnWcbx|&GIOf#&KdOJ%#s$aNHR|B0_y7IgXLeToUC> zN+w-pvW#gP*mqh3+Xiw08NQE)ZX-95N9WTHD6eI4hXvI4MLc`pNxaZ_X0O1I?kPg? zfd|czzrG)DJw(~pMYKj3UglGgF8HFsN=48!Crs~$NByZ zmeKq8@S4F$`)HV=lsGsvWnFMfHV1w(8PB|QOk&IWX+Fm19Wa2m?ew_8mzbK`ngwt8 zRP=lE>NvtZr-Z_wkcpJY>zXT@hXOkR7SHrr67zD$5i%GUJoLosEEQ@NZqhk`;)l+9 z%@t_mDxC;0lTyb;YfdVa7hl(e@pYV>&$>);bteRi!}LXkob72>t=oQYIB=04!wZ1? zuN2YLkVF~^_U%fo4_^>{ueeQvBs(QUhxIUBDIu0F$^f@YRJL?&6%OHBE=VuAkki}= zO~-Co_vu^W^x}mr4o)8NW7elFr*LU8>|ay*sb|Vts%M%0#uND%+cK3e72!u2Ww3v` zGFClzc0;dydiH>kvaZ3-pUX&VmsSOG;zH{Of5b5J1jhH&T=jhRqQ!I~oAyCssE_Sl z`Cm}BhNk%Yn;~*V1hYG9N6R4az#}#UFkTfXzWz4wGF8`VMf z(OTShp&)@6cA=Px!5#zoli6y;)K@y7F^iN`VP$IghM=BY6m$#Q#esSmBb|9XK$>! zF46dlaeTvme_8gU^#in_n5i9)Snd8g^q@1vVx!xP_LYsC2V)Pj71T?fr)f_B7Ia(z zvn9v3YF%(iH`o6m8q#36&>K}P@)ehX<*t)7HxXHz{@ul|3jV$86hC|*U9&&Ra@njJR%XjB5i8^Hoe)r3@6u+BTVZY+3~rTjm?id-ZZw7ZcaW)LYj8kaa)Oq z`${08IHl7i*VE2CJ(_ObtvX|c(}xbN$+2KpWyk6g_hJpnL~c|usQa=n!h zT2pUs_lJ&dzkgG>$UM>Gcfz%IAIR(3+_kX_Ar#l<3)dza7dc|1BIt4Ha8HM_J@NWa zf8dv&mXfmRi?#1s_hNB1L2H!RNc|T}RbG-+1(k%3#r0>Eqam?pgO@hqhhjMCYvn+o zuvg*WqHmP3Th?d0s#^vwA>25|Yj#lOTy7*^H_<>bJiPgoCt~u=n!NL_&>kEGMd$*o_i;w-eUAX*YU3~=zDgz zal*UHc^mqR|4{jB5iezO5VE`5D-oYJTBh2pYbAaE?zK4he_>DCZ`*<0fdl3LpLrF` z)Be{h4zQoDj-ouKe=T@_7Byl#pZ@BK|18|rrA`GhjKa*@Cf4*F{yZMwVG!HKfSCh9 zDIr@!Ps2Jx|NVRSAEd#~relYoyC(<>q^zO-x5ev{cuy8a*!<@)Gbmsc-80d#Q25`^ z|FpRg>bX%5-ySLb^`S{LkowaObn36B4G4r#uzfVk9v}7*H=S+&HLe)=?WdjSDYyT9 rf7DNp$#Jpo4@}w%SLb4fk~ZU0E2Fv(N_}wV0WMoB2g_=hU*dlNm&*9t literal 0 HcmV?d00001 diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 42bddca955..28f8acd21c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4453,6 +4453,42 @@ "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models", "supports_tool_choice": true }, + "gemini-2.5-pro-exp-03-25": { + "max_tokens": 65536, + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_images_per_prompt": 3000, + "max_videos_per_prompt": 10, + "max_video_length": 1, + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_pdf_size_mb": 30, + "input_cost_per_image": 0, + "input_cost_per_video_per_second": 0, + "input_cost_per_audio_per_second": 0, + "input_cost_per_token": 0, + "input_cost_per_character": 0, + "input_cost_per_token_above_128k_tokens": 0, + "input_cost_per_character_above_128k_tokens": 0, + "input_cost_per_image_above_128k_tokens": 0, + "input_cost_per_video_per_second_above_128k_tokens": 0, + "input_cost_per_audio_per_second_above_128k_tokens": 0, + "output_cost_per_token": 0, + "output_cost_per_character": 0, + "output_cost_per_token_above_128k_tokens": 0, + "output_cost_per_character_above_128k_tokens": 0, + "litellm_provider": "vertex_ai-language-models", + "mode": "chat", + "supports_system_messages": true, + "supports_function_calling": true, + "supports_vision": true, + "supports_audio_input": true, + "supports_video_input": true, + "supports_pdf_input": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" + }, "gemini-2.0-pro-exp-02-05": { "max_tokens": 8192, "max_input_tokens": 2097152, From 5965680176a00f80cc99cbefd9c18730c8a516ae Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 1 Apr 2025 12:02:45 -0700 Subject: [PATCH 22/22] fix dev release.txt --- cookbook/misc/dev_release.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cookbook/misc/dev_release.txt b/cookbook/misc/dev_release.txt index 717a6da546..bd40f89e6f 100644 --- a/cookbook/misc/dev_release.txt +++ b/cookbook/misc/dev_release.txt @@ -1,2 +1,11 @@ python3 -m build -twine upload --verbose dist/litellm-1.18.13.dev4.tar.gz -u __token__ - \ No newline at end of file +twine upload --verbose dist/litellm-1.18.13.dev4.tar.gz -u __token__ - + + +Note: You might need to make a MANIFEST.ini file on root for build process incase it fails + +Place this in MANIFEST.ini +recursive-exclude venv * +recursive-exclude myenv * +recursive-exclude py313_env * +recursive-exclude **/.venv *