From 1c3dcd4b2589615f35826e2e46f272ab8700a123 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Thu, 14 Nov 2024 00:42:37 +0530 Subject: [PATCH 01/15] Litellm key update fix (#6710) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(caching): convert arg to equivalent kwargs in llm caching handler prevent unexpected errors * fix(caching_handler.py): don't pass args to caching * fix(caching): remove all *args from caching.py * fix(caching): consistent function signatures + abc method * test(caching_unit_tests.py): add unit tests for llm caching ensures coverage for common caching scenarios across different implementations * refactor(litellm_logging.py): move to using cache key from hidden params instead of regenerating one * fix(router.py): drop redis password requirement * fix(proxy_server.py): fix faulty slack alerting check * fix(langfuse.py): avoid copying functions/thread lock objects in metadata fixes metadata copy error when parent otel span in metadata * test: update test * fix(key_management_endpoints.py): fix /key/update with metadata update * fix(key_management_endpoints.py): fix key_prepare_update helper * fix(key_management_endpoints.py): reset value to none if set in key update * fix: update test ' * Litellm dev 11 11 2024 (#6693) * fix(__init__.py): add 'watsonx_text' as mapped llm api route Fixes https://github.com/BerriAI/litellm/issues/6663 * fix(opentelemetry.py): fix passing parallel tool calls to otel Fixes https://github.com/BerriAI/litellm/issues/6677 * refactor(test_opentelemetry_unit_tests.py): create a base set of unit tests for all logging integrations - test for parallel tool call handling reduces bugs in repo * fix(__init__.py): update provider-model mapping to include all known provider-model mappings Fixes https://github.com/BerriAI/litellm/issues/6669 * feat(anthropic): support passing document in llm api call * docs(anthropic.md): add pdf anthropic call to docs + expose new 'supports_pdf_input' function * fix(factory.py): fix linting error * add clear doc string for GCS bucket logging * Add docs to export logs to Laminar (#6674) * Add docs to export logs to Laminar * minor fix: newline at end of file * place laminar after http and grpc * (Feat) Add langsmith key based logging (#6682) * add langsmith_api_key to StandardCallbackDynamicParams * create a file for langsmith types * langsmith add key / team based logging * add key based logging for langsmith * fix langsmith key based logging * fix linting langsmith * remove NOQA violation * add unit test coverage for all helpers in test langsmith * test_langsmith_key_based_logging * docs langsmith key based logging * run langsmith tests in logging callback tests * fix logging testing * test_langsmith_key_based_logging * test_add_callback_via_key_litellm_pre_call_utils_langsmith * add debug statement langsmith key based logging * test_langsmith_key_based_logging * (fix) OpenAI's optional messages[].name does not work with Mistral API (#6701) * use helper for _transform_messages mistral * add test_message_with_name to base LLMChat test * fix linting * add xAI on Admin UI (#6680) * (docs) add benchmarks on 1K RPS (#6704) * docs litellm proxy benchmarks * docs GCS bucket * doc fix - reduce clutter on logging doc title * (feat) add cost tracking stable diffusion 3 on Bedrock (#6676) * add cost tracking for sd3 * test_image_generation_bedrock * fix get model info for image cost * add cost_calculator for stability 1 models * add unit testing for bedrock image cost calc * test_cost_calculator_with_no_optional_params * add test_cost_calculator_basic * correctly allow size Optional * fix cost_calculator * sd3 unit tests cost calc * fix raise correct error 404 when /key/info is called on non-existent key (#6653) * fix raise correct error on /key/info * add not_found_error error * fix key not found in DB error * use 1 helper for checking token hash * fix error code on key info * fix test key gen prisma * test_generate_and_call_key_info * test fix test_call_with_valid_model_using_all_models * fix key info tests * bump: version 1.52.4 → 1.52.5 * add defaults used for GCS logging * LiteLLM Minor Fixes & Improvements (11/12/2024) (#6705) * fix(caching): convert arg to equivalent kwargs in llm caching handler prevent unexpected errors * fix(caching_handler.py): don't pass args to caching * fix(caching): remove all *args from caching.py * fix(caching): consistent function signatures + abc method * test(caching_unit_tests.py): add unit tests for llm caching ensures coverage for common caching scenarios across different implementations * refactor(litellm_logging.py): move to using cache key from hidden params instead of regenerating one * fix(router.py): drop redis password requirement * fix(proxy_server.py): fix faulty slack alerting check * fix(langfuse.py): avoid copying functions/thread lock objects in metadata fixes metadata copy error when parent otel span in metadata * test: update test * bump: version 1.52.5 → 1.52.6 * (feat) helm hook to sync db schema (#6715) * v0 migration job * fix job * fix migrations job.yml * handle standalone DB on helm hook * fix argo cd annotations * fix db migration helm hook * fix migration job * doc fix Using Http/2 with Hypercorn * (fix proxy redis) Add redis sentinel support (#6154) * add sentinel_password support * add doc for setting redis sentinel password * fix redis sentinel - use sentinel password * Fix: Update gpt-4o costs to that of gpt-4o-2024-08-06 (#6714) Fixes #6713 * (fix) using Anthropic `response_format={"type": "json_object"}` (#6721) * add support for response_format=json anthropic * add test_json_response_format to baseLLM ChatTest * fix test_litellm_anthropic_prompt_caching_tools * fix test_anthropic_function_call_with_no_schema * test test_create_json_tool_call_for_response_format * (feat) Add cost tracking for Azure Dall-e-3 Image Generation + use base class to ensure basic image generation tests pass (#6716) * add BaseImageGenTest * use 1 class for unit testing * add debugging to BaseImageGenTest * TestAzureOpenAIDalle3 * fix response_cost_calculator * test_basic_image_generation * fix img gen basic test * fix _select_model_name_for_cost_calc * fix test_aimage_generation_bedrock_with_optional_params * fix undo changes cost tracking * fix response_cost_calculator * fix test_cost_azure_gpt_35 * fix remove dup test (#6718) * (build) update db helm hook * (build) helm db pre sync hook * (build) helm db sync hook * test: run test_team_logging firdst --------- Co-authored-by: Ishaan Jaff Co-authored-by: Dinmukhamed Mailibay <47117969+dinmukhamedm@users.noreply.github.com> Co-authored-by: Kilian Lieret --- litellm/proxy/_new_secret_config.yaml | 2 +- .../key_management_endpoints.py | 16 +++++-------- tests/proxy_unit_tests/test_proxy_utils.py | 20 ++++++++++++++++ tests/test_keys.py | 23 ++++++++++++++----- tests/test_team_logging.py | 4 ++-- 5 files changed, 46 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 806b55994..911f15b86 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -68,7 +68,7 @@ model_list: litellm_settings: fallbacks: [{ "claude-3-5-sonnet-20240620": ["claude-3-5-sonnet-aihubmix"] }] - callbacks: ["otel", "prometheus"] + # callbacks: ["otel", "prometheus"] default_redis_batch_cache_expiry: 10 # default_team_settings: # - team_id: "dbe2f686-a686-4896-864a-4c3924458709" diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index c50aa0f9f..01baa5a43 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -303,21 +303,17 @@ async def generate_key_fn( # noqa: PLR0915 ) -async def prepare_key_update_data( +def prepare_key_update_data( data: Union[UpdateKeyRequest, RegenerateKeyRequest], existing_key_row ): - data_json: dict = data.dict(exclude_unset=True) + data_json: dict = data.model_dump(exclude_unset=True) data_json.pop("key", None) _metadata_fields = ["model_rpm_limit", "model_tpm_limit", "guardrails"] non_default_values = {} for k, v in data_json.items(): if k in _metadata_fields: continue - if v is not None: - if not isinstance(v, bool) and v in ([], {}, 0): - pass - else: - non_default_values[k] = v + non_default_values[k] = v if "duration" in non_default_values: duration = non_default_values.pop("duration") @@ -379,7 +375,7 @@ async def update_key_fn( ) try: - data_json: dict = data.json() + data_json: dict = data.model_dump(exclude_unset=True) key = data_json.pop("key") # get the row from db if prisma_client is None: @@ -395,7 +391,7 @@ async def update_key_fn( detail={"error": f"Team not found, passed team_id={data.team_id}"}, ) - non_default_values = await prepare_key_update_data( + non_default_values = prepare_key_update_data( data=data, existing_key_row=existing_key_row ) @@ -1144,7 +1140,7 @@ async def regenerate_key_fn( non_default_values = {} if data is not None: # Update with any provided parameters from GenerateKeyRequest - non_default_values = await prepare_key_update_data( + non_default_values = prepare_key_update_data( data=data, existing_key_row=_key_in_db ) diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index f3f33bad6..2e857808d 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -510,3 +510,23 @@ async def test_proxy_config_update_from_db(): "success_callback": "langfuse", } } + + +def test_prepare_key_update_data(): + from litellm.proxy.management_endpoints.key_management_endpoints import ( + prepare_key_update_data, + ) + from litellm.proxy._types import UpdateKeyRequest + + existing_key_row = MagicMock() + data = UpdateKeyRequest(key="test_key", models=["gpt-4"], duration="120s") + updated_data = prepare_key_update_data(data, existing_key_row) + assert "expires" in updated_data + + data = UpdateKeyRequest(key="test_key", metadata={}) + updated_data = prepare_key_update_data(data, existing_key_row) + assert updated_data["metadata"] == {} + + data = UpdateKeyRequest(key="test_key", metadata=None) + updated_data = prepare_key_update_data(data, existing_key_row) + assert updated_data["metadata"] == None diff --git a/tests/test_keys.py b/tests/test_keys.py index 437afc336..a569634bc 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -66,6 +66,7 @@ async def generate_key( max_parallel_requests: Optional[int] = None, user_id: Optional[str] = None, team_id: Optional[str] = None, + metadata: Optional[dict] = None, calling_key="sk-1234", ): url = "http://0.0.0.0:4000/key/generate" @@ -82,6 +83,7 @@ async def generate_key( "max_parallel_requests": max_parallel_requests, "user_id": user_id, "team_id": team_id, + "metadata": metadata, } print(f"data: {data}") @@ -136,16 +138,21 @@ async def test_key_gen_bad_key(): pass -async def update_key(session, get_key): +async def update_key(session, get_key, metadata: Optional[dict] = None): """ Make sure only models user has access to are returned """ url = "http://0.0.0.0:4000/key/update" headers = { - "Authorization": f"Bearer sk-1234", + "Authorization": "Bearer sk-1234", "Content-Type": "application/json", } - data = {"key": get_key, "models": ["gpt-4"], "duration": "120s"} + data = {"key": get_key} + + if metadata is not None: + data["metadata"] = metadata + else: + data.update({"models": ["gpt-4"], "duration": "120s"}) async with session.post(url, headers=headers, json=data) as response: status = response.status @@ -276,20 +283,24 @@ async def chat_completion_streaming(session, key, model="gpt-4"): return prompt_tokens, completion_tokens +@pytest.mark.parametrize("metadata", [{"test": "new"}, {}]) @pytest.mark.asyncio -async def test_key_update(): +async def test_key_update(metadata): """ Create key Update key with new model Test key w/ model """ async with aiohttp.ClientSession() as session: - key_gen = await generate_key(session=session, i=0) + key_gen = await generate_key(session=session, i=0, metadata={"test": "test"}) key = key_gen["key"] - await update_key( + assert key_gen["metadata"]["test"] == "test" + updated_key = await update_key( session=session, get_key=key, + metadata=metadata, ) + assert updated_key["metadata"] == metadata await update_proxy_budget(session=session) # resets proxy spend await chat_completion(session=session, key=key) diff --git a/tests/test_team_logging.py b/tests/test_team_logging.py index 97f18b42e..cf0fa6354 100644 --- a/tests/test_team_logging.py +++ b/tests/test_team_logging.py @@ -62,8 +62,8 @@ async def chat_completion(session, key, model="azure-gpt-3.5", request_metadata= @pytest.mark.asyncio -@pytest.mark.flaky(retries=6, delay=1) -async def test_team_logging(): +@pytest.mark.flaky(retries=12, delay=2) +async def test_aaateam_logging(): """ -> Team 1 logs to project 1 -> Create Key From 44709dd31d94ba68256e6df9dc191d6e979c67e1 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 14 Nov 2024 01:25:31 +0530 Subject: [PATCH 02/15] =?UTF-8?q?bump:=20version=201.52.6=20=E2=86=92=201.?= =?UTF-8?q?52.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 17d37c0ce..aed832f24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.52.6" +version = "1.52.7" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -91,7 +91,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.52.6" +version = "1.52.7" version_files = [ "pyproject.toml:^version" ] From b8b899f5d72c044a9b8a8956c5505662b977e249 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 13 Nov 2024 15:02:54 -0800 Subject: [PATCH 03/15] docs proxy_budget_rescheduler_min_time --- docs/my-website/docs/proxy/configs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index c6b9f2d45..888f424b4 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -791,9 +791,9 @@ general_settings: | store_model_in_db | boolean | If true, allows `/model/new` endpoint to store model information in db. Endpoint disabled by default. [Doc on `/model/new` endpoint](./model_management.md#create-a-new-model) | | max_request_size_mb | int | The maximum size for requests in MB. Requests above this size will be rejected. | | max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. | -| proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. | -| proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. | -| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. | +| proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. **Default is 597 seconds** | +| proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. **Default is 605 seconds** | +| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 10 seconds** | | alerting_args | dict | Args for Slack Alerting [Doc on Slack Alerting](./alerting.md) | | custom_key_generate | str | Custom function for key generation [Doc on custom key generation](./virtual_keys.md#custom--key-generate) | | allowed_ips | List[str] | List of IPs allowed to access the proxy. If not set, all IPs are allowed. | From e77ceec949b9deb0b957695bd53d14b7e882b796 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 13 Nov 2024 15:28:07 -0800 Subject: [PATCH 04/15] helm run DISABLE_SCHEMA_UPDATE --- deploy/charts/litellm-helm/templates/migrations-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index fc1aacf16..2ab361350 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -25,6 +25,6 @@ spec: value: {{ .Values.db.url | quote }} {{- end }} - name: DISABLE_SCHEMA_UPDATE - value: "{{ .Values.migrationJob.disableSchemaUpdate }}" + value: "false" # always run the migration from the Helm PreSync hook, override the value set restartPolicy: OnFailure backoffLimit: {{ .Values.migrationJob.backoffLimit }} From 49cda71c55522b98cd3a9c8a65bf32582095551d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 13 Nov 2024 15:33:43 -0800 Subject: [PATCH 05/15] docs helm pre sync hook --- docs/my-website/docs/proxy/prod.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index 66c719e5d..12cfdafc1 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -112,7 +112,27 @@ general_settings: disable_spend_logs: True ``` -## 7. Set LiteLLM Salt Key +## 7. Use Helm PreSync Hook for Database Migrations + +To ensure only one service manages database migrations, use our [Helm PreSync hook for Database Migrations](https://github.com/BerriAI/litellm/blob/main/deploy/charts/litellm-helm/templates/migrations-job.yaml). This ensures migrations are handled during `helm upgrade` or `helm install`, while LiteLLM pods explicitly disable migrations. + + +1. **Helm PreSync Hook**: + - The Helm PreSync hook is configured in the chart to run database migrations during deployments. + - The hook always sets `DISABLE_SCHEMA_UPDATE=false`, ensuring migrations are executed reliably. + +2. **LiteLLM Pods**: + - Set `DISABLE_SCHEMA_UPDATE=true` in LiteLLM pod configurations to prevent them from running migrations. + + Example configuration for LiteLLM pod: + ```yaml + env: + - name: DISABLE_SCHEMA_UPDATE + value: "true" + ``` + + +## 8. Set LiteLLM Salt Key If you plan on using the DB, set a salt key for encrypting/decrypting variables in the DB. From db9d9dde0a2dd60dd014f5133abc440466ddb567 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 13 Nov 2024 16:18:11 -0800 Subject: [PATCH 06/15] fix migration job.yaml --- deploy/charts/litellm-helm/templates/migrations-job.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index 2ab361350..4f8bca1e3 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -17,13 +17,8 @@ spec: command: ["python", "litellm/proxy/prisma_migration.py"] workingDir: "/app" env: - {{- if .Values.db.deployStandalone }} - - name: DATABASE_URL - value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }} - {{- else if .Values.db.useExisting }} - name: DATABASE_URL value: {{ .Values.db.url | quote }} - {{- end }} - name: DISABLE_SCHEMA_UPDATE value: "false" # always run the migration from the Helm PreSync hook, override the value set restartPolicy: OnFailure From aa82a88c5fbc7ca78f27ba541ccf65ca60f14600 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 13 Nov 2024 16:19:37 -0800 Subject: [PATCH 07/15] fix DATABASE_URL --- deploy/charts/litellm-helm/templates/migrations-job.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index 4f8bca1e3..d998592e2 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -18,7 +18,10 @@ spec: workingDir: "/app" env: - name: DATABASE_URL - value: {{ .Values.db.url | quote }} + valueFrom: + secretKeyRef: + name: db-secret + key: DATABASE_URL - name: DISABLE_SCHEMA_UPDATE value: "false" # always run the migration from the Helm PreSync hook, override the value set restartPolicy: OnFailure From 4dc23cf997338fa57d2177b3196ec6db712cc95f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 13 Nov 2024 16:43:26 -0800 Subject: [PATCH 08/15] use existing spec for migrations job --- deploy/charts/litellm-helm/templates/migrations-job.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index d998592e2..06d45d5fc 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -17,11 +17,12 @@ spec: command: ["python", "litellm/proxy/prisma_migration.py"] workingDir: "/app" env: + {{- if .Values.db.deployStandalone }} - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: db-secret - key: DATABASE_URL + value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }} + {{- else if .Values.db.useExisting }} + - name: DATABASE_URL + value: {{ .Values.db.url | quote }} - name: DISABLE_SCHEMA_UPDATE value: "false" # always run the migration from the Helm PreSync hook, override the value set restartPolicy: OnFailure From da5da64d27ca4c69daa8b6b24cd086fb737edea1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 13 Nov 2024 16:48:22 -0800 Subject: [PATCH 09/15] fix yaml on migrations job --- deploy/charts/litellm-helm/templates/migrations-job.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index 06d45d5fc..78684bcab 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -25,5 +25,6 @@ spec: value: {{ .Values.db.url | quote }} - name: DISABLE_SCHEMA_UPDATE value: "false" # always run the migration from the Helm PreSync hook, override the value set + {{- end }} restartPolicy: OnFailure backoffLimit: {{ .Values.migrationJob.backoffLimit }} From b5183ce31b76c311139883d98593cf0a6e7fba41 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 13 Nov 2024 16:56:09 -0800 Subject: [PATCH 10/15] fix migration job --- .../charts/litellm-helm/templates/migrations-job.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index 78684bcab..486072690 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -17,14 +17,14 @@ spec: command: ["python", "litellm/proxy/prisma_migration.py"] workingDir: "/app" env: - {{- if .Values.db.deployStandalone }} - - name: DATABASE_URL - value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }} - {{- else if .Values.db.useExisting }} + {{- if .Values.db.useExisting }} - name: DATABASE_URL value: {{ .Values.db.url | quote }} + {{- else }} + - name: DATABASE_URL + value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }} + {{- end }} - name: DISABLE_SCHEMA_UPDATE value: "false" # always run the migration from the Helm PreSync hook, override the value set - {{- end }} restartPolicy: OnFailure backoffLimit: {{ .Values.migrationJob.backoffLimit }} From 894b29565858cdbc601987cdaf42de62a97dc107 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 13 Nov 2024 16:56:55 -0800 Subject: [PATCH 11/15] update doc on pre sync hook --- docs/my-website/docs/proxy/prod.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/my-website/docs/proxy/prod.md b/docs/my-website/docs/proxy/prod.md index 12cfdafc1..14b88e684 100644 --- a/docs/my-website/docs/proxy/prod.md +++ b/docs/my-website/docs/proxy/prod.md @@ -1,5 +1,6 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +import Image from '@theme/IdealImage'; # ⚡ Best Practices for Production @@ -120,6 +121,14 @@ To ensure only one service manages database migrations, use our [Helm PreSync ho 1. **Helm PreSync Hook**: - The Helm PreSync hook is configured in the chart to run database migrations during deployments. - The hook always sets `DISABLE_SCHEMA_UPDATE=false`, ensuring migrations are executed reliably. + + Reference Settings to set on ArgoCD for `values.yaml` + + ```yaml + db: + useExisting: true # use existing Postgres DB + url: postgresql://ishaanjaffer0324:3rnwpOBau6hT@ep-withered-mud-a5dkdpke.us-east-2.aws.neon.tech/test-argo-cd?sslmode=require # url of existing Postgres DB + ``` 2. **LiteLLM Pods**: - Set `DISABLE_SCHEMA_UPDATE=true` in LiteLLM pod configurations to prevent them from running migrations. From b56b5dce7f966c2096def302d96b83aad40386d8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 13 Nov 2024 16:59:34 -0800 Subject: [PATCH 12/15] fix migrations-job.yaml --- deploy/charts/litellm-helm/templates/migrations-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index 486072690..94d1d06ec 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -13,7 +13,7 @@ spec: spec: containers: - name: prisma-migrations - image: "ghcr.io/berriai/litellm:main-stable" + image: "ghcr.io/berriai/litellm-database" command: ["python", "litellm/proxy/prisma_migration.py"] workingDir: "/app" env: From 0e2c16e948505a60c3a789f9a3db4d8338a10810 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 13 Nov 2024 17:02:06 -0800 Subject: [PATCH 13/15] fix migration job --- deploy/charts/litellm-helm/templates/migrations-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index 94d1d06ec..0472aaf40 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -13,7 +13,7 @@ spec: spec: containers: - name: prisma-migrations - image: "ghcr.io/berriai/litellm-database" + image: ghcr.io/berriai/litellm-database command: ["python", "litellm/proxy/prisma_migration.py"] workingDir: "/app" env: From f2e6025c659f7bf1725fa7e9123045e10ea413ca Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 13 Nov 2024 17:04:58 -0800 Subject: [PATCH 14/15] fix prisma migration --- deploy/charts/litellm-helm/templates/migrations-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/charts/litellm-helm/templates/migrations-job.yaml b/deploy/charts/litellm-helm/templates/migrations-job.yaml index 0472aaf40..010d2d1b5 100644 --- a/deploy/charts/litellm-helm/templates/migrations-job.yaml +++ b/deploy/charts/litellm-helm/templates/migrations-job.yaml @@ -13,7 +13,7 @@ spec: spec: containers: - name: prisma-migrations - image: ghcr.io/berriai/litellm-database + image: ghcr.io/berriai/litellm-database:main-latest command: ["python", "litellm/proxy/prisma_migration.py"] workingDir: "/app" env: From 914cec3ab5e0c931172c812a6d8a9214a5f49bba Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 13 Nov 2024 19:37:34 -0800 Subject: [PATCH 15/15] test - handle eol model claude-2, use claude-2.1 instead --- tests/local_testing/test_router_fallbacks.py | 14 +++++++------- tests/local_testing/test_streaming.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/local_testing/test_router_fallbacks.py b/tests/local_testing/test_router_fallbacks.py index cad640a54..7f4d318bf 100644 --- a/tests/local_testing/test_router_fallbacks.py +++ b/tests/local_testing/test_router_fallbacks.py @@ -1138,9 +1138,9 @@ async def test_router_content_policy_fallbacks( router = Router( model_list=[ { - "model_name": "claude-2", + "model_name": "claude-2.1", "litellm_params": { - "model": "claude-2", + "model": "claude-2.1", "api_key": "", "mock_response": mock_response, }, @@ -1164,7 +1164,7 @@ async def test_router_content_policy_fallbacks( { "model_name": "my-general-model", "litellm_params": { - "model": "claude-2", + "model": "claude-2.1", "api_key": "", "mock_response": Exception("Should not have called this."), }, @@ -1172,14 +1172,14 @@ async def test_router_content_policy_fallbacks( { "model_name": "my-context-window-model", "litellm_params": { - "model": "claude-2", + "model": "claude-2.1", "api_key": "", "mock_response": Exception("Should not have called this."), }, }, ], content_policy_fallbacks=( - [{"claude-2": ["my-fallback-model"]}] + [{"claude-2.1": ["my-fallback-model"]}] if fallback_type == "model-specific" else None ), @@ -1190,12 +1190,12 @@ async def test_router_content_policy_fallbacks( if sync_mode is True: response = router.completion( - model="claude-2", + model="claude-2.1", messages=[{"role": "user", "content": "Hey, how's it going?"}], ) else: response = await router.acompletion( - model="claude-2", + model="claude-2.1", messages=[{"role": "user", "content": "Hey, how's it going?"}], ) diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py index fcdc6b60d..7b3dbd8d6 100644 --- a/tests/local_testing/test_streaming.py +++ b/tests/local_testing/test_streaming.py @@ -718,7 +718,7 @@ async def test_acompletion_claude_2_stream(): try: litellm.set_verbose = True response = await litellm.acompletion( - model="claude-2", + model="claude-2.1", messages=[{"role": "user", "content": "hello from litellm"}], stream=True, )