From 833aa0ebd8ebc4344d983fd4dc7f39521630c2d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Tue, 14 Oct 2025 14:45:45 +0200 Subject: [PATCH] tests: adapt embedding tests for watsonx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setting the dimension is not supported see: ``` openai.BadRequestError: Error code: 400 - {'detail': "litellm.UnsupportedParamsError: watsonx does not support parameters: {'dimensions': 384} ``` Successful run: ``` INFO 2025-10-14 14:32:20,353 tests.integration.conftest:50 tests: Test stack config type: library_client (stack_config=None) ======================================================== test session starts ========================================================= platform darwin -- Python 3.12.8, pytest-8.4.2, pluggy-1.6.0 -- /Users/leseb/Documents/AI/llama-stack/.venv/bin/python3 cachedir: .pytest_cache metadata: {'Python': '3.12.8', 'Platform': 'macOS-26.0.1-arm64-arm-64bit', 'Packages': {'pytest': '8.4.2', 'pluggy': '1.6.0'}, 'Plugins': {'anyio': '4.9.0', 'html': '4.1.1', 'socket': '0.7.0', 'asyncio': '1.1.0', 'json-report': '1.5.0', 'timeout': '2.4.0', 'metadata': '3.1.1', 'cov': '6.2.1', 'nbval': '0.11.0'}} rootdir: /Users/leseb/Documents/AI/llama-stack configfile: pyproject.toml plugins: anyio-4.9.0, html-4.1.1, socket-0.7.0, asyncio-1.1.0, json-report-1.5.0, timeout-2.4.0, metadata-3.1.1, cov-6.2.1, nbval-0.11.0 asyncio: mode=Mode.AUTO, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function collected 20 items tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_single_string[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 5%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_multiple_strings[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 10%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_with_encoding_format_float[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 15%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_with_dimensions[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] SKIPPED [ 20%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_with_user_parameter[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 25%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_empty_list_error[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 30%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_invalid_model_error[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 35%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_different_inputs_different_outputs[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 40%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_with_encoding_format_base64[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] SKIPPED [ 45%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_base64_batch_processing[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 50%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_single_string[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 55%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_multiple_strings[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 60%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_with_encoding_format_float[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 65%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_with_dimensions[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] SKIPPED [ 70%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_with_user_parameter[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 75%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_empty_list_error[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 80%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_invalid_model_error[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 85%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_different_inputs_different_outputs[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [ 90%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_with_encoding_format_base64[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] SKIPPED [ 95%] tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_base64_batch_processing[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] PASSED [100%] ======================================================== slowest 10 durations ======================================================== 1.84s call tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_multiple_strings[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] 1.62s call tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_empty_list_error[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] 1.23s call tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_empty_list_error[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] 0.70s call tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_different_inputs_different_outputs[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] 0.69s call tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_different_inputs_different_outputs[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] 0.61s call tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_single_string[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] 0.41s call tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_with_user_parameter[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] 0.41s call tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_with_encoding_format_float[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] 0.41s call tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_base64_batch_processing[llama_stack_client-emb=watsonx/ibm/slate-30m-english-rtrvr] 0.38s call tests/integration/inference/test_openai_embeddings.py::test_openai_embeddings_multiple_strings[openai_client-emb=watsonx/ibm/slate-30m-english-rtrvr] ====================================================== short test summary info ======================================================= SKIPPED [4] tests/integration/inference/test_openai_embeddings.py:63: Model watsonx/ibm/slate-30m-english-rtrvr hosted by remote::watsonx does not support variable output embedding dimensions. ============================================= 16 passed, 4 skipped, 1 warning in 10.23s ============================================== ``` Signed-off-by: Sébastien Han --- .../inference/test_openai_embeddings.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/integration/inference/test_openai_embeddings.py b/tests/integration/inference/test_openai_embeddings.py index 84e92706a..0c1d4d08e 100644 --- a/tests/integration/inference/test_openai_embeddings.py +++ b/tests/integration/inference/test_openai_embeddings.py @@ -50,11 +50,15 @@ def skip_if_model_doesnt_support_encoding_format_base64(client, model_id): def skip_if_model_doesnt_support_variable_dimensions(client_with_models, model_id): provider = provider_from_model(client_with_models, model_id) - if provider.provider_type in ( - "remote::together", # returns 400 - "inline::sentence-transformers", - # Error code: 400 - {'error_code': 'BAD_REQUEST', 'message': 'Bad request: json: unknown field "dimensions"\n'} - "remote::databricks", + if ( + provider.provider_type + in ( + "remote::together", # returns 400 + "inline::sentence-transformers", + # Error code: 400 - {'error_code': 'BAD_REQUEST', 'message': 'Bad request: json: unknown field "dimensions"\n'} + "remote::databricks", + "remote::watsonx", # openai.BadRequestError: Error code: 400 - {'detail': "litellm.UnsupportedParamsError: watsonx does not support parameters: {'dimensions': 384} + ) ): pytest.skip( f"Model {model_id} hosted by {provider.provider_type} does not support variable output embedding dimensions."