forked from phoenix-oss/llama-stack-mirror
67450e4024
# What does this PR do?
Fixes two issues on providers/test/inference
- [ ] Addresses issue (#issue)
## Test Plan
### Before
```
===================================================================================== FAILURES =====================================================================================
__________________________________ TestVisionModelInference.test_vision_chat_completion_streaming[llama_vision-fireworks][llama_vision] ___________________________________
providers/tests/inference/test_vision_inference.py:145: in test_vision_chat_completion_streaming
content = "".join(
E TypeError: sequence item 0: expected str instance, TextDelta found
------------------------------------------------------------------------------ Captured log teardown -------------------------------------------------------------------------------
ERROR asyncio:base_events.py:1858 Task was destroyed but it is pending!
task: <Task pending name='Task-5' coro=<<async_generator_athrow without __name__>()>>
============================================================================= short test summary info ==============================================================================
FAILED providers/tests/inference/test_vision_inference.py::TestVisionModelInference::test_vision_chat_completion_streaming[llama_vision-fireworks] - TypeError: sequence item 0: expected str instance, TextDelta found
============================================================== 1 failed, 2 passed, 33 deselected, 7 warnings in 3.59s ==============================================================
(base) sxyi@sxyi-mbp llama_stack %
```
### After
```
(base) sxyi@sxyi-mbp llama_stack % pytest -k "fireworks" /Users/sxyi/llama-stack/llama_stack/providers/tests/inference/test_vision_inference.py
/Library/Frameworks/Python.framework/Versions/3.13/lib/python3.13/site-packages/pytest_asyncio/plugin.py:208: PytestDeprecationWarning: The configuration option "asyncio_default_fixture_loop_scope" is unset.
The event loop scope for asynchronous fixtures will default to the fixture caching scope. Future versions of pytest-asyncio will default the loop scope for asynchronous fixtures to function scope. Set the default fixture loop scope explicitly in order to avoid unexpected behavior in the future. Valid fixture loop scopes are: "function", "class", "module", "package", "session"
warnings.warn(PytestDeprecationWarning(_DEFAULT_FIXTURE_LOOP_SCOPE_UNSET))
=============================================================================== test session starts ================================================================================
platform darwin -- Python 3.13.0, pytest-8.3.3, pluggy-1.5.0
rootdir: /Users/sxyi/llama-stack
configfile: pyproject.toml
plugins: asyncio-0.24.0, html-4.1.1, metadata-3.1.1, dependency-0.6.0, anyio-4.6.2.post1
asyncio: mode=Mode.STRICT, default_loop_scope=None
collected 36 items / 33 deselected / 3 selected
providers/tests/inference/test_vision_inference.py ... [100%]
=================================================================== 3 passed, 33 deselected, 7 warnings in 3.75s ===================================================================
(base) sxyi@sxyi-mbp llama_stack %
```
## Before submitting
- [ ] This PR fixes a typo or improves the docs (you can dismiss the
other checks if that's the case).
- [ ] Ran pre-commit to handle lint / formatting issues.
- [ ] Read the [contributor
guideline](https://github.com/meta-llama/llama-stack/blob/main/CONTRIBUTING.md),
Pull Request section?
- [ ] Updated relevant documentation.
- [ ] Wrote necessary unit or integration tests.
63 lines
2.2 KiB
Python
63 lines
2.2 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the terms described in the LICENSE file in
|
|
# the root directory of this source tree.
|
|
|
|
import pytest
|
|
|
|
from llama_stack.apis.inference import EmbeddingsResponse
|
|
from llama_stack.apis.models import ModelType
|
|
|
|
# How to run this test:
|
|
# pytest -v -s llama_stack/providers/tests/inference/test_embeddings.py
|
|
|
|
|
|
class TestEmbeddings:
|
|
@pytest.mark.asyncio
|
|
async def test_embeddings(self, inference_model, inference_stack):
|
|
inference_impl, models_impl = inference_stack
|
|
model = await models_impl.get_model(inference_model)
|
|
|
|
if model.model_type != ModelType.embedding:
|
|
pytest.skip("This test is only applicable for embedding models")
|
|
|
|
response = await inference_impl.embeddings(
|
|
model_id=inference_model,
|
|
contents=["Hello, world!"],
|
|
)
|
|
assert isinstance(response, EmbeddingsResponse)
|
|
assert len(response.embeddings) > 0
|
|
assert all(isinstance(embedding, list) for embedding in response.embeddings)
|
|
assert all(
|
|
isinstance(value, float)
|
|
for embedding in response.embeddings
|
|
for value in embedding
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_batch_embeddings(self, inference_model, inference_stack):
|
|
inference_impl, models_impl = inference_stack
|
|
model = await models_impl.get_model(inference_model)
|
|
|
|
if model.model_type != ModelType.embedding:
|
|
pytest.skip("This test is only applicable for embedding models")
|
|
|
|
texts = ["Hello, world!", "This is a test", "Testing embeddings"]
|
|
|
|
response = await inference_impl.embeddings(
|
|
model_id=inference_model,
|
|
contents=texts,
|
|
)
|
|
|
|
assert isinstance(response, EmbeddingsResponse)
|
|
assert len(response.embeddings) == len(texts)
|
|
assert all(isinstance(embedding, list) for embedding in response.embeddings)
|
|
assert all(
|
|
isinstance(value, float)
|
|
for embedding in response.embeddings
|
|
for value in embedding
|
|
)
|
|
|
|
embedding_dim = len(response.embeddings[0])
|
|
assert all(len(embedding) == embedding_dim for embedding in response.embeddings)
|