diff --git a/llama_stack/templates/fireworks/report.md b/llama_stack/templates/fireworks/report.md index 5ca65c62e..ac6fab6eb 100644 --- a/llama_stack/templates/fireworks/report.md +++ b/llama_stack/templates/fireworks/report.md @@ -22,9 +22,9 @@ | Model | API | Capability | Test | Status | |:----- |:-----|:-----|:-----|:-----| | Text | /chat_completion | streaming | test_text_chat_completion_streaming | ✅ | -| Vision | /chat_completion | streaming | test_image_chat_completion_streaming | Passed | +| Vision | /chat_completion | streaming | test_image_chat_completion_streaming | ✅ | +| Vision | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | ✅ | | Text | /chat_completion | non_streaming | test_text_chat_completion_non_streaming | ✅ | -| Vision | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | Passed | | Text | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_streaming | ✅ | | Text | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_non_streaming | ✅ | | Text | /completion | streaming | test_text_completion_streaming | ✅ | diff --git a/tests/client-sdk/conftest.py b/tests/client-sdk/conftest.py index c19546887..0b5324c0e 100644 --- a/tests/client-sdk/conftest.py +++ b/tests/client-sdk/conftest.py @@ -32,7 +32,6 @@ def pytest_addoption(parser): TEXT_MODEL = "meta-llama/Llama-3.1-8B-Instruct" INFERENCE_MODEL = "meta-llama/Llama-3.2-11B-Vision-Instruct" - @pytest.fixture(scope="session") def provider_data(): # check env for tavily secret, brave secret and inject all into provider data diff --git a/tests/client-sdk/report.py b/tests/client-sdk/report.py index a2ff07e4f..22aa98935 100644 --- a/tests/client-sdk/report.py +++ b/tests/client-sdk/report.py @@ -135,24 +135,15 @@ class Report: ] for api, capa_map in API_MAPS["inference"].items(): for capa, tests in capa_map.items(): - vision_tests = filter(lambda test_name: "image" in test_name, tests) - text_tests = filter(lambda test_name: "text" in test_name, tests) - - for test_name in text_tests: + for test_name in tests: + model_type = "Text" if "text" in test_name else "Vision" test_nodeids = self.test_name_to_nodeid[test_name] assert len(test_nodeids) > 0 # There might be more than one parametrizations for the same test function. We take # the result of the first one for now. Ideally we should mark the test as failed if # any of the parametrizations failed. test_table.append( - f"| Text | /{api} | {capa} | {test_name} | {self._print_result_icon(self.test_data[test_nodeids[0]])} |" - ) - - for test_name in vision_tests: - test_nodeids = self.test_name_to_nodeid[test_name] - assert len(test_nodeids) > 0 - test_table.append( - f"| Vision | /{api} | {capa} | {test_name} | {self.test_data[test_nodeids[0]]} |" + f"| {model_type} | /{api} | {capa} | {test_name} | {self._print_result_icon(self.test_data[test_nodeids[0]])} |" ) report.extend(test_table)