From 35a00d004ab9063d8bc286971d0a6a2010d88e3e Mon Sep 17 00:00:00 2001 From: Sixian Yi Date: Tue, 21 Jan 2025 21:44:06 -0800 Subject: [PATCH] bug fix for distro report generation (#836) # What does this PR do? Minor bug fix and simplify code - [ ] Addresses issue (#issue) ## Test Plan See the updated `llama_stack/templates/fireworks/report.md` ## Sources Please link relevant resources if necessary. ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Ran pre-commit to handle lint / formatting issues. - [ ] Read the [contributor guideline](https://github.com/meta-llama/llama-stack/blob/main/CONTRIBUTING.md), Pull Request section? - [ ] Updated relevant documentation. - [ ] Wrote necessary unit or integration tests. --- llama_stack/templates/fireworks/report.md | 4 ++-- tests/client-sdk/conftest.py | 1 - tests/client-sdk/report.py | 15 +++------------ 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/llama_stack/templates/fireworks/report.md b/llama_stack/templates/fireworks/report.md index 5ca65c62e..ac6fab6eb 100644 --- a/llama_stack/templates/fireworks/report.md +++ b/llama_stack/templates/fireworks/report.md @@ -22,9 +22,9 @@ | Model | API | Capability | Test | Status | |:----- |:-----|:-----|:-----|:-----| | Text | /chat_completion | streaming | test_text_chat_completion_streaming | ✅ | -| Vision | /chat_completion | streaming | test_image_chat_completion_streaming | Passed | +| Vision | /chat_completion | streaming | test_image_chat_completion_streaming | ✅ | +| Vision | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | ✅ | | Text | /chat_completion | non_streaming | test_text_chat_completion_non_streaming | ✅ | -| Vision | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | Passed | | Text | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_streaming | ✅ | | Text | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_non_streaming | ✅ | | Text | /completion | streaming | test_text_completion_streaming | ✅ | diff --git a/tests/client-sdk/conftest.py b/tests/client-sdk/conftest.py index c19546887..0b5324c0e 100644 --- a/tests/client-sdk/conftest.py +++ b/tests/client-sdk/conftest.py @@ -32,7 +32,6 @@ def pytest_addoption(parser): TEXT_MODEL = "meta-llama/Llama-3.1-8B-Instruct" INFERENCE_MODEL = "meta-llama/Llama-3.2-11B-Vision-Instruct" - @pytest.fixture(scope="session") def provider_data(): # check env for tavily secret, brave secret and inject all into provider data diff --git a/tests/client-sdk/report.py b/tests/client-sdk/report.py index a2ff07e4f..22aa98935 100644 --- a/tests/client-sdk/report.py +++ b/tests/client-sdk/report.py @@ -135,24 +135,15 @@ class Report: ] for api, capa_map in API_MAPS["inference"].items(): for capa, tests in capa_map.items(): - vision_tests = filter(lambda test_name: "image" in test_name, tests) - text_tests = filter(lambda test_name: "text" in test_name, tests) - - for test_name in text_tests: + for test_name in tests: + model_type = "Text" if "text" in test_name else "Vision" test_nodeids = self.test_name_to_nodeid[test_name] assert len(test_nodeids) > 0 # There might be more than one parametrizations for the same test function. We take # the result of the first one for now. Ideally we should mark the test as failed if # any of the parametrizations failed. test_table.append( - f"| Text | /{api} | {capa} | {test_name} | {self._print_result_icon(self.test_data[test_nodeids[0]])} |" - ) - - for test_name in vision_tests: - test_nodeids = self.test_name_to_nodeid[test_name] - assert len(test_nodeids) > 0 - test_table.append( - f"| Vision | /{api} | {capa} | {test_name} | {self.test_data[test_nodeids[0]]} |" + f"| {model_type} | /{api} | {capa} | {test_name} | {self._print_result_icon(self.test_data[test_nodeids[0]])} |" ) report.extend(test_table)