mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-06-28 10:54:19 +00:00
bug fix for distro report generation (#836)
# What does this PR do? Minor bug fix and simplify code - [ ] Addresses issue (#issue) ## Test Plan See the updated `llama_stack/templates/fireworks/report.md` ## Sources Please link relevant resources if necessary. ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Ran pre-commit to handle lint / formatting issues. - [ ] Read the [contributor guideline](https://github.com/meta-llama/llama-stack/blob/main/CONTRIBUTING.md), Pull Request section? - [ ] Updated relevant documentation. - [ ] Wrote necessary unit or integration tests.
This commit is contained in:
parent
edf56884a7
commit
35a00d004a
3 changed files with 5 additions and 15 deletions
|
@ -22,9 +22,9 @@
|
||||||
| Model | API | Capability | Test | Status |
|
| Model | API | Capability | Test | Status |
|
||||||
|:----- |:-----|:-----|:-----|:-----|
|
|:----- |:-----|:-----|:-----|:-----|
|
||||||
| Text | /chat_completion | streaming | test_text_chat_completion_streaming | ✅ |
|
| Text | /chat_completion | streaming | test_text_chat_completion_streaming | ✅ |
|
||||||
| Vision | /chat_completion | streaming | test_image_chat_completion_streaming | Passed |
|
| Vision | /chat_completion | streaming | test_image_chat_completion_streaming | ✅ |
|
||||||
|
| Vision | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | ✅ |
|
||||||
| Text | /chat_completion | non_streaming | test_text_chat_completion_non_streaming | ✅ |
|
| Text | /chat_completion | non_streaming | test_text_chat_completion_non_streaming | ✅ |
|
||||||
| Vision | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | Passed |
|
|
||||||
| Text | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_streaming | ✅ |
|
| Text | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_streaming | ✅ |
|
||||||
| Text | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_non_streaming | ✅ |
|
| Text | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_non_streaming | ✅ |
|
||||||
| Text | /completion | streaming | test_text_completion_streaming | ✅ |
|
| Text | /completion | streaming | test_text_completion_streaming | ✅ |
|
||||||
|
|
|
@ -32,7 +32,6 @@ def pytest_addoption(parser):
|
||||||
TEXT_MODEL = "meta-llama/Llama-3.1-8B-Instruct"
|
TEXT_MODEL = "meta-llama/Llama-3.1-8B-Instruct"
|
||||||
INFERENCE_MODEL = "meta-llama/Llama-3.2-11B-Vision-Instruct"
|
INFERENCE_MODEL = "meta-llama/Llama-3.2-11B-Vision-Instruct"
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="session")
|
@pytest.fixture(scope="session")
|
||||||
def provider_data():
|
def provider_data():
|
||||||
# check env for tavily secret, brave secret and inject all into provider data
|
# check env for tavily secret, brave secret and inject all into provider data
|
||||||
|
|
|
@ -135,24 +135,15 @@ class Report:
|
||||||
]
|
]
|
||||||
for api, capa_map in API_MAPS["inference"].items():
|
for api, capa_map in API_MAPS["inference"].items():
|
||||||
for capa, tests in capa_map.items():
|
for capa, tests in capa_map.items():
|
||||||
vision_tests = filter(lambda test_name: "image" in test_name, tests)
|
for test_name in tests:
|
||||||
text_tests = filter(lambda test_name: "text" in test_name, tests)
|
model_type = "Text" if "text" in test_name else "Vision"
|
||||||
|
|
||||||
for test_name in text_tests:
|
|
||||||
test_nodeids = self.test_name_to_nodeid[test_name]
|
test_nodeids = self.test_name_to_nodeid[test_name]
|
||||||
assert len(test_nodeids) > 0
|
assert len(test_nodeids) > 0
|
||||||
# There might be more than one parametrizations for the same test function. We take
|
# There might be more than one parametrizations for the same test function. We take
|
||||||
# the result of the first one for now. Ideally we should mark the test as failed if
|
# the result of the first one for now. Ideally we should mark the test as failed if
|
||||||
# any of the parametrizations failed.
|
# any of the parametrizations failed.
|
||||||
test_table.append(
|
test_table.append(
|
||||||
f"| Text | /{api} | {capa} | {test_name} | {self._print_result_icon(self.test_data[test_nodeids[0]])} |"
|
f"| {model_type} | /{api} | {capa} | {test_name} | {self._print_result_icon(self.test_data[test_nodeids[0]])} |"
|
||||||
)
|
|
||||||
|
|
||||||
for test_name in vision_tests:
|
|
||||||
test_nodeids = self.test_name_to_nodeid[test_name]
|
|
||||||
assert len(test_nodeids) > 0
|
|
||||||
test_table.append(
|
|
||||||
f"| Vision | /{api} | {capa} | {test_name} | {self.test_data[test_nodeids[0]]} |"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
report.extend(test_table)
|
report.extend(test_table)
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue