phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-06 10:42:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix report

Browse source
This commit is contained in:
Sixian Yi 2025-01-22 19:15:44 -08:00
parent a239a39cfa
commit bd32ebb8f8
2 changed files with 24 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

2
llama_stack/templates/cerebras/report.md
View file

@ -40,5 +40,5 @@
| API | Capability | Test | Status | | API | Capability | Test | Status |
|:-----|:-----|:-----|:-----| |:-----|:-----|:-----|:-----|
| /create_agent_turn | rag | test_rag_agent | ✅ | | /create_agent_turn | rag | test_rag_agent | ✅ |
| /create_agent_turn | custom_tool | test_custom_tool | ✅ | | /create_agent_turn | custom_tool | test_custom_tool | ❌ |
| /create_agent_turn | code_execution | test_code_interpreter_for_attachments | ✅ | | /create_agent_turn | code_execution | test_code_interpreter_for_attachments | ✅ |

72
tests/client-sdk/report.py
View file

@ -41,7 +41,7 @@ def featured_models_repo_names():
*llama3_3_instruct_models(), *llama3_3_instruct_models(),
*safety_models(), *safety_models(),
] ]
return [model for model in models if not model.variant] return [model.huggingface_repo for model in models if not model.variant]
SUPPORTED_MODELS = { SUPPORTED_MODELS = {
@ -66,33 +66,6 @@ SUPPORTED_MODELS = {
CoreModelId.llama_guard_3_1b.value, CoreModelId.llama_guard_3_1b.value,
] ]
), ),
"fireworks": set(
[
CoreModelId.llama3_1_8b_instruct.value,
CoreModelId.llama3_1_70b_instruct.value,
CoreModelId.llama3_1_405b_instruct.value,
CoreModelId.llama3_2_1b_instruct.value,
CoreModelId.llama3_2_3b_instruct.value,
CoreModelId.llama3_2_11b_vision_instruct.value,
CoreModelId.llama3_2_90b_vision_instruct.value,
CoreModelId.llama3_3_70b_instruct.value,
CoreModelId.llama_guard_3_8b.value,
CoreModelId.llama_guard_3_11b_vision.value,
]
),
"together": set(
[
CoreModelId.llama3_1_8b_instruct.value,
CoreModelId.llama3_1_70b_instruct.value,
CoreModelId.llama3_1_405b_instruct.value,
CoreModelId.llama3_2_3b_instruct.value,
CoreModelId.llama3_2_11b_vision_instruct.value,
CoreModelId.llama3_2_90b_vision_instruct.value,
CoreModelId.llama3_3_70b_instruct.value,
CoreModelId.llama_guard_3_8b.value,
CoreModelId.llama_guard_3_11b_vision.value,
]
),
"tgi": set( "tgi": set(
[ [
model.core_model_id.value model.core_model_id.value
@ -107,12 +80,6 @@ SUPPORTED_MODELS = {
if model.huggingface_repo if model.huggingface_repo
] ]
), ),
"cerebras": set(
[
CoreModelId.llama3_1_8b_instruct.value,
CoreModelId.llama3_3_70b_instruct.value,
]
),
} }
@ -178,9 +145,6 @@ class Report:
self.test_data[report.nodeid] = outcome self.test_data[report.nodeid] = outcome
def pytest_sessionfinish(self, session): def pytest_sessionfinish(self, session):
if len(self.test_name_to_nodeid) == 0:
return
report = [] report = []
report.append(f"# Report for {self.image_name} distribution") report.append(f"# Report for {self.image_name} distribution")
report.append("\n## Supported Models:") report.append("\n## Supported Models:")
@ -192,18 +156,28 @@ class Report:
report.append(dividor) report.append(dividor)
rows = [] rows = []
for model in all_registered_models(): if self.image_name in SUPPORTED_MODELS:
if ( for model in all_registered_models():
"Instruct" not in model.core_model_id.value if (
and "Guard" not in model.core_model_id.value "Instruct" not in model.core_model_id.value
) or (model.variant): and "Guard" not in model.core_model_id.value
continue ) or (model.variant):
row = f"| {model.core_model_id.value} |" continue
if model.core_model_id.value in SUPPORTED_MODELS[self.image_name]: row = f"| {model.core_model_id.value} |"
row += " ✅ |" if model.core_model_id.value in SUPPORTED_MODELS[self.image_name]:
else: row += " ✅ |"
row += " ❌ |" else:
rows.append(row) row += " ❌ |"
rows.append(row)
else:
supported_models = {m.identifier for m in self.client.models.list()}
for model in featured_models_repo_names():
row = f"| {model} |"
if model in supported_models:
row += " ✅ |"
else:
row += " ❌ |"
rows.append(row)
report.extend(rows) report.extend(rows)
report.append("\n## Inference:") report.append("\n## Inference:")