phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-30 16:53:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
llama-stack-mirror/tests/verifications/test_results/fireworks_1744264202.json
ehhuang 14146e4b3f
feat(verification): various improvements (#1921) 
# What does this PR do?
- provider and their models now live in config.yaml
- better distinguish different cases within a test
- add model key to surface provider's model_id
- include example command to rerun single test case

## Test Plan
<img width="1173" alt="image"
src="https://github.com/user-attachments/assets/b414baf0-c768-451f-8c3b-c2905cf36fac"
/>
2025-04-10 10:26:19 -07:00

1329 lines
49 KiB
JSON
Raw Blame History

{
"created": 1744264258.730061,
"duration": 53.86071586608887,
"exitcode": 1,
"root": "/Users/erichuang/projects/llama-stack",
"environment": {},
"summary": {
"passed": 28,
"skipped": 2,
"failed": 3,
"total": 33,
"collected": 33
},
"collectors": [
{
"nodeid": "",
"outcome": "passed",
"result": [
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py",
"type": "Module"
}
]
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py",
"outcome": "passed",
"result": [
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
"type": "Function",
"lineno": 72
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
"type": "Function",
"lineno": 72
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
"type": "Function",
"lineno": 72
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
"type": "Function",
"lineno": 72
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
"type": "Function",
"lineno": 72
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
"type": "Function",
"lineno": 72
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
"type": "Function",
"lineno": 91
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
"type": "Function",
"lineno": 91
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
"type": "Function",
"lineno": 91
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
"type": "Function",
"lineno": 91
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
"type": "Function",
"lineno": 91
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
"type": "Function",
"lineno": 91
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
"type": "Function",
"lineno": 115
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
"type": "Function",
"lineno": 115
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
"type": "Function",
"lineno": 115
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
"type": "Function",
"lineno": 134
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
"type": "Function",
"lineno": 134
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
"type": "Function",
"lineno": 134
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
"type": "Function",
"lineno": 158
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
"type": "Function",
"lineno": 158
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
"type": "Function",
"lineno": 158
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
"type": "Function",
"lineno": 158
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
"type": "Function",
"lineno": 158
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
"type": "Function",
"lineno": 158
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
"type": "Function",
"lineno": 181
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
"type": "Function",
"lineno": 181
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
"type": "Function",
"lineno": 181
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
"type": "Function",
"lineno": 181
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
"type": "Function",
"lineno": 181
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
"type": "Function",
"lineno": 181
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
"type": "Function",
"lineno": 203
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
"type": "Function",
"lineno": 203
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
"type": "Function",
"lineno": 203
}
]
}
],
"tests": [
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
"lineno": 72,
"outcome": "passed",
"keywords": [
"test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama-v3p3-70b-instruct-earth",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
"case_id": "earth"
},
"setup": {
"duration": 0.05236550001427531,
"outcome": "passed"
},
"call": {
"duration": 0.5364967910572886,
"outcome": "passed"
},
"teardown": {
"duration": 0.00015075004193931818,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
"lineno": 72,
"outcome": "passed",
"keywords": [
"test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama-v3p3-70b-instruct-saturn",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
"case_id": "saturn"
},
"setup": {
"duration": 0.00699599995277822,
"outcome": "passed"
},
"call": {
"duration": 0.5843954589217901,
"outcome": "passed"
},
"teardown": {
"duration": 0.0003858329728245735,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
"lineno": 72,
"outcome": "passed",
"keywords": [
"test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-scout-instruct-basic-earth",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
"case_id": "earth"
},
"setup": {
"duration": 0.009176500025205314,
"outcome": "passed"
},
"call": {
"duration": 0.9258683329680935,
"outcome": "passed"
},
"teardown": {
"duration": 0.00015787500888109207,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
"lineno": 72,
"outcome": "passed",
"keywords": [
"test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-scout-instruct-basic-saturn",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
"case_id": "saturn"
},
"setup": {
"duration": 0.011275375029072165,
"outcome": "passed"
},
"call": {
"duration": 0.6890578339807689,
"outcome": "passed"
},
"teardown": {
"duration": 0.0004926669644191861,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
"lineno": 72,
"outcome": "passed",
"keywords": [
"test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-maverick-instruct-basic-earth",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
"case_id": "earth"
},
"setup": {
"duration": 0.007520624902099371,
"outcome": "passed"
},
"call": {
"duration": 0.6675686669768766,
"outcome": "passed"
},
"teardown": {
"duration": 0.00016137503553181887,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
"lineno": 72,
"outcome": "passed",
"keywords": [
"test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-maverick-instruct-basic-saturn",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
"case_id": "saturn"
},
"setup": {
"duration": 0.0076431670458987355,
"outcome": "passed"
},
"call": {
"duration": 1.6813415409997106,
"outcome": "passed"
},
"teardown": {
"duration": 0.0004928340204060078,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
"lineno": 91,
"outcome": "passed",
"keywords": [
"test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama-v3p3-70b-instruct-earth",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
"case_id": "earth"
},
"setup": {
"duration": 0.01302404107991606,
"outcome": "passed"
},
"call": {
"duration": 1.3206909999717027,
"outcome": "passed"
},
"teardown": {
"duration": 0.0002220839960500598,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
"lineno": 91,
"outcome": "passed",
"keywords": [
"test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama-v3p3-70b-instruct-saturn",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
"case_id": "saturn"
},
"setup": {
"duration": 0.0071772499941289425,
"outcome": "passed"
},
"call": {
"duration": 0.4109888339880854,
"outcome": "passed"
},
"teardown": {
"duration": 0.0005431669997051358,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
"lineno": 91,
"outcome": "passed",
"keywords": [
"test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-scout-instruct-basic-earth",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
"case_id": "earth"
},
"setup": {
"duration": 0.012043708004057407,
"outcome": "passed"
},
"call": {
"duration": 0.4509220840409398,
"outcome": "passed"
},
"teardown": {
"duration": 0.00016408402007073164,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
"lineno": 91,
"outcome": "passed",
"keywords": [
"test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-scout-instruct-basic-saturn",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
"case_id": "saturn"
},
"setup": {
"duration": 0.007165874936617911,
"outcome": "passed"
},
"call": {
"duration": 0.6527335830032825,
"outcome": "passed"
},
"teardown": {
"duration": 0.0006419579731300473,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
"lineno": 91,
"outcome": "passed",
"keywords": [
"test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-maverick-instruct-basic-earth",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
"case_id": "earth"
},
"setup": {
"duration": 0.007546542095951736,
"outcome": "passed"
},
"call": {
"duration": 0.9360042089829221,
"outcome": "passed"
},
"teardown": {
"duration": 0.00020483299158513546,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
"lineno": 91,
"outcome": "passed",
"keywords": [
"test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-maverick-instruct-basic-saturn",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
"case_id": "saturn"
},
"setup": {
"duration": 0.046697250101715326,
"outcome": "passed"
},
"call": {
"duration": 0.668349124956876,
"outcome": "passed"
},
"teardown": {
"duration": 0.0005031249020248652,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
"lineno": 115,
"outcome": "skipped",
"keywords": [
"test_chat_non_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
"case_id": "case0"
},
"setup": {
"duration": 0.012287458986975253,
"outcome": "passed"
},
"call": {
"duration": 0.00015287497080862522,
"outcome": "skipped",
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai_api/test_chat_completion.py', 124, 'Skipped: Skipping test_chat_non_streaming_image for model accounts/fireworks/models/llama-v3p3-70b-instruct on provider fireworks based on config.')"
},
"teardown": {
"duration": 0.00012162502389401197,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
"lineno": 115,
"outcome": "passed",
"keywords": [
"test_chat_non_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-scout-instruct-basic-case0",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
"case_id": "case0"
},
"setup": {
"duration": 0.007204124936833978,
"outcome": "passed"
},
"call": {
"duration": 1.8676417920505628,
"outcome": "passed"
},
"teardown": {
"duration": 0.0001557499635964632,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
"lineno": 115,
"outcome": "passed",
"keywords": [
"test_chat_non_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
"case_id": "case0"
},
"setup": {
"duration": 0.008226625039242208,
"outcome": "passed"
},
"call": {
"duration": 3.2724285409785807,
"outcome": "passed"
},
"teardown": {
"duration": 0.0002898330567404628,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
"lineno": 134,
"outcome": "skipped",
"keywords": [
"test_chat_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
"case_id": "case0"
},
"setup": {
"duration": 0.011927249957807362,
"outcome": "passed"
},
"call": {
"duration": 0.00017358292825520039,
"outcome": "skipped",
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai_api/test_chat_completion.py', 143, 'Skipped: Skipping test_chat_streaming_image for model accounts/fireworks/models/llama-v3p3-70b-instruct on provider fireworks based on config.')"
},
"teardown": {
"duration": 0.00014037499204277992,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
"lineno": 134,
"outcome": "passed",
"keywords": [
"test_chat_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-scout-instruct-basic-case0",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
"case_id": "case0"
},
"setup": {
"duration": 0.008731417008675635,
"outcome": "passed"
},
"call": {
"duration": 2.8333610829431564,
"outcome": "passed"
},
"teardown": {
"duration": 0.0005132080987095833,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
"lineno": 134,
"outcome": "passed",
"keywords": [
"test_chat_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
"case_id": "case0"
},
"setup": {
"duration": 0.016569208004511893,
"outcome": "passed"
},
"call": {
"duration": 2.302010750048794,
"outcome": "passed"
},
"teardown": {
"duration": 0.00016108399722725153,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
"lineno": 158,
"outcome": "passed",
"keywords": [
"test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama-v3p3-70b-instruct-calendar",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
"case_id": "calendar"
},
"setup": {
"duration": 0.039960999973118305,
"outcome": "passed"
},
"call": {
"duration": 7.661373125039972,
"outcome": "passed"
},
"teardown": {
"duration": 0.00015833403449505568,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
"lineno": 158,
"outcome": "passed",
"keywords": [
"test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama-v3p3-70b-instruct-math",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
"case_id": "math"
},
"setup": {
"duration": 0.006928625050932169,
"outcome": "passed"
},
"call": {
"duration": 2.762534625013359,
"outcome": "passed"
},
"teardown": {
"duration": 0.0006561250193044543,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
"lineno": 158,
"outcome": "passed",
"keywords": [
"test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-scout-instruct-basic-calendar",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
"case_id": "calendar"
},
"setup": {
"duration": 0.008602249901741743,
"outcome": "passed"
},
"call": {
"duration": 0.8311484589939937,
"outcome": "passed"
},
"teardown": {
"duration": 0.0005021670367568731,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
"lineno": 158,
"outcome": "passed",
"keywords": [
"test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-scout-instruct-basic-math",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
"case_id": "math"
},
"setup": {
"duration": 0.015500334091484547,
"outcome": "passed"
},
"call": {
"duration": 2.505719291046262,
"outcome": "passed"
},
"teardown": {
"duration": 0.0002619170118123293,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
"lineno": 158,
"outcome": "passed",
"keywords": [
"test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-maverick-instruct-basic-calendar",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
"case_id": "calendar"
},
"setup": {
"duration": 0.01948041608557105,
"outcome": "passed"
},
"call": {
"duration": 0.6336237500654534,
"outcome": "passed"
},
"teardown": {
"duration": 0.00016637507360428572,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
"lineno": 158,
"outcome": "passed",
"keywords": [
"test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-maverick-instruct-basic-math",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
"case_id": "math"
},
"setup": {
"duration": 0.006810749997384846,
"outcome": "passed"
},
"call": {
"duration": 1.9086956249084324,
"outcome": "passed"
},
"teardown": {
"duration": 0.00018824997823685408,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
"lineno": 181,
"outcome": "passed",
"keywords": [
"test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama-v3p3-70b-instruct-calendar",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
"case_id": "calendar"
},
"setup": {
"duration": 0.007881582947447896,
"outcome": "passed"
},
"call": {
"duration": 0.7142562499502674,
"outcome": "passed"
},
"teardown": {
"duration": 0.0007035828894004226,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
"lineno": 181,
"outcome": "passed",
"keywords": [
"test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama-v3p3-70b-instruct-math",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
"case_id": "math"
},
"setup": {
"duration": 0.00848070892971009,
"outcome": "passed"
},
"call": {
"duration": 1.5210869159782305,
"outcome": "passed"
},
"teardown": {
"duration": 0.00021216599270701408,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
"lineno": 181,
"outcome": "passed",
"keywords": [
"test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-scout-instruct-basic-calendar",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
"case_id": "calendar"
},
"setup": {
"duration": 0.009669666993431747,
"outcome": "passed"
},
"call": {
"duration": 1.3105999580584466,
"outcome": "passed"
},
"teardown": {
"duration": 0.000588166993111372,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
"lineno": 181,
"outcome": "passed",
"keywords": [
"test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-scout-instruct-basic-math",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
"case_id": "math"
},
"setup": {
"duration": 0.007745541981421411,
"outcome": "passed"
},
"call": {
"duration": 3.250162083073519,
"outcome": "passed"
},
"teardown": {
"duration": 0.0001455000601708889,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
"lineno": 181,
"outcome": "passed",
"keywords": [
"test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-maverick-instruct-basic-calendar",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
"case_id": "calendar"
},
"setup": {
"duration": 0.009726207936182618,
"outcome": "passed"
},
"call": {
"duration": 0.5564592910232022,
"outcome": "passed"
},
"teardown": {
"duration": 0.00019470800179988146,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
"lineno": 181,
"outcome": "passed",
"keywords": [
"test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-maverick-instruct-basic-math",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
"case_id": "math"
},
"setup": {
"duration": 0.018431040924042463,
"outcome": "passed"
},
"call": {
"duration": 3.8501765420660377,
"outcome": "passed"
},
"teardown": {
"duration": 0.00015279196668416262,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
"lineno": 203,
"outcome": "failed",
"keywords": [
"test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
"case_id": "case0"
},
"setup": {
"duration": 0.007509749964810908,
"outcome": "passed"
},
"call": {
"duration": 0.4906975000631064,
"outcome": "failed",
"crash": {
"path": "/Users/erichuang/projects/llama-stack/tests/verifications/openai_api/test_chat_completion.py",
"lineno": 222,
"message": "TypeError: object of type 'NoneType' has no len()"
},
"traceback": [
{
"path": "tests/verifications/openai_api/test_chat_completion.py",
"lineno": 222,
"message": "TypeError"
}
],
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]>>\nopenai_client = <openai.OpenAI object at 0x117081540>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...el_display_names': {'gpt-4o': 'gpt-4o', 'gpt-4o-mini': 'gpt-4o-mini'}, 'models': ['gpt-4o', 'gpt-4o-mini'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n stream=False,\n )\n \n assert response.choices[0].message.role == \"assistant\"\n> assert len(response.choices[0].message.tool_calls) > 0\nE TypeError: object of type 'NoneType' has no len()\n\ntests/verifications/openai_api/test_chat_completion.py:222: TypeError"
},
"teardown": {
"duration": 0.00023995805531740189,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
"lineno": 203,
"outcome": "failed",
"keywords": [
"test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-scout-instruct-basic-case0",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
"case_id": "case0"
},
"setup": {
"duration": 0.007144959061406553,
"outcome": "passed"
},
"call": {
"duration": 3.818257624981925,
"outcome": "failed",
"crash": {
"path": "/Users/erichuang/projects/llama-stack/tests/verifications/openai_api/test_chat_completion.py",
"lineno": 222,
"message": "TypeError: object of type 'NoneType' has no len()"
},
"traceback": [
{
"path": "tests/verifications/openai_api/test_chat_completion.py",
"lineno": 222,
"message": "TypeError"
}
],
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x11701c1c0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...el_display_names': {'gpt-4o': 'gpt-4o', 'gpt-4o-mini': 'gpt-4o-mini'}, 'models': ['gpt-4o', 'gpt-4o-mini'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n stream=False,\n )\n \n assert response.choices[0].message.role == \"assistant\"\n> assert len(response.choices[0].message.tool_calls) > 0\nE TypeError: object of type 'NoneType' has no len()\n\ntests/verifications/openai_api/test_chat_completion.py:222: TypeError"
},
"teardown": {
"duration": 0.0002668750239536166,
"outcome": "passed"
}
},
{
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
"lineno": 203,
"outcome": "failed",
"keywords": [
"test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
"parametrize",
"pytestmark",
"accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
"test_chat_completion.py",
"openai_api",
"verifications",
"tests",
"llama-stack",
""
],
"metadata": {
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
"case_id": "case0"
},
"setup": {
"duration": 0.015290249953977764,
"outcome": "passed"
},
"call": {
"duration": 1.5883799999719486,
"outcome": "failed",
"crash": {
"path": "/Users/erichuang/projects/llama-stack/tests/verifications/openai_api/test_chat_completion.py",
"lineno": 222,
"message": "TypeError: object of type 'NoneType' has no len()"
},
"traceback": [
{
"path": "tests/verifications/openai_api/test_chat_completion.py",
"lineno": 222,
"message": "TypeError"
}
],
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x115f17fa0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...el_display_names': {'gpt-4o': 'gpt-4o', 'gpt-4o-mini': 'gpt-4o-mini'}, 'models': ['gpt-4o', 'gpt-4o-mini'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n stream=False,\n )\n \n assert response.choices[0].message.role == \"assistant\"\n> assert len(response.choices[0].message.tool_calls) > 0\nE TypeError: object of type 'NoneType' has no len()\n\ntests/verifications/openai_api/test_chat_completion.py:222: TypeError"
},
"teardown": {
"duration": 0.0008049579337239265,
"outcome": "passed"
}
}
]
}
Reference in a new issue View git blame Copy permalink