mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-17 07:07:19 +00:00
bcbc56baa2
# What does this PR do? ## Test Plan pytest verifications/openai/test_chat_completion.py --provider together
2744 lines
96 KiB
JSON
2744 lines
96 KiB
JSON
{
|
|
"created": 1744154399.039055,
|
|
"duration": 87.73799800872803,
|
|
"exitcode": 1,
|
|
"root": "/Users/erichuang/projects/llama-stack",
|
|
"environment": {},
|
|
"summary": {
|
|
"skipped": 52,
|
|
"passed": 28,
|
|
"failed": 3,
|
|
"total": 83,
|
|
"collected": 83
|
|
},
|
|
"collectors": [
|
|
{
|
|
"nodeid": "",
|
|
"outcome": "passed",
|
|
"result": [
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py",
|
|
"type": "Module"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py",
|
|
"outcome": "passed",
|
|
"result": [
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-3.3-8B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-3.3-70B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Scout-17B-16E]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-gpt-4o]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-gpt-4o-mini]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-3.3-8B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-3.3-70B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Scout-17B-16E]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Scout-17B-16E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Maverick-17B-128E]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-gpt-4o]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-gpt-4o-mini]",
|
|
"type": "Function",
|
|
"lineno": 25
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-3.3-8B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-3.3-70B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Scout-17B-16E]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-gpt-4o]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-gpt-4o-mini]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-3.3-8B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-3.3-70B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Scout-17B-16E]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Scout-17B-16E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Maverick-17B-128E]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-gpt-4o]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-gpt-4o-mini]",
|
|
"type": "Function",
|
|
"lineno": 40
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Scout-17B-16E]",
|
|
"type": "Function",
|
|
"lineno": 60
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 60
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"type": "Function",
|
|
"lineno": 60
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 60
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-gpt-4o]",
|
|
"type": "Function",
|
|
"lineno": 60
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-gpt-4o-mini]",
|
|
"type": "Function",
|
|
"lineno": 60
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Scout-17B-16E]",
|
|
"type": "Function",
|
|
"lineno": 75
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 75
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"type": "Function",
|
|
"lineno": 75
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 75
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-gpt-4o]",
|
|
"type": "Function",
|
|
"lineno": 75
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-gpt-4o-mini]",
|
|
"type": "Function",
|
|
"lineno": 75
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-3.3-8B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-3.3-70B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-gpt-4o]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-gpt-4o-mini]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-3.3-8B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-3.3-70B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-gpt-4o]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-gpt-4o-mini]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-3.3-8B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-3.3-70B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-gpt-4o]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-gpt-4o-mini]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-3.3-8B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-3.3-70B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-gpt-4o]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-gpt-4o-mini]",
|
|
"type": "Function",
|
|
"lineno": 117
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-3.3-70B-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 138
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Scout-17B-16E]",
|
|
"type": "Function",
|
|
"lineno": 138
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 138
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"type": "Function",
|
|
"lineno": 138
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"type": "Function",
|
|
"lineno": 138
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-gpt-4o]",
|
|
"type": "Function",
|
|
"lineno": 138
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-gpt-4o-mini]",
|
|
"type": "Function",
|
|
"lineno": 138
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"tests": [
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-3.3-8B-Instruct]",
|
|
"lineno": 25,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output0-Llama-3.3-8B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-3.3-8B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.17320987500716,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.000177707988768816,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-3.3-70B-Instruct]",
|
|
"lineno": 25,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output0-Llama-3.3-70B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-3.3-70B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.009193749981932342,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 1.1473859580000862,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00043337501119822264,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Scout-17B-16E]",
|
|
"lineno": 25,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output0-Llama-4-Scout-17B-16E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Scout-17B-16E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.01645291701424867,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0002898749662563205,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"lineno": 25,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Scout-17B-16E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.01562033302616328,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.8782661251025274,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0002795408945530653,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"lineno": 25,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Maverick-17B-128E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.008571124984882772,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003043749602511525,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"lineno": 25,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Maverick-17B-128E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.00842841702979058,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 1.3863223339430988,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0009970410028472543,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-gpt-4o]",
|
|
"lineno": 25,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output0-gpt-4o]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-gpt-4o",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.007089875056408346,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model gpt-4o')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00017958390526473522,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-gpt-4o-mini]",
|
|
"lineno": 25,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output0-gpt-4o-mini]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-gpt-4o-mini",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.005809499998576939,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model gpt-4o-mini')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00016495899762958288,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-3.3-8B-Instruct]",
|
|
"lineno": 25,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output1-Llama-3.3-8B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-3.3-8B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.0119722920935601,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00016962504014372826,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-3.3-70B-Instruct]",
|
|
"lineno": 25,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output1-Llama-3.3-70B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-3.3-70B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.005716291954740882,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.6822018750244752,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0005292498972266912,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Scout-17B-16E]",
|
|
"lineno": 25,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output1-Llama-4-Scout-17B-16E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Scout-17B-16E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.025827708072029054,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.000295999925583601,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Scout-17B-16E-Instruct]",
|
|
"lineno": 25,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output1-Llama-4-Scout-17B-16E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Scout-17B-16E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.010980832972563803,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.7537062909686938,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0008091670460999012,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Maverick-17B-128E]",
|
|
"lineno": 25,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output1-Llama-4-Maverick-17B-128E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Maverick-17B-128E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.006567832897417247,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0001545000122860074,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"lineno": 25,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output1-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Maverick-17B-128E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.005985083989799023,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.7263387079583481,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0006324589485302567,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-gpt-4o]",
|
|
"lineno": 25,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output1-gpt-4o]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-gpt-4o",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.0171962499152869,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model gpt-4o')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.000780042028054595,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-gpt-4o-mini]",
|
|
"lineno": 25,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[input_output1-gpt-4o-mini]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-gpt-4o-mini",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.01365620899014175,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model gpt-4o-mini')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00016758404672145844,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-3.3-8B-Instruct]",
|
|
"lineno": 40,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output0-Llama-3.3-8B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-3.3-8B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.0064070840599015355,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0002031669719144702,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-3.3-70B-Instruct]",
|
|
"lineno": 40,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output0-Llama-3.3-70B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-3.3-70B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.010951624950394034,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.5433399169705808,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0013178749941289425,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Scout-17B-16E]",
|
|
"lineno": 40,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output0-Llama-4-Scout-17B-16E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Scout-17B-16E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.022056750021874905,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0006570409750565886,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"lineno": 40,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Scout-17B-16E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.008314333041198552,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.7779882500180975,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0006799160037189722,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"lineno": 40,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Maverick-17B-128E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.03601404093205929,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.000610582996159792,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"lineno": 40,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Maverick-17B-128E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.014321292052045465,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 1.0243758750148118,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0010485410457476974,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-gpt-4o]",
|
|
"lineno": 40,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output0-gpt-4o]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-gpt-4o",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.021133000031113625,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model gpt-4o')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0005400830414146185,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-gpt-4o-mini]",
|
|
"lineno": 40,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output0-gpt-4o-mini]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-gpt-4o-mini",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.007212458993308246,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model gpt-4o-mini')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00026770797558128834,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-3.3-8B-Instruct]",
|
|
"lineno": 40,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output1-Llama-3.3-8B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-3.3-8B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.012334750033915043,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00042683398351073265,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-3.3-70B-Instruct]",
|
|
"lineno": 40,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output1-Llama-3.3-70B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-3.3-70B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.011477917083539069,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 1.670572166913189,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0005759169580414891,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Scout-17B-16E]",
|
|
"lineno": 40,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output1-Llama-4-Scout-17B-16E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Scout-17B-16E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.024620208074338734,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0005166250048205256,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Scout-17B-16E-Instruct]",
|
|
"lineno": 40,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output1-Llama-4-Scout-17B-16E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Scout-17B-16E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.008708957931958139,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.6654335829662159,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0002927089808508754,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Maverick-17B-128E]",
|
|
"lineno": 40,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output1-Llama-4-Maverick-17B-128E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Maverick-17B-128E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.018128167022950947,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0001929170684888959,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"lineno": 40,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output1-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Maverick-17B-128E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.0063874589977785945,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.8047525839647278,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00039245898369699717,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-gpt-4o]",
|
|
"lineno": 40,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output1-gpt-4o]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-gpt-4o",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.01366533397231251,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model gpt-4o')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00028241705149412155,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-gpt-4o-mini]",
|
|
"lineno": 40,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[input_output1-gpt-4o-mini]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-gpt-4o-mini",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.010844790958799422,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model gpt-4o-mini')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.000258082989603281,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Scout-17B-16E]",
|
|
"lineno": 60,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_image[input_output0-Llama-4-Scout-17B-16E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Scout-17B-16E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.00936354196164757,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 61, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00020533299539238214,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"lineno": 60,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_image[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Scout-17B-16E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.008578249951824546,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 2.6288582499837503,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0006052498938515782,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"lineno": 60,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_image[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Maverick-17B-128E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.02061279199551791,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 61, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00029320805333554745,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"lineno": 60,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_image[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Maverick-17B-128E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.00995812495239079,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 3.0904540000483394,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003214169992133975,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-gpt-4o]",
|
|
"lineno": 60,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_image[input_output0-gpt-4o]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-gpt-4o",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.0261635419446975,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 61, 'Skipped: Provider fireworks does not support model gpt-4o')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00032716698478907347,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-gpt-4o-mini]",
|
|
"lineno": 60,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_image[input_output0-gpt-4o-mini]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-gpt-4o-mini",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.027220541960559785,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 61, 'Skipped: Provider fireworks does not support model gpt-4o-mini')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003192499279975891,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Scout-17B-16E]",
|
|
"lineno": 75,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_image[input_output0-Llama-4-Scout-17B-16E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Scout-17B-16E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.010883458075113595,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 76, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0002687909873202443,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"lineno": 75,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_image[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Scout-17B-16E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.0171177500160411,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 1.6752691670553759,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004877089522778988,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"lineno": 75,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_image[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Maverick-17B-128E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.011608208995312452,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 76, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00017137499526143074,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"lineno": 75,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_image[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Maverick-17B-128E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.009284624946303666,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 3.537356249988079,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0005068340105935931,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-gpt-4o]",
|
|
"lineno": 75,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_image[input_output0-gpt-4o]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-gpt-4o",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.016660499968566,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 76, 'Skipped: Provider fireworks does not support model gpt-4o')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00029341597110033035,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-gpt-4o-mini]",
|
|
"lineno": 75,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_image[input_output0-gpt-4o-mini]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-gpt-4o-mini",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.01374066702555865,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 76, 'Skipped: Provider fireworks does not support model gpt-4o-mini')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0002625000197440386,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-3.3-8B-Instruct]",
|
|
"lineno": 95,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output0-Llama-3.3-8B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-3.3-8B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.013120374991558492,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00021954195108264685,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-3.3-70B-Instruct]",
|
|
"lineno": 95,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output0-Llama-3.3-70B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-3.3-70B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.015080374898388982,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 1.157175041968003,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.000495875021442771,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E]",
|
|
"lineno": 95,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Scout-17B-16E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.013946042046882212,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0002954580122604966,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"lineno": 95,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Scout-17B-16E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.011617792071774602,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.9537639999762177,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004819999448955059,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"lineno": 95,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Maverick-17B-128E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.027436082949861884,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00030274991877377033,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"lineno": 95,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Maverick-17B-128E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.016110333963297307,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.8493227910948917,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004883749643340707,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-gpt-4o]",
|
|
"lineno": 95,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output0-gpt-4o]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-gpt-4o",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.017850833013653755,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model gpt-4o')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003287500003352761,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-gpt-4o-mini]",
|
|
"lineno": 95,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output0-gpt-4o-mini]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-gpt-4o-mini",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.012523208046332002,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model gpt-4o-mini')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00023500004317611456,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-3.3-8B-Instruct]",
|
|
"lineno": 95,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output1-Llama-3.3-8B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-3.3-8B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.007516667013987899,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00018912507221102715,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-3.3-70B-Instruct]",
|
|
"lineno": 95,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output1-Llama-3.3-70B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-3.3-70B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.007337165996432304,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 3.124099582899362,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0006703329272568226,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E]",
|
|
"lineno": 95,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Scout-17B-16E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.014259999967180192,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00030262500513345003,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E-Instruct]",
|
|
"lineno": 95,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Scout-17B-16E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.010863124975003302,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 1.3330956250429153,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00018679199274629354,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E]",
|
|
"lineno": 95,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Maverick-17B-128E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.005797958001494408,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00017529097385704517,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"lineno": 95,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Maverick-17B-128E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.005647709011100233,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 3.2295467499643564,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0005654999986290932,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-gpt-4o]",
|
|
"lineno": 95,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output1-gpt-4o]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-gpt-4o",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.007151791942305863,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model gpt-4o')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00015316694043576717,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-gpt-4o-mini]",
|
|
"lineno": 95,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[input_output1-gpt-4o-mini]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-gpt-4o-mini",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.006435790914110839,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model gpt-4o-mini')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00015954102855175734,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-3.3-8B-Instruct]",
|
|
"lineno": 117,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output0-Llama-3.3-8B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-3.3-8B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.006164791993796825,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00014074996579438448,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-3.3-70B-Instruct]",
|
|
"lineno": 117,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output0-Llama-3.3-70B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-3.3-70B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.010064583038911223,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 1.1676458748988807,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0002513329964131117,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E]",
|
|
"lineno": 117,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Scout-17B-16E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.011011417023837566,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00020608294289559126,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"lineno": 117,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Scout-17B-16E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.011654542060568929,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.7950789160095155,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0002690000692382455,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"lineno": 117,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Maverick-17B-128E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.0066834589233621955,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00017270795069634914,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"lineno": 117,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Maverick-17B-128E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.011390416999347508,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.7844940840732306,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.000511458027176559,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-gpt-4o]",
|
|
"lineno": 117,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output0-gpt-4o]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-gpt-4o",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.005813500029034913,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model gpt-4o')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00015495799016207457,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-gpt-4o-mini]",
|
|
"lineno": 117,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output0-gpt-4o-mini]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-gpt-4o-mini",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.0075639160349965096,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model gpt-4o-mini')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00014358304906636477,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-3.3-8B-Instruct]",
|
|
"lineno": 117,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output1-Llama-3.3-8B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-3.3-8B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.008526541059836745,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00015841599088162184,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-3.3-70B-Instruct]",
|
|
"lineno": 117,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output1-Llama-3.3-70B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-3.3-70B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.007805416011251509,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 13.25898533302825,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py",
|
|
"lineno": 196,
|
|
"message": "assert None is not None"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai/test_chat_completion.py",
|
|
"lineno": 136,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "tests/verifications/openai/test_chat_completion.py",
|
|
"lineno": 196,
|
|
"message": "AssertionError"
|
|
}
|
|
],
|
|
"longrepr": "openai_client = <openai.OpenAI object at 0x105cc8e50>\ninput_output = {'input': {'messages': [{'content': 'You are a helpful math tutor. Guide the user through the solution step by step.',... ['steps', 'final_answer'], 'title': 'MathReasoning', ...}}, 'type': 'json_schema'}}, 'output': 'valid_math_reasoning'}\ncorrect_model_name = 'accounts/fireworks/models/llama-v3p1-70b-instruct'\n\n @pytest.mark.parametrize(\n \"model\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"model\"],\n )\n @pytest.mark.parametrize(\n \"input_output\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"input_output\"],\n )\n def test_chat_streaming_structured_output(openai_client, input_output, correct_model_name):\n response = openai_client.chat.completions.create(\n model=correct_model_name,\n messages=input_output[\"input\"][\"messages\"],\n response_format=input_output[\"input\"][\"response_format\"],\n stream=True,\n )\n maybe_json_content = \"\"\n for chunk in response:\n maybe_json_content += chunk.choices[0].delta.content or \"\"\n> validate_structured_output(maybe_json_content, input_output[\"output\"])\n\ntests/verifications/openai/test_chat_completion.py:136: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nmaybe_json_content = '{ \"final_answer\": \"}To solve the equation 8x + 7 = -23, we need to isolate the variable x. We can do this by followin...tassistantassistantassistantassistantassistantassistantassistantassistantassistantassistantassistantassistantassistant'\nschema_name = 'valid_math_reasoning'\n\n def validate_structured_output(maybe_json_content: str, schema_name: str) -> None:\n structured_output = get_structured_output(maybe_json_content, schema_name)\n> assert structured_output is not None\nE assert None is not None\n\ntests/verifications/openai/test_chat_completion.py:196: AssertionError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00022583396639674902,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E]",
|
|
"lineno": 117,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Scout-17B-16E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.006412541959434748,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0001449589617550373,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E-Instruct]",
|
|
"lineno": 117,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Scout-17B-16E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.010353000019676983,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 4.559281209018081,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00021179206669330597,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E]",
|
|
"lineno": 117,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Maverick-17B-128E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.011320417048409581,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0001623749267309904,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"lineno": 117,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-Llama-4-Maverick-17B-128E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.005637791007757187,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 2.9282109580235556,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00019149994477629662,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-gpt-4o]",
|
|
"lineno": 117,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output1-gpt-4o]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-gpt-4o",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.021475916961207986,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model gpt-4o')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0002605828922241926,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-gpt-4o-mini]",
|
|
"lineno": 117,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[input_output1-gpt-4o-mini]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output1-gpt-4o-mini",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.012046082993037999,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model gpt-4o-mini')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00016966694965958595,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-3.3-70B-Instruct]",
|
|
"lineno": 138,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_calling[input_output0-Llama-3.3-70B-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-3.3-70B-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.00782629195600748,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.9290615000063553,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004110001027584076,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Scout-17B-16E]",
|
|
"lineno": 138,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_calling[input_output0-Llama-4-Scout-17B-16E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Scout-17B-16E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.00842183397617191,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 139, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00023745803628116846,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"lineno": 138,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_calling[input_output0-Llama-4-Scout-17B-16E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Scout-17B-16E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.010762874968349934,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 23.62101216695737,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py",
|
|
"lineno": 156,
|
|
"message": "TypeError: object of type 'NoneType' has no len()"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai/test_chat_completion.py",
|
|
"lineno": 156,
|
|
"message": "TypeError"
|
|
}
|
|
],
|
|
"longrepr": "openai_client = <openai.OpenAI object at 0x105d11f90>\ninput_output = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\ncorrect_model_name = 'accounts/fireworks/models/llama4-scout-instruct-basic'\n\n @pytest.mark.parametrize(\n \"model\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"model\"],\n )\n @pytest.mark.parametrize(\n \"input_output\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"input_output\"],\n )\n def test_chat_non_streaming_tool_calling(openai_client, input_output, correct_model_name):\n response = openai_client.chat.completions.create(\n model=correct_model_name,\n messages=input_output[\"input\"][\"messages\"],\n tools=input_output[\"input\"][\"tools\"],\n stream=False,\n )\n \n assert response.choices[0].message.role == \"assistant\"\n> assert len(response.choices[0].message.tool_calls) > 0\nE TypeError: object of type 'NoneType' has no len()\n\ntests/verifications/openai/test_chat_completion.py:156: TypeError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004520840011537075,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"lineno": 138,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_calling[input_output0-Llama-4-Maverick-17B-128E]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Maverick-17B-128E",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.00953104195650667,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 139, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00017912499606609344,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"lineno": 138,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_calling[input_output0-Llama-4-Maverick-17B-128E-Instruct]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-Llama-4-Maverick-17B-128E-Instruct",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.010302042006514966,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 5.55651158397086,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py",
|
|
"lineno": 156,
|
|
"message": "TypeError: object of type 'NoneType' has no len()"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai/test_chat_completion.py",
|
|
"lineno": 156,
|
|
"message": "TypeError"
|
|
}
|
|
],
|
|
"longrepr": "openai_client = <openai.OpenAI object at 0x1062017b0>\ninput_output = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\ncorrect_model_name = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\n\n @pytest.mark.parametrize(\n \"model\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"model\"],\n )\n @pytest.mark.parametrize(\n \"input_output\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"input_output\"],\n )\n def test_chat_non_streaming_tool_calling(openai_client, input_output, correct_model_name):\n response = openai_client.chat.completions.create(\n model=correct_model_name,\n messages=input_output[\"input\"][\"messages\"],\n tools=input_output[\"input\"][\"tools\"],\n stream=False,\n )\n \n assert response.choices[0].message.role == \"assistant\"\n> assert len(response.choices[0].message.tool_calls) > 0\nE TypeError: object of type 'NoneType' has no len()\n\ntests/verifications/openai/test_chat_completion.py:156: TypeError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003929579397663474,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-gpt-4o]",
|
|
"lineno": 138,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_calling[input_output0-gpt-4o]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-gpt-4o",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.01593891705852002,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 139, 'Skipped: Provider fireworks does not support model gpt-4o')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003579579060897231,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-gpt-4o-mini]",
|
|
"lineno": 138,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_calling[input_output0-gpt-4o-mini]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"input_output0-gpt-4o-mini",
|
|
"test_chat_completion.py",
|
|
"openai",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"setup": {
|
|
"duration": 0.01874550001230091,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 139, 'Skipped: Provider fireworks does not support model gpt-4o-mini')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00031995808240026236,
|
|
"outcome": "passed"
|
|
}
|
|
}
|
|
]
|
|
}
|