{ "created": 1744154399.039055, "duration": 87.73799800872803, "exitcode": 1, "root": "/Users/erichuang/projects/llama-stack", "environment": {}, "summary": { "skipped": 52, "passed": 28, "failed": 3, "total": 83, "collected": 83 }, "collectors": [ { "nodeid": "", "outcome": "passed", "result": [ { "nodeid": "tests/verifications/openai/test_chat_completion.py", "type": "Module" } ] }, { "nodeid": "tests/verifications/openai/test_chat_completion.py", "outcome": "passed", "result": [ { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-3.3-8B-Instruct]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-3.3-70B-Instruct]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Scout-17B-16E]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Scout-17B-16E-Instruct]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Maverick-17B-128E]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-gpt-4o]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-gpt-4o-mini]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-3.3-8B-Instruct]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-3.3-70B-Instruct]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Scout-17B-16E]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Scout-17B-16E-Instruct]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Maverick-17B-128E]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Maverick-17B-128E-Instruct]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-gpt-4o]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-gpt-4o-mini]", "type": "Function", "lineno": 25 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-3.3-8B-Instruct]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-3.3-70B-Instruct]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Scout-17B-16E]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Scout-17B-16E-Instruct]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Maverick-17B-128E]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-gpt-4o]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-gpt-4o-mini]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-3.3-8B-Instruct]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-3.3-70B-Instruct]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Scout-17B-16E]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Scout-17B-16E-Instruct]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Maverick-17B-128E]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Maverick-17B-128E-Instruct]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-gpt-4o]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-gpt-4o-mini]", "type": "Function", "lineno": 40 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Scout-17B-16E]", "type": "Function", "lineno": 60 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Scout-17B-16E-Instruct]", "type": "Function", "lineno": 60 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Maverick-17B-128E]", "type": "Function", "lineno": 60 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "type": "Function", "lineno": 60 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-gpt-4o]", "type": "Function", "lineno": 60 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-gpt-4o-mini]", "type": "Function", "lineno": 60 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Scout-17B-16E]", "type": "Function", "lineno": 75 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Scout-17B-16E-Instruct]", "type": "Function", "lineno": 75 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Maverick-17B-128E]", "type": "Function", "lineno": 75 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "type": "Function", "lineno": 75 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-gpt-4o]", "type": "Function", "lineno": 75 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-gpt-4o-mini]", "type": "Function", "lineno": 75 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-3.3-8B-Instruct]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-3.3-70B-Instruct]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E-Instruct]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-gpt-4o]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-gpt-4o-mini]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-3.3-8B-Instruct]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-3.3-70B-Instruct]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E-Instruct]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E-Instruct]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-gpt-4o]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-gpt-4o-mini]", "type": "Function", "lineno": 95 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-3.3-8B-Instruct]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-3.3-70B-Instruct]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E-Instruct]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-gpt-4o]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-gpt-4o-mini]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-3.3-8B-Instruct]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-3.3-70B-Instruct]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E-Instruct]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E-Instruct]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-gpt-4o]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-gpt-4o-mini]", "type": "Function", "lineno": 117 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-3.3-70B-Instruct]", "type": "Function", "lineno": 138 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Scout-17B-16E]", "type": "Function", "lineno": 138 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Scout-17B-16E-Instruct]", "type": "Function", "lineno": 138 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Maverick-17B-128E]", "type": "Function", "lineno": 138 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "type": "Function", "lineno": 138 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-gpt-4o]", "type": "Function", "lineno": 138 }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-gpt-4o-mini]", "type": "Function", "lineno": 138 } ] } ], "tests": [ { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-3.3-8B-Instruct]", "lineno": 25, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_basic[input_output0-Llama-3.3-8B-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-3.3-8B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.17320987500716, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')" }, "teardown": { "duration": 0.000177707988768816, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-3.3-70B-Instruct]", "lineno": 25, "outcome": "passed", "keywords": [ "test_chat_non_streaming_basic[input_output0-Llama-3.3-70B-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-3.3-70B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.009193749981932342, "outcome": "passed" }, "call": { "duration": 1.1473859580000862, "outcome": "passed" }, "teardown": { "duration": 0.00043337501119822264, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Scout-17B-16E]", "lineno": 25, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_basic[input_output0-Llama-4-Scout-17B-16E]", "parametrize", "pytestmark", "input_output0-Llama-4-Scout-17B-16E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.01645291701424867, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')" }, "teardown": { "duration": 0.0002898749662563205, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Scout-17B-16E-Instruct]", "lineno": 25, "outcome": "passed", "keywords": [ "test_chat_non_streaming_basic[input_output0-Llama-4-Scout-17B-16E-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-4-Scout-17B-16E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.01562033302616328, "outcome": "passed" }, "call": { "duration": 0.8782661251025274, "outcome": "passed" }, "teardown": { "duration": 0.0002795408945530653, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Maverick-17B-128E]", "lineno": 25, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_basic[input_output0-Llama-4-Maverick-17B-128E]", "parametrize", "pytestmark", "input_output0-Llama-4-Maverick-17B-128E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.008571124984882772, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')" }, "teardown": { "duration": 0.0003043749602511525, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "lineno": 25, "outcome": "passed", "keywords": [ "test_chat_non_streaming_basic[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-4-Maverick-17B-128E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.00842841702979058, "outcome": "passed" }, "call": { "duration": 1.3863223339430988, "outcome": "passed" }, "teardown": { "duration": 0.0009970410028472543, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-gpt-4o]", "lineno": 25, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_basic[input_output0-gpt-4o]", "parametrize", "pytestmark", "input_output0-gpt-4o", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.007089875056408346, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model gpt-4o')" }, "teardown": { "duration": 0.00017958390526473522, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output0-gpt-4o-mini]", "lineno": 25, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_basic[input_output0-gpt-4o-mini]", "parametrize", "pytestmark", "input_output0-gpt-4o-mini", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.005809499998576939, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model gpt-4o-mini')" }, "teardown": { "duration": 0.00016495899762958288, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-3.3-8B-Instruct]", "lineno": 25, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_basic[input_output1-Llama-3.3-8B-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-3.3-8B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.0119722920935601, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')" }, "teardown": { "duration": 0.00016962504014372826, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-3.3-70B-Instruct]", "lineno": 25, "outcome": "passed", "keywords": [ "test_chat_non_streaming_basic[input_output1-Llama-3.3-70B-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-3.3-70B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.005716291954740882, "outcome": "passed" }, "call": { "duration": 0.6822018750244752, "outcome": "passed" }, "teardown": { "duration": 0.0005292498972266912, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Scout-17B-16E]", "lineno": 25, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_basic[input_output1-Llama-4-Scout-17B-16E]", "parametrize", "pytestmark", "input_output1-Llama-4-Scout-17B-16E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.025827708072029054, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')" }, "teardown": { "duration": 0.000295999925583601, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Scout-17B-16E-Instruct]", "lineno": 25, "outcome": "passed", "keywords": [ "test_chat_non_streaming_basic[input_output1-Llama-4-Scout-17B-16E-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-4-Scout-17B-16E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.010980832972563803, "outcome": "passed" }, "call": { "duration": 0.7537062909686938, "outcome": "passed" }, "teardown": { "duration": 0.0008091670460999012, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Maverick-17B-128E]", "lineno": 25, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_basic[input_output1-Llama-4-Maverick-17B-128E]", "parametrize", "pytestmark", "input_output1-Llama-4-Maverick-17B-128E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.006567832897417247, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')" }, "teardown": { "duration": 0.0001545000122860074, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-Llama-4-Maverick-17B-128E-Instruct]", "lineno": 25, "outcome": "passed", "keywords": [ "test_chat_non_streaming_basic[input_output1-Llama-4-Maverick-17B-128E-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-4-Maverick-17B-128E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.005985083989799023, "outcome": "passed" }, "call": { "duration": 0.7263387079583481, "outcome": "passed" }, "teardown": { "duration": 0.0006324589485302567, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-gpt-4o]", "lineno": 25, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_basic[input_output1-gpt-4o]", "parametrize", "pytestmark", "input_output1-gpt-4o", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.0171962499152869, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model gpt-4o')" }, "teardown": { "duration": 0.000780042028054595, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_basic[input_output1-gpt-4o-mini]", "lineno": 25, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_basic[input_output1-gpt-4o-mini]", "parametrize", "pytestmark", "input_output1-gpt-4o-mini", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.01365620899014175, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 26, 'Skipped: Provider fireworks does not support model gpt-4o-mini')" }, "teardown": { "duration": 0.00016758404672145844, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-3.3-8B-Instruct]", "lineno": 40, "outcome": "skipped", "keywords": [ "test_chat_streaming_basic[input_output0-Llama-3.3-8B-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-3.3-8B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.0064070840599015355, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')" }, "teardown": { "duration": 0.0002031669719144702, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-3.3-70B-Instruct]", "lineno": 40, "outcome": "passed", "keywords": [ "test_chat_streaming_basic[input_output0-Llama-3.3-70B-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-3.3-70B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.010951624950394034, "outcome": "passed" }, "call": { "duration": 0.5433399169705808, "outcome": "passed" }, "teardown": { "duration": 0.0013178749941289425, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Scout-17B-16E]", "lineno": 40, "outcome": "skipped", "keywords": [ "test_chat_streaming_basic[input_output0-Llama-4-Scout-17B-16E]", "parametrize", "pytestmark", "input_output0-Llama-4-Scout-17B-16E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.022056750021874905, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')" }, "teardown": { "duration": 0.0006570409750565886, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Scout-17B-16E-Instruct]", "lineno": 40, "outcome": "passed", "keywords": [ "test_chat_streaming_basic[input_output0-Llama-4-Scout-17B-16E-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-4-Scout-17B-16E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.008314333041198552, "outcome": "passed" }, "call": { "duration": 0.7779882500180975, "outcome": "passed" }, "teardown": { "duration": 0.0006799160037189722, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Maverick-17B-128E]", "lineno": 40, "outcome": "skipped", "keywords": [ "test_chat_streaming_basic[input_output0-Llama-4-Maverick-17B-128E]", "parametrize", "pytestmark", "input_output0-Llama-4-Maverick-17B-128E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.03601404093205929, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')" }, "teardown": { "duration": 0.000610582996159792, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "lineno": 40, "outcome": "passed", "keywords": [ "test_chat_streaming_basic[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-4-Maverick-17B-128E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.014321292052045465, "outcome": "passed" }, "call": { "duration": 1.0243758750148118, "outcome": "passed" }, "teardown": { "duration": 0.0010485410457476974, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-gpt-4o]", "lineno": 40, "outcome": "skipped", "keywords": [ "test_chat_streaming_basic[input_output0-gpt-4o]", "parametrize", "pytestmark", "input_output0-gpt-4o", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.021133000031113625, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model gpt-4o')" }, "teardown": { "duration": 0.0005400830414146185, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output0-gpt-4o-mini]", "lineno": 40, "outcome": "skipped", "keywords": [ "test_chat_streaming_basic[input_output0-gpt-4o-mini]", "parametrize", "pytestmark", "input_output0-gpt-4o-mini", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.007212458993308246, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model gpt-4o-mini')" }, "teardown": { "duration": 0.00026770797558128834, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-3.3-8B-Instruct]", "lineno": 40, "outcome": "skipped", "keywords": [ "test_chat_streaming_basic[input_output1-Llama-3.3-8B-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-3.3-8B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.012334750033915043, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')" }, "teardown": { "duration": 0.00042683398351073265, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-3.3-70B-Instruct]", "lineno": 40, "outcome": "passed", "keywords": [ "test_chat_streaming_basic[input_output1-Llama-3.3-70B-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-3.3-70B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.011477917083539069, "outcome": "passed" }, "call": { "duration": 1.670572166913189, "outcome": "passed" }, "teardown": { "duration": 0.0005759169580414891, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Scout-17B-16E]", "lineno": 40, "outcome": "skipped", "keywords": [ "test_chat_streaming_basic[input_output1-Llama-4-Scout-17B-16E]", "parametrize", "pytestmark", "input_output1-Llama-4-Scout-17B-16E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.024620208074338734, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')" }, "teardown": { "duration": 0.0005166250048205256, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Scout-17B-16E-Instruct]", "lineno": 40, "outcome": "passed", "keywords": [ "test_chat_streaming_basic[input_output1-Llama-4-Scout-17B-16E-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-4-Scout-17B-16E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.008708957931958139, "outcome": "passed" }, "call": { "duration": 0.6654335829662159, "outcome": "passed" }, "teardown": { "duration": 0.0002927089808508754, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Maverick-17B-128E]", "lineno": 40, "outcome": "skipped", "keywords": [ "test_chat_streaming_basic[input_output1-Llama-4-Maverick-17B-128E]", "parametrize", "pytestmark", "input_output1-Llama-4-Maverick-17B-128E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.018128167022950947, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')" }, "teardown": { "duration": 0.0001929170684888959, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-Llama-4-Maverick-17B-128E-Instruct]", "lineno": 40, "outcome": "passed", "keywords": [ "test_chat_streaming_basic[input_output1-Llama-4-Maverick-17B-128E-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-4-Maverick-17B-128E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.0063874589977785945, "outcome": "passed" }, "call": { "duration": 0.8047525839647278, "outcome": "passed" }, "teardown": { "duration": 0.00039245898369699717, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-gpt-4o]", "lineno": 40, "outcome": "skipped", "keywords": [ "test_chat_streaming_basic[input_output1-gpt-4o]", "parametrize", "pytestmark", "input_output1-gpt-4o", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.01366533397231251, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model gpt-4o')" }, "teardown": { "duration": 0.00028241705149412155, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_basic[input_output1-gpt-4o-mini]", "lineno": 40, "outcome": "skipped", "keywords": [ "test_chat_streaming_basic[input_output1-gpt-4o-mini]", "parametrize", "pytestmark", "input_output1-gpt-4o-mini", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.010844790958799422, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 41, 'Skipped: Provider fireworks does not support model gpt-4o-mini')" }, "teardown": { "duration": 0.000258082989603281, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Scout-17B-16E]", "lineno": 60, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_image[input_output0-Llama-4-Scout-17B-16E]", "parametrize", "pytestmark", "input_output0-Llama-4-Scout-17B-16E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.00936354196164757, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 61, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')" }, "teardown": { "duration": 0.00020533299539238214, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Scout-17B-16E-Instruct]", "lineno": 60, "outcome": "passed", "keywords": [ "test_chat_non_streaming_image[input_output0-Llama-4-Scout-17B-16E-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-4-Scout-17B-16E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.008578249951824546, "outcome": "passed" }, "call": { "duration": 2.6288582499837503, "outcome": "passed" }, "teardown": { "duration": 0.0006052498938515782, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Maverick-17B-128E]", "lineno": 60, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_image[input_output0-Llama-4-Maverick-17B-128E]", "parametrize", "pytestmark", "input_output0-Llama-4-Maverick-17B-128E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.02061279199551791, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 61, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')" }, "teardown": { "duration": 0.00029320805333554745, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "lineno": 60, "outcome": "passed", "keywords": [ "test_chat_non_streaming_image[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-4-Maverick-17B-128E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.00995812495239079, "outcome": "passed" }, "call": { "duration": 3.0904540000483394, "outcome": "passed" }, "teardown": { "duration": 0.0003214169992133975, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-gpt-4o]", "lineno": 60, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_image[input_output0-gpt-4o]", "parametrize", "pytestmark", "input_output0-gpt-4o", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.0261635419446975, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 61, 'Skipped: Provider fireworks does not support model gpt-4o')" }, "teardown": { "duration": 0.00032716698478907347, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_image[input_output0-gpt-4o-mini]", "lineno": 60, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_image[input_output0-gpt-4o-mini]", "parametrize", "pytestmark", "input_output0-gpt-4o-mini", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.027220541960559785, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 61, 'Skipped: Provider fireworks does not support model gpt-4o-mini')" }, "teardown": { "duration": 0.0003192499279975891, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Scout-17B-16E]", "lineno": 75, "outcome": "skipped", "keywords": [ "test_chat_streaming_image[input_output0-Llama-4-Scout-17B-16E]", "parametrize", "pytestmark", "input_output0-Llama-4-Scout-17B-16E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.010883458075113595, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 76, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')" }, "teardown": { "duration": 0.0002687909873202443, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Scout-17B-16E-Instruct]", "lineno": 75, "outcome": "passed", "keywords": [ "test_chat_streaming_image[input_output0-Llama-4-Scout-17B-16E-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-4-Scout-17B-16E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.0171177500160411, "outcome": "passed" }, "call": { "duration": 1.6752691670553759, "outcome": "passed" }, "teardown": { "duration": 0.0004877089522778988, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Maverick-17B-128E]", "lineno": 75, "outcome": "skipped", "keywords": [ "test_chat_streaming_image[input_output0-Llama-4-Maverick-17B-128E]", "parametrize", "pytestmark", "input_output0-Llama-4-Maverick-17B-128E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.011608208995312452, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 76, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')" }, "teardown": { "duration": 0.00017137499526143074, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "lineno": 75, "outcome": "passed", "keywords": [ "test_chat_streaming_image[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-4-Maverick-17B-128E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.009284624946303666, "outcome": "passed" }, "call": { "duration": 3.537356249988079, "outcome": "passed" }, "teardown": { "duration": 0.0005068340105935931, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-gpt-4o]", "lineno": 75, "outcome": "skipped", "keywords": [ "test_chat_streaming_image[input_output0-gpt-4o]", "parametrize", "pytestmark", "input_output0-gpt-4o", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.016660499968566, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 76, 'Skipped: Provider fireworks does not support model gpt-4o')" }, "teardown": { "duration": 0.00029341597110033035, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_image[input_output0-gpt-4o-mini]", "lineno": 75, "outcome": "skipped", "keywords": [ "test_chat_streaming_image[input_output0-gpt-4o-mini]", "parametrize", "pytestmark", "input_output0-gpt-4o-mini", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.01374066702555865, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 76, 'Skipped: Provider fireworks does not support model gpt-4o-mini')" }, "teardown": { "duration": 0.0002625000197440386, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-3.3-8B-Instruct]", "lineno": 95, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_structured_output[input_output0-Llama-3.3-8B-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-3.3-8B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.013120374991558492, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')" }, "teardown": { "duration": 0.00021954195108264685, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-3.3-70B-Instruct]", "lineno": 95, "outcome": "passed", "keywords": [ "test_chat_non_streaming_structured_output[input_output0-Llama-3.3-70B-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-3.3-70B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.015080374898388982, "outcome": "passed" }, "call": { "duration": 1.157175041968003, "outcome": "passed" }, "teardown": { "duration": 0.000495875021442771, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E]", "lineno": 95, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E]", "parametrize", "pytestmark", "input_output0-Llama-4-Scout-17B-16E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.013946042046882212, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')" }, "teardown": { "duration": 0.0002954580122604966, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E-Instruct]", "lineno": 95, "outcome": "passed", "keywords": [ "test_chat_non_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-4-Scout-17B-16E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.011617792071774602, "outcome": "passed" }, "call": { "duration": 0.9537639999762177, "outcome": "passed" }, "teardown": { "duration": 0.0004819999448955059, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E]", "lineno": 95, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E]", "parametrize", "pytestmark", "input_output0-Llama-4-Maverick-17B-128E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.027436082949861884, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')" }, "teardown": { "duration": 0.00030274991877377033, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "lineno": 95, "outcome": "passed", "keywords": [ "test_chat_non_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-4-Maverick-17B-128E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.016110333963297307, "outcome": "passed" }, "call": { "duration": 0.8493227910948917, "outcome": "passed" }, "teardown": { "duration": 0.0004883749643340707, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-gpt-4o]", "lineno": 95, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_structured_output[input_output0-gpt-4o]", "parametrize", "pytestmark", "input_output0-gpt-4o", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.017850833013653755, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model gpt-4o')" }, "teardown": { "duration": 0.0003287500003352761, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output0-gpt-4o-mini]", "lineno": 95, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_structured_output[input_output0-gpt-4o-mini]", "parametrize", "pytestmark", "input_output0-gpt-4o-mini", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.012523208046332002, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model gpt-4o-mini')" }, "teardown": { "duration": 0.00023500004317611456, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-3.3-8B-Instruct]", "lineno": 95, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_structured_output[input_output1-Llama-3.3-8B-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-3.3-8B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.007516667013987899, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')" }, "teardown": { "duration": 0.00018912507221102715, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-3.3-70B-Instruct]", "lineno": 95, "outcome": "passed", "keywords": [ "test_chat_non_streaming_structured_output[input_output1-Llama-3.3-70B-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-3.3-70B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.007337165996432304, "outcome": "passed" }, "call": { "duration": 3.124099582899362, "outcome": "passed" }, "teardown": { "duration": 0.0006703329272568226, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E]", "lineno": 95, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E]", "parametrize", "pytestmark", "input_output1-Llama-4-Scout-17B-16E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.014259999967180192, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')" }, "teardown": { "duration": 0.00030262500513345003, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E-Instruct]", "lineno": 95, "outcome": "passed", "keywords": [ "test_chat_non_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-4-Scout-17B-16E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.010863124975003302, "outcome": "passed" }, "call": { "duration": 1.3330956250429153, "outcome": "passed" }, "teardown": { "duration": 0.00018679199274629354, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E]", "lineno": 95, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E]", "parametrize", "pytestmark", "input_output1-Llama-4-Maverick-17B-128E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.005797958001494408, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')" }, "teardown": { "duration": 0.00017529097385704517, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E-Instruct]", "lineno": 95, "outcome": "passed", "keywords": [ "test_chat_non_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-4-Maverick-17B-128E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.005647709011100233, "outcome": "passed" }, "call": { "duration": 3.2295467499643564, "outcome": "passed" }, "teardown": { "duration": 0.0005654999986290932, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-gpt-4o]", "lineno": 95, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_structured_output[input_output1-gpt-4o]", "parametrize", "pytestmark", "input_output1-gpt-4o", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.007151791942305863, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model gpt-4o')" }, "teardown": { "duration": 0.00015316694043576717, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_structured_output[input_output1-gpt-4o-mini]", "lineno": 95, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_structured_output[input_output1-gpt-4o-mini]", "parametrize", "pytestmark", "input_output1-gpt-4o-mini", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.006435790914110839, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 96, 'Skipped: Provider fireworks does not support model gpt-4o-mini')" }, "teardown": { "duration": 0.00015954102855175734, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-3.3-8B-Instruct]", "lineno": 117, "outcome": "skipped", "keywords": [ "test_chat_streaming_structured_output[input_output0-Llama-3.3-8B-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-3.3-8B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.006164791993796825, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')" }, "teardown": { "duration": 0.00014074996579438448, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-3.3-70B-Instruct]", "lineno": 117, "outcome": "passed", "keywords": [ "test_chat_streaming_structured_output[input_output0-Llama-3.3-70B-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-3.3-70B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.010064583038911223, "outcome": "passed" }, "call": { "duration": 1.1676458748988807, "outcome": "passed" }, "teardown": { "duration": 0.0002513329964131117, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E]", "lineno": 117, "outcome": "skipped", "keywords": [ "test_chat_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E]", "parametrize", "pytestmark", "input_output0-Llama-4-Scout-17B-16E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.011011417023837566, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')" }, "teardown": { "duration": 0.00020608294289559126, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E-Instruct]", "lineno": 117, "outcome": "passed", "keywords": [ "test_chat_streaming_structured_output[input_output0-Llama-4-Scout-17B-16E-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-4-Scout-17B-16E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.011654542060568929, "outcome": "passed" }, "call": { "duration": 0.7950789160095155, "outcome": "passed" }, "teardown": { "duration": 0.0002690000692382455, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E]", "lineno": 117, "outcome": "skipped", "keywords": [ "test_chat_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E]", "parametrize", "pytestmark", "input_output0-Llama-4-Maverick-17B-128E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.0066834589233621955, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')" }, "teardown": { "duration": 0.00017270795069634914, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "lineno": 117, "outcome": "passed", "keywords": [ "test_chat_streaming_structured_output[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-4-Maverick-17B-128E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.011390416999347508, "outcome": "passed" }, "call": { "duration": 0.7844940840732306, "outcome": "passed" }, "teardown": { "duration": 0.000511458027176559, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-gpt-4o]", "lineno": 117, "outcome": "skipped", "keywords": [ "test_chat_streaming_structured_output[input_output0-gpt-4o]", "parametrize", "pytestmark", "input_output0-gpt-4o", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.005813500029034913, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model gpt-4o')" }, "teardown": { "duration": 0.00015495799016207457, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output0-gpt-4o-mini]", "lineno": 117, "outcome": "skipped", "keywords": [ "test_chat_streaming_structured_output[input_output0-gpt-4o-mini]", "parametrize", "pytestmark", "input_output0-gpt-4o-mini", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.0075639160349965096, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model gpt-4o-mini')" }, "teardown": { "duration": 0.00014358304906636477, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-3.3-8B-Instruct]", "lineno": 117, "outcome": "skipped", "keywords": [ "test_chat_streaming_structured_output[input_output1-Llama-3.3-8B-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-3.3-8B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.008526541059836745, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model Llama-3.3-8B-Instruct')" }, "teardown": { "duration": 0.00015841599088162184, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-3.3-70B-Instruct]", "lineno": 117, "outcome": "failed", "keywords": [ "test_chat_streaming_structured_output[input_output1-Llama-3.3-70B-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-3.3-70B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.007805416011251509, "outcome": "passed" }, "call": { "duration": 13.25898533302825, "outcome": "failed", "crash": { "path": "/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py", "lineno": 196, "message": "assert None is not None" }, "traceback": [ { "path": "tests/verifications/openai/test_chat_completion.py", "lineno": 136, "message": "" }, { "path": "tests/verifications/openai/test_chat_completion.py", "lineno": 196, "message": "AssertionError" } ], "longrepr": "openai_client = \ninput_output = {'input': {'messages': [{'content': 'You are a helpful math tutor. Guide the user through the solution step by step.',... ['steps', 'final_answer'], 'title': 'MathReasoning', ...}}, 'type': 'json_schema'}}, 'output': 'valid_math_reasoning'}\ncorrect_model_name = 'accounts/fireworks/models/llama-v3p1-70b-instruct'\n\n @pytest.mark.parametrize(\n \"model\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"model\"],\n )\n @pytest.mark.parametrize(\n \"input_output\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"input_output\"],\n )\n def test_chat_streaming_structured_output(openai_client, input_output, correct_model_name):\n response = openai_client.chat.completions.create(\n model=correct_model_name,\n messages=input_output[\"input\"][\"messages\"],\n response_format=input_output[\"input\"][\"response_format\"],\n stream=True,\n )\n maybe_json_content = \"\"\n for chunk in response:\n maybe_json_content += chunk.choices[0].delta.content or \"\"\n> validate_structured_output(maybe_json_content, input_output[\"output\"])\n\ntests/verifications/openai/test_chat_completion.py:136: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nmaybe_json_content = '{ \"final_answer\": \"}To solve the equation 8x + 7 = -23, we need to isolate the variable x. We can do this by followin...tassistantassistantassistantassistantassistantassistantassistantassistantassistantassistantassistantassistantassistant'\nschema_name = 'valid_math_reasoning'\n\n def validate_structured_output(maybe_json_content: str, schema_name: str) -> None:\n structured_output = get_structured_output(maybe_json_content, schema_name)\n> assert structured_output is not None\nE assert None is not None\n\ntests/verifications/openai/test_chat_completion.py:196: AssertionError" }, "teardown": { "duration": 0.00022583396639674902, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E]", "lineno": 117, "outcome": "skipped", "keywords": [ "test_chat_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E]", "parametrize", "pytestmark", "input_output1-Llama-4-Scout-17B-16E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.006412541959434748, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')" }, "teardown": { "duration": 0.0001449589617550373, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E-Instruct]", "lineno": 117, "outcome": "passed", "keywords": [ "test_chat_streaming_structured_output[input_output1-Llama-4-Scout-17B-16E-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-4-Scout-17B-16E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.010353000019676983, "outcome": "passed" }, "call": { "duration": 4.559281209018081, "outcome": "passed" }, "teardown": { "duration": 0.00021179206669330597, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E]", "lineno": 117, "outcome": "skipped", "keywords": [ "test_chat_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E]", "parametrize", "pytestmark", "input_output1-Llama-4-Maverick-17B-128E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.011320417048409581, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')" }, "teardown": { "duration": 0.0001623749267309904, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E-Instruct]", "lineno": 117, "outcome": "passed", "keywords": [ "test_chat_streaming_structured_output[input_output1-Llama-4-Maverick-17B-128E-Instruct]", "parametrize", "pytestmark", "input_output1-Llama-4-Maverick-17B-128E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.005637791007757187, "outcome": "passed" }, "call": { "duration": 2.9282109580235556, "outcome": "passed" }, "teardown": { "duration": 0.00019149994477629662, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-gpt-4o]", "lineno": 117, "outcome": "skipped", "keywords": [ "test_chat_streaming_structured_output[input_output1-gpt-4o]", "parametrize", "pytestmark", "input_output1-gpt-4o", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.021475916961207986, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model gpt-4o')" }, "teardown": { "duration": 0.0002605828922241926, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_streaming_structured_output[input_output1-gpt-4o-mini]", "lineno": 117, "outcome": "skipped", "keywords": [ "test_chat_streaming_structured_output[input_output1-gpt-4o-mini]", "parametrize", "pytestmark", "input_output1-gpt-4o-mini", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.012046082993037999, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 118, 'Skipped: Provider fireworks does not support model gpt-4o-mini')" }, "teardown": { "duration": 0.00016966694965958595, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-3.3-70B-Instruct]", "lineno": 138, "outcome": "passed", "keywords": [ "test_chat_non_streaming_tool_calling[input_output0-Llama-3.3-70B-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-3.3-70B-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.00782629195600748, "outcome": "passed" }, "call": { "duration": 0.9290615000063553, "outcome": "passed" }, "teardown": { "duration": 0.0004110001027584076, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Scout-17B-16E]", "lineno": 138, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_tool_calling[input_output0-Llama-4-Scout-17B-16E]", "parametrize", "pytestmark", "input_output0-Llama-4-Scout-17B-16E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.00842183397617191, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 139, 'Skipped: Provider fireworks does not support model Llama-4-Scout-17B-16E')" }, "teardown": { "duration": 0.00023745803628116846, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Scout-17B-16E-Instruct]", "lineno": 138, "outcome": "failed", "keywords": [ "test_chat_non_streaming_tool_calling[input_output0-Llama-4-Scout-17B-16E-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-4-Scout-17B-16E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.010762874968349934, "outcome": "passed" }, "call": { "duration": 23.62101216695737, "outcome": "failed", "crash": { "path": "/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py", "lineno": 156, "message": "TypeError: object of type 'NoneType' has no len()" }, "traceback": [ { "path": "tests/verifications/openai/test_chat_completion.py", "lineno": 156, "message": "TypeError" } ], "longrepr": "openai_client = \ninput_output = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\ncorrect_model_name = 'accounts/fireworks/models/llama4-scout-instruct-basic'\n\n @pytest.mark.parametrize(\n \"model\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"model\"],\n )\n @pytest.mark.parametrize(\n \"input_output\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"input_output\"],\n )\n def test_chat_non_streaming_tool_calling(openai_client, input_output, correct_model_name):\n response = openai_client.chat.completions.create(\n model=correct_model_name,\n messages=input_output[\"input\"][\"messages\"],\n tools=input_output[\"input\"][\"tools\"],\n stream=False,\n )\n \n assert response.choices[0].message.role == \"assistant\"\n> assert len(response.choices[0].message.tool_calls) > 0\nE TypeError: object of type 'NoneType' has no len()\n\ntests/verifications/openai/test_chat_completion.py:156: TypeError" }, "teardown": { "duration": 0.0004520840011537075, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Maverick-17B-128E]", "lineno": 138, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_tool_calling[input_output0-Llama-4-Maverick-17B-128E]", "parametrize", "pytestmark", "input_output0-Llama-4-Maverick-17B-128E", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.00953104195650667, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 139, 'Skipped: Provider fireworks does not support model Llama-4-Maverick-17B-128E')" }, "teardown": { "duration": 0.00017912499606609344, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "lineno": 138, "outcome": "failed", "keywords": [ "test_chat_non_streaming_tool_calling[input_output0-Llama-4-Maverick-17B-128E-Instruct]", "parametrize", "pytestmark", "input_output0-Llama-4-Maverick-17B-128E-Instruct", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.010302042006514966, "outcome": "passed" }, "call": { "duration": 5.55651158397086, "outcome": "failed", "crash": { "path": "/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py", "lineno": 156, "message": "TypeError: object of type 'NoneType' has no len()" }, "traceback": [ { "path": "tests/verifications/openai/test_chat_completion.py", "lineno": 156, "message": "TypeError" } ], "longrepr": "openai_client = \ninput_output = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\ncorrect_model_name = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\n\n @pytest.mark.parametrize(\n \"model\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"model\"],\n )\n @pytest.mark.parametrize(\n \"input_output\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"input_output\"],\n )\n def test_chat_non_streaming_tool_calling(openai_client, input_output, correct_model_name):\n response = openai_client.chat.completions.create(\n model=correct_model_name,\n messages=input_output[\"input\"][\"messages\"],\n tools=input_output[\"input\"][\"tools\"],\n stream=False,\n )\n \n assert response.choices[0].message.role == \"assistant\"\n> assert len(response.choices[0].message.tool_calls) > 0\nE TypeError: object of type 'NoneType' has no len()\n\ntests/verifications/openai/test_chat_completion.py:156: TypeError" }, "teardown": { "duration": 0.0003929579397663474, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-gpt-4o]", "lineno": 138, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_tool_calling[input_output0-gpt-4o]", "parametrize", "pytestmark", "input_output0-gpt-4o", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.01593891705852002, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 139, 'Skipped: Provider fireworks does not support model gpt-4o')" }, "teardown": { "duration": 0.0003579579060897231, "outcome": "passed" } }, { "nodeid": "tests/verifications/openai/test_chat_completion.py::test_chat_non_streaming_tool_calling[input_output0-gpt-4o-mini]", "lineno": 138, "outcome": "skipped", "keywords": [ "test_chat_non_streaming_tool_calling[input_output0-gpt-4o-mini]", "parametrize", "pytestmark", "input_output0-gpt-4o-mini", "test_chat_completion.py", "openai", "verifications", "tests", "llama-stack", "" ], "setup": { "duration": 0.01874550001230091, "outcome": "skipped", "longrepr": "('/Users/erichuang/projects/llama-stack/tests/verifications/openai/test_chat_completion.py', 139, 'Skipped: Provider fireworks does not support model gpt-4o-mini')" }, "teardown": { "duration": 0.00031995808240026236, "outcome": "passed" } } ] }