mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-07-27 14:38:49 +00:00
2fcb70b789
# What does this PR do? ## Test Plan (myenv) ➜ llama-stack python tests/verifications/generate_report.py --providers fireworks,together,openai --run-tests
945 lines
25 KiB
JSON
945 lines
25 KiB
JSON
{
|
|
"created": 1744328898.0248861,
|
|
"duration": 47.561042070388794,
|
|
"exitcode": 0,
|
|
"root": "/Users/erichuang/projects/llama-stack",
|
|
"environment": {},
|
|
"summary": {
|
|
"passed": 24,
|
|
"total": 24,
|
|
"collected": 24
|
|
},
|
|
"collectors": [
|
|
{
|
|
"nodeid": "",
|
|
"outcome": "passed",
|
|
"result": [
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"type": "Module"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"outcome": "passed",
|
|
"result": [
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[gpt-4o-earth]",
|
|
"type": "Function",
|
|
"lineno": 73
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[gpt-4o-saturn]",
|
|
"type": "Function",
|
|
"lineno": 73
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[gpt-4o-mini-earth]",
|
|
"type": "Function",
|
|
"lineno": 73
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[gpt-4o-mini-saturn]",
|
|
"type": "Function",
|
|
"lineno": 73
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[gpt-4o-earth]",
|
|
"type": "Function",
|
|
"lineno": 92
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[gpt-4o-saturn]",
|
|
"type": "Function",
|
|
"lineno": 92
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[gpt-4o-mini-earth]",
|
|
"type": "Function",
|
|
"lineno": 92
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[gpt-4o-mini-saturn]",
|
|
"type": "Function",
|
|
"lineno": 92
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[gpt-4o-case0]",
|
|
"type": "Function",
|
|
"lineno": 116
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[gpt-4o-mini-case0]",
|
|
"type": "Function",
|
|
"lineno": 116
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[gpt-4o-case0]",
|
|
"type": "Function",
|
|
"lineno": 135
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[gpt-4o-mini-case0]",
|
|
"type": "Function",
|
|
"lineno": 135
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[gpt-4o-calendar]",
|
|
"type": "Function",
|
|
"lineno": 159
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[gpt-4o-math]",
|
|
"type": "Function",
|
|
"lineno": 159
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[gpt-4o-mini-calendar]",
|
|
"type": "Function",
|
|
"lineno": 159
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[gpt-4o-mini-math]",
|
|
"type": "Function",
|
|
"lineno": 159
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[gpt-4o-calendar]",
|
|
"type": "Function",
|
|
"lineno": 182
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[gpt-4o-math]",
|
|
"type": "Function",
|
|
"lineno": 182
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[gpt-4o-mini-calendar]",
|
|
"type": "Function",
|
|
"lineno": 182
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[gpt-4o-mini-math]",
|
|
"type": "Function",
|
|
"lineno": 182
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[gpt-4o-case0]",
|
|
"type": "Function",
|
|
"lineno": 204
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[gpt-4o-mini-case0]",
|
|
"type": "Function",
|
|
"lineno": 204
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[gpt-4o-case0]",
|
|
"type": "Function",
|
|
"lineno": 228
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[gpt-4o-mini-case0]",
|
|
"type": "Function",
|
|
"lineno": 228
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"tests": [
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[gpt-4o-earth]",
|
|
"lineno": 73,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[gpt-4o-earth]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-earth",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o",
|
|
"case_id": "earth"
|
|
},
|
|
"setup": {
|
|
"duration": 0.0694252080284059,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.5709165419684723,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0007626248989254236,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[gpt-4o-saturn]",
|
|
"lineno": 73,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[gpt-4o-saturn]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-saturn",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o",
|
|
"case_id": "saturn"
|
|
},
|
|
"setup": {
|
|
"duration": 0.010281750001013279,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.6309260830748826,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0001824579667299986,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[gpt-4o-mini-earth]",
|
|
"lineno": 73,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[gpt-4o-mini-earth]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-mini-earth",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o-mini",
|
|
"case_id": "earth"
|
|
},
|
|
"setup": {
|
|
"duration": 0.007922374992631376,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.31756504194345325,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0005268750246614218,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[gpt-4o-mini-saturn]",
|
|
"lineno": 73,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[gpt-4o-mini-saturn]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-mini-saturn",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o-mini",
|
|
"case_id": "saturn"
|
|
},
|
|
"setup": {
|
|
"duration": 0.01643404201604426,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.7479908330133185,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004037501057609916,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[gpt-4o-earth]",
|
|
"lineno": 92,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[gpt-4o-earth]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-earth",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o",
|
|
"case_id": "earth"
|
|
},
|
|
"setup": {
|
|
"duration": 0.021671707974746823,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.6701172919711098,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0005569590721279383,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[gpt-4o-saturn]",
|
|
"lineno": 92,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[gpt-4o-saturn]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-saturn",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o",
|
|
"case_id": "saturn"
|
|
},
|
|
"setup": {
|
|
"duration": 0.015847125090658665,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.636536999954842,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00029395800083875656,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[gpt-4o-mini-earth]",
|
|
"lineno": 92,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[gpt-4o-mini-earth]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-mini-earth",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o-mini",
|
|
"case_id": "earth"
|
|
},
|
|
"setup": {
|
|
"duration": 0.011792832985520363,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.5610962919890881,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003578749019652605,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[gpt-4o-mini-saturn]",
|
|
"lineno": 92,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[gpt-4o-mini-saturn]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-mini-saturn",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o-mini",
|
|
"case_id": "saturn"
|
|
},
|
|
"setup": {
|
|
"duration": 0.016500207944773138,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.8060244580265135,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0005296670133247972,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[gpt-4o-case0]",
|
|
"lineno": 116,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_image[gpt-4o-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.008338792016729712,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 7.009252917021513,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003042910248041153,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[gpt-4o-mini-case0]",
|
|
"lineno": 116,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_image[gpt-4o-mini-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-mini-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o-mini",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.007238540914840996,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 3.134693874977529,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003104590578004718,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[gpt-4o-case0]",
|
|
"lineno": 135,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_image[gpt-4o-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.0161851670127362,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 3.0745719589758664,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00022620800882577896,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[gpt-4o-mini-case0]",
|
|
"lineno": 135,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_image[gpt-4o-mini-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-mini-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o-mini",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.013220708002336323,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 3.624867417034693,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00020633300300687551,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[gpt-4o-calendar]",
|
|
"lineno": 159,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[gpt-4o-calendar]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-calendar",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o",
|
|
"case_id": "calendar"
|
|
},
|
|
"setup": {
|
|
"duration": 0.017596833989955485,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 1.248568250099197,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004248750628903508,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[gpt-4o-math]",
|
|
"lineno": 159,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[gpt-4o-math]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-math",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o",
|
|
"case_id": "math"
|
|
},
|
|
"setup": {
|
|
"duration": 0.01512012502644211,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 8.170285542029887,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00043537491001188755,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[gpt-4o-mini-calendar]",
|
|
"lineno": 159,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[gpt-4o-mini-calendar]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-mini-calendar",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o-mini",
|
|
"case_id": "calendar"
|
|
},
|
|
"setup": {
|
|
"duration": 0.010376665974035859,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.756480542011559,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00025695806834846735,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[gpt-4o-mini-math]",
|
|
"lineno": 159,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[gpt-4o-mini-math]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-mini-math",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o-mini",
|
|
"case_id": "math"
|
|
},
|
|
"setup": {
|
|
"duration": 0.006846625008620322,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 2.6833953330060467,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00022558309137821198,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[gpt-4o-calendar]",
|
|
"lineno": 182,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[gpt-4o-calendar]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-calendar",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o",
|
|
"case_id": "calendar"
|
|
},
|
|
"setup": {
|
|
"duration": 0.009646040969528258,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.6117532079806551,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00015258300118148327,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[gpt-4o-math]",
|
|
"lineno": 182,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[gpt-4o-math]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-math",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o",
|
|
"case_id": "math"
|
|
},
|
|
"setup": {
|
|
"duration": 0.012024458032101393,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 4.522625041077845,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004230838967487216,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[gpt-4o-mini-calendar]",
|
|
"lineno": 182,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[gpt-4o-mini-calendar]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-mini-calendar",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o-mini",
|
|
"case_id": "calendar"
|
|
},
|
|
"setup": {
|
|
"duration": 0.009566582972183824,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 2.5591942919418216,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0007555419579148293,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[gpt-4o-mini-math]",
|
|
"lineno": 182,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[gpt-4o-mini-math]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-mini-math",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o-mini",
|
|
"case_id": "math"
|
|
},
|
|
"setup": {
|
|
"duration": 0.010828875005245209,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 2.495122667052783,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0002802090020850301,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[gpt-4o-case0]",
|
|
"lineno": 204,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_calling[gpt-4o-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.012762792059220374,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.5655921660363674,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00022304197773337364,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[gpt-4o-mini-case0]",
|
|
"lineno": 204,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_calling[gpt-4o-mini-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-mini-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o-mini",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.03188708401285112,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.6159415419679135,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0005549580091610551,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[gpt-4o-case0]",
|
|
"lineno": 228,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_tool_calling[gpt-4o-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.014768208027817309,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.47373537498060614,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0005811670562252402,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[gpt-4o-mini-case0]",
|
|
"lineno": 228,
|
|
"outcome": "passed",
|
|
"keywords": [
|
|
"test_chat_streaming_tool_calling[gpt-4o-mini-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"gpt-4o-mini-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "gpt-4o-mini",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.010271625011228025,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.5656027499353513,
|
|
"outcome": "passed"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0025699170073494315,
|
|
"outcome": "passed"
|
|
}
|
|
}
|
|
],
|
|
"run_timestamp": 1744328848
|
|
}
|