mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-12-28 04:02:00 +00:00
6270 lines
758 KiB
JSON
6270 lines
758 KiB
JSON
{
|
|
"created": 1744915698.8314075,
|
|
"duration": 25.619409322738647,
|
|
"exitcode": 1,
|
|
"root": "/home/erichuang/llama-stack",
|
|
"environment": {},
|
|
"summary": {
|
|
"failed": 80,
|
|
"skipped": 4,
|
|
"total": 84,
|
|
"collected": 84
|
|
},
|
|
"collectors": [
|
|
{
|
|
"nodeid": "",
|
|
"outcome": "passed",
|
|
"result": [
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"type": "Module"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"outcome": "passed",
|
|
"result": [
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
|
|
"type": "Function",
|
|
"lineno": 95
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
|
|
"type": "Function",
|
|
"lineno": 114
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
|
|
"type": "Function",
|
|
"lineno": 114
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
|
|
"type": "Function",
|
|
"lineno": 114
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
|
|
"type": "Function",
|
|
"lineno": 114
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
|
|
"type": "Function",
|
|
"lineno": 114
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
|
|
"type": "Function",
|
|
"lineno": 114
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"type": "Function",
|
|
"lineno": 138
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 138
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 138
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"type": "Function",
|
|
"lineno": 157
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 157
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 157
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
|
|
"type": "Function",
|
|
"lineno": 181
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
|
|
"type": "Function",
|
|
"lineno": 181
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
|
|
"type": "Function",
|
|
"lineno": 181
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
|
|
"type": "Function",
|
|
"lineno": 181
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
|
|
"type": "Function",
|
|
"lineno": 181
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
|
|
"type": "Function",
|
|
"lineno": 181
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
|
|
"type": "Function",
|
|
"lineno": 204
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
|
|
"type": "Function",
|
|
"lineno": 204
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
|
|
"type": "Function",
|
|
"lineno": 204
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
|
|
"type": "Function",
|
|
"lineno": 204
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
|
|
"type": "Function",
|
|
"lineno": 204
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
|
|
"type": "Function",
|
|
"lineno": 204
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"type": "Function",
|
|
"lineno": 226
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 226
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 226
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"type": "Function",
|
|
"lineno": 250
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 250
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 250
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"type": "Function",
|
|
"lineno": 278
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 278
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 278
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"type": "Function",
|
|
"lineno": 302
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 302
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 302
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"type": "Function",
|
|
"lineno": 329
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 329
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 329
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"type": "Function",
|
|
"lineno": 352
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 352
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"type": "Function",
|
|
"lineno": 352
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]",
|
|
"type": "Function",
|
|
"lineno": 380
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]",
|
|
"type": "Function",
|
|
"lineno": 471
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama-v3p3-70b-instruct-stream=False]",
|
|
"type": "Function",
|
|
"lineno": 554
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama-v3p3-70b-instruct-stream=True]",
|
|
"type": "Function",
|
|
"lineno": 554
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=False]",
|
|
"type": "Function",
|
|
"lineno": 554
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=True]",
|
|
"type": "Function",
|
|
"lineno": 554
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=False]",
|
|
"type": "Function",
|
|
"lineno": 554
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=True]",
|
|
"type": "Function",
|
|
"lineno": 554
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"tests": [
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
|
|
"lineno": 95,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-earth",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "earth"
|
|
},
|
|
"setup": {
|
|
"duration": 0.12150054518133402,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.35746899526566267,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 106,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]>>\nopenai_client = <openai.OpenAI object at 0x7fa60ff6d630>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'earth', 'input': {'messages': [{'content': 'Which planet do humans live on?', 'role': 'user'}]}, 'output': 'Earth'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_basic(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:106: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60ff6d630>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00032798200845718384,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
|
|
"lineno": 95,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-saturn",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "saturn"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07205227017402649,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.1848590886220336,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 106,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc4fd60>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'saturn', 'input': {'messages': [{'content': 'Which planet has rings around it with a name starting with letter S?', 'role': 'user'}]}, 'output': 'Saturn'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_basic(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:106: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc4fd60>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003115283325314522,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
|
|
"lineno": 95,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-earth",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "earth"
|
|
},
|
|
"setup": {
|
|
"duration": 0.06999052409082651,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.20786387100815773,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 106,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc18dc0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'earth', 'input': {'messages': [{'content': 'Which planet do humans live on?', 'role': 'user'}]}, 'output': 'Earth'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_basic(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:106: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc18dc0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.000301288440823555,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
|
|
"lineno": 95,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-saturn",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "saturn"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07327916752547026,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.26050146389752626,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 106,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc18310>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'saturn', 'input': {'messages': [{'content': 'Which planet has rings around it with a name starting with letter S?', 'role': 'user'}]}, 'output': 'Saturn'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_basic(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:106: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc18310>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004022866487503052,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
|
|
"lineno": 95,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-earth",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "earth"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07078082300722599,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12057740241289139,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 106,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]>>\nopenai_client = <openai.OpenAI object at 0x7fa60ff28280>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'earth', 'input': {'messages': [{'content': 'Which planet do humans live on?', 'role': 'user'}]}, 'output': 'Earth'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_basic(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:106: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60ff28280>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003570122644305229,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
|
|
"lineno": 95,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-saturn",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "saturn"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07103450503200293,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.1985377622768283,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 106,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fedb280>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'saturn', 'input': {'messages': [{'content': 'Which planet has rings around it with a name starting with letter S?', 'role': 'user'}]}, 'output': 'Saturn'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_basic(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:106: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fedb280>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00040165428072214127,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
|
|
"lineno": 114,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-earth",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "earth"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07413783948868513,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.11946621909737587,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 125,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fcdb850>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'earth', 'input': {'messages': [{'content': 'Which planet do humans live on?', 'role': 'user'}]}, 'output': 'Earth'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_basic(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:125: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fcdb850>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.000340278260409832,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
|
|
"lineno": 114,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-saturn",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "saturn"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07317963056266308,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12889465410262346,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 125,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd5bd90>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'saturn', 'input': {'messages': [{'content': 'Which planet has rings around it with a name starting with letter S?', 'role': 'user'}]}, 'output': 'Saturn'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_basic(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:125: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd5bd90>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00041081756353378296,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
|
|
"lineno": 114,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-earth",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "earth"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07138469163328409,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.15935677476227283,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 125,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc47c40>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'earth', 'input': {'messages': [{'content': 'Which planet do humans live on?', 'role': 'user'}]}, 'output': 'Earth'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_basic(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:125: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc47c40>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004040272906422615,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
|
|
"lineno": 114,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-saturn",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "saturn"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07108956202864647,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.16156401950865984,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 125,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc44070>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'saturn', 'input': {'messages': [{'content': 'Which planet has rings around it with a name starting with letter S?', 'role': 'user'}]}, 'output': 'Saturn'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_basic(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:125: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc44070>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0005099587142467499,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
|
|
"lineno": 114,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-earth",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "earth"
|
|
},
|
|
"setup": {
|
|
"duration": 0.10071694944053888,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.10996749810874462,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 125,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc506a0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'earth', 'input': {'messages': [{'content': 'Which planet do humans live on?', 'role': 'user'}]}, 'output': 'Earth'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_basic(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:125: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc506a0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003982819616794586,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
|
|
"lineno": 114,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-saturn",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "saturn"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07027470227330923,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.2695386055856943,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 125,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbc1f60>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'saturn', 'input': {'messages': [{'content': 'Which planet has rings around it with a name starting with letter S?', 'role': 'user'}]}, 'output': 'Saturn'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_basic(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:125: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbc1f60>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00039549078792333603,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"lineno": 138,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_non_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07210677769035101,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.0004008617252111435,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/home/erichuang/llama-stack/tests/verifications/openai_api/test_chat_completion.py', 147, 'Skipped: Skipping test_chat_non_streaming_image for model accounts/fireworks/models/llama-v3p3-70b-instruct on provider fireworks based on config.')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00022324267774820328,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"lineno": 138,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07092681247740984,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.1368834748864174,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 149,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc51570>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': [{'text': 'What is in this image?', 'type': 'text'}, {'image_url': {...}, 'type': 'image_url'}], 'role': 'user'}]}, 'output': 'llama'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_image\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_image(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:149: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc51570>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00034329574555158615,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"lineno": 138,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07233018893748522,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.2651740964502096,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 149,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb2b940>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': [{'text': 'What is in this image?', 'type': 'text'}, {'image_url': {...}, 'type': 'image_url'}], 'role': 'user'}]}, 'output': 'llama'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_image\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_image(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:149: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb2b940>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00043803267180919647,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"lineno": 157,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07322083134204149,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.0002896450459957123,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/home/erichuang/llama-stack/tests/verifications/openai_api/test_chat_completion.py', 166, 'Skipped: Skipping test_chat_streaming_image for model accounts/fireworks/models/llama-v3p3-70b-instruct on provider fireworks based on config.')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003790464252233505,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"lineno": 157,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07414108049124479,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.11951867491006851,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 168,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fea8c70>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': [{'text': 'What is in this image?', 'type': 'text'}, {'image_url': {...}, 'type': 'image_url'}], 'role': 'user'}]}, 'output': 'llama'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_image\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_image(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:168: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fea8c70>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00031629856675863266,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"lineno": 157,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07023830153048038,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.27065565437078476,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 168,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd7c490>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': [{'text': 'What is in this image?', 'type': 'text'}, {'image_url': {...}, 'type': 'image_url'}], 'role': 'user'}]}, 'output': 'llama'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_image\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_image(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:168: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd7c490>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00030570197850465775,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
|
|
"lineno": 181,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-calendar",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "calendar"
|
|
},
|
|
"setup": {
|
|
"duration": 0.06930147111415863,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12505585234612226,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 192,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd56470>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'calendar', 'input': {'messages': [{'content': 'Extract the event information.', 'role': 'system'}, {'cont...articipants'], 'title': 'CalendarEvent', 'type': 'object'}}, 'type': 'json_schema'}}, 'output': 'valid_calendar_event'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n response_format=case[\"input\"][\"response_format\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:192: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd56470>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00032288581132888794,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
|
|
"lineno": 181,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-math",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "math"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07256390806287527,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.19006201811134815,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 192,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd5a1d0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'math', 'input': {'messages': [{'content': 'You are a helpful math tutor. Guide the user through the solut... ['steps', 'final_answer'], 'title': 'MathReasoning', ...}}, 'type': 'json_schema'}}, 'output': 'valid_math_reasoning'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n response_format=case[\"input\"][\"response_format\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:192: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd5a1d0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003082631155848503,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
|
|
"lineno": 181,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-calendar",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "calendar"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07006069924682379,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.15021017380058765,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 192,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb01000>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'calendar', 'input': {'messages': [{'content': 'Extract the event information.', 'role': 'system'}, {'cont...articipants'], 'title': 'CalendarEvent', 'type': 'object'}}, 'type': 'json_schema'}}, 'output': 'valid_calendar_event'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n response_format=case[\"input\"][\"response_format\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:192: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb01000>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003278367221355438,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
|
|
"lineno": 181,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-math",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "math"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07383340876549482,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.2492945184931159,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 192,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc188b0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'math', 'input': {'messages': [{'content': 'You are a helpful math tutor. Guide the user through the solut... ['steps', 'final_answer'], 'title': 'MathReasoning', ...}}, 'type': 'json_schema'}}, 'output': 'valid_math_reasoning'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n response_format=case[\"input\"][\"response_format\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:192: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc188b0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00041339918971061707,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
|
|
"lineno": 181,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-calendar",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "calendar"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07140334881842136,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.3302767900750041,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 192,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]>>\nopenai_client = <openai.OpenAI object at 0x7fa60ff6d480>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'calendar', 'input': {'messages': [{'content': 'Extract the event information.', 'role': 'system'}, {'cont...articipants'], 'title': 'CalendarEvent', 'type': 'object'}}, 'type': 'json_schema'}}, 'output': 'valid_calendar_event'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n response_format=case[\"input\"][\"response_format\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:192: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60ff6d480>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004214206710457802,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
|
|
"lineno": 181,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-math",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "math"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07027470134198666,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.13318416848778725,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 192,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb01c30>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'math', 'input': {'messages': [{'content': 'You are a helpful math tutor. Guide the user through the solut... ['steps', 'final_answer'], 'title': 'MathReasoning', ...}}, 'type': 'json_schema'}}, 'output': 'valid_math_reasoning'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n response_format=case[\"input\"][\"response_format\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:192: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb01c30>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003965655341744423,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
|
|
"lineno": 204,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-calendar",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "calendar"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07036527991294861,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.10335173550993204,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 215,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc4ffa0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'calendar', 'input': {'messages': [{'content': 'Extract the event information.', 'role': 'system'}, {'cont...articipants'], 'title': 'CalendarEvent', 'type': 'object'}}, 'type': 'json_schema'}}, 'output': 'valid_calendar_event'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n response_format=case[\"input\"][\"response_format\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc4ffa0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003379611298441887,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
|
|
"lineno": 204,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-math",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "math"
|
|
},
|
|
"setup": {
|
|
"duration": 0.06978577468544245,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12087872251868248,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 215,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fe19cf0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'math', 'input': {'messages': [{'content': 'You are a helpful math tutor. Guide the user through the solut... ['steps', 'final_answer'], 'title': 'MathReasoning', ...}}, 'type': 'json_schema'}}, 'output': 'valid_math_reasoning'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n response_format=case[\"input\"][\"response_format\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fe19cf0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00042513664811849594,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
|
|
"lineno": 204,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-calendar",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "calendar"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07085503917187452,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.11609443742781878,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 215,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbca530>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'calendar', 'input': {'messages': [{'content': 'Extract the event information.', 'role': 'system'}, {'cont...articipants'], 'title': 'CalendarEvent', 'type': 'object'}}, 'type': 'json_schema'}}, 'output': 'valid_calendar_event'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n response_format=case[\"input\"][\"response_format\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbca530>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.000426730141043663,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
|
|
"lineno": 204,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-math",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "math"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07437158096581697,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12889361381530762,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 215,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc52500>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'math', 'input': {'messages': [{'content': 'You are a helpful math tutor. Guide the user through the solut... ['steps', 'final_answer'], 'title': 'MathReasoning', ...}}, 'type': 'json_schema'}}, 'output': 'valid_math_reasoning'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n response_format=case[\"input\"][\"response_format\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc52500>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00043479073792696,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
|
|
"lineno": 204,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-calendar",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "calendar"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07079631183296442,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.17871549632400274,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 215,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbf54e0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'calendar', 'input': {'messages': [{'content': 'Extract the event information.', 'role': 'system'}, {'cont...articipants'], 'title': 'CalendarEvent', 'type': 'object'}}, 'type': 'json_schema'}}, 'output': 'valid_calendar_event'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n response_format=case[\"input\"][\"response_format\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbf54e0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003268783912062645,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
|
|
"lineno": 204,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-math",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "math"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07951002009212971,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.1458642790094018,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 215,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fe1a230>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'math', 'input': {'messages': [{'content': 'You are a helpful math tutor. Guide the user through the solut... ['steps', 'final_answer'], 'title': 'MathReasoning', ...}}, 'type': 'json_schema'}}, 'output': 'valid_math_reasoning'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n response_format=case[\"input\"][\"response_format\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fe1a230>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004075644537806511,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"lineno": 226,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07230154424905777,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.13512122631072998,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 237,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb9c6d0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:237: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb9c6d0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003347489982843399,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"lineno": 226,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07080346252769232,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.1290199300274253,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 237,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb6e3e0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:237: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb6e3e0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003189612179994583,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"lineno": 226,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07067843340337276,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12150294054299593,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 237,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb01ba0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_non_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:237: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb01ba0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00032811518758535385,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"lineno": 250,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.0696528134867549,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.2847281629219651,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 261,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb7b2b0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> stream = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:261: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb7b2b0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004156995564699173,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"lineno": 250,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07187621854245663,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12863421067595482,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 261,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb9c3a0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> stream = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:261: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb9c3a0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00036760419607162476,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"lineno": 250,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07296357583254576,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12340501230210066,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 261,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fea83d0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n ids=case_id_generator,\n )\n def test_chat_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> stream = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:261: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fea83d0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00042413268238306046,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"lineno": 278,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07447731029242277,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.13054667692631483,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 289,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd71450>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"], # Reusing existing case for now\n ids=case_id_generator,\n )\n def test_chat_non_streaming_tool_choice_required(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n tool_choice=\"required\", # Force tool call\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:289: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd71450>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00034614279866218567,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"lineno": 278,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07243293151259422,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12261831760406494,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 289,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fe198d0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"], # Reusing existing case for now\n ids=case_id_generator,\n )\n def test_chat_non_streaming_tool_choice_required(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n tool_choice=\"required\", # Force tool call\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:289: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fe198d0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004268251359462738,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"lineno": 278,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07407685182988644,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.09860698413103819,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 289,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd70a60>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"], # Reusing existing case for now\n ids=case_id_generator,\n )\n def test_chat_non_streaming_tool_choice_required(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n tool_choice=\"required\", # Force tool call\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:289: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd70a60>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00039894692599773407,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"lineno": 302,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07139755226671696,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.27306741289794445,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 313,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbc8580>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"], # Reusing existing case for now\n ids=case_id_generator,\n )\n def test_chat_streaming_tool_choice_required(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> stream = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n tool_choice=\"required\", # Force tool call\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:313: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbc8580>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00032315682619810104,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"lineno": 302,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.0701784947887063,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12367013934999704,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 313,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fe0b580>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"], # Reusing existing case for now\n ids=case_id_generator,\n )\n def test_chat_streaming_tool_choice_required(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> stream = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n tool_choice=\"required\", # Force tool call\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:313: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fe0b580>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003095511347055435,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"lineno": 302,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07112481445074081,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.11879229731857777,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 313,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb7b6a0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"], # Reusing existing case for now\n ids=case_id_generator,\n )\n def test_chat_streaming_tool_choice_required(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> stream = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n tool_choice=\"required\", # Force tool call\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:313: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb7b6a0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00032928306609392166,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"lineno": 329,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.0733236288651824,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.20418731309473515,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 340,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb198d0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"], # Reusing existing case for now\n ids=case_id_generator,\n )\n def test_chat_non_streaming_tool_choice_none(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n tool_choice=\"none\",\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:340: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb198d0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003160899505019188,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"lineno": 329,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07103190571069717,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.13796625938266516,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 340,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd5afe0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"], # Reusing existing case for now\n ids=case_id_generator,\n )\n def test_chat_non_streaming_tool_choice_none(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n tool_choice=\"none\",\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:340: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd5afe0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003080805763602257,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"lineno": 329,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07084846775978804,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.11544281151145697,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 340,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb21d80>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"], # Reusing existing case for now\n ids=case_id_generator,\n )\n def test_chat_non_streaming_tool_choice_none(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> response = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n tool_choice=\"none\",\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:340: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb21d80>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00031726714223623276,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"lineno": 352,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07118451129645109,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.3085783813148737,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 363,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb01150>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"], # Reusing existing case for now\n ids=case_id_generator,\n )\n def test_chat_streaming_tool_choice_none(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> stream = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n tool_choice=\"none\",\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:363: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb01150>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003956826403737068,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"lineno": 352,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07257829792797565,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.09841848351061344,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 363,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbf5600>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"], # Reusing existing case for now\n ids=case_id_generator,\n )\n def test_chat_streaming_tool_choice_none(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> stream = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n tool_choice=\"none\",\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:363: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbf5600>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0005597397685050964,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"lineno": 352,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "case0"
|
|
},
|
|
"setup": {
|
|
"duration": 0.0763206360861659,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.24762020073831081,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 363,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd585b0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"], # Reusing existing case for now\n ids=case_id_generator,\n )\n def test_chat_streaming_tool_choice_none(request, openai_client, model, provider, verification_config, case):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n> stream = openai_client.chat.completions.create(\n model=model,\n messages=case[\"input\"][\"messages\"],\n tools=case[\"input\"][\"tools\"],\n tool_choice=\"none\",\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:363: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd585b0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003156941384077072,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "text_then_weather_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07102795876562595,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.11488684639334679,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fab8fa0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'text_then_weather_tool', 'expected': [{'answer': ['sol'], 'num_tool_calls': 0}, {'num_tool_calls': 1, 'to...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fab8fa0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00033565983176231384,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "weather_tool_then_text"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07645629066973925,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.11238154675811529,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fea93c0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'weather_tool_then_text', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'location': 'San Francisco...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fea93c0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004070783033967018,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "add_product_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07554292771965265,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.2664942145347595,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbcb010>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'add_product_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'inStock': True, 'name': 'Widget...}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': 'Successfully added product with id: 123'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbcb010>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00040212273597717285,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "get_then_create_event_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07388069480657578,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12607386708259583,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd7ed10>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'get_then_create_event_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'date': '2025-03-03', ...ents found for 2025-03-03 at 10:00'}\"}, {'response': \"{'response': 'Successfully created new event with id: e_123'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd7ed10>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003165826201438904,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "compare_monthly_expense_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.0708252303302288,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.24374851863831282,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb798d0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'compare_monthly_expense_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'month': 1, 'year': ... 'Total expenses for January 2025: $1000'}\"}, {'response': \"{'response': 'Total expenses for February 2024: $2000'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb798d0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003087436780333519,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "text_then_weather_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07189069222658873,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.1129898214712739,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb7dd80>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'text_then_weather_tool', 'expected': [{'answer': ['sol'], 'num_tool_calls': 0}, {'num_tool_calls': 1, 'to...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb7dd80>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00041724275797605515,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "weather_tool_then_text"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07218896970152855,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.2670225091278553,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc45e70>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'weather_tool_then_text', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'location': 'San Francisco...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc45e70>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003341538831591606,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "add_product_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07147200033068657,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.11082868836820126,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fcdb850>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'add_product_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'inStock': True, 'name': 'Widget...}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': 'Successfully added product with id: 123'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fcdb850>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00040625128895044327,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "get_then_create_event_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07236841041594744,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.27738126553595066,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fdb0f70>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'get_then_create_event_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'date': '2025-03-03', ...ents found for 2025-03-03 at 10:00'}\"}, {'response': \"{'response': 'Successfully created new event with id: e_123'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fdb0f70>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003846520557999611,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "compare_monthly_expense_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.06980593129992485,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12573269568383694,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd7e980>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'compare_monthly_expense_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'month': 1, 'year': ... 'Total expenses for January 2025: $1000'}\"}, {'response': \"{'response': 'Total expenses for February 2024: $2000'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd7e980>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003903098404407501,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "text_then_weather_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07204260025173426,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.15989400260150433,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fdab4f0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'text_then_weather_tool', 'expected': [{'answer': ['sol'], 'num_tool_calls': 0}, {'num_tool_calls': 1, 'to...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fdab4f0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00032292958348989487,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "weather_tool_then_text"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07113024219870567,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.11847934126853943,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fcf4160>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'weather_tool_then_text', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'location': 'San Francisco...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fcf4160>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004066070541739464,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "add_product_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07287267316132784,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.2623152956366539,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb973d0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'add_product_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'inStock': True, 'name': 'Widget...}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': 'Successfully added product with id: 123'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb973d0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00032747630029916763,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "get_then_create_event_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.0714963860809803,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.1075747637078166,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbd2620>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'get_then_create_event_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'date': '2025-03-03', ...ents found for 2025-03-03 at 10:00'}\"}, {'response': \"{'response': 'Successfully created new event with id: e_123'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbd2620>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003954702988266945,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]",
|
|
"lineno": 380,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "compare_monthly_expense_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07068679295480251,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12899171095341444,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 421,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc09570>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'compare_monthly_expense_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'month': 1, 'year': ... 'Total expenses for January 2025: $1000'}\"}, {'response': \"{'response': 'Total expenses for February 2024: $2000'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\"\n Test cases for multi-turn tool calling.\n Tool calls are asserted.\n Tool responses are provided in the test case.\n Final response is asserted.\n \"\"\"\n \n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n # Create a copy of the messages list to avoid modifying the original\n messages = []\n tools = case[\"input\"][\"tools\"]\n # Use deepcopy to prevent modification across runs/parametrization\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n # keep going until either\n # 1. we have messages to test in multi-turn\n # 2. no messages but last message is tool response\n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n # do not take new messages if last message is tool response\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n # Ensure new_messages is a list of message objects\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n # If it's a single message object, add it directly\n messages.append(new_messages)\n \n # --- API Call ---\n> response = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=False,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc09570>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00031181517988443375,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "text_then_weather_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.06948941852897406,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.1193860862404108,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc06560>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'text_then_weather_tool', 'expected': [{'answer': ['sol'], 'num_tool_calls': 0}, {'num_tool_calls': 1, 'to...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc06560>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004093386232852936,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "weather_tool_then_text"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07101139053702354,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.1606877325102687,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fe09630>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'weather_tool_then_text', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'location': 'San Francisco...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fe09630>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00041847582906484604,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "add_product_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07783220708370209,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.11765131819993258,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fe19f00>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'add_product_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'inStock': True, 'name': 'Widget...}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': 'Successfully added product with id: 123'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fe19f00>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004519466310739517,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "get_then_create_event_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07041152473539114,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.13473773282021284,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fa59b40>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'get_then_create_event_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'date': '2025-03-03', ...ents found for 2025-03-03 at 10:00'}\"}, {'response': \"{'response': 'Successfully created new event with id: e_123'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fa59b40>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00039084814488887787,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "compare_monthly_expense_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07150810118764639,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.2046368457376957,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fed82b0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'compare_monthly_expense_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'month': 1, 'year': ... 'Total expenses for January 2025: $1000'}\"}, {'response': \"{'response': 'Total expenses for February 2024: $2000'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fed82b0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003378065302968025,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "text_then_weather_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07012568973004818,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12623131182044744,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc1abf0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'text_then_weather_tool', 'expected': [{'answer': ['sol'], 'num_tool_calls': 0}, {'num_tool_calls': 1, 'to...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc1abf0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00034175068140029907,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "weather_tool_then_text"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07349637430161238,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.11243469640612602,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbc3ac0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'weather_tool_then_text', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'location': 'San Francisco...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbc3ac0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003092559054493904,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "add_product_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07129209581762552,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.13334522116929293,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fab85b0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'add_product_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'inStock': True, 'name': 'Widget...}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': 'Successfully added product with id: 123'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fab85b0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004090704023838043,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "get_then_create_event_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07043681107461452,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.10430899448692799,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fdb30a0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'get_then_create_event_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'date': '2025-03-03', ...ents found for 2025-03-03 at 10:00'}\"}, {'response': \"{'response': 'Successfully created new event with id: e_123'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fdb30a0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003421120345592499,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "compare_monthly_expense_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.0733558852225542,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.10938013903796673,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd708e0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'compare_monthly_expense_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'month': 1, 'year': ... 'Total expenses for January 2025: $1000'}\"}, {'response': \"{'response': 'Total expenses for February 2024: $2000'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd708e0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00038493238389492035,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "text_then_weather_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07225227076560259,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.11426256597042084,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb2ebc0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'text_then_weather_tool', 'expected': [{'answer': ['sol'], 'num_tool_calls': 0}, {'num_tool_calls': 1, 'to...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb2ebc0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003271028399467468,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "weather_tool_then_text"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07055194210261106,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.1077765729278326,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb85e10>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'weather_tool_then_text', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'location': 'San Francisco...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb85e10>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003206087276339531,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "add_product_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.08054796047508717,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12442617677152157,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fde9d20>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'add_product_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'inStock': True, 'name': 'Widget...}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': 'Successfully added product with id: 123'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fde9d20>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00040055252611637115,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "get_then_create_event_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07104168552905321,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.10808593034744263,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60f9bc700>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'get_then_create_event_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'date': '2025-03-03', ...ents found for 2025-03-03 at 10:00'}\"}, {'response': \"{'response': 'Successfully created new event with id: e_123'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60f9bc700>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00033656321465969086,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]",
|
|
"lineno": 471,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "compare_monthly_expense_tool"
|
|
},
|
|
"setup": {
|
|
"duration": 0.06945569068193436,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.12593147810548544,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 498,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc066b0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'compare_monthly_expense_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'month': 1, 'year': ... 'Total expenses for January 2025: $1000'}\"}, {'response': \"{'response': 'Total expenses for February 2024: $2000'}\"}]}\n\n @pytest.mark.parametrize(\n \"case\",\n chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n ids=case_id_generator,\n )\n def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n \"\"\" \"\"\"\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages = []\n tools = case[\"input\"][\"tools\"]\n expected_results = copy.deepcopy(case[\"expected\"])\n tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n \n while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n new_messages = input_messages_turns.pop(0)\n if isinstance(new_messages, list):\n messages.extend(new_messages)\n else:\n messages.append(new_messages)\n \n # --- API Call (Streaming) ---\n> stream = openai_client.chat.completions.create(\n model=model,\n messages=messages,\n tools=tools,\n stream=True,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc066b0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0004069330170750618,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama-v3p3-70b-instruct-stream=False]",
|
|
"lineno": 554,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama-v3p3-70b-instruct-stream=False]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-stream=False",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "stream=False"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07031089067459106,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.00027627311646938324,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/home/erichuang/llama-stack/tests/verifications/openai_api/test_chat_completion.py', 561, 'Skipped: Skipping test_chat_multi_turn_multiple_images for model accounts/fireworks/models/llama-v3p3-70b-instruct on provider fireworks based on config.')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003817221149802208,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama-v3p3-70b-instruct-stream=True]",
|
|
"lineno": 554,
|
|
"outcome": "skipped",
|
|
"keywords": [
|
|
"test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama-v3p3-70b-instruct-stream=True]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama-v3p3-70b-instruct-stream=True",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
|
|
"case_id": "stream=True"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07004163973033428,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.000285550020635128,
|
|
"outcome": "skipped",
|
|
"longrepr": "('/home/erichuang/llama-stack/tests/verifications/openai_api/test_chat_completion.py', 561, 'Skipped: Skipping test_chat_multi_turn_multiple_images for model accounts/fireworks/models/llama-v3p3-70b-instruct on provider fireworks based on config.')"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00021260324865579605,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=False]",
|
|
"lineno": 554,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=False]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-stream=False",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "stream=False"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07160913478583097,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.23755338042974472,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 588,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=False]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbf4190>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\nmulti_image_data = ['data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGC...6pH9jaTzNv7vfRRXzubfxj9f8Pv8AkTz/AMX/ALbEz5Ly38lfMk/5Z/u64PxhqEZh+z/6rzvn2UUV5EvgPuzy/wAc6p5dt5ccibJpNkkdFFFec27mZ//Z']\nstream = False\n\n @pytest.mark.parametrize(\"stream\", [False, True], ids=[\"stream=False\", \"stream=True\"])\n def test_chat_multi_turn_multiple_images(\n request, openai_client, model, provider, verification_config, multi_image_data, stream\n ):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages_turn1 = [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": multi_image_data[0],\n },\n },\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": multi_image_data[1],\n },\n },\n {\n \"type\": \"text\",\n \"text\": \"What furniture is in the first image that is not in the second image?\",\n },\n ],\n },\n ]\n \n # First API call\n> response1 = openai_client.chat.completions.create(\n model=model,\n messages=messages_turn1,\n stream=stream,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:588: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbf4190>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.00033799000084400177,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=True]",
|
|
"lineno": 554,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=True]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-scout-instruct-basic-stream=True",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-scout-instruct-basic",
|
|
"case_id": "stream=True"
|
|
},
|
|
"setup": {
|
|
"duration": 0.0742298774421215,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.28080874867737293,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 588,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=True]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb6e920>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\nmulti_image_data = ['data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGC...6pH9jaTzNv7vfRRXzubfxj9f8Pv8AkTz/AMX/ALbEz5Ly38lfMk/5Z/u64PxhqEZh+z/6rzvn2UUV5EvgPuzy/wAc6p5dt5ccibJpNkkdFFFec27mZ//Z']\nstream = True\n\n @pytest.mark.parametrize(\"stream\", [False, True], ids=[\"stream=False\", \"stream=True\"])\n def test_chat_multi_turn_multiple_images(\n request, openai_client, model, provider, verification_config, multi_image_data, stream\n ):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages_turn1 = [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": multi_image_data[0],\n },\n },\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": multi_image_data[1],\n },\n },\n {\n \"type\": \"text\",\n \"text\": \"What furniture is in the first image that is not in the second image?\",\n },\n ],\n },\n ]\n \n # First API call\n> response1 = openai_client.chat.completions.create(\n model=model,\n messages=messages_turn1,\n stream=stream,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:588: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb6e920>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003352127969264984,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=False]",
|
|
"lineno": 554,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=False]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-stream=False",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "stream=False"
|
|
},
|
|
"setup": {
|
|
"duration": 0.07394346781075001,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.2838349239900708,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 588,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=False]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fabb6d0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\nmulti_image_data = ['data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGC...6pH9jaTzNv7vfRRXzubfxj9f8Pv8AkTz/AMX/ALbEz5Ly38lfMk/5Z/u64PxhqEZh+z/6rzvn2UUV5EvgPuzy/wAc6p5dt5ccibJpNkkdFFFec27mZ//Z']\nstream = False\n\n @pytest.mark.parametrize(\"stream\", [False, True], ids=[\"stream=False\", \"stream=True\"])\n def test_chat_multi_turn_multiple_images(\n request, openai_client, model, provider, verification_config, multi_image_data, stream\n ):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages_turn1 = [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": multi_image_data[0],\n },\n },\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": multi_image_data[1],\n },\n },\n {\n \"type\": \"text\",\n \"text\": \"What furniture is in the first image that is not in the second image?\",\n },\n ],\n },\n ]\n \n # First API call\n> response1 = openai_client.chat.completions.create(\n model=model,\n messages=messages_turn1,\n stream=stream,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:588: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fabb6d0>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0003282083198428154,
|
|
"outcome": "passed"
|
|
}
|
|
},
|
|
{
|
|
"nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=True]",
|
|
"lineno": 554,
|
|
"outcome": "failed",
|
|
"keywords": [
|
|
"test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=True]",
|
|
"parametrize",
|
|
"pytestmark",
|
|
"accounts/fireworks/models/llama4-maverick-instruct-basic-stream=True",
|
|
"test_chat_completion.py",
|
|
"openai_api",
|
|
"verifications",
|
|
"tests",
|
|
"llama-stack",
|
|
""
|
|
],
|
|
"metadata": {
|
|
"model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
|
|
"case_id": "stream=True"
|
|
},
|
|
"setup": {
|
|
"duration": 0.0713854730129242,
|
|
"outcome": "passed"
|
|
},
|
|
"call": {
|
|
"duration": 0.20238025579601526,
|
|
"outcome": "failed",
|
|
"crash": {
|
|
"path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
|
|
},
|
|
"traceback": [
|
|
{
|
|
"path": "tests/verifications/openai_api/test_chat_completion.py",
|
|
"lineno": 588,
|
|
"message": ""
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
|
|
"lineno": 279,
|
|
"message": "in wrapper"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
|
|
"lineno": 914,
|
|
"message": "in create"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1242,
|
|
"message": "in post"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 919,
|
|
"message": "in request"
|
|
},
|
|
{
|
|
"path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
|
|
"lineno": 1023,
|
|
"message": "PermissionDeniedError"
|
|
}
|
|
],
|
|
"longrepr": "request = <FixtureRequest for <Function test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=True]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc46d40>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\nmulti_image_data = ['data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGC...6pH9jaTzNv7vfRRXzubfxj9f8Pv8AkTz/AMX/ALbEz5Ly38lfMk/5Z/u64PxhqEZh+z/6rzvn2UUV5EvgPuzy/wAc6p5dt5ccibJpNkkdFFFec27mZ//Z']\nstream = True\n\n @pytest.mark.parametrize(\"stream\", [False, True], ids=[\"stream=False\", \"stream=True\"])\n def test_chat_multi_turn_multiple_images(\n request, openai_client, model, provider, verification_config, multi_image_data, stream\n ):\n test_name_base = get_base_test_name(request)\n if should_skip_test(verification_config, provider, model, test_name_base):\n pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n \n messages_turn1 = [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": multi_image_data[0],\n },\n },\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": multi_image_data[1],\n },\n },\n {\n \"type\": \"text\",\n \"text\": \"What furniture is in the first image that is not in the second image?\",\n },\n ],\n },\n ]\n \n # First API call\n> response1 = openai_client.chat.completions.create(\n model=model,\n messages=messages_turn1,\n stream=stream,\n )\n\ntests/verifications/openai_api/test_chat_completion.py:588: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc46d40>\n\n def _request(\n self,\n *,\n cast_to: Type[ResponseT],\n options: FinalRequestOptions,\n retries_taken: int,\n stream: bool,\n stream_cls: type[_StreamT] | None,\n ) -> ResponseT | _StreamT:\n # create a copy of the options we were given so that if the\n # options are mutated later & we then retry, the retries are\n # given the original options\n input_options = model_copy(options)\n \n cast_to = self._maybe_override_cast_to(cast_to, options)\n options = self._prepare_options(options)\n \n remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n request = self._build_request(options, retries_taken=retries_taken)\n self._prepare_request(request)\n \n kwargs: HttpxSendArgs = {}\n if self.custom_auth is not None:\n kwargs[\"auth\"] = self.custom_auth\n \n log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n \n try:\n response = self._client.send(\n request,\n stream=stream or self._should_stream_response_body(request=request),\n **kwargs,\n )\n except httpx.TimeoutException as err:\n log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising timeout error\")\n raise APITimeoutError(request=request) from err\n except Exception as err:\n log.debug(\"Encountered Exception\", exc_info=True)\n \n if remaining_retries > 0:\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n stream=stream,\n stream_cls=stream_cls,\n response_headers=None,\n )\n \n log.debug(\"Raising connection error\")\n raise APIConnectionError(request=request) from err\n \n log.debug(\n 'HTTP Response: %s %s \"%i %s\" %s',\n request.method,\n request.url,\n response.status_code,\n response.reason_phrase,\n response.headers,\n )\n log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n \n try:\n response.raise_for_status()\n except httpx.HTTPStatusError as err: # thrown on 4xx and 5xx status code\n log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n \n if remaining_retries > 0 and self._should_retry(err.response):\n err.response.close()\n return self._retry_request(\n input_options,\n cast_to,\n retries_taken=retries_taken,\n response_headers=err.response.headers,\n stream=stream,\n stream_cls=stream_cls,\n )\n \n # If the response is streamed then we need to explicitly read the response\n # to completion before attempting to access the response text.\n if not err.response.is_closed:\n err.response.read()\n \n log.debug(\"Re-raising status error\")\n> raise self._make_status_error_from_response(err.response) from None\nE openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
|
|
},
|
|
"teardown": {
|
|
"duration": 0.0019415868446230888,
|
|
"outcome": "passed"
|
|
}
|
|
}
|
|
],
|
|
"run_timestamp": 1744915672
|
|
}
|