{
  "created": 1744915698.8314075,
  "duration": 25.619409322738647,
  "exitcode": 1,
  "root": "/home/erichuang/llama-stack",
  "environment": {},
  "summary": {
    "failed": 80,
    "skipped": 4,
    "total": 84,
    "collected": 84
  },
  "collectors": [
    {
      "nodeid": "",
      "outcome": "passed",
      "result": [
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py",
          "type": "Module"
        }
      ]
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py",
      "outcome": "passed",
      "result": [
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
          "type": "Function",
          "lineno": 95
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
          "type": "Function",
          "lineno": 95
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
          "type": "Function",
          "lineno": 95
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
          "type": "Function",
          "lineno": 95
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
          "type": "Function",
          "lineno": 95
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
          "type": "Function",
          "lineno": 95
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
          "type": "Function",
          "lineno": 114
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
          "type": "Function",
          "lineno": 114
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
          "type": "Function",
          "lineno": 114
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
          "type": "Function",
          "lineno": 114
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
          "type": "Function",
          "lineno": 114
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
          "type": "Function",
          "lineno": 114
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
          "type": "Function",
          "lineno": 138
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
          "type": "Function",
          "lineno": 138
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
          "type": "Function",
          "lineno": 138
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
          "type": "Function",
          "lineno": 157
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
          "type": "Function",
          "lineno": 157
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
          "type": "Function",
          "lineno": 157
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
          "type": "Function",
          "lineno": 181
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
          "type": "Function",
          "lineno": 181
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
          "type": "Function",
          "lineno": 181
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
          "type": "Function",
          "lineno": 181
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
          "type": "Function",
          "lineno": 181
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
          "type": "Function",
          "lineno": 181
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
          "type": "Function",
          "lineno": 204
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
          "type": "Function",
          "lineno": 204
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
          "type": "Function",
          "lineno": 204
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
          "type": "Function",
          "lineno": 204
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
          "type": "Function",
          "lineno": 204
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
          "type": "Function",
          "lineno": 204
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
          "type": "Function",
          "lineno": 226
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
          "type": "Function",
          "lineno": 226
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
          "type": "Function",
          "lineno": 226
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
          "type": "Function",
          "lineno": 250
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
          "type": "Function",
          "lineno": 250
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
          "type": "Function",
          "lineno": 250
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
          "type": "Function",
          "lineno": 278
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
          "type": "Function",
          "lineno": 278
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
          "type": "Function",
          "lineno": 278
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
          "type": "Function",
          "lineno": 302
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
          "type": "Function",
          "lineno": 302
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
          "type": "Function",
          "lineno": 302
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
          "type": "Function",
          "lineno": 329
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
          "type": "Function",
          "lineno": 329
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
          "type": "Function",
          "lineno": 329
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
          "type": "Function",
          "lineno": 352
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
          "type": "Function",
          "lineno": 352
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
          "type": "Function",
          "lineno": 352
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]",
          "type": "Function",
          "lineno": 380
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]",
          "type": "Function",
          "lineno": 471
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama-v3p3-70b-instruct-stream=False]",
          "type": "Function",
          "lineno": 554
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama-v3p3-70b-instruct-stream=True]",
          "type": "Function",
          "lineno": 554
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=False]",
          "type": "Function",
          "lineno": 554
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=True]",
          "type": "Function",
          "lineno": 554
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=False]",
          "type": "Function",
          "lineno": 554
        },
        {
          "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=True]",
          "type": "Function",
          "lineno": 554
        }
      ]
    }
  ],
  "tests": [
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
      "lineno": 95,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-earth",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "earth"
      },
      "setup": {
        "duration": 0.12150054518133402,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.35746899526566267,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 106,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]>>\nopenai_client = <openai.OpenAI object at 0x7fa60ff6d630>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'earth', 'input': {'messages': [{'content': 'Which planet do humans live on?', 'role': 'user'}]}, 'output': 'Earth'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_basic(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:106: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60ff6d630>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00032798200845718384,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
      "lineno": 95,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-saturn",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "saturn"
      },
      "setup": {
        "duration": 0.07205227017402649,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.1848590886220336,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 106,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc4fd60>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'saturn', 'input': {'messages': [{'content': 'Which planet has rings around it with a name starting with letter S?', 'role': 'user'}]}, 'output': 'Saturn'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_basic(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:106: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc4fd60>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003115283325314522,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
      "lineno": 95,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-earth",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "earth"
      },
      "setup": {
        "duration": 0.06999052409082651,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.20786387100815773,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 106,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc18dc0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'earth', 'input': {'messages': [{'content': 'Which planet do humans live on?', 'role': 'user'}]}, 'output': 'Earth'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_basic(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:106: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc18dc0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.000301288440823555,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
      "lineno": 95,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-saturn",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "saturn"
      },
      "setup": {
        "duration": 0.07327916752547026,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.26050146389752626,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 106,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc18310>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'saturn', 'input': {'messages': [{'content': 'Which planet has rings around it with a name starting with letter S?', 'role': 'user'}]}, 'output': 'Saturn'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_basic(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:106: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc18310>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0004022866487503052,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
      "lineno": 95,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-earth",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "earth"
      },
      "setup": {
        "duration": 0.07078082300722599,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12057740241289139,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 106,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]>>\nopenai_client = <openai.OpenAI object at 0x7fa60ff28280>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'earth', 'input': {'messages': [{'content': 'Which planet do humans live on?', 'role': 'user'}]}, 'output': 'Earth'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_basic(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:106: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60ff28280>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003570122644305229,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
      "lineno": 95,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-saturn",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "saturn"
      },
      "setup": {
        "duration": 0.07103450503200293,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.1985377622768283,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 106,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fedb280>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'saturn', 'input': {'messages': [{'content': 'Which planet has rings around it with a name starting with letter S?', 'role': 'user'}]}, 'output': 'Saturn'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_basic(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:106: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fedb280>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00040165428072214127,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
      "lineno": 114,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-earth",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "earth"
      },
      "setup": {
        "duration": 0.07413783948868513,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.11946621909737587,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 125,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-earth]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fcdb850>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'earth', 'input': {'messages': [{'content': 'Which planet do humans live on?', 'role': 'user'}]}, 'output': 'Earth'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_basic(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:125: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fcdb850>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.000340278260409832,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
      "lineno": 114,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-saturn",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "saturn"
      },
      "setup": {
        "duration": 0.07317963056266308,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12889465410262346,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 125,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_basic[accounts/fireworks/models/llama-v3p3-70b-instruct-saturn]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd5bd90>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'saturn', 'input': {'messages': [{'content': 'Which planet has rings around it with a name starting with letter S?', 'role': 'user'}]}, 'output': 'Saturn'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_basic(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:125: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd5bd90>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00041081756353378296,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
      "lineno": 114,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-earth",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "earth"
      },
      "setup": {
        "duration": 0.07138469163328409,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.15935677476227283,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 125,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-earth]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc47c40>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'earth', 'input': {'messages': [{'content': 'Which planet do humans live on?', 'role': 'user'}]}, 'output': 'Earth'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_basic(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:125: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc47c40>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0004040272906422615,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
      "lineno": 114,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-saturn",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "saturn"
      },
      "setup": {
        "duration": 0.07108956202864647,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.16156401950865984,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 125,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_basic[accounts/fireworks/models/llama4-scout-instruct-basic-saturn]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc44070>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'saturn', 'input': {'messages': [{'content': 'Which planet has rings around it with a name starting with letter S?', 'role': 'user'}]}, 'output': 'Saturn'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_basic(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:125: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc44070>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0005099587142467499,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
      "lineno": 114,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-earth",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "earth"
      },
      "setup": {
        "duration": 0.10071694944053888,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.10996749810874462,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 125,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-earth]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc506a0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'earth', 'input': {'messages': [{'content': 'Which planet do humans live on?', 'role': 'user'}]}, 'output': 'Earth'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_basic(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:125: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc506a0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003982819616794586,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
      "lineno": 114,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-saturn",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "saturn"
      },
      "setup": {
        "duration": 0.07027470227330923,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.2695386055856943,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 125,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_basic[accounts/fireworks/models/llama4-maverick-instruct-basic-saturn]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbc1f60>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'saturn', 'input': {'messages': [{'content': 'Which planet has rings around it with a name starting with letter S?', 'role': 'user'}]}, 'output': 'Saturn'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_basic\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_basic(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:125: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbc1f60>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00039549078792333603,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
      "lineno": 138,
      "outcome": "skipped",
      "keywords": [
        "test_chat_non_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07210677769035101,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.0004008617252111435,
        "outcome": "skipped",
        "longrepr": "('/home/erichuang/llama-stack/tests/verifications/openai_api/test_chat_completion.py', 147, 'Skipped: Skipping test_chat_non_streaming_image for model accounts/fireworks/models/llama-v3p3-70b-instruct on provider fireworks based on config.')"
      },
      "teardown": {
        "duration": 0.00022324267774820328,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
      "lineno": 138,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07092681247740984,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.1368834748864174,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 149,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc51570>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': [{'text': 'What is in this image?', 'type': 'text'}, {'image_url': {...}, 'type': 'image_url'}], 'role': 'user'}]}, 'output': 'llama'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_image\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_image(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:149: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc51570>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00034329574555158615,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
      "lineno": 138,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07233018893748522,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.2651740964502096,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 149,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb2b940>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': [{'text': 'What is in this image?', 'type': 'text'}, {'image_url': {...}, 'type': 'image_url'}], 'role': 'user'}]}, 'output': 'llama'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_image\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_image(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:149: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb2b940>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00043803267180919647,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
      "lineno": 157,
      "outcome": "skipped",
      "keywords": [
        "test_chat_streaming_image[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07322083134204149,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.0002896450459957123,
        "outcome": "skipped",
        "longrepr": "('/home/erichuang/llama-stack/tests/verifications/openai_api/test_chat_completion.py', 166, 'Skipped: Skipping test_chat_streaming_image for model accounts/fireworks/models/llama-v3p3-70b-instruct on provider fireworks based on config.')"
      },
      "teardown": {
        "duration": 0.0003790464252233505,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
      "lineno": 157,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07414108049124479,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.11951867491006851,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 168,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_image[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fea8c70>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': [{'text': 'What is in this image?', 'type': 'text'}, {'image_url': {...}, 'type': 'image_url'}], 'role': 'user'}]}, 'output': 'llama'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_image\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_image(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:168: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fea8c70>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00031629856675863266,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
      "lineno": 157,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07023830153048038,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.27065565437078476,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 168,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_image[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd7c490>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': [{'text': 'What is in this image?', 'type': 'text'}, {'image_url': {...}, 'type': 'image_url'}], 'role': 'user'}]}, 'output': 'llama'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_image\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_image(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:168: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd7c490>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00030570197850465775,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
      "lineno": 181,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-calendar",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "calendar"
      },
      "setup": {
        "duration": 0.06930147111415863,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12505585234612226,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 192,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd56470>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'calendar', 'input': {'messages': [{'content': 'Extract the event information.', 'role': 'system'}, {'cont...articipants'], 'title': 'CalendarEvent', 'type': 'object'}}, 'type': 'json_schema'}}, 'output': 'valid_calendar_event'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            response_format=case[\"input\"][\"response_format\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:192: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd56470>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00032288581132888794,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
      "lineno": 181,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-math",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "math"
      },
      "setup": {
        "duration": 0.07256390806287527,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.19006201811134815,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 192,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd5a1d0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'math', 'input': {'messages': [{'content': 'You are a helpful math tutor. Guide the user through the solut... ['steps', 'final_answer'], 'title': 'MathReasoning', ...}}, 'type': 'json_schema'}}, 'output': 'valid_math_reasoning'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            response_format=case[\"input\"][\"response_format\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:192: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd5a1d0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003082631155848503,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
      "lineno": 181,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-calendar",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "calendar"
      },
      "setup": {
        "duration": 0.07006069924682379,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.15021017380058765,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 192,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb01000>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'calendar', 'input': {'messages': [{'content': 'Extract the event information.', 'role': 'system'}, {'cont...articipants'], 'title': 'CalendarEvent', 'type': 'object'}}, 'type': 'json_schema'}}, 'output': 'valid_calendar_event'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            response_format=case[\"input\"][\"response_format\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:192: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb01000>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003278367221355438,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
      "lineno": 181,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-math",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "math"
      },
      "setup": {
        "duration": 0.07383340876549482,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.2492945184931159,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 192,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc188b0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'math', 'input': {'messages': [{'content': 'You are a helpful math tutor. Guide the user through the solut... ['steps', 'final_answer'], 'title': 'MathReasoning', ...}}, 'type': 'json_schema'}}, 'output': 'valid_math_reasoning'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            response_format=case[\"input\"][\"response_format\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:192: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc188b0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00041339918971061707,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
      "lineno": 181,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-calendar",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "calendar"
      },
      "setup": {
        "duration": 0.07140334881842136,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.3302767900750041,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 192,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]>>\nopenai_client = <openai.OpenAI object at 0x7fa60ff6d480>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'calendar', 'input': {'messages': [{'content': 'Extract the event information.', 'role': 'system'}, {'cont...articipants'], 'title': 'CalendarEvent', 'type': 'object'}}, 'type': 'json_schema'}}, 'output': 'valid_calendar_event'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            response_format=case[\"input\"][\"response_format\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:192: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60ff6d480>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0004214206710457802,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
      "lineno": 181,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-math",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "math"
      },
      "setup": {
        "duration": 0.07027470134198666,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.13318416848778725,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 192,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb01c30>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'math', 'input': {'messages': [{'content': 'You are a helpful math tutor. Guide the user through the solut... ['steps', 'final_answer'], 'title': 'MathReasoning', ...}}, 'type': 'json_schema'}}, 'output': 'valid_math_reasoning'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            response_format=case[\"input\"][\"response_format\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:192: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb01c30>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003965655341744423,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
      "lineno": 204,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-calendar",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "calendar"
      },
      "setup": {
        "duration": 0.07036527991294861,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.10335173550993204,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 215,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-calendar]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc4ffa0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'calendar', 'input': {'messages': [{'content': 'Extract the event information.', 'role': 'system'}, {'cont...articipants'], 'title': 'CalendarEvent', 'type': 'object'}}, 'type': 'json_schema'}}, 'output': 'valid_calendar_event'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            response_format=case[\"input\"][\"response_format\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc4ffa0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003379611298441887,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
      "lineno": 204,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-math",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "math"
      },
      "setup": {
        "duration": 0.06978577468544245,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12087872251868248,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 215,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_structured_output[accounts/fireworks/models/llama-v3p3-70b-instruct-math]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fe19cf0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'math', 'input': {'messages': [{'content': 'You are a helpful math tutor. Guide the user through the solut... ['steps', 'final_answer'], 'title': 'MathReasoning', ...}}, 'type': 'json_schema'}}, 'output': 'valid_math_reasoning'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            response_format=case[\"input\"][\"response_format\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fe19cf0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00042513664811849594,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
      "lineno": 204,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-calendar",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "calendar"
      },
      "setup": {
        "duration": 0.07085503917187452,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.11609443742781878,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 215,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-calendar]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbca530>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'calendar', 'input': {'messages': [{'content': 'Extract the event information.', 'role': 'system'}, {'cont...articipants'], 'title': 'CalendarEvent', 'type': 'object'}}, 'type': 'json_schema'}}, 'output': 'valid_calendar_event'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            response_format=case[\"input\"][\"response_format\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbca530>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.000426730141043663,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
      "lineno": 204,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-math",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "math"
      },
      "setup": {
        "duration": 0.07437158096581697,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12889361381530762,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 215,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_structured_output[accounts/fireworks/models/llama4-scout-instruct-basic-math]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc52500>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'math', 'input': {'messages': [{'content': 'You are a helpful math tutor. Guide the user through the solut... ['steps', 'final_answer'], 'title': 'MathReasoning', ...}}, 'type': 'json_schema'}}, 'output': 'valid_math_reasoning'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            response_format=case[\"input\"][\"response_format\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc52500>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00043479073792696,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
      "lineno": 204,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-calendar",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "calendar"
      },
      "setup": {
        "duration": 0.07079631183296442,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.17871549632400274,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 215,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-calendar]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbf54e0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'calendar', 'input': {'messages': [{'content': 'Extract the event information.', 'role': 'system'}, {'cont...articipants'], 'title': 'CalendarEvent', 'type': 'object'}}, 'type': 'json_schema'}}, 'output': 'valid_calendar_event'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            response_format=case[\"input\"][\"response_format\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbf54e0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003268783912062645,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
      "lineno": 204,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-math",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "math"
      },
      "setup": {
        "duration": 0.07951002009212971,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.1458642790094018,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 215,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_structured_output[accounts/fireworks/models/llama4-maverick-instruct-basic-math]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fe1a230>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'math', 'input': {'messages': [{'content': 'You are a helpful math tutor. Guide the user through the solut... ['steps', 'final_answer'], 'title': 'MathReasoning', ...}}, 'type': 'json_schema'}}, 'output': 'valid_math_reasoning'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_chat_structured_output\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_structured_output(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            response_format=case[\"input\"][\"response_format\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fe1a230>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0004075644537806511,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
      "lineno": 226,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07230154424905777,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.13512122631072998,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 237,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb9c6d0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:237: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb9c6d0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003347489982843399,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
      "lineno": 226,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07080346252769232,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.1290199300274253,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 237,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb6e3e0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:237: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb6e3e0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003189612179994583,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
      "lineno": 226,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07067843340337276,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12150294054299593,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 237,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb01ba0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:237: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb01ba0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00032811518758535385,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
      "lineno": 250,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.0696528134867549,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.2847281629219651,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 261,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb7b2b0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       stream = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:261: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb7b2b0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0004156995564699173,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
      "lineno": 250,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07187621854245663,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12863421067595482,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 261,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb9c3a0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       stream = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:261: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb9c3a0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00036760419607162476,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
      "lineno": 250,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07296357583254576,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12340501230210066,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 261,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fea83d0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_tool_calling(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       stream = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:261: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fea83d0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00042413268238306046,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
      "lineno": 278,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07447731029242277,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.13054667692631483,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 289,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd71450>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],  # Reusing existing case for now\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_tool_choice_required(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            tool_choice=\"required\",  # Force tool call\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:289: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd71450>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00034614279866218567,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
      "lineno": 278,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07243293151259422,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12261831760406494,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 289,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fe198d0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],  # Reusing existing case for now\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_tool_choice_required(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            tool_choice=\"required\",  # Force tool call\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:289: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fe198d0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0004268251359462738,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
      "lineno": 278,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07407685182988644,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.09860698413103819,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 289,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd70a60>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],  # Reusing existing case for now\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_tool_choice_required(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            tool_choice=\"required\",  # Force tool call\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:289: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd70a60>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00039894692599773407,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
      "lineno": 302,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07139755226671696,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.27306741289794445,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 313,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbc8580>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],  # Reusing existing case for now\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_tool_choice_required(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       stream = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            tool_choice=\"required\",  # Force tool call\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:313: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbc8580>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00032315682619810104,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
      "lineno": 302,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.0701784947887063,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12367013934999704,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 313,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fe0b580>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],  # Reusing existing case for now\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_tool_choice_required(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       stream = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            tool_choice=\"required\",  # Force tool call\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:313: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fe0b580>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003095511347055435,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
      "lineno": 302,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07112481445074081,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.11879229731857777,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 313,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_choice_required[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb7b6a0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],  # Reusing existing case for now\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_tool_choice_required(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       stream = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            tool_choice=\"required\",  # Force tool call\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:313: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb7b6a0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00032928306609392166,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
      "lineno": 329,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.0733236288651824,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.20418731309473515,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 340,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb198d0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],  # Reusing existing case for now\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_tool_choice_none(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            tool_choice=\"none\",\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:340: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb198d0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003160899505019188,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
      "lineno": 329,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07103190571069717,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.13796625938266516,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 340,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd5afe0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],  # Reusing existing case for now\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_tool_choice_none(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            tool_choice=\"none\",\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:340: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd5afe0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003080805763602257,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
      "lineno": 329,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07084846775978804,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.11544281151145697,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 340,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb21d80>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],  # Reusing existing case for now\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_tool_choice_none(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       response = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            tool_choice=\"none\",\n            stream=False,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:340: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb21d80>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00031726714223623276,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
      "lineno": 352,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07118451129645109,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.3085783813148737,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 363,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama-v3p3-70b-instruct-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb01150>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],  # Reusing existing case for now\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_tool_choice_none(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       stream = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            tool_choice=\"none\",\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:363: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb01150>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003956826403737068,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
      "lineno": 352,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.07257829792797565,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.09841848351061344,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 363,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-scout-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbf5600>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],  # Reusing existing case for now\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_tool_choice_none(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       stream = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            tool_choice=\"none\",\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:363: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbf5600>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0005597397685050964,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
      "lineno": 352,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-case0",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "case0"
      },
      "setup": {
        "duration": 0.0763206360861659,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.24762020073831081,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 363,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_tool_choice_none[accounts/fireworks/models/llama4-maverick-instruct-basic-case0]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd585b0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'input': {'messages': [{'content': 'You are a helpful assistant that can use tools to get information.', 'role': 'sys..., 'properties': {...}, 'required': [...], 'type': 'object'}}, 'type': 'function'}]}, 'output': 'get_weather_tool_call'}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases[\"test_tool_calling\"][\"test_params\"][\"case\"],  # Reusing existing case for now\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_tool_choice_none(request, openai_client, model, provider, verification_config, case):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n>       stream = openai_client.chat.completions.create(\n            model=model,\n            messages=case[\"input\"][\"messages\"],\n            tools=case[\"input\"][\"tools\"],\n            tool_choice=\"none\",\n            stream=True,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:363: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd585b0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003156941384077072,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "text_then_weather_tool"
      },
      "setup": {
        "duration": 0.07102795876562595,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.11488684639334679,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fab8fa0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'text_then_weather_tool', 'expected': [{'answer': ['sol'], 'num_tool_calls': 0}, {'num_tool_calls': 1, 'to...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fab8fa0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00033565983176231384,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "weather_tool_then_text"
      },
      "setup": {
        "duration": 0.07645629066973925,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.11238154675811529,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fea93c0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'weather_tool_then_text', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'location': 'San Francisco...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fea93c0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0004070783033967018,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "add_product_tool"
      },
      "setup": {
        "duration": 0.07554292771965265,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.2664942145347595,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbcb010>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'add_product_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'inStock': True, 'name': 'Widget...}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': 'Successfully added product with id: 123'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbcb010>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00040212273597717285,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "get_then_create_event_tool"
      },
      "setup": {
        "duration": 0.07388069480657578,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12607386708259583,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd7ed10>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'get_then_create_event_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'date': '2025-03-03', ...ents found for 2025-03-03 at 10:00'}\"}, {'response': \"{'response': 'Successfully created new event with id: e_123'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd7ed10>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003165826201438904,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "compare_monthly_expense_tool"
      },
      "setup": {
        "duration": 0.0708252303302288,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.24374851863831282,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb798d0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'compare_monthly_expense_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'month': 1, 'year': ... 'Total expenses for January 2025: $1000'}\"}, {'response': \"{'response': 'Total expenses for February 2024: $2000'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb798d0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003087436780333519,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "text_then_weather_tool"
      },
      "setup": {
        "duration": 0.07189069222658873,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.1129898214712739,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb7dd80>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'text_then_weather_tool', 'expected': [{'answer': ['sol'], 'num_tool_calls': 0}, {'num_tool_calls': 1, 'to...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb7dd80>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00041724275797605515,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "weather_tool_then_text"
      },
      "setup": {
        "duration": 0.07218896970152855,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.2670225091278553,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc45e70>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'weather_tool_then_text', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'location': 'San Francisco...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc45e70>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003341538831591606,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "add_product_tool"
      },
      "setup": {
        "duration": 0.07147200033068657,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.11082868836820126,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fcdb850>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'add_product_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'inStock': True, 'name': 'Widget...}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': 'Successfully added product with id: 123'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fcdb850>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00040625128895044327,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "get_then_create_event_tool"
      },
      "setup": {
        "duration": 0.07236841041594744,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.27738126553595066,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fdb0f70>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'get_then_create_event_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'date': '2025-03-03', ...ents found for 2025-03-03 at 10:00'}\"}, {'response': \"{'response': 'Successfully created new event with id: e_123'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fdb0f70>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003846520557999611,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "compare_monthly_expense_tool"
      },
      "setup": {
        "duration": 0.06980593129992485,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12573269568383694,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd7e980>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'compare_monthly_expense_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'month': 1, 'year': ... 'Total expenses for January 2025: $1000'}\"}, {'response': \"{'response': 'Total expenses for February 2024: $2000'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd7e980>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003903098404407501,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "text_then_weather_tool"
      },
      "setup": {
        "duration": 0.07204260025173426,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.15989400260150433,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fdab4f0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'text_then_weather_tool', 'expected': [{'answer': ['sol'], 'num_tool_calls': 0}, {'num_tool_calls': 1, 'to...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fdab4f0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00032292958348989487,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "weather_tool_then_text"
      },
      "setup": {
        "duration": 0.07113024219870567,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.11847934126853943,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fcf4160>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'weather_tool_then_text', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'location': 'San Francisco...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fcf4160>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0004066070541739464,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "add_product_tool"
      },
      "setup": {
        "duration": 0.07287267316132784,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.2623152956366539,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb973d0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'add_product_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'inStock': True, 'name': 'Widget...}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': 'Successfully added product with id: 123'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb973d0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00032747630029916763,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "get_then_create_event_tool"
      },
      "setup": {
        "duration": 0.0714963860809803,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.1075747637078166,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbd2620>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'get_then_create_event_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'date': '2025-03-03', ...ents found for 2025-03-03 at 10:00'}\"}, {'response': \"{'response': 'Successfully created new event with id: e_123'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbd2620>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003954702988266945,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]",
      "lineno": 380,
      "outcome": "failed",
      "keywords": [
        "test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "compare_monthly_expense_tool"
      },
      "setup": {
        "duration": 0.07068679295480251,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12899171095341444,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 421,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_non_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc09570>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'compare_monthly_expense_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'month': 1, 'year': ... 'Total expenses for January 2025: $1000'}\"}, {'response': \"{'response': 'Total expenses for February 2024: $2000'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_non_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\"\n        Test cases for multi-turn tool calling.\n        Tool calls are asserted.\n        Tool responses are provided in the test case.\n        Final response is asserted.\n        \"\"\"\n    \n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        # Create a copy of the messages list to avoid modifying the original\n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        # Use deepcopy to prevent modification across runs/parametrization\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        # keep going until either\n        # 1. we have messages to test in multi-turn\n        # 2. no messages but last message is tool response\n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            # do not take new messages if last message is tool response\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                # Ensure new_messages is a list of message objects\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    # If it's a single message object, add it directly\n                    messages.append(new_messages)\n    \n            # --- API Call ---\n>           response = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=False,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:421: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc09570>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00031181517988443375,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "text_then_weather_tool"
      },
      "setup": {
        "duration": 0.06948941852897406,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.1193860862404108,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-text_then_weather_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc06560>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'text_then_weather_tool', 'expected': [{'answer': ['sol'], 'num_tool_calls': 0}, {'num_tool_calls': 1, 'to...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc06560>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0004093386232852936,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "weather_tool_then_text"
      },
      "setup": {
        "duration": 0.07101139053702354,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.1606877325102687,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-weather_tool_then_text]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fe09630>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'weather_tool_then_text', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'location': 'San Francisco...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fe09630>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00041847582906484604,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "add_product_tool"
      },
      "setup": {
        "duration": 0.07783220708370209,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.11765131819993258,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-add_product_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fe19f00>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'add_product_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'inStock': True, 'name': 'Widget...}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': 'Successfully added product with id: 123'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fe19f00>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0004519466310739517,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "get_then_create_event_tool"
      },
      "setup": {
        "duration": 0.07041152473539114,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.13473773282021284,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-get_then_create_event_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fa59b40>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'get_then_create_event_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'date': '2025-03-03', ...ents found for 2025-03-03 at 10:00'}\"}, {'response': \"{'response': 'Successfully created new event with id: e_123'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fa59b40>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00039084814488887787,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "compare_monthly_expense_tool"
      },
      "setup": {
        "duration": 0.07150810118764639,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.2046368457376957,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama-v3p3-70b-instruct-compare_monthly_expense_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fed82b0>\nmodel = 'accounts/fireworks/models/llama-v3p3-70b-instruct'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'compare_monthly_expense_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'month': 1, 'year': ... 'Total expenses for January 2025: $1000'}\"}, {'response': \"{'response': 'Total expenses for February 2024: $2000'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fed82b0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003378065302968025,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "text_then_weather_tool"
      },
      "setup": {
        "duration": 0.07012568973004818,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12623131182044744,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-text_then_weather_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc1abf0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'text_then_weather_tool', 'expected': [{'answer': ['sol'], 'num_tool_calls': 0}, {'num_tool_calls': 1, 'to...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc1abf0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00034175068140029907,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "weather_tool_then_text"
      },
      "setup": {
        "duration": 0.07349637430161238,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.11243469640612602,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-weather_tool_then_text]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbc3ac0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'weather_tool_then_text', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'location': 'San Francisco...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbc3ac0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003092559054493904,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "add_product_tool"
      },
      "setup": {
        "duration": 0.07129209581762552,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.13334522116929293,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-add_product_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fab85b0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'add_product_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'inStock': True, 'name': 'Widget...}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': 'Successfully added product with id: 123'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fab85b0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0004090704023838043,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "get_then_create_event_tool"
      },
      "setup": {
        "duration": 0.07043681107461452,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.10430899448692799,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-get_then_create_event_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fdb30a0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'get_then_create_event_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'date': '2025-03-03', ...ents found for 2025-03-03 at 10:00'}\"}, {'response': \"{'response': 'Successfully created new event with id: e_123'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fdb30a0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003421120345592499,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "compare_monthly_expense_tool"
      },
      "setup": {
        "duration": 0.0733558852225542,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.10938013903796673,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-scout-instruct-basic-compare_monthly_expense_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fd708e0>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'compare_monthly_expense_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'month': 1, 'year': ... 'Total expenses for January 2025: $1000'}\"}, {'response': \"{'response': 'Total expenses for February 2024: $2000'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fd708e0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00038493238389492035,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "text_then_weather_tool"
      },
      "setup": {
        "duration": 0.07225227076560259,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.11426256597042084,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-text_then_weather_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb2ebc0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'text_then_weather_tool', 'expected': [{'answer': ['sol'], 'num_tool_calls': 0}, {'num_tool_calls': 1, 'to...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb2ebc0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003271028399467468,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "weather_tool_then_text"
      },
      "setup": {
        "duration": 0.07055194210261106,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.1077765729278326,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-weather_tool_then_text]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb85e10>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'weather_tool_then_text', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'location': 'San Francisco...], 'type': 'object'}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': '70 degrees and foggy'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb85e10>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003206087276339531,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "add_product_tool"
      },
      "setup": {
        "duration": 0.08054796047508717,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12442617677152157,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-add_product_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fde9d20>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'add_product_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'inStock': True, 'name': 'Widget...}}, 'type': 'function'}]}, 'tool_responses': [{'response': \"{'response': 'Successfully added product with id: 123'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fde9d20>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00040055252611637115,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "get_then_create_event_tool"
      },
      "setup": {
        "duration": 0.07104168552905321,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.10808593034744263,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-get_then_create_event_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60f9bc700>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'get_then_create_event_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'date': '2025-03-03', ...ents found for 2025-03-03 at 10:00'}\"}, {'response': \"{'response': 'Successfully created new event with id: e_123'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60f9bc700>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00033656321465969086,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]",
      "lineno": 471,
      "outcome": "failed",
      "keywords": [
        "test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "compare_monthly_expense_tool"
      },
      "setup": {
        "duration": 0.06945569068193436,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.12593147810548544,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 498,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_streaming_multi_turn_tool_calling[accounts/fireworks/models/llama4-maverick-instruct-basic-compare_monthly_expense_tool]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc066b0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\ncase = {'case_id': 'compare_monthly_expense_tool', 'expected': [{'num_tool_calls': 1, 'tool_arguments': {'month': 1, 'year': ... 'Total expenses for January 2025: $1000'}\"}, {'response': \"{'response': 'Total expenses for February 2024: $2000'}\"}]}\n\n    @pytest.mark.parametrize(\n        \"case\",\n        chat_completion_test_cases.get(\"test_chat_multi_turn_tool_calling\", {}).get(\"test_params\", {}).get(\"case\", []),\n        ids=case_id_generator,\n    )\n    def test_chat_streaming_multi_turn_tool_calling(request, openai_client, model, provider, verification_config, case):\n        \"\"\" \"\"\"\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages = []\n        tools = case[\"input\"][\"tools\"]\n        expected_results = copy.deepcopy(case[\"expected\"])\n        tool_responses = copy.deepcopy(case.get(\"tool_responses\", []))\n        input_messages_turns = copy.deepcopy(case[\"input\"][\"messages\"])\n    \n        while len(input_messages_turns) > 0 or (len(messages) > 0 and messages[-1][\"role\"] == \"tool\"):\n            if len(messages) == 0 or messages[-1][\"role\"] != \"tool\":\n                new_messages = input_messages_turns.pop(0)\n                if isinstance(new_messages, list):\n                    messages.extend(new_messages)\n                else:\n                    messages.append(new_messages)\n    \n            # --- API Call (Streaming) ---\n>           stream = openai_client.chat.completions.create(\n                model=model,\n                messages=messages,\n                tools=tools,\n                stream=True,\n            )\n\ntests/verifications/openai_api/test_chat_completion.py:498: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc066b0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0004069330170750618,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama-v3p3-70b-instruct-stream=False]",
      "lineno": 554,
      "outcome": "skipped",
      "keywords": [
        "test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama-v3p3-70b-instruct-stream=False]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-stream=False",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "stream=False"
      },
      "setup": {
        "duration": 0.07031089067459106,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.00027627311646938324,
        "outcome": "skipped",
        "longrepr": "('/home/erichuang/llama-stack/tests/verifications/openai_api/test_chat_completion.py', 561, 'Skipped: Skipping test_chat_multi_turn_multiple_images for model accounts/fireworks/models/llama-v3p3-70b-instruct on provider fireworks based on config.')"
      },
      "teardown": {
        "duration": 0.0003817221149802208,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama-v3p3-70b-instruct-stream=True]",
      "lineno": 554,
      "outcome": "skipped",
      "keywords": [
        "test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama-v3p3-70b-instruct-stream=True]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama-v3p3-70b-instruct-stream=True",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama-v3p3-70b-instruct",
        "case_id": "stream=True"
      },
      "setup": {
        "duration": 0.07004163973033428,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.000285550020635128,
        "outcome": "skipped",
        "longrepr": "('/home/erichuang/llama-stack/tests/verifications/openai_api/test_chat_completion.py', 561, 'Skipped: Skipping test_chat_multi_turn_multiple_images for model accounts/fireworks/models/llama-v3p3-70b-instruct on provider fireworks based on config.')"
      },
      "teardown": {
        "duration": 0.00021260324865579605,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=False]",
      "lineno": 554,
      "outcome": "failed",
      "keywords": [
        "test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=False]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-stream=False",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "stream=False"
      },
      "setup": {
        "duration": 0.07160913478583097,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.23755338042974472,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 588,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=False]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fbf4190>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\nmulti_image_data = ['data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGC...6pH9jaTzNv7vfRRXzubfxj9f8Pv8AkTz/AMX/ALbEz5Ly38lfMk/5Z/u64PxhqEZh+z/6rzvn2UUV5EvgPuzy/wAc6p5dt5ccibJpNkkdFFFec27mZ//Z']\nstream = False\n\n    @pytest.mark.parametrize(\"stream\", [False, True], ids=[\"stream=False\", \"stream=True\"])\n    def test_chat_multi_turn_multiple_images(\n        request, openai_client, model, provider, verification_config, multi_image_data, stream\n    ):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages_turn1 = [\n            {\n                \"role\": \"user\",\n                \"content\": [\n                    {\n                        \"type\": \"image_url\",\n                        \"image_url\": {\n                            \"url\": multi_image_data[0],\n                        },\n                    },\n                    {\n                        \"type\": \"image_url\",\n                        \"image_url\": {\n                            \"url\": multi_image_data[1],\n                        },\n                    },\n                    {\n                        \"type\": \"text\",\n                        \"text\": \"What furniture is in the first image that is not in the second image?\",\n                    },\n                ],\n            },\n        ]\n    \n        # First API call\n>       response1 = openai_client.chat.completions.create(\n            model=model,\n            messages=messages_turn1,\n            stream=stream,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:588: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fbf4190>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.00033799000084400177,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=True]",
      "lineno": 554,
      "outcome": "failed",
      "keywords": [
        "test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=True]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-scout-instruct-basic-stream=True",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-scout-instruct-basic",
        "case_id": "stream=True"
      },
      "setup": {
        "duration": 0.0742298774421215,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.28080874867737293,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 588,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-scout-instruct-basic-stream=True]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fb6e920>\nmodel = 'accounts/fireworks/models/llama4-scout-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\nmulti_image_data = ['data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGC...6pH9jaTzNv7vfRRXzubfxj9f8Pv8AkTz/AMX/ALbEz5Ly38lfMk/5Z/u64PxhqEZh+z/6rzvn2UUV5EvgPuzy/wAc6p5dt5ccibJpNkkdFFFec27mZ//Z']\nstream = True\n\n    @pytest.mark.parametrize(\"stream\", [False, True], ids=[\"stream=False\", \"stream=True\"])\n    def test_chat_multi_turn_multiple_images(\n        request, openai_client, model, provider, verification_config, multi_image_data, stream\n    ):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages_turn1 = [\n            {\n                \"role\": \"user\",\n                \"content\": [\n                    {\n                        \"type\": \"image_url\",\n                        \"image_url\": {\n                            \"url\": multi_image_data[0],\n                        },\n                    },\n                    {\n                        \"type\": \"image_url\",\n                        \"image_url\": {\n                            \"url\": multi_image_data[1],\n                        },\n                    },\n                    {\n                        \"type\": \"text\",\n                        \"text\": \"What furniture is in the first image that is not in the second image?\",\n                    },\n                ],\n            },\n        ]\n    \n        # First API call\n>       response1 = openai_client.chat.completions.create(\n            model=model,\n            messages=messages_turn1,\n            stream=stream,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:588: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fb6e920>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003352127969264984,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=False]",
      "lineno": 554,
      "outcome": "failed",
      "keywords": [
        "test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=False]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-stream=False",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "stream=False"
      },
      "setup": {
        "duration": 0.07394346781075001,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.2838349239900708,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 588,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=False]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fabb6d0>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\nmulti_image_data = ['data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGC...6pH9jaTzNv7vfRRXzubfxj9f8Pv8AkTz/AMX/ALbEz5Ly38lfMk/5Z/u64PxhqEZh+z/6rzvn2UUV5EvgPuzy/wAc6p5dt5ccibJpNkkdFFFec27mZ//Z']\nstream = False\n\n    @pytest.mark.parametrize(\"stream\", [False, True], ids=[\"stream=False\", \"stream=True\"])\n    def test_chat_multi_turn_multiple_images(\n        request, openai_client, model, provider, verification_config, multi_image_data, stream\n    ):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages_turn1 = [\n            {\n                \"role\": \"user\",\n                \"content\": [\n                    {\n                        \"type\": \"image_url\",\n                        \"image_url\": {\n                            \"url\": multi_image_data[0],\n                        },\n                    },\n                    {\n                        \"type\": \"image_url\",\n                        \"image_url\": {\n                            \"url\": multi_image_data[1],\n                        },\n                    },\n                    {\n                        \"type\": \"text\",\n                        \"text\": \"What furniture is in the first image that is not in the second image?\",\n                    },\n                ],\n            },\n        ]\n    \n        # First API call\n>       response1 = openai_client.chat.completions.create(\n            model=model,\n            messages=messages_turn1,\n            stream=stream,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:588: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fabb6d0>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0003282083198428154,
        "outcome": "passed"
      }
    },
    {
      "nodeid": "tests/verifications/openai_api/test_chat_completion.py::test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=True]",
      "lineno": 554,
      "outcome": "failed",
      "keywords": [
        "test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=True]",
        "parametrize",
        "pytestmark",
        "accounts/fireworks/models/llama4-maverick-instruct-basic-stream=True",
        "test_chat_completion.py",
        "openai_api",
        "verifications",
        "tests",
        "llama-stack",
        ""
      ],
      "metadata": {
        "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
        "case_id": "stream=True"
      },
      "setup": {
        "duration": 0.0713854730129242,
        "outcome": "passed"
      },
      "call": {
        "duration": 0.20238025579601526,
        "outcome": "failed",
        "crash": {
          "path": "/home/erichuang/.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
          "lineno": 1023,
          "message": "openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}"
        },
        "traceback": [
          {
            "path": "tests/verifications/openai_api/test_chat_completion.py",
            "lineno": 588,
            "message": ""
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py",
            "lineno": 279,
            "message": "in wrapper"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py",
            "lineno": 914,
            "message": "in create"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1242,
            "message": "in post"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 919,
            "message": "in request"
          },
          {
            "path": "../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py",
            "lineno": 1023,
            "message": "PermissionDeniedError"
          }
        ],
        "longrepr": "request = <FixtureRequest for <Function test_chat_multi_turn_multiple_images[accounts/fireworks/models/llama4-maverick-instruct-basic-stream=True]>>\nopenai_client = <openai.OpenAI object at 0x7fa60fc46d40>\nmodel = 'accounts/fireworks/models/llama4-maverick-instruct-basic'\nprovider = 'fireworks'\nverification_config = {'providers': {'cerebras': {'api_key_var': 'CEREBRAS_API_KEY', 'base_url': 'https://api.cerebras.ai/v1', 'model_displa...-versatile', 'meta-llama/llama-4-scout-17b-16e-instruct', 'meta-llama/llama-4-maverick-17b-128e-instruct'], ...}, ...}}\nmulti_image_data = ['data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAIBAQEBAQIBAQECAgICAgQDAgICAgUEBAMEBgUGBgYFBgYGBwkIBgcJBwYGC...6pH9jaTzNv7vfRRXzubfxj9f8Pv8AkTz/AMX/ALbEz5Ly38lfMk/5Z/u64PxhqEZh+z/6rzvn2UUV5EvgPuzy/wAc6p5dt5ccibJpNkkdFFFec27mZ//Z']\nstream = True\n\n    @pytest.mark.parametrize(\"stream\", [False, True], ids=[\"stream=False\", \"stream=True\"])\n    def test_chat_multi_turn_multiple_images(\n        request, openai_client, model, provider, verification_config, multi_image_data, stream\n    ):\n        test_name_base = get_base_test_name(request)\n        if should_skip_test(verification_config, provider, model, test_name_base):\n            pytest.skip(f\"Skipping {test_name_base} for model {model} on provider {provider} based on config.\")\n    \n        messages_turn1 = [\n            {\n                \"role\": \"user\",\n                \"content\": [\n                    {\n                        \"type\": \"image_url\",\n                        \"image_url\": {\n                            \"url\": multi_image_data[0],\n                        },\n                    },\n                    {\n                        \"type\": \"image_url\",\n                        \"image_url\": {\n                            \"url\": multi_image_data[1],\n                        },\n                    },\n                    {\n                        \"type\": \"text\",\n                        \"text\": \"What furniture is in the first image that is not in the second image?\",\n                    },\n                ],\n            },\n        ]\n    \n        # First API call\n>       response1 = openai_client.chat.completions.create(\n            model=model,\n            messages=messages_turn1,\n            stream=stream,\n        )\n\ntests/verifications/openai_api/test_chat_completion.py:588: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_utils/_utils.py:279: in wrapper\n    return func(*args, **kwargs)\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/resources/chat/completions/completions.py:914: in create\n    return self._post(\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1242: in post\n    return cast(ResponseT, self.request(cast_to, opts, stream=stream, stream_cls=stream_cls))\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:919: in request\n    return self._request(\n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <openai.OpenAI object at 0x7fa60fc46d40>\n\n    def _request(\n        self,\n        *,\n        cast_to: Type[ResponseT],\n        options: FinalRequestOptions,\n        retries_taken: int,\n        stream: bool,\n        stream_cls: type[_StreamT] | None,\n    ) -> ResponseT | _StreamT:\n        # create a copy of the options we were given so that if the\n        # options are mutated later & we then retry, the retries are\n        # given the original options\n        input_options = model_copy(options)\n    \n        cast_to = self._maybe_override_cast_to(cast_to, options)\n        options = self._prepare_options(options)\n    \n        remaining_retries = options.get_max_retries(self.max_retries) - retries_taken\n        request = self._build_request(options, retries_taken=retries_taken)\n        self._prepare_request(request)\n    \n        kwargs: HttpxSendArgs = {}\n        if self.custom_auth is not None:\n            kwargs[\"auth\"] = self.custom_auth\n    \n        log.debug(\"Sending HTTP Request: %s %s\", request.method, request.url)\n    \n        try:\n            response = self._client.send(\n                request,\n                stream=stream or self._should_stream_response_body(request=request),\n                **kwargs,\n            )\n        except httpx.TimeoutException as err:\n            log.debug(\"Encountered httpx.TimeoutException\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising timeout error\")\n            raise APITimeoutError(request=request) from err\n        except Exception as err:\n            log.debug(\"Encountered Exception\", exc_info=True)\n    \n            if remaining_retries > 0:\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                    response_headers=None,\n                )\n    \n            log.debug(\"Raising connection error\")\n            raise APIConnectionError(request=request) from err\n    \n        log.debug(\n            'HTTP Response: %s %s \"%i %s\" %s',\n            request.method,\n            request.url,\n            response.status_code,\n            response.reason_phrase,\n            response.headers,\n        )\n        log.debug(\"request_id: %s\", response.headers.get(\"x-request-id\"))\n    \n        try:\n            response.raise_for_status()\n        except httpx.HTTPStatusError as err:  # thrown on 4xx and 5xx status code\n            log.debug(\"Encountered httpx.HTTPStatusError\", exc_info=True)\n    \n            if remaining_retries > 0 and self._should_retry(err.response):\n                err.response.close()\n                return self._retry_request(\n                    input_options,\n                    cast_to,\n                    retries_taken=retries_taken,\n                    response_headers=err.response.headers,\n                    stream=stream,\n                    stream_cls=stream_cls,\n                )\n    \n            # If the response is streamed then we need to explicitly read the response\n            # to completion before attempting to access the response text.\n            if not err.response.is_closed:\n                err.response.read()\n    \n            log.debug(\"Re-raising status error\")\n>           raise self._make_status_error_from_response(err.response) from None\nE           openai.PermissionDeniedError: Error code: 403 - {'error': 'unauthorized'}\n\n../.conda/envs/myenv/lib/python3.10/site-packages/openai/_base_client.py:1023: PermissionDeniedError"
      },
      "teardown": {
        "duration": 0.0019415868446230888,
        "outcome": "passed"
      }
    }
  ],
  "run_timestamp": 1744915672
}