llama-stack/llama_stack/providers/tests/test_cases/inference/chat_completion.json

{
  "non_streaming_01": {
    "data": {
      "question": "Which planet do humans live on?",
      "expected": "Earth"
    }
  },
  "non_streaming_02": {
    "data": {
      "question": "Which planet has rings around it with a name starting with letter S?",
      "expected": "Saturn"
    }
  },
  "sample_messages": {
    "data": {
      "messages": [
        {
          "role": "system",
          "content": "You are a helpful assistant."
        },
        {
          "role": "user",
          "content": "What's the weather like today?"
        }
      ]
    }
  },
  "streaming_01": {
    "data": {
      "question": "What's the name of the Sun in latin?",
      "expected": "Sol"
    }
  },
  "streaming_02": {
    "data": {
      "question": "What is the name of the US captial?",
      "expected": "Washington"
    }
  },
  "tool_calling": {
    "data": {
      "messages": [
        {"role": "system", "content": "Pretend you are a weather assistant."},
        {"role": "user", "content": "What's the weather like in San Francisco?"}
      ],
      "tools": [
        {
          "tool_name": "get_weather",
          "description": "Get the current weather",
          "parameters": {
            "location": {
              "param_type": "string",
              "description": "The city and state, e.g. San Francisco, CA"
            }
          }
        }
      ],
      "expected": {
        "location": "San Francisco, CA"
      }
    }
  },
  "sample_messages_tool_calling": {
    "data": {
      "messages": [
        {
          "role": "system",
          "content": "Pretend you are a weather assistant."
        },
        {
          "role": "user",
          "content": "What's the weather like today?"
        },
        {
          "role": "user",
          "content": "What's the weather like in San Francisco?"
        }
      ],
      "tools": [
        {
          "tool_name": "get_weather",
          "description": "Get the current weather",
          "parameters": {
            "location": {
                "param_type": "string",
                "description": "The city and state, e.g. San Francisco, CA",
                "required": true
            }
          }
        }
      ],
      "expected": {
        "location": "San Francisco"
      }
    }
  },
  "structured_output": {
    "data": {
      "notes": "We include context about Michael Jordan in the prompt so that the test is focused on the funtionality of the model and not on the information embedded in the model. Llama 3.2 3B Instruct tends to think MJ played for 14 seasons.",
      "messages": [
        {
          "role": "system",
          "content": "You are a helpful assistant. Michael Jordan was born in 1963. He played basketball for the Chicago Bulls for 15 seasons."
        },
        {
          "role": "user",
          "content": "Please give me information about Michael Jordan."
        }
      ],
      "expected": {
        "first_name": "Michael",
        "last_name": "Jordan",
        "year_of_birth": 1963,
        "num_seasons_in_nba": 15
      }
    }
  },
  "tool_calling_tools_absent": {
    "data": {
      "messages": [
        {
          "role": "system",
          "content": "You are a helpful assistant."
        },
        {
          "role": "user",
          "content": "What pods are in the namespace openshift-lightspeed?"
        },
        {
          "role": "assistant",
          "content": "",
          "stop_reason": "end_of_turn",
          "tool_calls": [
            {
              "call_id": "1",
              "tool_name": "get_object_namespace_list",
              "arguments": {
                "kind": "pod",
                "namespace": "openshift-lightspeed"
              }
            }
          ]
        },
        {
          "role": "tool",
          "call_id": "1",
          "tool_name": "get_object_namespace_list",
          "content": "the objects are pod1, pod2, pod3"
        }
      ],
      "tools": [
        {
          "tool_name": "get_object_namespace_list",
          "description": "Get the list of objects in a namespace",
          "parameters": {
            "kind": {
                "param_type": "string",
                "description": "the type of object",
                "required": true
            },
            "namespace": {
                "param_type": "string",
                "description": "the name of the namespace",
                "required": true
            }
          }
        }
      ]
    }
  }
}