mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-11 05:38:38 +00:00
fix(tests): ensure test isolation in server mode (#3737)
Propagate test IDs from client to server via HTTP headers to maintain proper test isolation when running with server-based stack configs. Without this, recorded/replayed inference requests in server mode would leak across tests. Changes: - Patch client _prepare_request to inject test ID into provider data header - Sync test context from provider data on server side before storage operations - Set LLAMA_STACK_TEST_STACK_CONFIG_TYPE env var based on stack config - Configure console width for cleaner log output in CI - Add SQLITE_STORE_DIR temp directory for test data isolation
This commit is contained in:
Ashwin Bharambe
GitHub
parent
96886afaca
commit
79bed44b04
419 changed files with 106801 additions and 35909 deletions
|
|
@ -1,4 +1,5 @@
|
|||
{
|
||||
"test_id": "tests/integration/eval/test_eval.py::test_evaluate_rows[txt=ollama/llama3.2:3b-instruct-fp16-basic::equality]",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "http://0.0.0.0:11434/v1/v1/chat/completions",
|
||||
|
|
@ -20,7 +21,7 @@
|
|||
"body": {
|
||||
"__type__": "openai.types.chat.chat_completion.ChatCompletion",
|
||||
"__data__": {
|
||||
"id": "rec-8d035e153b6f",
|
||||
"id": "rec-171c4dcb3dc8",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@
|
|||
"index": 0,
|
||||
"logprobs": null,
|
||||
"message": {
|
||||
"content": "The official currency of Japan is the yen (JPY). It is denoted by the ISO code \"JPY\" and subdivided into 100 sen, although sen are no longer used in everyday transactions. The Japanese government currently plans to phase out cash payments for certain types of transactions and aims to move fully away from paper money by year-end 2031.",
|
||||
"content": "The currency of Japan is the Japanese yen (\u00a5).",
|
||||
"refusal": null,
|
||||
"role": "assistant",
|
||||
"annotations": null,
|
||||
|
|
@ -44,9 +44,9 @@
|
|||
"service_tier": null,
|
||||
"system_fingerprint": "fp_ollama",
|
||||
"usage": {
|
||||
"completion_tokens": 74,
|
||||
"completion_tokens": 12,
|
||||
"prompt_tokens": 32,
|
||||
"total_tokens": 106,
|
||||
"total_tokens": 44,
|
||||
"completion_tokens_details": null,
|
||||
"prompt_tokens_details": null
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@
|
|||
"index": 0,
|
||||
"logprobs": null,
|
||||
"message": {
|
||||
"content": "The smallest country in the world is officially Vatican City, with an area of approximately 0.44 km\u00b2 (0.17 sq mi). It is located within Rome, Italy, and is home to the Pope and the central government of the Catholic Church.\n\nVatican City has a population of around 800 people, making it the smallest internationally recognized sovereign state in the world, both by area and population. Despite its small size, Vatican City has its own government, currency, postal system, and even its own police force.\n\nInterestingly, Vatican City is also the headquarters of the Catholic Church and is home to numerous iconic landmarks, including St. Peter's Basilica, the Sistine Chapel, and the Vatican Museums.",
|
||||
"content": "The smallest country in the world is the Vatican City, with an area of approximately 0.44 km\u00b2 (0.17 square miles). It is an independent city-state located within Rome, Italy, and serves as the headquarters of the Catholic Church. The Vatican City has a population of around 800 people and is home to numerous iconic landmarks, including St. Peter's Basilica and the Sistine Chapel.",
|
||||
"refusal": null,
|
||||
"role": "assistant",
|
||||
"annotations": null,
|
||||
|
|
@ -44,9 +44,9 @@
|
|||
"service_tier": null,
|
||||
"system_fingerprint": "fp_ollama",
|
||||
"usage": {
|
||||
"completion_tokens": 146,
|
||||
"completion_tokens": 84,
|
||||
"prompt_tokens": 34,
|
||||
"total_tokens": 180,
|
||||
"total_tokens": 118,
|
||||
"completion_tokens_details": null,
|
||||
"prompt_tokens_details": null
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
{
|
||||
"test_id": "tests/integration/eval/test_eval.py::test_evaluate_benchmark[txt=ollama/llama3.2:3b-instruct-fp16-basic::subset_of]",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "http://0.0.0.0:11434/v1/v1/chat/completions",
|
||||
|
|
@ -8,7 +9,7 @@
|
|||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is the currency of Japan?"
|
||||
"content": "Who is the CEO of Meta?"
|
||||
}
|
||||
],
|
||||
"max_tokens": 0
|
||||
|
|
@ -20,14 +21,14 @@
|
|||
"body": {
|
||||
"__type__": "openai.types.chat.chat_completion.ChatCompletion",
|
||||
"__data__": {
|
||||
"id": "rec-92a9a916ef02",
|
||||
"id": "rec-9ebe1e04fc3a",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"logprobs": null,
|
||||
"message": {
|
||||
"content": "The currency of Japan is the Japanese yen (\u00a5). It is represented by the symbol \u00a5. In some contexts, it's also abbreviated as \"JPY\" or written as \"yen\". The Bank of Japan is responsible for managing the country's monetary policy and issuing new yen banknotes and coins.",
|
||||
"content": "Mark Zuckerberg is the founder, chairman and CEO of Meta, which he originally founded as Facebook in 2004.",
|
||||
"refusal": null,
|
||||
"role": "assistant",
|
||||
"annotations": null,
|
||||
|
|
@ -43,9 +44,9 @@
|
|||
"service_tier": null,
|
||||
"system_fingerprint": "fp_ollama",
|
||||
"usage": {
|
||||
"completion_tokens": 61,
|
||||
"completion_tokens": 24,
|
||||
"prompt_tokens": 32,
|
||||
"total_tokens": 93,
|
||||
"total_tokens": 56,
|
||||
"completion_tokens_details": null,
|
||||
"prompt_tokens_details": null
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
{
|
||||
"test_id": "tests/integration/eval/test_eval.py::test_evaluate_benchmark[txt=ollama/llama3.2:3b-instruct-fp16-basic::subset_of]",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "http://0.0.0.0:11434/v1/v1/chat/completions",
|
||||
|
|
@ -8,7 +9,7 @@
|
|||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is the smallest country in the world?"
|
||||
"content": "What is the largest planet in our solar system?"
|
||||
}
|
||||
],
|
||||
"max_tokens": 0
|
||||
|
|
@ -20,14 +21,14 @@
|
|||
"body": {
|
||||
"__type__": "openai.types.chat.chat_completion.ChatCompletion",
|
||||
"__data__": {
|
||||
"id": "rec-c62eb5d7115e",
|
||||
"id": "rec-aa20023c358a",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"logprobs": null,
|
||||
"message": {
|
||||
"content": "The smallest country in the world is the Vatican City, with an area of approximately 0.44 km\u00b2 (0.17 sq mi). It is an independent city-state located within Rome, Italy, and is the headquarters of the Catholic Church. Despite its small size, the Vatican City has a population of around 800 people, including the Pope and other high-ranking officials.",
|
||||
"content": "The largest planet in our solar system is Jupiter. It has a diameter of approximately 142,984 kilometers (88,846 miles), which is more than 11 times the diameter of the Earth! Jupiter is a gas giant and is known for its massive size, stormy atmosphere, and numerous moons.",
|
||||
"refusal": null,
|
||||
"role": "assistant",
|
||||
"annotations": null,
|
||||
|
|
@ -43,9 +44,9 @@
|
|||
"service_tier": null,
|
||||
"system_fingerprint": "fp_ollama",
|
||||
"usage": {
|
||||
"completion_tokens": 77,
|
||||
"prompt_tokens": 34,
|
||||
"total_tokens": 111,
|
||||
"completion_tokens": 62,
|
||||
"prompt_tokens": 35,
|
||||
"total_tokens": 97,
|
||||
"completion_tokens_details": null,
|
||||
"prompt_tokens_details": null
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
{
|
||||
"test_id": "tests/integration/eval/test_eval.py::test_evaluate_rows[txt=ollama/llama3.2:3b-instruct-fp16-basic::equality]",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "http://0.0.0.0:11434/v1/v1/chat/completions",
|
||||
|
|
@ -20,7 +21,7 @@
|
|||
"body": {
|
||||
"__type__": "openai.types.chat.chat_completion.ChatCompletion",
|
||||
"__data__": {
|
||||
"id": "rec-e25ab43491af",
|
||||
"id": "rec-b52a054b314c",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
{
|
||||
"test_id": "tests/integration/eval/test_eval.py::test_evaluate_rows[txt=ollama/llama3.2:3b-instruct-fp16-basic::equality]",
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "http://0.0.0.0:11434/v1/v1/chat/completions",
|
||||
|
|
@ -20,14 +21,14 @@
|
|||
"body": {
|
||||
"__type__": "openai.types.chat.chat_completion.ChatCompletion",
|
||||
"__data__": {
|
||||
"id": "rec-f28a44c97ea7",
|
||||
"id": "rec-c07b01fe9946",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"logprobs": null,
|
||||
"message": {
|
||||
"content": "The largest planet in our solar system is Jupiter. It is a gas giant and has a diameter of approximately 142,984 kilometers (88,846 miles). Jupiter is more than 1,300 times the size of Earth and is the fifth planet from the Sun.",
|
||||
"content": "Jupiter is the largest planet in our solar system, with a diameter of approximately 142,984 kilometers (88,846 miles). It is a gas giant and composed mainly of hydrogen and helium. Jupiter's large size and mass are more than 2.5 times that of all the other planets in our solar system combined.",
|
||||
"refusal": null,
|
||||
"role": "assistant",
|
||||
"annotations": null,
|
||||
|
|
@ -43,9 +44,9 @@
|
|||
"service_tier": null,
|
||||
"system_fingerprint": "fp_ollama",
|
||||
"usage": {
|
||||
"completion_tokens": 55,
|
||||
"completion_tokens": 67,
|
||||
"prompt_tokens": 35,
|
||||
"total_tokens": 90,
|
||||
"total_tokens": 102,
|
||||
"completion_tokens_details": null,
|
||||
"prompt_tokens_details": null
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue