phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-15 06:37:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
llama-stack-mirror/tests/integration/responses/recordings/1dbd102b81c7dee87b189664823f2e43f6105ae3111ad9cf843899f495f8bf76.json
Ashwin Bharambe f50ce11a3b
feat(tests): make inference_recorder into api_recorder (include tool_invoke) (#3403) 
Renames `inference_recorder.py` to `api_recorder.py` and extends it to
support recording/replaying tool invocations in addition to inference
calls.

This allows us to record web-search, etc. tool calls and thereafter
apply recordings for `tests/integration/responses`

## Test Plan

```
export OPENAI_API_KEY=...
export TAVILY_SEARCH_API_KEY=...

./scripts/integration-tests.sh --stack-config ci-tests \
   --suite responses --inference-mode record-if-missing
```
2025-10-09 14:27:51 -07:00

779 lines
26 KiB
JSON
Raw Blame History

{
"test_id": "tests/integration/responses/test_file_search.py::test_response_file_search_filter_compound_or[openai_client-txt=openai/gpt-4o]",
"request": {
"method": "POST",
"url": "https://api.openai.com/v1/v1/chat/completions",
"headers": {},
"body": {
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": "Show me marketing and sales documents"
},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"index": 0,
"id": "call_Fvpcub7CINR4M4mne3NrqPSh",
"type": "function",
"function": {
"name": "knowledge_search",
"arguments": "{\"query\": \"marketing documents examples\"}"
}
},
{
"index": 1,
"id": "call_nNONu48z4o4psZIlHNDbWIe6",
"type": "function",
"function": {
"name": "knowledge_search",
"arguments": "{\"query\": \"sales documents examples\"}"
}
}
]
},
{
"role": "tool",
"tool_call_id": "call_Fvpcub7CINR4M4mne3NrqPSh",
"content": [
{
"type": "text",
"text": "knowledge_search tool found 3 chunks:\nBEGIN of knowledge_search tool results.\n"
},
{
"type": "text",
"text": "[1] document_id: file-862946202236, score: 0.7656461893973538, attributes: {'region': 'eu', 'category': 'marketing', 'date': 1672531200.0, 'filename': 'eu_marketing_q1.txt', 'document_id': 'file-862946202236', 'token_count': 17.0, 'metadata_token_count': 32.0} (cite as <|file-862946202236|>)\nEuropean advertising campaign results for Q1 2023. Strong growth in EU markets.\n"
},
{
"type": "text",
"text": "[2] document_id: file-862946202234, score: 0.729409669664322, attributes: {'region': 'us', 'category': 'marketing', 'date': 1672531200.0, 'filename': 'us_marketing_q1.txt', 'document_id': 'file-862946202234', 'token_count': 21.0, 'metadata_token_count': 32.0} (cite as <|file-862946202234|>)\nUS promotional campaigns for Q1 2023. Revenue increased by 15% in the US region.\n"
},
{
"type": "text",
"text": "[3] document_id: file-862946202237, score: 0.5528548752393603, attributes: {'region': 'asia', 'category': 'sales', 'date': 1688169600.0, 'filename': 'asia_sales_q3.txt', 'document_id': 'file-862946202237', 'token_count': 17.0, 'metadata_token_count': 31.0} (cite as <|file-862946202237|>)\nAsia Pacific revenue figures for Q3 2023. Record breaking quarter in Asia.\n"
},
{
"type": "text",
"text": "END of knowledge_search tool results.\n"
},
{
"type": "text",
"text": "The above results were retrieved to help answer the user's query: \"marketing documents examples\". Use them as supporting information only in answering this query. Cite sources immediately at the end of sentences before punctuation, using `<|file-id|>` format (e.g., 'This is a fact <|file-Cn3MSNn72ENTiiq11Qda4A|>.'). Do not add extra punctuation. Use only the file IDs provided (do not invent new ones).\n"
}
]
},
{
"role": "tool",
"tool_call_id": "call_nNONu48z4o4psZIlHNDbWIe6",
"content": [
{
"type": "text",
"text": "knowledge_search tool found 3 chunks:\nBEGIN of knowledge_search tool results.\n"
},
{
"type": "text",
"text": "[1] document_id: file-862946202234, score: 0.6624775971970099, attributes: {'region': 'us', 'category': 'marketing', 'date': 1672531200.0, 'filename': 'us_marketing_q1.txt', 'document_id': 'file-862946202234', 'token_count': 21.0, 'metadata_token_count': 32.0} (cite as <|file-862946202234|>)\nUS promotional campaigns for Q1 2023. Revenue increased by 15% in the US region.\n"
},
{
"type": "text",
"text": "[2] document_id: file-862946202236, score: 0.6241908355663578, attributes: {'region': 'eu', 'category': 'marketing', 'date': 1672531200.0, 'filename': 'eu_marketing_q1.txt', 'document_id': 'file-862946202236', 'token_count': 17.0, 'metadata_token_count': 32.0} (cite as <|file-862946202236|>)\nEuropean advertising campaign results for Q1 2023. Strong growth in EU markets.\n"
},
{
"type": "text",
"text": "[3] document_id: file-862946202237, score: 0.5761999414500201, attributes: {'region': 'asia', 'category': 'sales', 'date': 1688169600.0, 'filename': 'asia_sales_q3.txt', 'document_id': 'file-862946202237', 'token_count': 17.0, 'metadata_token_count': 31.0} (cite as <|file-862946202237|>)\nAsia Pacific revenue figures for Q3 2023. Record breaking quarter in Asia.\n"
},
{
"type": "text",
"text": "END of knowledge_search tool results.\n"
},
{
"type": "text",
"text": "The above results were retrieved to help answer the user's query: \"sales documents examples\". Use them as supporting information only in answering this query. Cite sources immediately at the end of sentences before punctuation, using `<|file-id|>` format (e.g., 'This is a fact <|file-Cn3MSNn72ENTiiq11Qda4A|>.'). Do not add extra punctuation. Use only the file IDs provided (do not invent new ones).\n"
}
]
}
],
"stream": true,
"tools": [
{
"type": "function",
"function": {
"name": "knowledge_search",
"description": "Search for information in a database.",
"parameters": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "The query to search for. Can be a natural language sentence or keywords."
}
},
"required": [
"query"
]
}
}
}
]
},
"endpoint": "/v1/chat/completions",
"model": "gpt-4o"
},
"response": {
"body": [
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": "",
"function_call": null,
"refusal": null,
"role": "assistant",
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "R6uA5YGaVXy"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": "Here are some examples of marketing and sales documents:\n\n",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "fvEM0zzsT"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": "### Marketing Documents:\n",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "fM8pbIH2Du"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": "1.",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "ywGbbKNyMkSs"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": " **European Advertising Campaign**:",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "9Df30NePsv"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": " This document reports on the results of European advertising campaigns for Q1 ",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "1KDOAYrg"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": "2023,",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "Ur6YZUx8o2"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": " highlighting strong growth in EU markets <|file-862946202236|>.\n",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": ""
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": "2.",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "bGNEay3i7CbT"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": " **US Promotional Campaigns**:",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "BcVg7O8xUL"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": " It details the marketing activities in the US for Q1 ",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "lJMGEMwQQW"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": "2023,",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "fWLN9vPA4l"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": " showing a ",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "hcpod"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": "15% revenue increase in the region <|file-862946202234|>",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "zSOh8pStGzP"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": ".\n\n",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "4FFmHXhr"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": "### Sales Documents:\n",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "f2CxS7gaPY"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": "1.",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "hKaxDGxBdyNL"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": " **Asia Pacific Revenue Figures**:",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "MBZgU7ywir"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": " This document provides sales figures for the Asia Pacific region for Q3 ",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "6qKqjjyo"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": "2023,",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "KRSPEkdDeP"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": " noting a record-breaking quarter in Asia <|file-862946202237|>",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "wCkeab"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": ".\n\n",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "0FVU0V3j"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": "These documents provide insights into regional marketing strategies and sales performance over specific periods.",
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": null,
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "F5QDS9Hw"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-1dbd102b81c7",
"choices": [
{
"delta": {
"content": null,
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": "stop",
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_f64f290af2",
"usage": null,
"obfuscation": "rOg1C0x"
}
}
],
"is_streaming": true
},
"id_normalization_mapping": {}
}
Reference in a new issue View git blame Copy permalink