mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-15 06:37:58 +00:00
f50ce11a3b
Renames `inference_recorder.py` to `api_recorder.py` and extends it to support recording/replaying tool invocations in addition to inference calls. This allows us to record web-search, etc. tool calls and thereafter apply recordings for `tests/integration/responses` ## Test Plan ``` export OPENAI_API_KEY=... export TAVILY_SEARCH_API_KEY=... ./scripts/integration-tests.sh --stack-config ci-tests \ --suite responses --inference-mode record-if-missing ```
354 lines
11 KiB
JSON
354 lines
11 KiB
JSON
{
|
|
"test_id": "tests/integration/responses/test_file_search.py::test_response_file_search_filter_compound_and[openai_client-txt=openai/gpt-4o]",
|
|
"request": {
|
|
"method": "POST",
|
|
"url": "https://api.openai.com/v1/v1/chat/completions",
|
|
"headers": {},
|
|
"body": {
|
|
"model": "gpt-4o",
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": "What are the engineering updates from the US?"
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"index": 0,
|
|
"id": "call_xmBCIHb34NLfvnRnt1wDXjie",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "knowledge_search",
|
|
"arguments": "{\"query\":\"engineering updates US October 2023\"}"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"tool_call_id": "call_xmBCIHb34NLfvnRnt1wDXjie",
|
|
"content": [
|
|
{
|
|
"type": "text",
|
|
"text": "knowledge_search tool found 1 chunks:\nBEGIN of knowledge_search tool results.\n"
|
|
},
|
|
{
|
|
"type": "text",
|
|
"text": "[1] document_id: file-90357487492, score: 1.708989789892816, attributes: {'region': 'us', 'category': 'engineering', 'date': 1680307200.0, 'filename': 'us_engineering_q2.txt', 'document_id': 'file-90357487492', 'token_count': 18.0, 'metadata_token_count': 32.0} (cite as <|file-90357487492|>)\nUS technical updates for Q2 2023. New features deployed in the US region.\n"
|
|
},
|
|
{
|
|
"type": "text",
|
|
"text": "END of knowledge_search tool results.\n"
|
|
},
|
|
{
|
|
"type": "text",
|
|
"text": "The above results were retrieved to help answer the user's query: \"engineering updates US October 2023\". Use them as supporting information only in answering this query. Cite sources immediately at the end of sentences before punctuation, using `<|file-id|>` format (e.g., 'This is a fact <|file-Cn3MSNn72ENTiiq11Qda4A|>.'). Do not add extra punctuation. Use only the file IDs provided (do not invent new ones).\n"
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"stream": true,
|
|
"tools": [
|
|
{
|
|
"type": "function",
|
|
"function": {
|
|
"name": "knowledge_search",
|
|
"description": "Search for information in a database.",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {
|
|
"query": {
|
|
"type": "string",
|
|
"description": "The query to search for. Can be a natural language sentence or keywords."
|
|
}
|
|
},
|
|
"required": [
|
|
"query"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"endpoint": "/v1/chat/completions",
|
|
"model": "gpt-4o"
|
|
},
|
|
"response": {
|
|
"body": [
|
|
{
|
|
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
|
|
"__data__": {
|
|
"id": "rec-c060e0e12e3b",
|
|
"choices": [
|
|
{
|
|
"delta": {
|
|
"content": "",
|
|
"function_call": null,
|
|
"refusal": null,
|
|
"role": "assistant",
|
|
"tool_calls": null
|
|
},
|
|
"finish_reason": null,
|
|
"index": 0,
|
|
"logprobs": null
|
|
}
|
|
],
|
|
"created": 0,
|
|
"model": "gpt-4o-2024-08-06",
|
|
"object": "chat.completion.chunk",
|
|
"service_tier": "default",
|
|
"system_fingerprint": "fp_f64f290af2",
|
|
"usage": null,
|
|
"obfuscation": "QL1wD7Bp6RT"
|
|
}
|
|
},
|
|
{
|
|
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
|
|
"__data__": {
|
|
"id": "rec-c060e0e12e3b",
|
|
"choices": [
|
|
{
|
|
"delta": {
|
|
"content": "The search for specific engineering updates from the US for October ",
|
|
"function_call": null,
|
|
"refusal": null,
|
|
"role": null,
|
|
"tool_calls": null
|
|
},
|
|
"finish_reason": null,
|
|
"index": 0,
|
|
"logprobs": null
|
|
}
|
|
],
|
|
"created": 0,
|
|
"model": "gpt-4o-2024-08-06",
|
|
"object": "chat.completion.chunk",
|
|
"service_tier": "default",
|
|
"system_fingerprint": "fp_f64f290af2",
|
|
"usage": null,
|
|
"obfuscation": "fzZkmiEW1g"
|
|
}
|
|
},
|
|
{
|
|
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
|
|
"__data__": {
|
|
"id": "rec-c060e0e12e3b",
|
|
"choices": [
|
|
{
|
|
"delta": {
|
|
"content": "2023 did not return the desired results.",
|
|
"function_call": null,
|
|
"refusal": null,
|
|
"role": null,
|
|
"tool_calls": null
|
|
},
|
|
"finish_reason": null,
|
|
"index": 0,
|
|
"logprobs": null
|
|
}
|
|
],
|
|
"created": 0,
|
|
"model": "gpt-4o-2024-08-06",
|
|
"object": "chat.completion.chunk",
|
|
"service_tier": "default",
|
|
"system_fingerprint": "fp_f64f290af2",
|
|
"usage": null,
|
|
"obfuscation": "cJGY5dUAi5"
|
|
}
|
|
},
|
|
{
|
|
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
|
|
"__data__": {
|
|
"id": "rec-c060e0e12e3b",
|
|
"choices": [
|
|
{
|
|
"delta": {
|
|
"content": " It seems that there are only technical updates available for earlier periods,",
|
|
"function_call": null,
|
|
"refusal": null,
|
|
"role": null,
|
|
"tool_calls": null
|
|
},
|
|
"finish_reason": null,
|
|
"index": 0,
|
|
"logprobs": null
|
|
}
|
|
],
|
|
"created": 0,
|
|
"model": "gpt-4o-2024-08-06",
|
|
"object": "chat.completion.chunk",
|
|
"service_tier": "default",
|
|
"system_fingerprint": "fp_f64f290af2",
|
|
"usage": null,
|
|
"obfuscation": "KzI4vF1Wib"
|
|
}
|
|
},
|
|
{
|
|
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
|
|
"__data__": {
|
|
"id": "rec-c060e0e12e3b",
|
|
"choices": [
|
|
{
|
|
"delta": {
|
|
"content": " such as Q2 ",
|
|
"function_call": null,
|
|
"refusal": null,
|
|
"role": null,
|
|
"tool_calls": null
|
|
},
|
|
"finish_reason": null,
|
|
"index": 0,
|
|
"logprobs": null
|
|
}
|
|
],
|
|
"created": 0,
|
|
"model": "gpt-4o-2024-08-06",
|
|
"object": "chat.completion.chunk",
|
|
"service_tier": "default",
|
|
"system_fingerprint": "fp_f64f290af2",
|
|
"usage": null,
|
|
"obfuscation": "4vlbZJ3Y"
|
|
}
|
|
},
|
|
{
|
|
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
|
|
"__data__": {
|
|
"id": "rec-c060e0e12e3b",
|
|
"choices": [
|
|
{
|
|
"delta": {
|
|
"content": "2023,",
|
|
"function_call": null,
|
|
"refusal": null,
|
|
"role": null,
|
|
"tool_calls": null
|
|
},
|
|
"finish_reason": null,
|
|
"index": 0,
|
|
"logprobs": null
|
|
}
|
|
],
|
|
"created": 0,
|
|
"model": "gpt-4o-2024-08-06",
|
|
"object": "chat.completion.chunk",
|
|
"service_tier": "default",
|
|
"system_fingerprint": "fp_f64f290af2",
|
|
"usage": null,
|
|
"obfuscation": "fdLgA1uk9A"
|
|
}
|
|
},
|
|
{
|
|
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
|
|
"__data__": {
|
|
"id": "rec-c060e0e12e3b",
|
|
"choices": [
|
|
{
|
|
"delta": {
|
|
"content": " where new features were deployed in the US region <|file-90357487492|>.",
|
|
"function_call": null,
|
|
"refusal": null,
|
|
"role": null,
|
|
"tool_calls": null
|
|
},
|
|
"finish_reason": null,
|
|
"index": 0,
|
|
"logprobs": null
|
|
}
|
|
],
|
|
"created": 0,
|
|
"model": "gpt-4o-2024-08-06",
|
|
"object": "chat.completion.chunk",
|
|
"service_tier": "default",
|
|
"system_fingerprint": "fp_f64f290af2",
|
|
"usage": null,
|
|
"obfuscation": "1NITGWv"
|
|
}
|
|
},
|
|
{
|
|
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
|
|
"__data__": {
|
|
"id": "rec-c060e0e12e3b",
|
|
"choices": [
|
|
{
|
|
"delta": {
|
|
"content": " For the most current information,",
|
|
"function_call": null,
|
|
"refusal": null,
|
|
"role": null,
|
|
"tool_calls": null
|
|
},
|
|
"finish_reason": null,
|
|
"index": 0,
|
|
"logprobs": null
|
|
}
|
|
],
|
|
"created": 0,
|
|
"model": "gpt-4o-2024-08-06",
|
|
"object": "chat.completion.chunk",
|
|
"service_tier": "default",
|
|
"system_fingerprint": "fp_f64f290af2",
|
|
"usage": null,
|
|
"obfuscation": "ElPjxSzYB"
|
|
}
|
|
},
|
|
{
|
|
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
|
|
"__data__": {
|
|
"id": "rec-c060e0e12e3b",
|
|
"choices": [
|
|
{
|
|
"delta": {
|
|
"content": " checking engineering news websites or official announcements is recommended.",
|
|
"function_call": null,
|
|
"refusal": null,
|
|
"role": null,
|
|
"tool_calls": null
|
|
},
|
|
"finish_reason": null,
|
|
"index": 0,
|
|
"logprobs": null
|
|
}
|
|
],
|
|
"created": 0,
|
|
"model": "gpt-4o-2024-08-06",
|
|
"object": "chat.completion.chunk",
|
|
"service_tier": "default",
|
|
"system_fingerprint": "fp_f64f290af2",
|
|
"usage": null,
|
|
"obfuscation": "SpqI"
|
|
}
|
|
},
|
|
{
|
|
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
|
|
"__data__": {
|
|
"id": "rec-c060e0e12e3b",
|
|
"choices": [
|
|
{
|
|
"delta": {
|
|
"content": null,
|
|
"function_call": null,
|
|
"refusal": null,
|
|
"role": null,
|
|
"tool_calls": null
|
|
},
|
|
"finish_reason": "stop",
|
|
"index": 0,
|
|
"logprobs": null
|
|
}
|
|
],
|
|
"created": 0,
|
|
"model": "gpt-4o-2024-08-06",
|
|
"object": "chat.completion.chunk",
|
|
"service_tier": "default",
|
|
"system_fingerprint": "fp_f64f290af2",
|
|
"usage": null,
|
|
"obfuscation": "RT09BI6"
|
|
}
|
|
}
|
|
],
|
|
"is_streaming": true
|
|
},
|
|
"id_normalization_mapping": {}
|
|
}
|