phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-22 12:56:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
llama-stack-mirror/tests/integration/responses/recordings/a6c5e729150f78535d09a5b59e3a28b0309eb2a756d8b99d109fce66bd2230df.json
Shabana Baig 805abf573f
Some checks failed
SqlStore Integration Tests / test-postgres (3.12) (push) Failing after 0s
Details
Integration Auth Tests / test-matrix (oauth2_token) (push) Failing after 1s
Details
Test External Providers Installed via Module / test-external-providers-from-module (venv) (push) Has been skipped
Details
Integration Tests (Replay) / generate-matrix (push) Successful in 3s
Details
SqlStore Integration Tests / test-postgres (3.13) (push) Failing after 3s
Details
API Conformance Tests / check-schema-compatibility (push) Successful in 15s
Details
Python Package Build Test / build (3.12) (push) Successful in 17s
Details
Python Package Build Test / build (3.13) (push) Successful in 18s
Details
Test External API and Providers / test-external (venv) (push) Failing after 28s
Details
Vector IO Integration Tests / test-matrix (push) Failing after 43s
Details
UI Tests / ui-tests (22) (push) Successful in 52s
Details
Unit Tests / unit-tests (3.13) (push) Failing after 1m45s
Details
Unit Tests / unit-tests (3.12) (push) Failing after 1m58s
Details
Pre-commit / pre-commit (22) (push) Successful in 3m9s
Details
Integration Tests (Replay) / Integration Tests (, , , client=, ) (push) Failing after 4m5s
Details
feat!: Implement include parameter specifically for adding logprobs in the output message (#4261) 
# Problem
As an Application Developer, I want to use the include parameter with
the value message.output_text.logprobs, so that I can receive log
probabilities for output tokens to assess the model's confidence in its
response.

# What does this PR do?

- Updates the include parameter in various resource definitions
- Updates the inline provider to return logprobs when
"message.output_text.logprobs" is passed in the include parameter
- Converts the logprobs returned by the inference provider from chat
completion format to responses format

Closes #[4260](https://github.com/llamastack/llama-stack/issues/4260)

## Test Plan

- Created a script to explore OpenAI behavior:
https://github.com/s-akhtar-baig/llama-stack-examples/blob/main/responses/src/include.py
- Added integration tests and new recordings

---------

Co-authored-by: Matthew Farrellee <matt@cs.wisc.edu>
Co-authored-by: Ashwin Bharambe <ashwin.bharambe@gmail.com>
2025-12-11 11:11:21 -08:00

345 lines
9.7 KiB
JSON
Generated
Raw Blame History

{
"test_id": "tests/integration/responses/test_basic_responses.py::test_include_logprobs_with_function_tools[txt=openai/gpt-4o]",
"request": {
"method": "POST",
"url": "https://api.openai.com/v1/v1/chat/completions",
"headers": {},
"body": {
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": "What is the weather in Paris?"
}
],
"logprobs": true,
"stream": true,
"stream_options": {
"include_usage": true
},
"tools": [
{
"type": "function",
"function": {
"type": "function",
"name": "get_weather",
"description": "Get weather information for a specified location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name (e.g., 'New York', 'London')"
}
}
},
"strict": null
}
}
]
},
"endpoint": "/v1/chat/completions",
"model": "gpt-4o"
},
"response": {
"body": [
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-a6c5e729150f",
"choices": [
{
"delta": {
"content": null,
"function_call": null,
"refusal": null,
"role": "assistant",
"tool_calls": [
{
"index": 0,
"id": "call_8T1tTdWOywKWs4tYSzyflftR",
"function": {
"arguments": "",
"name": "get_weather"
},
"type": "function"
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": {
"content": null,
"refusal": null
}
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_e819e3438b",
"usage": null,
"obfuscation": "2U536HpxwFMNHw"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-a6c5e729150f",
"choices": [
{
"delta": {
"content": null,
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": [
{
"index": 0,
"id": null,
"function": {
"arguments": "{\"",
"name": null
},
"type": null
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": {
"content": null,
"refusal": null
}
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_e819e3438b",
"usage": null,
"obfuscation": "LDpoCusy"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-a6c5e729150f",
"choices": [
{
"delta": {
"content": null,
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": [
{
"index": 0,
"id": null,
"function": {
"arguments": "location",
"name": null
},
"type": null
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": {
"content": null,
"refusal": null
}
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_e819e3438b",
"usage": null,
"obfuscation": "X7q"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-a6c5e729150f",
"choices": [
{
"delta": {
"content": null,
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": [
{
"index": 0,
"id": null,
"function": {
"arguments": "\":\"",
"name": null
},
"type": null
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": {
"content": null,
"refusal": null
}
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_e819e3438b",
"usage": null,
"obfuscation": "TMTYo8"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-a6c5e729150f",
"choices": [
{
"delta": {
"content": null,
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": [
{
"index": 0,
"id": null,
"function": {
"arguments": "Paris",
"name": null
},
"type": null
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": {
"content": null,
"refusal": null
}
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_e819e3438b",
"usage": null,
"obfuscation": "8EmcxX"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-a6c5e729150f",
"choices": [
{
"delta": {
"content": null,
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": [
{
"index": 0,
"id": null,
"function": {
"arguments": "\"}",
"name": null
},
"type": null
}
]
},
"finish_reason": null,
"index": 0,
"logprobs": {
"content": null,
"refusal": null
}
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_e819e3438b",
"usage": null,
"obfuscation": "cnMSe1WI"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-a6c5e729150f",
"choices": [
{
"delta": {
"content": null,
"function_call": null,
"refusal": null,
"role": null,
"tool_calls": null
},
"finish_reason": "tool_calls",
"index": 0,
"logprobs": null
}
],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_e819e3438b",
"usage": null,
"obfuscation": "fs5p"
}
},
{
"__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk",
"__data__": {
"id": "rec-a6c5e729150f",
"choices": [],
"created": 0,
"model": "gpt-4o-2024-08-06",
"object": "chat.completion.chunk",
"service_tier": "default",
"system_fingerprint": "fp_e819e3438b",
"usage": {
"completion_tokens": 14,
"prompt_tokens": 67,
"total_tokens": 81,
"completion_tokens_details": {
"accepted_prediction_tokens": 0,
"audio_tokens": 0,
"reasoning_tokens": 0,
"rejected_prediction_tokens": 0
},
"prompt_tokens_details": {
"audio_tokens": 0,
"cached_tokens": 0
}
},
"obfuscation": "xFLK91cW1si1hsA"
}
}
],
"is_streaming": true
},
"id_normalization_mapping": {}
}
Reference in a new issue View git blame Copy permalink