diff --git a/client-sdks/stainless/openapi.yml b/client-sdks/stainless/openapi.yml index 93049a14a..99df93572 100644 --- a/client-sdks/stainless/openapi.yml +++ b/client-sdks/stainless/openapi.yml @@ -5574,11 +5574,44 @@ components: oneOf: - $ref: '#/components/schemas/OpenAIResponseInputMessageContentText' - $ref: '#/components/schemas/OpenAIResponseInputMessageContentImage' + - $ref: '#/components/schemas/OpenAIResponseInputMessageContentFile' discriminator: propertyName: type mapping: input_text: '#/components/schemas/OpenAIResponseInputMessageContentText' input_image: '#/components/schemas/OpenAIResponseInputMessageContentImage' + input_file: '#/components/schemas/OpenAIResponseInputMessageContentFile' + OpenAIResponseInputMessageContentFile: + type: object + properties: + type: + type: string + const: input_file + default: input_file + description: >- + The type of the input item. Always `input_file`. + file_data: + type: string + description: >- + The data of the file to be sent to the model. + file_id: + type: string + description: >- + (Optional) The ID of the file to be sent to the model. + file_url: + type: string + description: >- + The URL of the file to be sent to the model. + filename: + type: string + description: >- + The name of the file to be sent to the model. + additionalProperties: false + required: + - type + title: OpenAIResponseInputMessageContentFile + description: >- + File content for input messages in OpenAI response format. OpenAIResponseInputMessageContentImage: type: object properties: @@ -5599,6 +5632,10 @@ components: default: input_image description: >- Content type identifier, always "input_image" + file_id: + type: string + description: >- + (Optional) The ID of the file to be sent to the model. image_url: type: string description: (Optional) URL of the image content @@ -6998,6 +7035,10 @@ components: type: string description: >- (Optional) ID of the previous response in a conversation + prompt: + $ref: '#/components/schemas/Prompt' + description: >- + (Optional) Prompt object with ID, version, and variables status: type: string description: >- @@ -7315,6 +7356,29 @@ components: title: OpenAIResponseInputToolMCP description: >- Model Context Protocol (MCP) tool configuration for OpenAI response inputs. + OpenAIResponsePromptParam: + type: object + properties: + id: + type: string + description: Unique identifier of the prompt template + variables: + type: object + additionalProperties: + $ref: '#/components/schemas/OpenAIResponseInputMessageContent' + description: >- + Dictionary of variable names to OpenAIResponseInputMessageContent structure + for template substitution + version: + type: string + description: >- + Version number of the prompt to use (defaults to latest if not specified) + additionalProperties: false + required: + - id + title: OpenAIResponsePromptParam + description: >- + Prompt object that is used for OpenAI responses. CreateOpenaiResponseRequest: type: object properties: @@ -7328,6 +7392,10 @@ components: model: type: string description: The underlying LLM used for completions. + prompt: + $ref: '#/components/schemas/OpenAIResponsePromptParam' + description: >- + Prompt object with ID, version, and variables. instructions: type: string previous_response_id: @@ -7405,6 +7473,10 @@ components: type: string description: >- (Optional) ID of the previous response in a conversation + prompt: + $ref: '#/components/schemas/Prompt' + description: >- + (Optional) Prompt object with ID, version, and variables status: type: string description: >- diff --git a/docs/static/deprecated-llama-stack-spec.html b/docs/static/deprecated-llama-stack-spec.html index d920317cf..eb3e57ccc 100644 --- a/docs/static/deprecated-llama-stack-spec.html +++ b/docs/static/deprecated-llama-stack-spec.html @@ -8593,16 +8593,53 @@ }, { "$ref": "#/components/schemas/OpenAIResponseInputMessageContentImage" + }, + { + "$ref": "#/components/schemas/OpenAIResponseInputMessageContentFile" } ], "discriminator": { "propertyName": "type", "mapping": { "input_text": "#/components/schemas/OpenAIResponseInputMessageContentText", - "input_image": "#/components/schemas/OpenAIResponseInputMessageContentImage" + "input_image": "#/components/schemas/OpenAIResponseInputMessageContentImage", + "input_file": "#/components/schemas/OpenAIResponseInputMessageContentFile" } } }, + "OpenAIResponseInputMessageContentFile": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "input_file", + "default": "input_file", + "description": "The type of the input item. Always `input_file`." + }, + "file_data": { + "type": "string", + "description": "The data of the file to be sent to the model." + }, + "file_id": { + "type": "string", + "description": "(Optional) The ID of the file to be sent to the model." + }, + "file_url": { + "type": "string", + "description": "The URL of the file to be sent to the model." + }, + "filename": { + "type": "string", + "description": "The name of the file to be sent to the model." + } + }, + "additionalProperties": false, + "required": [ + "type" + ], + "title": "OpenAIResponseInputMessageContentFile", + "description": "File content for input messages in OpenAI response format." + }, "OpenAIResponseInputMessageContentImage": { "type": "object", "properties": { @@ -8630,6 +8667,10 @@ "default": "input_image", "description": "Content type identifier, always \"input_image\"" }, + "file_id": { + "type": "string", + "description": "(Optional) The ID of the file to be sent to the model." + }, "image_url": { "type": "string", "description": "(Optional) URL of the image content" @@ -8993,6 +9034,10 @@ "type": "string", "description": "(Optional) ID of the previous response in a conversation" }, + "prompt": { + "$ref": "#/components/schemas/Prompt", + "description": "(Optional) Prompt object with ID, version, and variables" + }, "status": { "type": "string", "description": "Current status of the response generation" @@ -9610,6 +9655,44 @@ "title": "OpenAIResponseUsage", "description": "Usage information for OpenAI response." }, + "Prompt": { + "type": "object", + "properties": { + "prompt": { + "type": "string", + "description": "The system prompt text with variable placeholders. Variables are only supported when using the Responses API." + }, + "version": { + "type": "integer", + "description": "Version (integer starting at 1, incremented on save)" + }, + "prompt_id": { + "type": "string", + "description": "Unique identifier formatted as 'pmpt_<48-digit-hash>'" + }, + "variables": { + "type": "array", + "items": { + "type": "string" + }, + "description": "List of prompt variable names that can be used in the prompt template" + }, + "is_default": { + "type": "boolean", + "default": false, + "description": "Boolean indicating whether this version is the default version for this prompt" + } + }, + "additionalProperties": false, + "required": [ + "version", + "prompt_id", + "variables", + "is_default" + ], + "title": "Prompt", + "description": "A prompt resource representing a stored OpenAI Compatible prompt template in Llama Stack." + }, "ResponseGuardrailSpec": { "type": "object", "properties": { @@ -9766,6 +9849,32 @@ "title": "OpenAIResponseInputToolMCP", "description": "Model Context Protocol (MCP) tool configuration for OpenAI response inputs." }, + "OpenAIResponsePromptParam": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the prompt template" + }, + "variables": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/OpenAIResponseInputMessageContent" + }, + "description": "Dictionary of variable names to OpenAIResponseInputMessageContent structure for template substitution" + }, + "version": { + "type": "string", + "description": "Version number of the prompt to use (defaults to latest if not specified)" + } + }, + "additionalProperties": false, + "required": [ + "id" + ], + "title": "OpenAIResponsePromptParam", + "description": "Prompt object that is used for OpenAI responses." + }, "CreateOpenaiResponseRequest": { "type": "object", "properties": { @@ -9787,6 +9896,10 @@ "type": "string", "description": "The underlying LLM used for completions." }, + "prompt": { + "$ref": "#/components/schemas/OpenAIResponsePromptParam", + "description": "Prompt object with ID, version, and variables." + }, "instructions": { "type": "string" }, @@ -9875,6 +9988,10 @@ "type": "string", "description": "(Optional) ID of the previous response in a conversation" }, + "prompt": { + "$ref": "#/components/schemas/Prompt", + "description": "(Optional) Prompt object with ID, version, and variables" + }, "status": { "type": "string", "description": "Current status of the response generation" diff --git a/docs/static/deprecated-llama-stack-spec.yaml b/docs/static/deprecated-llama-stack-spec.yaml index 66b2caeca..b01118f34 100644 --- a/docs/static/deprecated-llama-stack-spec.yaml +++ b/docs/static/deprecated-llama-stack-spec.yaml @@ -6409,11 +6409,44 @@ components: oneOf: - $ref: '#/components/schemas/OpenAIResponseInputMessageContentText' - $ref: '#/components/schemas/OpenAIResponseInputMessageContentImage' + - $ref: '#/components/schemas/OpenAIResponseInputMessageContentFile' discriminator: propertyName: type mapping: input_text: '#/components/schemas/OpenAIResponseInputMessageContentText' input_image: '#/components/schemas/OpenAIResponseInputMessageContentImage' + input_file: '#/components/schemas/OpenAIResponseInputMessageContentFile' + OpenAIResponseInputMessageContentFile: + type: object + properties: + type: + type: string + const: input_file + default: input_file + description: >- + The type of the input item. Always `input_file`. + file_data: + type: string + description: >- + The data of the file to be sent to the model. + file_id: + type: string + description: >- + (Optional) The ID of the file to be sent to the model. + file_url: + type: string + description: >- + The URL of the file to be sent to the model. + filename: + type: string + description: >- + The name of the file to be sent to the model. + additionalProperties: false + required: + - type + title: OpenAIResponseInputMessageContentFile + description: >- + File content for input messages in OpenAI response format. OpenAIResponseInputMessageContentImage: type: object properties: @@ -6434,6 +6467,10 @@ components: default: input_image description: >- Content type identifier, always "input_image" + file_id: + type: string + description: >- + (Optional) The ID of the file to be sent to the model. image_url: type: string description: (Optional) URL of the image content @@ -6704,6 +6741,10 @@ components: type: string description: >- (Optional) ID of the previous response in a conversation + prompt: + $ref: '#/components/schemas/Prompt' + description: >- + (Optional) Prompt object with ID, version, and variables status: type: string description: >- @@ -7181,6 +7222,44 @@ components: - total_tokens title: OpenAIResponseUsage description: Usage information for OpenAI response. + Prompt: + type: object + properties: + prompt: + type: string + description: >- + The system prompt text with variable placeholders. Variables are only + supported when using the Responses API. + version: + type: integer + description: >- + Version (integer starting at 1, incremented on save) + prompt_id: + type: string + description: >- + Unique identifier formatted as 'pmpt_<48-digit-hash>' + variables: + type: array + items: + type: string + description: >- + List of prompt variable names that can be used in the prompt template + is_default: + type: boolean + default: false + description: >- + Boolean indicating whether this version is the default version for this + prompt + additionalProperties: false + required: + - version + - prompt_id + - variables + - is_default + title: Prompt + description: >- + A prompt resource representing a stored OpenAI Compatible prompt template + in Llama Stack. ResponseGuardrailSpec: type: object properties: @@ -7287,6 +7366,29 @@ components: title: OpenAIResponseInputToolMCP description: >- Model Context Protocol (MCP) tool configuration for OpenAI response inputs. + OpenAIResponsePromptParam: + type: object + properties: + id: + type: string + description: Unique identifier of the prompt template + variables: + type: object + additionalProperties: + $ref: '#/components/schemas/OpenAIResponseInputMessageContent' + description: >- + Dictionary of variable names to OpenAIResponseInputMessageContent structure + for template substitution + version: + type: string + description: >- + Version number of the prompt to use (defaults to latest if not specified) + additionalProperties: false + required: + - id + title: OpenAIResponsePromptParam + description: >- + Prompt object that is used for OpenAI responses. CreateOpenaiResponseRequest: type: object properties: @@ -7300,6 +7402,10 @@ components: model: type: string description: The underlying LLM used for completions. + prompt: + $ref: '#/components/schemas/OpenAIResponsePromptParam' + description: >- + Prompt object with ID, version, and variables. instructions: type: string previous_response_id: @@ -7377,6 +7483,10 @@ components: type: string description: >- (Optional) ID of the previous response in a conversation + prompt: + $ref: '#/components/schemas/Prompt' + description: >- + (Optional) Prompt object with ID, version, and variables status: type: string description: >- diff --git a/docs/static/llama-stack-spec.html b/docs/static/llama-stack-spec.html index 61deaec1e..e759f876d 100644 --- a/docs/static/llama-stack-spec.html +++ b/docs/static/llama-stack-spec.html @@ -5729,16 +5729,53 @@ }, { "$ref": "#/components/schemas/OpenAIResponseInputMessageContentImage" + }, + { + "$ref": "#/components/schemas/OpenAIResponseInputMessageContentFile" } ], "discriminator": { "propertyName": "type", "mapping": { "input_text": "#/components/schemas/OpenAIResponseInputMessageContentText", - "input_image": "#/components/schemas/OpenAIResponseInputMessageContentImage" + "input_image": "#/components/schemas/OpenAIResponseInputMessageContentImage", + "input_file": "#/components/schemas/OpenAIResponseInputMessageContentFile" } } }, + "OpenAIResponseInputMessageContentFile": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "input_file", + "default": "input_file", + "description": "The type of the input item. Always `input_file`." + }, + "file_data": { + "type": "string", + "description": "The data of the file to be sent to the model." + }, + "file_id": { + "type": "string", + "description": "(Optional) The ID of the file to be sent to the model." + }, + "file_url": { + "type": "string", + "description": "The URL of the file to be sent to the model." + }, + "filename": { + "type": "string", + "description": "The name of the file to be sent to the model." + } + }, + "additionalProperties": false, + "required": [ + "type" + ], + "title": "OpenAIResponseInputMessageContentFile", + "description": "File content for input messages in OpenAI response format." + }, "OpenAIResponseInputMessageContentImage": { "type": "object", "properties": { @@ -5766,6 +5803,10 @@ "default": "input_image", "description": "Content type identifier, always \"input_image\"" }, + "file_id": { + "type": "string", + "description": "(Optional) The ID of the file to be sent to the model." + }, "image_url": { "type": "string", "description": "(Optional) URL of the image content" @@ -7569,6 +7610,10 @@ "type": "string", "description": "(Optional) ID of the previous response in a conversation" }, + "prompt": { + "$ref": "#/components/schemas/Prompt", + "description": "(Optional) Prompt object with ID, version, and variables" + }, "status": { "type": "string", "description": "Current status of the response generation" @@ -8013,6 +8058,32 @@ "title": "OpenAIResponseInputToolMCP", "description": "Model Context Protocol (MCP) tool configuration for OpenAI response inputs." }, + "OpenAIResponsePromptParam": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the prompt template" + }, + "variables": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/OpenAIResponseInputMessageContent" + }, + "description": "Dictionary of variable names to OpenAIResponseInputMessageContent structure for template substitution" + }, + "version": { + "type": "string", + "description": "Version number of the prompt to use (defaults to latest if not specified)" + } + }, + "additionalProperties": false, + "required": [ + "id" + ], + "title": "OpenAIResponsePromptParam", + "description": "Prompt object that is used for OpenAI responses." + }, "CreateOpenaiResponseRequest": { "type": "object", "properties": { @@ -8034,6 +8105,10 @@ "type": "string", "description": "The underlying LLM used for completions." }, + "prompt": { + "$ref": "#/components/schemas/OpenAIResponsePromptParam", + "description": "Prompt object with ID, version, and variables." + }, "instructions": { "type": "string" }, @@ -8122,6 +8197,10 @@ "type": "string", "description": "(Optional) ID of the previous response in a conversation" }, + "prompt": { + "$ref": "#/components/schemas/Prompt", + "description": "(Optional) Prompt object with ID, version, and variables" + }, "status": { "type": "string", "description": "Current status of the response generation" diff --git a/docs/static/llama-stack-spec.yaml b/docs/static/llama-stack-spec.yaml index c6197b36f..f092ec29a 100644 --- a/docs/static/llama-stack-spec.yaml +++ b/docs/static/llama-stack-spec.yaml @@ -4361,11 +4361,44 @@ components: oneOf: - $ref: '#/components/schemas/OpenAIResponseInputMessageContentText' - $ref: '#/components/schemas/OpenAIResponseInputMessageContentImage' + - $ref: '#/components/schemas/OpenAIResponseInputMessageContentFile' discriminator: propertyName: type mapping: input_text: '#/components/schemas/OpenAIResponseInputMessageContentText' input_image: '#/components/schemas/OpenAIResponseInputMessageContentImage' + input_file: '#/components/schemas/OpenAIResponseInputMessageContentFile' + OpenAIResponseInputMessageContentFile: + type: object + properties: + type: + type: string + const: input_file + default: input_file + description: >- + The type of the input item. Always `input_file`. + file_data: + type: string + description: >- + The data of the file to be sent to the model. + file_id: + type: string + description: >- + (Optional) The ID of the file to be sent to the model. + file_url: + type: string + description: >- + The URL of the file to be sent to the model. + filename: + type: string + description: >- + The name of the file to be sent to the model. + additionalProperties: false + required: + - type + title: OpenAIResponseInputMessageContentFile + description: >- + File content for input messages in OpenAI response format. OpenAIResponseInputMessageContentImage: type: object properties: @@ -4386,6 +4419,10 @@ components: default: input_image description: >- Content type identifier, always "input_image" + file_id: + type: string + description: >- + (Optional) The ID of the file to be sent to the model. image_url: type: string description: (Optional) URL of the image content @@ -5785,6 +5822,10 @@ components: type: string description: >- (Optional) ID of the previous response in a conversation + prompt: + $ref: '#/components/schemas/Prompt' + description: >- + (Optional) Prompt object with ID, version, and variables status: type: string description: >- @@ -6102,6 +6143,29 @@ components: title: OpenAIResponseInputToolMCP description: >- Model Context Protocol (MCP) tool configuration for OpenAI response inputs. + OpenAIResponsePromptParam: + type: object + properties: + id: + type: string + description: Unique identifier of the prompt template + variables: + type: object + additionalProperties: + $ref: '#/components/schemas/OpenAIResponseInputMessageContent' + description: >- + Dictionary of variable names to OpenAIResponseInputMessageContent structure + for template substitution + version: + type: string + description: >- + Version number of the prompt to use (defaults to latest if not specified) + additionalProperties: false + required: + - id + title: OpenAIResponsePromptParam + description: >- + Prompt object that is used for OpenAI responses. CreateOpenaiResponseRequest: type: object properties: @@ -6115,6 +6179,10 @@ components: model: type: string description: The underlying LLM used for completions. + prompt: + $ref: '#/components/schemas/OpenAIResponsePromptParam' + description: >- + Prompt object with ID, version, and variables. instructions: type: string previous_response_id: @@ -6192,6 +6260,10 @@ components: type: string description: >- (Optional) ID of the previous response in a conversation + prompt: + $ref: '#/components/schemas/Prompt' + description: >- + (Optional) Prompt object with ID, version, and variables status: type: string description: >- diff --git a/docs/static/stainless-llama-stack-spec.html b/docs/static/stainless-llama-stack-spec.html index 38122ebc0..5cb04d710 100644 --- a/docs/static/stainless-llama-stack-spec.html +++ b/docs/static/stainless-llama-stack-spec.html @@ -7401,16 +7401,53 @@ }, { "$ref": "#/components/schemas/OpenAIResponseInputMessageContentImage" + }, + { + "$ref": "#/components/schemas/OpenAIResponseInputMessageContentFile" } ], "discriminator": { "propertyName": "type", "mapping": { "input_text": "#/components/schemas/OpenAIResponseInputMessageContentText", - "input_image": "#/components/schemas/OpenAIResponseInputMessageContentImage" + "input_image": "#/components/schemas/OpenAIResponseInputMessageContentImage", + "input_file": "#/components/schemas/OpenAIResponseInputMessageContentFile" } } }, + "OpenAIResponseInputMessageContentFile": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "input_file", + "default": "input_file", + "description": "The type of the input item. Always `input_file`." + }, + "file_data": { + "type": "string", + "description": "The data of the file to be sent to the model." + }, + "file_id": { + "type": "string", + "description": "(Optional) The ID of the file to be sent to the model." + }, + "file_url": { + "type": "string", + "description": "The URL of the file to be sent to the model." + }, + "filename": { + "type": "string", + "description": "The name of the file to be sent to the model." + } + }, + "additionalProperties": false, + "required": [ + "type" + ], + "title": "OpenAIResponseInputMessageContentFile", + "description": "File content for input messages in OpenAI response format." + }, "OpenAIResponseInputMessageContentImage": { "type": "object", "properties": { @@ -7438,6 +7475,10 @@ "default": "input_image", "description": "Content type identifier, always \"input_image\"" }, + "file_id": { + "type": "string", + "description": "(Optional) The ID of the file to be sent to the model." + }, "image_url": { "type": "string", "description": "(Optional) URL of the image content" @@ -9241,6 +9282,10 @@ "type": "string", "description": "(Optional) ID of the previous response in a conversation" }, + "prompt": { + "$ref": "#/components/schemas/Prompt", + "description": "(Optional) Prompt object with ID, version, and variables" + }, "status": { "type": "string", "description": "Current status of the response generation" @@ -9685,6 +9730,32 @@ "title": "OpenAIResponseInputToolMCP", "description": "Model Context Protocol (MCP) tool configuration for OpenAI response inputs." }, + "OpenAIResponsePromptParam": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the prompt template" + }, + "variables": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/OpenAIResponseInputMessageContent" + }, + "description": "Dictionary of variable names to OpenAIResponseInputMessageContent structure for template substitution" + }, + "version": { + "type": "string", + "description": "Version number of the prompt to use (defaults to latest if not specified)" + } + }, + "additionalProperties": false, + "required": [ + "id" + ], + "title": "OpenAIResponsePromptParam", + "description": "Prompt object that is used for OpenAI responses." + }, "CreateOpenaiResponseRequest": { "type": "object", "properties": { @@ -9706,6 +9777,10 @@ "type": "string", "description": "The underlying LLM used for completions." }, + "prompt": { + "$ref": "#/components/schemas/OpenAIResponsePromptParam", + "description": "Prompt object with ID, version, and variables." + }, "instructions": { "type": "string" }, @@ -9794,6 +9869,10 @@ "type": "string", "description": "(Optional) ID of the previous response in a conversation" }, + "prompt": { + "$ref": "#/components/schemas/Prompt", + "description": "(Optional) Prompt object with ID, version, and variables" + }, "status": { "type": "string", "description": "Current status of the response generation" diff --git a/docs/static/stainless-llama-stack-spec.yaml b/docs/static/stainless-llama-stack-spec.yaml index 93049a14a..99df93572 100644 --- a/docs/static/stainless-llama-stack-spec.yaml +++ b/docs/static/stainless-llama-stack-spec.yaml @@ -5574,11 +5574,44 @@ components: oneOf: - $ref: '#/components/schemas/OpenAIResponseInputMessageContentText' - $ref: '#/components/schemas/OpenAIResponseInputMessageContentImage' + - $ref: '#/components/schemas/OpenAIResponseInputMessageContentFile' discriminator: propertyName: type mapping: input_text: '#/components/schemas/OpenAIResponseInputMessageContentText' input_image: '#/components/schemas/OpenAIResponseInputMessageContentImage' + input_file: '#/components/schemas/OpenAIResponseInputMessageContentFile' + OpenAIResponseInputMessageContentFile: + type: object + properties: + type: + type: string + const: input_file + default: input_file + description: >- + The type of the input item. Always `input_file`. + file_data: + type: string + description: >- + The data of the file to be sent to the model. + file_id: + type: string + description: >- + (Optional) The ID of the file to be sent to the model. + file_url: + type: string + description: >- + The URL of the file to be sent to the model. + filename: + type: string + description: >- + The name of the file to be sent to the model. + additionalProperties: false + required: + - type + title: OpenAIResponseInputMessageContentFile + description: >- + File content for input messages in OpenAI response format. OpenAIResponseInputMessageContentImage: type: object properties: @@ -5599,6 +5632,10 @@ components: default: input_image description: >- Content type identifier, always "input_image" + file_id: + type: string + description: >- + (Optional) The ID of the file to be sent to the model. image_url: type: string description: (Optional) URL of the image content @@ -6998,6 +7035,10 @@ components: type: string description: >- (Optional) ID of the previous response in a conversation + prompt: + $ref: '#/components/schemas/Prompt' + description: >- + (Optional) Prompt object with ID, version, and variables status: type: string description: >- @@ -7315,6 +7356,29 @@ components: title: OpenAIResponseInputToolMCP description: >- Model Context Protocol (MCP) tool configuration for OpenAI response inputs. + OpenAIResponsePromptParam: + type: object + properties: + id: + type: string + description: Unique identifier of the prompt template + variables: + type: object + additionalProperties: + $ref: '#/components/schemas/OpenAIResponseInputMessageContent' + description: >- + Dictionary of variable names to OpenAIResponseInputMessageContent structure + for template substitution + version: + type: string + description: >- + Version number of the prompt to use (defaults to latest if not specified) + additionalProperties: false + required: + - id + title: OpenAIResponsePromptParam + description: >- + Prompt object that is used for OpenAI responses. CreateOpenaiResponseRequest: type: object properties: @@ -7328,6 +7392,10 @@ components: model: type: string description: The underlying LLM used for completions. + prompt: + $ref: '#/components/schemas/OpenAIResponsePromptParam' + description: >- + Prompt object with ID, version, and variables. instructions: type: string previous_response_id: @@ -7405,6 +7473,10 @@ components: type: string description: >- (Optional) ID of the previous response in a conversation + prompt: + $ref: '#/components/schemas/Prompt' + description: >- + (Optional) Prompt object with ID, version, and variables status: type: string description: >- diff --git a/llama_stack/apis/agents/agents.py b/llama_stack/apis/agents/agents.py index 6ad45cf99..2331c855c 100644 --- a/llama_stack/apis/agents/agents.py +++ b/llama_stack/apis/agents/agents.py @@ -38,6 +38,7 @@ from .openai_responses import ( OpenAIResponseInputTool, OpenAIResponseObject, OpenAIResponseObjectStream, + OpenAIResponsePromptParam, OpenAIResponseText, ) @@ -810,6 +811,7 @@ class Agents(Protocol): self, input: str | list[OpenAIResponseInput], model: str, + prompt: OpenAIResponsePromptParam | None = None, instructions: str | None = None, previous_response_id: str | None = None, conversation: str | None = None, @@ -831,6 +833,7 @@ class Agents(Protocol): :param input: Input message(s) to create the response. :param model: The underlying LLM used for completions. + :param prompt: Prompt object with ID, version, and variables. :param previous_response_id: (Optional) if specified, the new response will be a continuation of the previous response. This can be used to easily fork-off new responses from existing responses. :param conversation: (Optional) The ID of a conversation to add the response to. Must begin with 'conv_'. Input and output messages will be automatically added to the conversation. :param include: (Optional) Additional fields to include in the response. diff --git a/llama_stack/apis/agents/openai_responses.py b/llama_stack/apis/agents/openai_responses.py index 821d6a8af..1e7675a9b 100644 --- a/llama_stack/apis/agents/openai_responses.py +++ b/llama_stack/apis/agents/openai_responses.py @@ -6,9 +6,10 @@ from typing import Annotated, Any, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator from typing_extensions import TypedDict +from llama_stack.apis.prompts.prompts import Prompt from llama_stack.apis.vector_io import SearchRankingOptions as FileSearchRankingOptions from llama_stack.schema_utils import json_schema_type, register_schema @@ -46,18 +47,44 @@ class OpenAIResponseInputMessageContentImage(BaseModel): :param detail: Level of detail for image processing, can be "low", "high", or "auto" :param type: Content type identifier, always "input_image" + :param file_id: (Optional) The ID of the file to be sent to the model. :param image_url: (Optional) URL of the image content """ detail: Literal["low"] | Literal["high"] | Literal["auto"] = "auto" type: Literal["input_image"] = "input_image" - # TODO: handle file_id + file_id: str | None = None image_url: str | None = None -# TODO: handle file content types +@json_schema_type +class OpenAIResponseInputMessageContentFile(BaseModel): + """File content for input messages in OpenAI response format. + + :param type: The type of the input item. Always `input_file`. + :param file_data: The data of the file to be sent to the model. + :param file_id: (Optional) The ID of the file to be sent to the model. + :param file_url: The URL of the file to be sent to the model. + :param filename: The name of the file to be sent to the model. + """ + + type: Literal["input_file"] = "input_file" + file_data: str | None = None + file_id: str | None = None + file_url: str | None = None + filename: str | None = None + + @model_validator(mode="after") + def validate_file_source(self) -> "OpenAIResponseInputMessageContentFile": + if not any([self.file_id, self.file_data, self.file_url]): + raise ValueError("At least one of 'file_id', 'file_data', or 'file_url' must be provided for file content") + return self + + OpenAIResponseInputMessageContent = Annotated[ - OpenAIResponseInputMessageContentText | OpenAIResponseInputMessageContentImage, + OpenAIResponseInputMessageContentText + | OpenAIResponseInputMessageContentImage + | OpenAIResponseInputMessageContentFile, Field(discriminator="type"), ] register_schema(OpenAIResponseInputMessageContent, name="OpenAIResponseInputMessageContent") @@ -348,6 +375,20 @@ class OpenAIResponseTextFormat(TypedDict, total=False): strict: bool | None +@json_schema_type +class OpenAIResponsePromptParam(BaseModel): + """Prompt object that is used for OpenAI responses. + + :param id: Unique identifier of the prompt template + :param variables: Dictionary of variable names to OpenAIResponseInputMessageContent structure for template substitution + :param version: Version number of the prompt to use (defaults to latest if not specified) + """ + + id: str + variables: dict[str, OpenAIResponseInputMessageContent] | None = None + version: str | None = None + + @json_schema_type class OpenAIResponseText(BaseModel): """Text response configuration for OpenAI responses. @@ -537,6 +578,7 @@ class OpenAIResponseObject(BaseModel): :param object: Object type identifier, always "response" :param output: List of generated output items (messages, tool calls, etc.) :param parallel_tool_calls: Whether tool calls can be executed in parallel + :param prompt: (Optional) Prompt object with ID, version, and variables :param previous_response_id: (Optional) ID of the previous response in a conversation :param status: Current status of the response generation :param temperature: (Optional) Sampling temperature used for generation @@ -556,6 +598,7 @@ class OpenAIResponseObject(BaseModel): output: list[OpenAIResponseOutput] parallel_tool_calls: bool = False previous_response_id: str | None = None + prompt: Prompt | None = None status: str temperature: float | None = None # Default to text format to avoid breaking the loading of old responses diff --git a/llama_stack/distributions/ci-tests/run.yaml b/llama_stack/distributions/ci-tests/run.yaml index ecf9eed3b..1623bb678 100644 --- a/llama_stack/distributions/ci-tests/run.yaml +++ b/llama_stack/distributions/ci-tests/run.yaml @@ -247,6 +247,9 @@ storage: conversations: table_name: openai_conversations backend: sql_default + prompts: + table_name: prompts + backend: sql_default registered_resources: models: [] shields: diff --git a/llama_stack/distributions/dell/run-with-safety.yaml b/llama_stack/distributions/dell/run-with-safety.yaml index 2563f2f4b..c11583fea 100644 --- a/llama_stack/distributions/dell/run-with-safety.yaml +++ b/llama_stack/distributions/dell/run-with-safety.yaml @@ -109,6 +109,9 @@ storage: conversations: table_name: openai_conversations backend: sql_default + prompts: + table_name: prompts + backend: sql_default registered_resources: models: - metadata: {} diff --git a/llama_stack/distributions/dell/run.yaml b/llama_stack/distributions/dell/run.yaml index 7bada394f..4611179b8 100644 --- a/llama_stack/distributions/dell/run.yaml +++ b/llama_stack/distributions/dell/run.yaml @@ -105,6 +105,9 @@ storage: conversations: table_name: openai_conversations backend: sql_default + prompts: + table_name: prompts + backend: sql_default registered_resources: models: - metadata: {} diff --git a/llama_stack/distributions/meta-reference-gpu/run-with-safety.yaml b/llama_stack/distributions/meta-reference-gpu/run-with-safety.yaml index 01b5db4f9..ef28e4c60 100644 --- a/llama_stack/distributions/meta-reference-gpu/run-with-safety.yaml +++ b/llama_stack/distributions/meta-reference-gpu/run-with-safety.yaml @@ -122,6 +122,9 @@ storage: conversations: table_name: openai_conversations backend: sql_default + prompts: + table_name: prompts + backend: sql_default registered_resources: models: - metadata: {} diff --git a/llama_stack/distributions/meta-reference-gpu/run.yaml b/llama_stack/distributions/meta-reference-gpu/run.yaml index 87c33dde0..be9e86dfa 100644 --- a/llama_stack/distributions/meta-reference-gpu/run.yaml +++ b/llama_stack/distributions/meta-reference-gpu/run.yaml @@ -112,6 +112,9 @@ storage: conversations: table_name: openai_conversations backend: sql_default + prompts: + table_name: prompts + backend: sql_default registered_resources: models: - metadata: {} diff --git a/llama_stack/distributions/nvidia/run-with-safety.yaml b/llama_stack/distributions/nvidia/run-with-safety.yaml index c23d0f9cb..e07a1a3a1 100644 --- a/llama_stack/distributions/nvidia/run-with-safety.yaml +++ b/llama_stack/distributions/nvidia/run-with-safety.yaml @@ -111,6 +111,9 @@ storage: conversations: table_name: openai_conversations backend: sql_default + prompts: + table_name: prompts + backend: sql_default registered_resources: models: - metadata: {} diff --git a/llama_stack/distributions/nvidia/run.yaml b/llama_stack/distributions/nvidia/run.yaml index 81e744d53..015c3c6f9 100644 --- a/llama_stack/distributions/nvidia/run.yaml +++ b/llama_stack/distributions/nvidia/run.yaml @@ -100,6 +100,9 @@ storage: conversations: table_name: openai_conversations backend: sql_default + prompts: + table_name: prompts + backend: sql_default registered_resources: models: [] shields: [] diff --git a/llama_stack/distributions/open-benchmark/run.yaml b/llama_stack/distributions/open-benchmark/run.yaml index 4fd0e199b..70a691096 100644 --- a/llama_stack/distributions/open-benchmark/run.yaml +++ b/llama_stack/distributions/open-benchmark/run.yaml @@ -142,6 +142,9 @@ storage: conversations: table_name: openai_conversations backend: sql_default + prompts: + table_name: prompts + backend: sql_default registered_resources: models: - metadata: {} diff --git a/llama_stack/distributions/postgres-demo/run.yaml b/llama_stack/distributions/postgres-demo/run.yaml index 0d7ecff48..73d213501 100644 --- a/llama_stack/distributions/postgres-demo/run.yaml +++ b/llama_stack/distributions/postgres-demo/run.yaml @@ -87,6 +87,9 @@ storage: conversations: table_name: openai_conversations backend: sql_default + prompts: + table_name: prompts + backend: sql_default registered_resources: models: - metadata: {} diff --git a/llama_stack/distributions/starter-gpu/run.yaml b/llama_stack/distributions/starter-gpu/run.yaml index 92483c78e..7edc6a884 100644 --- a/llama_stack/distributions/starter-gpu/run.yaml +++ b/llama_stack/distributions/starter-gpu/run.yaml @@ -250,6 +250,9 @@ storage: conversations: table_name: openai_conversations backend: sql_default + prompts: + table_name: prompts + backend: sql_default registered_resources: models: [] shields: diff --git a/llama_stack/distributions/starter/run.yaml b/llama_stack/distributions/starter/run.yaml index 3b9d8f890..d7bdaa7d4 100644 --- a/llama_stack/distributions/starter/run.yaml +++ b/llama_stack/distributions/starter/run.yaml @@ -247,6 +247,9 @@ storage: conversations: table_name: openai_conversations backend: sql_default + prompts: + table_name: prompts + backend: sql_default registered_resources: models: [] shields: diff --git a/llama_stack/distributions/template.py b/llama_stack/distributions/template.py index 64f21e626..be068f804 100644 --- a/llama_stack/distributions/template.py +++ b/llama_stack/distributions/template.py @@ -257,6 +257,10 @@ class RunConfigSettings(BaseModel): backend="sql_default", table_name="openai_conversations", ).model_dump(exclude_none=True), + "prompts": SqlStoreReference( + backend="sql_default", + table_name="prompts", + ).model_dump(exclude_none=True), } storage_config = dict( diff --git a/llama_stack/distributions/watsonx/run.yaml b/llama_stack/distributions/watsonx/run.yaml index ca3c8402d..9be3f2c92 100644 --- a/llama_stack/distributions/watsonx/run.yaml +++ b/llama_stack/distributions/watsonx/run.yaml @@ -115,6 +115,9 @@ storage: conversations: table_name: openai_conversations backend: sql_default + prompts: + table_name: prompts + backend: sql_default registered_resources: models: [] shields: [] diff --git a/llama_stack/providers/inline/agents/meta_reference/__init__.py b/llama_stack/providers/inline/agents/meta_reference/__init__.py index 91287617a..e066b7b01 100644 --- a/llama_stack/providers/inline/agents/meta_reference/__init__.py +++ b/llama_stack/providers/inline/agents/meta_reference/__init__.py @@ -20,15 +20,17 @@ async def get_provider_impl( from .agents import MetaReferenceAgentsImpl impl = MetaReferenceAgentsImpl( - config, - deps[Api.inference], - deps[Api.vector_io], - deps[Api.safety], - deps[Api.tool_runtime], - deps[Api.tool_groups], - deps[Api.conversations], - policy, - telemetry_enabled, + config=config, + inference_api=deps[Api.inference], + vector_io_api=deps[Api.vector_io], + safety_api=deps[Api.safety], + tool_runtime_api=deps[Api.tool_runtime], + tool_groups_api=deps[Api.tool_groups], + conversations_api=deps[Api.conversations], + prompts_api=deps[Api.prompts], + files_api=deps[Api.files], + telemetry_enabled=Api.telemetry in deps, + policy=policy, ) await impl.initialize() return impl diff --git a/llama_stack/providers/inline/agents/meta_reference/agents.py b/llama_stack/providers/inline/agents/meta_reference/agents.py index c2f6ea640..9933815d1 100644 --- a/llama_stack/providers/inline/agents/meta_reference/agents.py +++ b/llama_stack/providers/inline/agents/meta_reference/agents.py @@ -29,9 +29,10 @@ from llama_stack.apis.agents import ( Turn, ) from llama_stack.apis.agents.agents import ResponseGuardrail -from llama_stack.apis.agents.openai_responses import OpenAIResponseText +from llama_stack.apis.agents.openai_responses import OpenAIResponsePromptParam, OpenAIResponseText from llama_stack.apis.common.responses import PaginatedResponse from llama_stack.apis.conversations import Conversations +from llama_stack.apis.files import Files from llama_stack.apis.inference import ( Inference, ToolConfig, @@ -39,6 +40,7 @@ from llama_stack.apis.inference import ( ToolResponseMessage, UserMessage, ) +from llama_stack.apis.prompts import Prompts from llama_stack.apis.safety import Safety from llama_stack.apis.tools import ToolGroups, ToolRuntime from llama_stack.apis.vector_io import VectorIO @@ -66,6 +68,8 @@ class MetaReferenceAgentsImpl(Agents): tool_runtime_api: ToolRuntime, tool_groups_api: ToolGroups, conversations_api: Conversations, + prompts_api: Prompts, + files_api: Files, policy: list[AccessRule], telemetry_enabled: bool = False, ): @@ -77,7 +81,8 @@ class MetaReferenceAgentsImpl(Agents): self.tool_groups_api = tool_groups_api self.conversations_api = conversations_api self.telemetry_enabled = telemetry_enabled - + self.prompts_api = prompts_api + self.files_api = files_api self.in_memory_store = InmemoryKVStoreImpl() self.openai_responses_impl: OpenAIResponsesImpl | None = None self.policy = policy @@ -94,6 +99,8 @@ class MetaReferenceAgentsImpl(Agents): vector_io_api=self.vector_io_api, safety_api=self.safety_api, conversations_api=self.conversations_api, + prompts_api=self.prompts_api, + files_api=self.files_api, ) async def create_agent( @@ -329,6 +336,7 @@ class MetaReferenceAgentsImpl(Agents): self, input: str | list[OpenAIResponseInput], model: str, + prompt: OpenAIResponsePromptParam | None = None, instructions: str | None = None, previous_response_id: str | None = None, conversation: str | None = None, @@ -344,6 +352,7 @@ class MetaReferenceAgentsImpl(Agents): return await self.openai_responses_impl.create_openai_response( input, model, + prompt, instructions, previous_response_id, conversation, diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py b/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py index 2360dafd9..547530717 100644 --- a/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py +++ b/llama_stack/providers/inline/agents/meta_reference/responses/openai_responses.py @@ -4,6 +4,7 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. +import re import time import uuid from collections.abc import AsyncIterator @@ -17,11 +18,14 @@ from llama_stack.apis.agents.openai_responses import ( ListOpenAIResponseObject, OpenAIDeleteResponseObject, OpenAIResponseInput, + OpenAIResponseInputMessageContentFile, + OpenAIResponseInputMessageContentImage, OpenAIResponseInputMessageContentText, OpenAIResponseInputTool, OpenAIResponseMessage, OpenAIResponseObject, OpenAIResponseObjectStream, + OpenAIResponsePromptParam, OpenAIResponseText, OpenAIResponseTextFormat, ) @@ -30,11 +34,17 @@ from llama_stack.apis.common.errors import ( ) from llama_stack.apis.conversations import Conversations from llama_stack.apis.conversations.conversations import ConversationItem +from llama_stack.apis.files import Files from llama_stack.apis.inference import ( Inference, + OpenAIChatCompletionContentPartParam, + OpenAIChatCompletionContentPartTextParam, OpenAIMessageParam, OpenAISystemMessageParam, + OpenAIUserMessageParam, ) +from llama_stack.apis.prompts import Prompts +from llama_stack.apis.prompts.prompts import Prompt from llama_stack.apis.safety import Safety from llama_stack.apis.tools import ToolGroups, ToolRuntime from llama_stack.apis.vector_io import VectorIO @@ -71,6 +81,8 @@ class OpenAIResponsesImpl: vector_io_api: VectorIO, # VectorIO safety_api: Safety, conversations_api: Conversations, + prompts_api: Prompts, + files_api: Files, ): self.inference_api = inference_api self.tool_groups_api = tool_groups_api @@ -84,6 +96,8 @@ class OpenAIResponsesImpl: tool_runtime_api=tool_runtime_api, vector_io_api=vector_io_api, ) + self.prompts_api = prompts_api + self.files_api = files_api async def _prepend_previous_response( self, @@ -123,11 +137,13 @@ class OpenAIResponsesImpl: # Use stored messages directly and convert only new input message_adapter = TypeAdapter(list[OpenAIMessageParam]) messages = message_adapter.validate_python(previous_response.messages) - new_messages = await convert_response_input_to_chat_messages(input, previous_messages=messages) + new_messages = await convert_response_input_to_chat_messages( + input, previous_messages=messages, files_api=self.files_api + ) messages.extend(new_messages) else: # Backward compatibility: reconstruct from inputs - messages = await convert_response_input_to_chat_messages(all_input) + messages = await convert_response_input_to_chat_messages(all_input, files_api=self.files_api) tool_context.recover_tools_from_previous_response(previous_response) elif conversation is not None: @@ -139,7 +155,7 @@ class OpenAIResponsesImpl: all_input = input if not conversation_items.data: # First turn - just convert the new input - messages = await convert_response_input_to_chat_messages(input) + messages = await convert_response_input_to_chat_messages(input, files_api=self.files_api) else: if not stored_messages: all_input = conversation_items.data @@ -155,14 +171,114 @@ class OpenAIResponsesImpl: all_input = input messages = stored_messages or [] - new_messages = await convert_response_input_to_chat_messages(all_input, previous_messages=messages) + new_messages = await convert_response_input_to_chat_messages( + all_input, previous_messages=messages, files_api=self.files_api + ) messages.extend(new_messages) else: all_input = input - messages = await convert_response_input_to_chat_messages(all_input) + messages = await convert_response_input_to_chat_messages(all_input, files_api=self.files_api) return all_input, messages, tool_context + async def _prepend_prompt( + self, + messages: list[OpenAIMessageParam], + prompt_params: OpenAIResponsePromptParam, + ) -> Prompt: + """Prepend prompt template to messages, resolving text/image/file variables. + + For text-only prompts: Inserts as system message + For prompts with media: Inserts text as system message + media into first user message + """ + if not prompt_params or not prompt_params.id: + return None + + prompt_version = int(prompt_params.version) if prompt_params.version else None + cur_prompt = await self.prompts_api.get_prompt(prompt_params.id, prompt_version) + + if not cur_prompt: + return None + + cur_prompt_text = cur_prompt.prompt + cur_prompt_variables = cur_prompt.variables + + if not prompt_params.variables: + messages.insert(0, OpenAISystemMessageParam(content=cur_prompt_text)) + return cur_prompt + + # Validate that all provided variables exist in the prompt + for name in prompt_params.variables.keys(): + if name not in cur_prompt_variables: + raise ValueError(f"Variable {name} not found in prompt {prompt_params.id}") + + # Separate text and media variables + text_substitutions = {} + media_content_parts = [] + + for name, value in prompt_params.variables.items(): + # Text variable found + if isinstance(value, OpenAIResponseInputMessageContentText): + text_substitutions[name] = value.text + + # Media variable found + elif isinstance(value, OpenAIResponseInputMessageContentImage | OpenAIResponseInputMessageContentFile): + # use existing converter to achieve OpenAI Chat Completion format + from .utils import convert_response_content_to_chat_content + + converted_parts = await convert_response_content_to_chat_content([value], files_api=self.files_api) + media_content_parts.extend(converted_parts) + + # Eg: {{product_photo}} becomes "[Image: product_photo]" + # This gives the model textual context about what media exists in the prompt + var_type = value.type.replace("input_", "").replace("_", " ").title() + text_substitutions[name] = f"[{var_type}: {name}]" + + def replace_variable(match: re.Match[str]) -> str: + var_name = match.group(1).strip() + return str(text_substitutions.get(var_name, match.group(0))) + + pattern = r"\{\{\s*(\w+)\s*\}\}" + resolved_prompt_text = re.sub(pattern, replace_variable, cur_prompt_text) + + # Insert system message with resolved text + messages.insert(0, OpenAISystemMessageParam(content=resolved_prompt_text)) + + # If we have media, prepend to first user message + if media_content_parts: + self._prepend_media_into_first_user_message(messages, media_content_parts) + + return cur_prompt + + def _prepend_media_into_first_user_message( + self, messages: list[OpenAIMessageParam], media_parts: list[OpenAIChatCompletionContentPartParam] + ) -> None: + """Prepend media content parts into the first user message.""" + + # Find first user message (skip the system message we just added) + first_user_msg_index = None + for i, message in enumerate(messages): + if isinstance(message, OpenAIUserMessageParam): + first_user_msg_index = i + break + + if first_user_msg_index is not None: + user_msg = messages[first_user_msg_index] + + # Convert string content to parts if needed, otherwise use existing parts directly + if isinstance(user_msg.content, str): + existing_parts = [OpenAIChatCompletionContentPartTextParam(text=user_msg.content)] + else: + existing_parts = user_msg.content + + # Prepend media before user's content + combined_parts = media_parts + existing_parts + + messages[first_user_msg_index] = OpenAIUserMessageParam(content=combined_parts, name=user_msg.name) + else: + # No user message exists - append one with just media + messages.append(OpenAIUserMessageParam(content=media_parts)) + async def get_openai_response( self, response_id: str, @@ -239,6 +355,7 @@ class OpenAIResponsesImpl: self, input: str | list[OpenAIResponseInput], model: str, + prompt: OpenAIResponsePromptParam | None = None, instructions: str | None = None, previous_response_id: str | None = None, conversation: str | None = None, @@ -269,6 +386,7 @@ class OpenAIResponsesImpl: input=input, conversation=conversation, model=model, + prompt=prompt, instructions=instructions, previous_response_id=previous_response_id, store=store, @@ -314,6 +432,7 @@ class OpenAIResponsesImpl: self, input: str | list[OpenAIResponseInput], model: str, + prompt: OpenAIResponsePromptParam | None = None, instructions: str | None = None, previous_response_id: str | None = None, conversation: str | None = None, @@ -332,6 +451,9 @@ class OpenAIResponsesImpl: if instructions: messages.insert(0, OpenAISystemMessageParam(content=instructions)) + # Prepend reusable prompt (if provided) + prompt_obj = await self._prepend_prompt(messages, prompt) + # Structured outputs response_format = await convert_response_text_to_chat_response_format(text) @@ -354,6 +476,7 @@ class OpenAIResponsesImpl: ctx=ctx, response_id=response_id, created_at=created_at, + prompt=prompt_obj, text=text, max_infer_iters=max_infer_iters, tool_executor=self.tool_executor, diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/streaming.py b/llama_stack/providers/inline/agents/meta_reference/responses/streaming.py index e80ffcdd1..59c082825 100644 --- a/llama_stack/providers/inline/agents/meta_reference/responses/streaming.py +++ b/llama_stack/providers/inline/agents/meta_reference/responses/streaming.py @@ -65,6 +65,7 @@ from llama_stack.apis.inference import ( OpenAIChoice, OpenAIMessageParam, ) +from llama_stack.apis.prompts.prompts import Prompt from llama_stack.log import get_logger from llama_stack.providers.utils.inference.prompt_adapter import interleaved_content_as_str from llama_stack.providers.utils.telemetry import tracing @@ -107,6 +108,7 @@ class StreamingResponseOrchestrator: ctx: ChatCompletionContext, response_id: str, created_at: int, + prompt: Prompt | None, text: OpenAIResponseText, max_infer_iters: int, tool_executor, # Will be the tool execution logic from the main class @@ -118,6 +120,7 @@ class StreamingResponseOrchestrator: self.ctx = ctx self.response_id = response_id self.created_at = created_at + self.prompt = prompt self.text = text self.max_infer_iters = max_infer_iters self.tool_executor = tool_executor @@ -175,6 +178,7 @@ class StreamingResponseOrchestrator: object="response", status=status, output=self._clone_outputs(outputs), + prompt=self.prompt, text=self.text, tools=self.ctx.available_tools(), error=error, diff --git a/llama_stack/providers/inline/agents/meta_reference/responses/utils.py b/llama_stack/providers/inline/agents/meta_reference/responses/utils.py index 7ca8af632..720403653 100644 --- a/llama_stack/providers/inline/agents/meta_reference/responses/utils.py +++ b/llama_stack/providers/inline/agents/meta_reference/responses/utils.py @@ -5,6 +5,7 @@ # the root directory of this source tree. import asyncio +import base64 import re import uuid @@ -14,6 +15,7 @@ from llama_stack.apis.agents.openai_responses import ( OpenAIResponseInput, OpenAIResponseInputFunctionToolCallOutput, OpenAIResponseInputMessageContent, + OpenAIResponseInputMessageContentFile, OpenAIResponseInputMessageContentImage, OpenAIResponseInputMessageContentText, OpenAIResponseInputTool, @@ -27,6 +29,7 @@ from llama_stack.apis.agents.openai_responses import ( OpenAIResponseOutputMessageMCPListTools, OpenAIResponseText, ) +from llama_stack.apis.files import Files from llama_stack.apis.inference import ( OpenAIAssistantMessageParam, OpenAIChatCompletionContentPartImageParam, @@ -36,6 +39,8 @@ from llama_stack.apis.inference import ( OpenAIChatCompletionToolCallFunction, OpenAIChoice, OpenAIDeveloperMessageParam, + OpenAIFile, + OpenAIFileFile, OpenAIImageURL, OpenAIJSONSchema, OpenAIMessageParam, @@ -50,6 +55,49 @@ from llama_stack.apis.inference import ( from llama_stack.apis.safety import Safety +async def extract_file_content(file_id: str, files_api: Files) -> bytes: + """ + Retrieve file content directly using the Files API. + + :param file_id: The file identifier (e.g., "file-abc123") + :param files_api: Files API instance + :returns: Raw file content as bytes + :raises: ValueError if file cannot be retrieved + """ + try: + response = await files_api.openai_retrieve_file_content(file_id) + if hasattr(response, "body"): + return response.body + elif hasattr(response, "content"): + return response.content + else: + raise AttributeError(f"Response object has no 'body' or 'content' attribute. Type: {type(response)}") + except Exception as e: + raise ValueError(f"Failed to retrieve file content for file_id '{file_id}': {str(e)}") from e + + +def get_mime_type_from_filename(filename: str | None) -> str: + """ + Determine MIME type from filename extension. + + :param filename: The filename to analyze + :returns: MIME type string (defaults to "application/octet-stream" if unknown) + """ + if not filename: + return "application/octet-stream" + + filename_lower = filename.lower() + if filename_lower.endswith(".pdf"): + return "application/pdf" + elif filename_lower.endswith((".png", ".jpg", ".jpeg")): + ext = filename_lower.split(".")[-1] + return f"image/{ext.replace('jpg', 'jpeg')}" + elif filename_lower.endswith(".txt"): + return "text/plain" + else: + return "application/octet-stream" + + async def convert_chat_choice_to_response_message( choice: OpenAIChoice, citation_files: dict[str, str] | None = None, @@ -79,11 +127,15 @@ async def convert_chat_choice_to_response_message( async def convert_response_content_to_chat_content( content: (str | list[OpenAIResponseInputMessageContent] | list[OpenAIResponseOutputMessageContent]), + files_api: Files, ) -> str | list[OpenAIChatCompletionContentPartParam]: """ Convert the content parts from an OpenAI Response API request into OpenAI Chat Completion content parts. The content schemas of each API look similar, but are not exactly the same. + + :param content: The content to convert + :param files_api: Files API for resolving file_id to raw file content (required) """ if isinstance(content, str): return content @@ -95,9 +147,69 @@ async def convert_response_content_to_chat_content( elif isinstance(content_part, OpenAIResponseOutputMessageContentOutputText): converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part.text)) elif isinstance(content_part, OpenAIResponseInputMessageContentImage): + detail = content_part.detail + if content_part.image_url: - image_url = OpenAIImageURL(url=content_part.image_url, detail=content_part.detail) + image_url = OpenAIImageURL(url=content_part.image_url, detail=detail) converted_parts.append(OpenAIChatCompletionContentPartImageParam(image_url=image_url)) + elif content_part.file_id: + file_content = await extract_file_content(content_part.file_id, files_api) + encoded_content = base64.b64encode(file_content).decode("utf-8") + data_url = f"data:image/png;base64,{encoded_content}" + image_url = OpenAIImageURL(url=data_url, detail=detail) + converted_parts.append(OpenAIChatCompletionContentPartImageParam(image_url=image_url)) + else: + raise ValueError( + f"Image content must have either 'image_url' or 'file_id'. " + f"Got image_url={content_part.image_url}, file_id={content_part.file_id}" + ) + elif isinstance(content_part, OpenAIResponseInputMessageContentFile): + file_data = getattr(content_part, "file_data", None) + file_id = getattr(content_part, "file_id", None) + file_url = getattr(content_part, "file_url", None) + filename = getattr(content_part, "filename", None) + + if not any([file_id, file_data, file_url]): + raise ValueError( + f"File content must have at least one of 'file_id', 'file_data', or 'file_url'. " + f"Got file_id={file_id}, file_data={'' if file_data else None}, file_url={file_url}" + ) + + resolved_file_data = None + + if file_id: + file_content = await extract_file_content(file_id, files_api) + + # If filename is not provided, fetch it from the Files API + if not filename: + file_metadata = await files_api.openai_retrieve_file(file_id) + filename = file_metadata.filename + + # Determine MIME type and encode as data URL + mime_type = get_mime_type_from_filename(filename) + base64_content = base64.b64encode(file_content).decode("utf-8") + resolved_file_data = f"data:{mime_type};base64,{base64_content}" + + elif file_data: + # If file_data provided directly + if file_data.startswith("data:"): + resolved_file_data = file_data + else: + # Raw base64 data, wrap in data URL format + mime_type = get_mime_type_from_filename(filename) + resolved_file_data = f"data:{mime_type};base64,{file_data}" + + elif file_url: + resolved_file_data = file_url + + converted_parts.append( + OpenAIFile( + file=OpenAIFileFile( + file_data=resolved_file_data, + filename=filename, + ) + ) + ) elif isinstance(content_part, str): converted_parts.append(OpenAIChatCompletionContentPartTextParam(text=content_part)) else: @@ -110,12 +222,14 @@ async def convert_response_content_to_chat_content( async def convert_response_input_to_chat_messages( input: str | list[OpenAIResponseInput], previous_messages: list[OpenAIMessageParam] | None = None, + files_api: Files | None = None, ) -> list[OpenAIMessageParam]: """ Convert the input from an OpenAI Response API request into OpenAI Chat Completion messages. :param input: The input to convert :param previous_messages: Optional previous messages to check for function_call references + :param files_api: Files API for resolving file_id to raw file content (optional, required for file/image content) """ messages: list[OpenAIMessageParam] = [] if isinstance(input, list): @@ -173,7 +287,7 @@ async def convert_response_input_to_chat_messages( # these are handled by the responses impl itself and not pass through to chat completions pass else: - content = await convert_response_content_to_chat_content(input_item.content) + content = await convert_response_content_to_chat_content(input_item.content, files_api) message_type = await get_message_type_by_role(input_item.role) if message_type is None: raise ValueError( diff --git a/llama_stack/providers/registry/agents.py b/llama_stack/providers/registry/agents.py index 1845d6f46..04fbfc18c 100644 --- a/llama_stack/providers/registry/agents.py +++ b/llama_stack/providers/registry/agents.py @@ -35,6 +35,8 @@ def available_providers() -> list[ProviderSpec]: Api.tool_runtime, Api.tool_groups, Api.conversations, + Api.prompts, + Api.files, ], description="Meta's reference implementation of an agent system that can use tools, access vector databases, and perform complex reasoning tasks.", ), diff --git a/tests/unit/providers/agent/test_meta_reference_agent.py b/tests/unit/providers/agent/test_meta_reference_agent.py index dfd9b6d52..839f4daee 100644 --- a/tests/unit/providers/agent/test_meta_reference_agent.py +++ b/tests/unit/providers/agent/test_meta_reference_agent.py @@ -16,7 +16,9 @@ from llama_stack.apis.agents import ( ) from llama_stack.apis.common.responses import PaginatedResponse from llama_stack.apis.conversations import Conversations +from llama_stack.apis.files import Files from llama_stack.apis.inference import Inference +from llama_stack.apis.prompts import Prompts from llama_stack.apis.safety import Safety from llama_stack.apis.tools import ListToolDefsResponse, ToolDef, ToolGroups, ToolRuntime from llama_stack.apis.vector_io import VectorIO @@ -49,6 +51,8 @@ def mock_apis(): "tool_runtime_api": AsyncMock(spec=ToolRuntime), "tool_groups_api": AsyncMock(spec=ToolGroups), "conversations_api": AsyncMock(spec=Conversations), + "prompts_api": AsyncMock(spec=Prompts), + "files_api": AsyncMock(spec=Files), } @@ -81,7 +85,9 @@ async def agents_impl(config, mock_apis): mock_apis["tool_runtime_api"], mock_apis["tool_groups_api"], mock_apis["conversations_api"], - [], + mock_apis["prompts_api"], + mock_apis["files_api"], + [], # policy (empty list for tests) ) await impl.initialize() yield impl diff --git a/tests/unit/providers/agents/meta_reference/test_openai_responses.py b/tests/unit/providers/agents/meta_reference/test_openai_responses.py index f31ec0c28..779aa7934 100644 --- a/tests/unit/providers/agents/meta_reference/test_openai_responses.py +++ b/tests/unit/providers/agents/meta_reference/test_openai_responses.py @@ -40,6 +40,7 @@ from llama_stack.apis.inference import ( OpenAIResponseFormatJSONSchema, OpenAIUserMessageParam, ) +from llama_stack.apis.prompts import Prompt from llama_stack.apis.tools.tools import ListToolDefsResponse, ToolDef, ToolGroups, ToolInvocationResult, ToolRuntime from llama_stack.core.access_control.access_control import default_policy from llama_stack.core.storage.datatypes import ResponsesStoreReference, SqliteSqlStoreConfig @@ -97,6 +98,19 @@ def mock_safety_api(): return safety_api +@pytest.fixture +def mock_prompts_api(): + prompts_api = AsyncMock() + return prompts_api + + +@pytest.fixture +def mock_files_api(): + """Mock files API for testing.""" + files_api = AsyncMock() + return files_api + + @pytest.fixture def openai_responses_impl( mock_inference_api, @@ -106,6 +120,8 @@ def openai_responses_impl( mock_vector_io_api, mock_safety_api, mock_conversations_api, + mock_prompts_api, + mock_files_api, ): return OpenAIResponsesImpl( inference_api=mock_inference_api, @@ -115,6 +131,8 @@ def openai_responses_impl( vector_io_api=mock_vector_io_api, safety_api=mock_safety_api, conversations_api=mock_conversations_api, + prompts_api=mock_prompts_api, + files_api=mock_files_api, ) @@ -498,7 +516,7 @@ async def test_create_openai_response_with_tool_call_function_arguments_none(ope mock_inference_api.openai_chat_completion.return_value = fake_stream_toolcall() -async def test_create_openai_response_with_multiple_messages(openai_responses_impl, mock_inference_api): +async def test_create_openai_response_with_multiple_messages(openai_responses_impl, mock_inference_api, mock_files_api): """Test creating an OpenAI response with multiple messages.""" # Setup input_messages = [ @@ -709,7 +727,7 @@ async def test_create_openai_response_with_instructions(openai_responses_impl, m async def test_create_openai_response_with_instructions_and_multiple_messages( - openai_responses_impl, mock_inference_api + openai_responses_impl, mock_inference_api, mock_files_api ): # Setup input_messages = [ @@ -1169,3 +1187,657 @@ async def test_create_openai_response_with_invalid_text_format(openai_responses_ model=model, text=OpenAIResponseText(format={"type": "invalid"}), ) + + +async def test_create_openai_response_with_prompt(openai_responses_impl, mock_inference_api, mock_prompts_api): + """Test creating an OpenAI response with a prompt.""" + input_text = "What is the capital of Ireland?" + model = "meta-llama/Llama-3.1-8B-Instruct" + prompt_id = "pmpt_1234567890abcdef1234567890abcdef1234567890abcdef" + prompt = Prompt( + prompt="You are a helpful {{ area_name }} assistant at {{ company_name }}. Always provide accurate information.", + prompt_id=prompt_id, + version=1, + variables=["area_name", "company_name"], + is_default=True, + ) + + from llama_stack.apis.agents.openai_responses import ( + OpenAIResponseInputMessageContentText, + OpenAIResponsePromptParam, + ) + + prompt_params_with_version_1 = OpenAIResponsePromptParam( + id=prompt_id, + version="1", + variables={ + "area_name": OpenAIResponseInputMessageContentText(text="geography"), + "company_name": OpenAIResponseInputMessageContentText(text="Dummy Company"), + }, + ) + + mock_prompts_api.get_prompt.return_value = prompt + mock_inference_api.openai_chat_completion.return_value = fake_stream() + + result = await openai_responses_impl.create_openai_response( + input=input_text, + model=model, + prompt=prompt_params_with_version_1, + ) + + mock_prompts_api.get_prompt.assert_called_with(prompt_id, 1) + mock_inference_api.openai_chat_completion.assert_called() + call_args = mock_inference_api.openai_chat_completion.call_args + sent_messages = call_args.args[0].messages + assert len(sent_messages) == 2 + + system_messages = [msg for msg in sent_messages if msg.role == "system"] + assert len(system_messages) == 1 + assert ( + system_messages[0].content + == "You are a helpful geography assistant at Dummy Company. Always provide accurate information." + ) + + user_messages = [msg for msg in sent_messages if msg.role == "user"] + assert len(user_messages) == 1 + assert user_messages[0].content == input_text + + assert result.model == model + assert result.status == "completed" + assert result.prompt.prompt_id == prompt_id + assert result.prompt.variables == ["area_name", "company_name"] + assert result.prompt.version == 1 + assert result.prompt.prompt == prompt.prompt + + +async def test_prepend_prompt_successful_without_variables(openai_responses_impl, mock_prompts_api): + """Test prepend_prompt function without variables.""" + # Setup + prompt_id = "pmpt_1234567890abcdef1234567890abcdef1234567890abcdef" + prompt = Prompt( + prompt="You are a helpful assistant. Always provide accurate information.", + prompt_id=prompt_id, + version=1, + variables=[], + is_default=True, + ) + + from llama_stack.apis.agents.openai_responses import OpenAIResponsePromptParam + from llama_stack.apis.inference import OpenAISystemMessageParam, OpenAIUserMessageParam + + prompt_params = OpenAIResponsePromptParam(id=prompt_id, version="1") + + mock_prompts_api.get_prompt.return_value = prompt + + # Initial messages + messages = [OpenAIUserMessageParam(content="Hello")] + + # Execute + result = await openai_responses_impl._prepend_prompt(messages, prompt_params) + + # Verify + mock_prompts_api.get_prompt.assert_called_once_with(prompt_id, 1) + + # Check that prompt was returned + assert result == prompt + + # Check that system message was prepended + assert len(messages) == 2 + assert isinstance(messages[0], OpenAISystemMessageParam) + assert messages[0].content == "You are a helpful assistant. Always provide accurate information." + + +async def test_prepend_prompt_no_version_specified(openai_responses_impl, mock_prompts_api): + """Test prepend_prompt function when no version is specified (should use None).""" + # Setup + prompt_id = "pmpt_1234567890abcdef1234567890abcdef1234567890abcdef" + prompt = Prompt( + prompt="Default prompt text.", + prompt_id=prompt_id, + version=3, + variables=[], + is_default=True, + ) + + from llama_stack.apis.agents.openai_responses import OpenAIResponsePromptParam + from llama_stack.apis.inference import OpenAIUserMessageParam + + prompt_params = OpenAIResponsePromptParam(id=prompt_id) # No version specified + + mock_prompts_api.get_prompt.return_value = prompt + + # Initial messages + messages = [OpenAIUserMessageParam(content="Test")] + + # Execute + result = await openai_responses_impl._prepend_prompt(messages, prompt_params) + + # Verify + mock_prompts_api.get_prompt.assert_called_once_with(prompt_id, None) + assert result == prompt + assert len(messages) == 2 + + +async def test_prepend_prompt_invalid_variable(openai_responses_impl, mock_prompts_api): + """Test error handling in prepend_prompt function when prompt parameters contain invalid variables.""" + # Setup + prompt_id = "pmpt_1234567890abcdef1234567890abcdef1234567890abcdef" + prompt = Prompt( + prompt="You are a {{ role }} assistant.", + prompt_id=prompt_id, + version=1, + variables=["role"], # Only "role" is valid + is_default=True, + ) + + from llama_stack.apis.agents.openai_responses import ( + OpenAIResponseInputMessageContentText, + OpenAIResponsePromptParam, + ) + from llama_stack.apis.inference import OpenAIUserMessageParam + + prompt_params = OpenAIResponsePromptParam( + id=prompt_id, + version="1", + variables={ + "role": OpenAIResponseInputMessageContentText(text="helpful"), + "company": OpenAIResponseInputMessageContentText( + text="Dummy Company" + ), # company is not in prompt.variables + }, + ) + + mock_prompts_api.get_prompt.return_value = prompt + + # Initial messages + messages = [OpenAIUserMessageParam(content="Test prompt")] + + # Execute - should raise ValueError for invalid variable + with pytest.raises(ValueError, match="Variable company not found in prompt"): + await openai_responses_impl._prepend_prompt(messages, prompt_params) + + # Verify + mock_prompts_api.get_prompt.assert_called_once_with(prompt_id, 1) + + +async def test_prepend_prompt_not_found(openai_responses_impl, mock_prompts_api): + """Test prepend_prompt function when prompt is not found.""" + # Setup + prompt_id = "pmpt_nonexistent" + + from llama_stack.apis.agents.openai_responses import OpenAIResponsePromptParam + from llama_stack.apis.inference import OpenAIUserMessageParam + + prompt_params = OpenAIResponsePromptParam(id=prompt_id, version="1") + + mock_prompts_api.get_prompt.return_value = None # Prompt not found + + # Initial messages + messages = [OpenAIUserMessageParam(content="Test prompt")] + initial_length = len(messages) + + # Execute + result = await openai_responses_impl._prepend_prompt(messages, prompt_params) + + # Verify + mock_prompts_api.get_prompt.assert_called_once_with(prompt_id, 1) + + # Should return None when prompt not found + assert result is None + + # Messages should not be modified + assert len(messages) == initial_length + assert messages[0].content == "Test prompt" + + +async def test_prepend_prompt_no_params(openai_responses_impl, mock_prompts_api): + """Test handling in prepend_prompt function when prompt_params is None.""" + # Setup + from llama_stack.apis.inference import OpenAIUserMessageParam + + messages = [OpenAIUserMessageParam(content="Test")] + initial_length = len(messages) + + # Execute + result = await openai_responses_impl._prepend_prompt(messages, None) + + # Verify + mock_prompts_api.get_prompt.assert_not_called() + + # Should return None when no prompt params + assert result is None + + # Messages should not be modified + assert len(messages) == initial_length + + +async def test_prepend_prompt_variable_substitution(openai_responses_impl, mock_prompts_api): + """Test complex variable substitution with multiple occurrences and special characters in prepend_prompt function.""" + # Setup + prompt_id = "pmpt_1234567890abcdef1234567890abcdef1234567890abcdef" + + # Support all whitespace variations: {{name}}, {{ name }}, {{ name}}, {{name }}, etc. + prompt = Prompt( + prompt="Hello {{name}}! You are working at {{ company}}. Your role is {{role}} at {{company}}. Remember, {{ name }}, to be {{ tone }}.", + prompt_id=prompt_id, + version=1, + variables=["name", "company", "role", "tone"], + is_default=True, + ) + + from llama_stack.apis.agents.openai_responses import ( + OpenAIResponseInputMessageContentText, + OpenAIResponsePromptParam, + ) + from llama_stack.apis.inference import OpenAISystemMessageParam, OpenAIUserMessageParam + + prompt_params = OpenAIResponsePromptParam( + id=prompt_id, + version="1", + variables={ + "name": OpenAIResponseInputMessageContentText(text="Alice"), + "company": OpenAIResponseInputMessageContentText(text="Dummy Company"), + "role": OpenAIResponseInputMessageContentText(text="AI Assistant"), + "tone": OpenAIResponseInputMessageContentText(text="professional"), + }, + ) + + mock_prompts_api.get_prompt.return_value = prompt + + # Initial messages + messages = [OpenAIUserMessageParam(content="Test")] + + # Execute + result = await openai_responses_impl._prepend_prompt(messages, prompt_params) + + # Verify + assert result == prompt + assert len(messages) == 2 + assert isinstance(messages[0], OpenAISystemMessageParam) + expected_content = "Hello Alice! You are working at Dummy Company. Your role is AI Assistant at Dummy Company. Remember, Alice, to be professional." + assert messages[0].content == expected_content + + +async def test_prepend_prompt_with_image_variable(openai_responses_impl, mock_prompts_api, mock_files_api): + """Test prepend_prompt with image variable - should create placeholder in system message and inject image into user message.""" + # Setup + prompt_id = "pmpt_1234567890abcdef1234567890abcdef1234567890abcdef" + prompt = Prompt( + prompt="Analyze this {{product_image}} and describe what you see.", + prompt_id=prompt_id, + version=1, + variables=["product_image"], + is_default=True, + ) + + from llama_stack.apis.agents.openai_responses import ( + OpenAIResponseInputMessageContentImage, + OpenAIResponsePromptParam, + ) + from llama_stack.apis.inference import ( + OpenAIChatCompletionContentPartImageParam, + OpenAISystemMessageParam, + OpenAIUserMessageParam, + ) + + # Mock file content + mock_file_content = b"fake_image_data" + mock_files_api.openai_retrieve_file_content.return_value = type("obj", (object,), {"body": mock_file_content})() + + prompt_params = OpenAIResponsePromptParam( + id=prompt_id, + version="1", + variables={ + "product_image": OpenAIResponseInputMessageContentImage( + file_id="file-abc123", + detail="high", + ) + }, + ) + + mock_prompts_api.get_prompt.return_value = prompt + + # Initial messages + messages = [OpenAIUserMessageParam(content="What do you think?")] + + # Execute + result = await openai_responses_impl._prepend_prompt(messages, prompt_params) + + # Verify + assert result == prompt + assert len(messages) == 2 + + # Check system message has placeholder + assert isinstance(messages[0], OpenAISystemMessageParam) + assert messages[0].content == "Analyze this [Image: product_image] and describe what you see." + + # Check user message has image prepended + assert isinstance(messages[1], OpenAIUserMessageParam) + assert isinstance(messages[1].content, list) + assert len(messages[1].content) == 2 # Image + original text + + # First part should be image with data URL + assert isinstance(messages[1].content[0], OpenAIChatCompletionContentPartImageParam) + assert messages[1].content[0].image_url.url.startswith("data:image/") + assert messages[1].content[0].image_url.detail == "high" + + # Second part should be original text + assert messages[1].content[1].text == "What do you think?" + + +async def test_prepend_prompt_with_file_variable(openai_responses_impl, mock_prompts_api, mock_files_api): + """Test prepend_prompt with file variable - should create placeholder in system message and inject file into user message.""" + # Setup + prompt_id = "pmpt_1234567890abcdef1234567890abcdef1234567890abcdef" + prompt = Prompt( + prompt="Review the document {{contract_file}} and summarize key points.", + prompt_id=prompt_id, + version=1, + variables=["contract_file"], + is_default=True, + ) + + from llama_stack.apis.agents.openai_responses import ( + OpenAIResponseInputMessageContentFile, + OpenAIResponsePromptParam, + ) + from llama_stack.apis.files import OpenAIFileObject + from llama_stack.apis.inference import ( + OpenAIFile, + OpenAISystemMessageParam, + OpenAIUserMessageParam, + ) + + # Mock file retrieval + mock_file_content = b"fake_pdf_content" + mock_files_api.openai_retrieve_file_content.return_value = type("obj", (object,), {"body": mock_file_content})() + mock_files_api.openai_retrieve_file.return_value = OpenAIFileObject( + object="file", + id="file-contract-789", + bytes=len(mock_file_content), + created_at=1234567890, + expires_at=1234567890, + filename="contract.pdf", + purpose="assistants", + ) + + prompt_params = OpenAIResponsePromptParam( + id=prompt_id, + version="1", + variables={ + "contract_file": OpenAIResponseInputMessageContentFile( + file_id="file-contract-789", + filename="contract.pdf", + ) + }, + ) + + mock_prompts_api.get_prompt.return_value = prompt + + # Initial messages + messages = [OpenAIUserMessageParam(content="Please review this.")] + + # Execute + result = await openai_responses_impl._prepend_prompt(messages, prompt_params) + + # Verify + assert result == prompt + assert len(messages) == 2 + + # Check system message has placeholder + assert isinstance(messages[0], OpenAISystemMessageParam) + assert messages[0].content == "Review the document [File: contract_file] and summarize key points." + + # Check user message has file prepended + assert isinstance(messages[1], OpenAIUserMessageParam) + assert isinstance(messages[1].content, list) + assert len(messages[1].content) == 2 # File + original text + + # First part should be file with data URL (not file_id) + assert isinstance(messages[1].content[0], OpenAIFile) + assert messages[1].content[0].file.file_data.startswith("data:application/pdf;base64,") + assert messages[1].content[0].file.filename == "contract.pdf" + # file_id should NOT be set in the OpenAI request + assert messages[1].content[0].file.file_id is None + + # Second part should be original text + assert messages[1].content[1].text == "Please review this." + + +async def test_prepend_prompt_with_mixed_variables(openai_responses_impl, mock_prompts_api, mock_files_api): + """Test prepend_prompt with text, image, and file variables mixed together.""" + # Setup + prompt_id = "pmpt_1234567890abcdef1234567890abcdef1234567890abcdef" + prompt = Prompt( + prompt="Hello {{name}}! Analyze {{photo}} and review {{document}}. Provide insights for {{company}}.", + prompt_id=prompt_id, + version=1, + variables=["name", "photo", "document", "company"], + is_default=True, + ) + + from llama_stack.apis.agents.openai_responses import ( + OpenAIResponseInputMessageContentFile, + OpenAIResponseInputMessageContentImage, + OpenAIResponseInputMessageContentText, + OpenAIResponsePromptParam, + ) + from llama_stack.apis.files import OpenAIFileObject + from llama_stack.apis.inference import ( + OpenAIChatCompletionContentPartImageParam, + OpenAIFile, + OpenAISystemMessageParam, + OpenAIUserMessageParam, + ) + + # Mock file retrieval for document + mock_file_content = b"fake_doc_content" + mock_files_api.openai_retrieve_file_content.return_value = type("obj", (object,), {"body": mock_file_content})() + mock_files_api.openai_retrieve_file.return_value = OpenAIFileObject( + object="file", + id="file-doc-456", + bytes=len(mock_file_content), + created_at=1234567890, + expires_at=1234567890, + filename="doc.pdf", + purpose="assistants", + ) + + prompt_params = OpenAIResponsePromptParam( + id=prompt_id, + version="1", + variables={ + "name": OpenAIResponseInputMessageContentText(text="Alice"), + "photo": OpenAIResponseInputMessageContentImage(file_id="file-photo-123", detail="auto"), + "document": OpenAIResponseInputMessageContentFile(file_id="file-doc-456", filename="doc.pdf"), + "company": OpenAIResponseInputMessageContentText(text="Acme Corp"), + }, + ) + + mock_prompts_api.get_prompt.return_value = prompt + + # Initial messages + messages = [OpenAIUserMessageParam(content="Here's my question.")] + + # Execute + result = await openai_responses_impl._prepend_prompt(messages, prompt_params) + + # Verify + assert result == prompt + assert len(messages) == 2 + + # Check system message has text and placeholders + assert isinstance(messages[0], OpenAISystemMessageParam) + expected_system = "Hello Alice! Analyze [Image: photo] and review [File: document]. Provide insights for Acme Corp." + assert messages[0].content == expected_system + + # Check user message has media prepended (2 media items + original text) + assert isinstance(messages[1], OpenAIUserMessageParam) + assert isinstance(messages[1].content, list) + assert len(messages[1].content) == 3 # Image + File + original text + + # First part should be image with data URL + assert isinstance(messages[1].content[0], OpenAIChatCompletionContentPartImageParam) + assert messages[1].content[0].image_url.url.startswith("data:image/") + + # Second part should be file with data URL + assert isinstance(messages[1].content[1], OpenAIFile) + assert messages[1].content[1].file.file_data.startswith("data:application/pdf;base64,") + assert messages[1].content[1].file.filename == "doc.pdf" + assert messages[1].content[1].file.file_id is None # file_id should NOT be sent + + # Third part should be original text + assert messages[1].content[2].text == "Here's my question." + + +async def test_prepend_prompt_with_image_using_image_url(openai_responses_impl, mock_prompts_api): + """Test prepend_prompt with image variable using image_url instead of file_id.""" + # Setup + prompt_id = "pmpt_1234567890abcdef1234567890abcdef1234567890abcdef" + prompt = Prompt( + prompt="Describe {{screenshot}}.", + prompt_id=prompt_id, + version=1, + variables=["screenshot"], + is_default=True, + ) + + from llama_stack.apis.agents.openai_responses import ( + OpenAIResponseInputMessageContentImage, + OpenAIResponsePromptParam, + ) + from llama_stack.apis.inference import ( + OpenAIChatCompletionContentPartImageParam, + OpenAISystemMessageParam, + OpenAIUserMessageParam, + ) + + prompt_params = OpenAIResponsePromptParam( + id=prompt_id, + version="1", + variables={ + "screenshot": OpenAIResponseInputMessageContentImage( + image_url="https://example.com/screenshot.png", + detail="low", + ) + }, + ) + + mock_prompts_api.get_prompt.return_value = prompt + + # Initial messages + messages = [OpenAIUserMessageParam(content="What is this?")] + + # Execute + result = await openai_responses_impl._prepend_prompt(messages, prompt_params) + + # Verify + assert result == prompt + assert len(messages) == 2 + + # Check system message has placeholder + assert isinstance(messages[0], OpenAISystemMessageParam) + assert messages[0].content == "Describe [Image: screenshot]." + + # Check user message has image with URL + assert isinstance(messages[1], OpenAIUserMessageParam) + assert isinstance(messages[1].content, list) + + # Image should use the provided URL + assert isinstance(messages[1].content[0], OpenAIChatCompletionContentPartImageParam) + assert messages[1].content[0].image_url.url == "https://example.com/screenshot.png" + assert messages[1].content[0].image_url.detail == "low" + + +async def test_prepend_prompt_with_media_no_user_message(openai_responses_impl, mock_prompts_api, mock_files_api): + """Test prepend_prompt with media when there's no existing user message - should create one.""" + # Setup + prompt_id = "pmpt_1234567890abcdef1234567890abcdef1234567890abcdef" + prompt = Prompt( + prompt="Analyze {{image}}.", + prompt_id=prompt_id, + version=1, + variables=["image"], + is_default=True, + ) + + from llama_stack.apis.agents.openai_responses import ( + OpenAIResponseInputMessageContentImage, + OpenAIResponsePromptParam, + ) + from llama_stack.apis.inference import ( + OpenAIAssistantMessageParam, + OpenAIChatCompletionContentPartImageParam, + OpenAISystemMessageParam, + OpenAIUserMessageParam, + ) + + # Mock file content + mock_file_content = b"fake_image_data" + mock_files_api.openai_retrieve_file_content.return_value = type("obj", (object,), {"body": mock_file_content})() + + prompt_params = OpenAIResponsePromptParam( + id=prompt_id, + version="1", + variables={"image": OpenAIResponseInputMessageContentImage(file_id="file-img-999")}, + ) + + mock_prompts_api.get_prompt.return_value = prompt + + # Initial messages - only assistant message, no user message + messages = [OpenAIAssistantMessageParam(content="Previous response")] + + # Execute + result = await openai_responses_impl._prepend_prompt(messages, prompt_params) + + # Verify + assert result == prompt + assert len(messages) == 3 # System + Assistant + New User + + # Check system message + assert isinstance(messages[0], OpenAISystemMessageParam) + assert messages[0].content == "Analyze [Image: image]." + + # Original assistant message should still be there + assert isinstance(messages[1], OpenAIAssistantMessageParam) + assert messages[1].content == "Previous response" + + # New user message with just the image should be appended + assert isinstance(messages[2], OpenAIUserMessageParam) + assert isinstance(messages[2].content, list) + assert len(messages[2].content) == 1 + assert isinstance(messages[2].content[0], OpenAIChatCompletionContentPartImageParam) + assert messages[2].content[0].image_url.url.startswith("data:image/") + + +async def test_prepend_prompt_image_variable_missing_required_fields(openai_responses_impl, mock_prompts_api): + """Test prepend_prompt with image variable that has neither file_id nor image_url - should raise error.""" + # Setup + prompt_id = "pmpt_1234567890abcdef1234567890abcdef1234567890abcdef" + prompt = Prompt( + prompt="Analyze {{bad_image}}.", + prompt_id=prompt_id, + version=1, + variables=["bad_image"], + is_default=True, + ) + + from llama_stack.apis.agents.openai_responses import ( + OpenAIResponseInputMessageContentImage, + OpenAIResponsePromptParam, + ) + from llama_stack.apis.inference import OpenAIUserMessageParam + + # Create image content with neither file_id nor image_url + prompt_params = OpenAIResponsePromptParam( + id=prompt_id, + version="1", + variables={"bad_image": OpenAIResponseInputMessageContentImage()}, # No file_id or image_url + ) + + mock_prompts_api.get_prompt.return_value = prompt + messages = [OpenAIUserMessageParam(content="Test")] + + # Execute - should raise ValueError + with pytest.raises(ValueError, match="Image content must have either 'image_url' or 'file_id'"): + await openai_responses_impl._prepend_prompt(messages, prompt_params) diff --git a/tests/unit/providers/agents/meta_reference/test_openai_responses_conversations.py b/tests/unit/providers/agents/meta_reference/test_openai_responses_conversations.py index 2ca350862..41ac816a3 100644 --- a/tests/unit/providers/agents/meta_reference/test_openai_responses_conversations.py +++ b/tests/unit/providers/agents/meta_reference/test_openai_responses_conversations.py @@ -39,6 +39,8 @@ def responses_impl_with_conversations( mock_vector_io_api, mock_conversations_api, mock_safety_api, + mock_prompts_api, + mock_files_api, ): """Create OpenAIResponsesImpl instance with conversations API.""" return OpenAIResponsesImpl( @@ -49,6 +51,8 @@ def responses_impl_with_conversations( vector_io_api=mock_vector_io_api, conversations_api=mock_conversations_api, safety_api=mock_safety_api, + prompts_api=mock_prompts_api, + files_api=mock_files_api, ) diff --git a/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py b/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py index 2698b88c8..186ff6ed6 100644 --- a/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py +++ b/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py @@ -5,6 +5,8 @@ # the root directory of this source tree. +from unittest.mock import AsyncMock + import pytest from llama_stack.apis.agents.openai_responses import ( @@ -46,6 +48,12 @@ from llama_stack.providers.inline.agents.meta_reference.responses.utils import ( ) +@pytest.fixture +def mock_files_api(): + """Mock files API for testing.""" + return AsyncMock() + + class TestConvertChatChoiceToResponseMessage: async def test_convert_string_content(self): choice = OpenAIChoice( @@ -78,17 +86,17 @@ class TestConvertChatChoiceToResponseMessage: class TestConvertResponseContentToChatContent: - async def test_convert_string_content(self): - result = await convert_response_content_to_chat_content("Simple string") + async def test_convert_string_content(self, mock_files_api): + result = await convert_response_content_to_chat_content("Simple string", mock_files_api) assert result == "Simple string" - async def test_convert_text_content_parts(self): + async def test_convert_text_content_parts(self, mock_files_api): content = [ OpenAIResponseInputMessageContentText(text="First part"), OpenAIResponseOutputMessageContentOutputText(text="Second part"), ] - result = await convert_response_content_to_chat_content(content) + result = await convert_response_content_to_chat_content(content, mock_files_api) assert len(result) == 2 assert isinstance(result[0], OpenAIChatCompletionContentPartTextParam) @@ -96,10 +104,10 @@ class TestConvertResponseContentToChatContent: assert isinstance(result[1], OpenAIChatCompletionContentPartTextParam) assert result[1].text == "Second part" - async def test_convert_image_content(self): + async def test_convert_image_content(self, mock_files_api): content = [OpenAIResponseInputMessageContentImage(image_url="https://example.com/image.jpg", detail="high")] - result = await convert_response_content_to_chat_content(content) + result = await convert_response_content_to_chat_content(content, mock_files_api) assert len(result) == 1 assert isinstance(result[0], OpenAIChatCompletionContentPartImageParam) diff --git a/tests/unit/providers/agents/meta_reference/test_responses_safety_utils.py b/tests/unit/providers/agents/meta_reference/test_responses_safety_utils.py index 9c5cc853c..8604ffa02 100644 --- a/tests/unit/providers/agents/meta_reference/test_responses_safety_utils.py +++ b/tests/unit/providers/agents/meta_reference/test_responses_safety_utils.py @@ -30,6 +30,8 @@ def mock_apis(): "vector_io_api": AsyncMock(), "conversations_api": AsyncMock(), "safety_api": AsyncMock(), + "prompts_api": AsyncMock(), + "files_api": AsyncMock(), }