diff --git a/docs/resources/llama-stack-spec.html b/docs/resources/llama-stack-spec.html index a9fb22b10..b1bef0882 100644 --- a/docs/resources/llama-stack-spec.html +++ b/docs/resources/llama-stack-spec.html @@ -462,6 +462,46 @@ } } }, + "/alpha/tool-runtime/discover": { + "post": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/jsonl": { + "schema": { + "$ref": "#/components/schemas/ToolDef" + } + } + } + } + }, + "tags": [ + "ToolRuntime" + ], + "parameters": [ + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DiscoverToolsRequest" + } + } + }, + "required": true + } + } + }, "/alpha/inference/embeddings": { "post": { "responses": { @@ -1118,6 +1158,82 @@ } } }, + "/alpha/tools/get": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Tool" + } + } + } + } + }, + "tags": [ + "ToolGroups" + ], + "parameters": [ + { + "name": "tool_name", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/alpha/toolgroups/get": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolGroup" + } + } + } + } + }, + "tags": [ + "ToolGroups" + ], + "parameters": [ + { + "name": "tool_group_id", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, "/alpha/post-training/job/artifacts": { "get": { "responses": { @@ -1301,6 +1417,47 @@ } } }, + "/alpha/tool-runtime/invoke": { + "post": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ToolInvocationResult" + } + } + } + } + }, + "tags": [ + "ToolRuntime" + ], + "summary": "Run a tool with the given arguments", + "parameters": [ + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InvokeToolRequest" + } + } + }, + "required": true + } + } + }, "/alpha/eval/job/cancel": { "post": { "responses": { @@ -1695,6 +1852,76 @@ ] } }, + "/alpha/toolgroups/list": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/jsonl": { + "schema": { + "$ref": "#/components/schemas/ToolGroup" + } + } + } + } + }, + "tags": [ + "ToolGroups" + ], + "summary": "List tool groups with optional provider", + "parameters": [ + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/alpha/tools/list": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/jsonl": { + "schema": { + "$ref": "#/components/schemas/Tool" + } + } + } + } + }, + "tags": [ + "ToolGroups" + ], + "summary": "List tools with optional tool group", + "parameters": [ + { + "name": "tool_group_id", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, "/alpha/telemetry/log-event": { "post": { "responses": { @@ -2096,6 +2323,40 @@ } } }, + "/alpha/toolgroups/register": { + "post": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "ToolGroups" + ], + "summary": "Register a tool group", + "parameters": [ + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterToolGroupRequest" + } + } + }, + "required": true + } + } + }, "/alpha/eval/run-eval": { "post": { "responses": { @@ -3444,29 +3705,16 @@ "type": "string" } }, - "tools": { + "available_tools": { "type": "array", "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/SearchToolDefinition" - }, - { - "$ref": "#/components/schemas/WolframAlphaToolDefinition" - }, - { - "$ref": "#/components/schemas/PhotogenToolDefinition" - }, - { - "$ref": "#/components/schemas/CodeInterpreterToolDefinition" - }, - { - "$ref": "#/components/schemas/FunctionCallToolDefinition" - }, - { - "$ref": "#/components/schemas/MemoryToolDefinition" - } - ] + "type": "string" + } + }, + "preprocessing_tools": { + "type": "array", + "items": { + "type": "string" } }, "tool_choice": { @@ -3499,479 +3747,6 @@ "enable_session_persistence" ] }, - "CodeInterpreterToolDefinition": { - "type": "object", - "properties": { - "input_shields": { - "type": "array", - "items": { - "type": "string" - } - }, - "output_shields": { - "type": "array", - "items": { - "type": "string" - } - }, - "type": { - "type": "string", - "const": "code_interpreter", - "default": "code_interpreter" - }, - "enable_inline_code_execution": { - "type": "boolean", - "default": true - }, - "remote_execution": { - "$ref": "#/components/schemas/RestAPIExecutionConfig" - } - }, - "additionalProperties": false, - "required": [ - "type", - "enable_inline_code_execution" - ] - }, - "FunctionCallToolDefinition": { - "type": "object", - "properties": { - "input_shields": { - "type": "array", - "items": { - "type": "string" - } - }, - "output_shields": { - "type": "array", - "items": { - "type": "string" - } - }, - "type": { - "type": "string", - "const": "function_call", - "default": "function_call" - }, - "function_name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "parameters": { - "type": "object", - "additionalProperties": { - "$ref": "#/components/schemas/ToolParamDefinition" - } - }, - "remote_execution": { - "$ref": "#/components/schemas/RestAPIExecutionConfig" - } - }, - "additionalProperties": false, - "required": [ - "type", - "function_name", - "description", - "parameters" - ] - }, - "MemoryToolDefinition": { - "type": "object", - "properties": { - "input_shields": { - "type": "array", - "items": { - "type": "string" - } - }, - "output_shields": { - "type": "array", - "items": { - "type": "string" - } - }, - "type": { - "type": "string", - "const": "memory", - "default": "memory" - }, - "memory_bank_configs": { - "type": "array", - "items": { - "oneOf": [ - { - "type": "object", - "properties": { - "bank_id": { - "type": "string" - }, - "type": { - "type": "string", - "const": "vector", - "default": "vector" - } - }, - "additionalProperties": false, - "required": [ - "bank_id", - "type" - ] - }, - { - "type": "object", - "properties": { - "bank_id": { - "type": "string" - }, - "type": { - "type": "string", - "const": "keyvalue", - "default": "keyvalue" - }, - "keys": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false, - "required": [ - "bank_id", - "type", - "keys" - ] - }, - { - "type": "object", - "properties": { - "bank_id": { - "type": "string" - }, - "type": { - "type": "string", - "const": "keyword", - "default": "keyword" - } - }, - "additionalProperties": false, - "required": [ - "bank_id", - "type" - ] - }, - { - "type": "object", - "properties": { - "bank_id": { - "type": "string" - }, - "type": { - "type": "string", - "const": "graph", - "default": "graph" - }, - "entities": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false, - "required": [ - "bank_id", - "type", - "entities" - ] - } - ] - } - }, - "query_generator_config": { - "oneOf": [ - { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "default", - "default": "default" - }, - "sep": { - "type": "string", - "default": " " - } - }, - "additionalProperties": false, - "required": [ - "type", - "sep" - ] - }, - { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "llm", - "default": "llm" - }, - "model": { - "type": "string" - }, - "template": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "type", - "model", - "template" - ] - }, - { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "custom", - "default": "custom" - } - }, - "additionalProperties": false, - "required": [ - "type" - ] - } - ] - }, - "max_tokens_in_context": { - "type": "integer", - "default": 4096 - }, - "max_chunks": { - "type": "integer", - "default": 10 - } - }, - "additionalProperties": false, - "required": [ - "type", - "memory_bank_configs", - "query_generator_config", - "max_tokens_in_context", - "max_chunks" - ] - }, - "PhotogenToolDefinition": { - "type": "object", - "properties": { - "input_shields": { - "type": "array", - "items": { - "type": "string" - } - }, - "output_shields": { - "type": "array", - "items": { - "type": "string" - } - }, - "type": { - "type": "string", - "const": "photogen", - "default": "photogen" - }, - "remote_execution": { - "$ref": "#/components/schemas/RestAPIExecutionConfig" - } - }, - "additionalProperties": false, - "required": [ - "type" - ] - }, - "RestAPIExecutionConfig": { - "type": "object", - "properties": { - "url": { - "$ref": "#/components/schemas/URL" - }, - "method": { - "$ref": "#/components/schemas/RestAPIMethod" - }, - "params": { - "type": "object", - "additionalProperties": { - "oneOf": [ - { - "type": "null" - }, - { - "type": "boolean" - }, - { - "type": "number" - }, - { - "type": "string" - }, - { - "type": "array" - }, - { - "type": "object" - } - ] - } - }, - "headers": { - "type": "object", - "additionalProperties": { - "oneOf": [ - { - "type": "null" - }, - { - "type": "boolean" - }, - { - "type": "number" - }, - { - "type": "string" - }, - { - "type": "array" - }, - { - "type": "object" - } - ] - } - }, - "body": { - "type": "object", - "additionalProperties": { - "oneOf": [ - { - "type": "null" - }, - { - "type": "boolean" - }, - { - "type": "number" - }, - { - "type": "string" - }, - { - "type": "array" - }, - { - "type": "object" - } - ] - } - } - }, - "additionalProperties": false, - "required": [ - "url", - "method" - ] - }, - "RestAPIMethod": { - "type": "string", - "enum": [ - "GET", - "POST", - "PUT", - "DELETE" - ] - }, - "SearchToolDefinition": { - "type": "object", - "properties": { - "input_shields": { - "type": "array", - "items": { - "type": "string" - } - }, - "output_shields": { - "type": "array", - "items": { - "type": "string" - } - }, - "type": { - "type": "string", - "const": "brave_search", - "default": "brave_search" - }, - "api_key": { - "type": "string" - }, - "engine": { - "type": "string", - "enum": [ - "bing", - "brave", - "tavily" - ], - "default": "brave" - }, - "remote_execution": { - "$ref": "#/components/schemas/RestAPIExecutionConfig" - } - }, - "additionalProperties": false, - "required": [ - "type", - "api_key", - "engine" - ] - }, - "WolframAlphaToolDefinition": { - "type": "object", - "properties": { - "input_shields": { - "type": "array", - "items": { - "type": "string" - } - }, - "output_shields": { - "type": "array", - "items": { - "type": "string" - } - }, - "type": { - "type": "string", - "const": "wolfram_alpha", - "default": "wolfram_alpha" - }, - "api_key": { - "type": "string" - }, - "remote_execution": { - "$ref": "#/components/schemas/RestAPIExecutionConfig" - } - }, - "additionalProperties": false, - "required": [ - "type", - "api_key" - ] - }, "CreateAgentRequest": { "type": "object", "properties": { @@ -4628,6 +4403,167 @@ "session_id" ] }, + "MCPToolGroupDef": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "model_context_protocol", + "default": "model_context_protocol" + }, + "endpoint": { + "$ref": "#/components/schemas/URL" + } + }, + "additionalProperties": false, + "required": [ + "type", + "endpoint" + ], + "title": "A tool group that is defined by in a model context protocol server. Refer to https://modelcontextprotocol.io/docs/concepts/tools for more information." + }, + "ToolDef": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "parameters": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ToolParameter" + } + }, + "metadata": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + }, + "tool_prompt_format": { + "$ref": "#/components/schemas/ToolPromptFormat", + "default": "json" + } + }, + "additionalProperties": false, + "required": [ + "name", + "description", + "parameters", + "metadata" + ] + }, + "ToolGroupDef": { + "oneOf": [ + { + "$ref": "#/components/schemas/MCPToolGroupDef" + }, + { + "$ref": "#/components/schemas/UserDefinedToolGroupDef" + } + ] + }, + "ToolParameter": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "parameter_type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "required": { + "type": "boolean" + }, + "default": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + }, + "additionalProperties": false, + "required": [ + "name", + "parameter_type", + "description", + "required" + ] + }, + "UserDefinedToolGroupDef": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "user_defined", + "default": "user_defined" + }, + "tools": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ToolDef" + } + } + }, + "additionalProperties": false, + "required": [ + "type", + "tools" + ] + }, + "DiscoverToolsRequest": { + "type": "object", + "properties": { + "tool_group": { + "$ref": "#/components/schemas/ToolGroupDef" + } + }, + "additionalProperties": false, + "required": [ + "tool_group" + ] + }, "EmbeddingsRequest": { "type": "object", "properties": { @@ -5841,6 +5777,101 @@ "start_time" ] }, + "Tool": { + "type": "object", + "properties": { + "identifier": { + "type": "string" + }, + "provider_resource_id": { + "type": "string" + }, + "provider_id": { + "type": "string" + }, + "type": { + "type": "string", + "const": "tool", + "default": "tool" + }, + "tool_group": { + "type": "string" + }, + "description": { + "type": "string" + }, + "parameters": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ToolParameter" + } + }, + "metadata": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + }, + "tool_prompt_format": { + "$ref": "#/components/schemas/ToolPromptFormat", + "default": "json" + } + }, + "additionalProperties": false, + "required": [ + "identifier", + "provider_resource_id", + "type", + "tool_group", + "description", + "parameters" + ] + }, + "ToolGroup": { + "type": "object", + "properties": { + "identifier": { + "type": "string" + }, + "provider_resource_id": { + "type": "string" + }, + "provider_id": { + "type": "string" + }, + "type": { + "type": "string", + "const": "tool_group", + "default": "tool_group" + } + }, + "additionalProperties": false, + "required": [ + "identifier", + "provider_resource_id", + "provider_id", + "type" + ] + }, "Checkpoint": { "description": "Checkpoint created during training runs" }, @@ -6041,6 +6072,62 @@ "documents" ] }, + "InvokeToolRequest": { + "type": "object", + "properties": { + "tool_name": { + "type": "string" + }, + "args": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + } + }, + "additionalProperties": false, + "required": [ + "tool_name", + "args" + ] + }, + "ToolInvocationResult": { + "type": "object", + "properties": { + "content": { + "$ref": "#/components/schemas/InterleavedContent" + }, + "error_message": { + "type": "string" + }, + "error_code": { + "type": "integer" + } + }, + "additionalProperties": false, + "required": [ + "content" + ] + }, "JobCancelRequest": { "type": "object", "properties": { @@ -7187,6 +7274,25 @@ "shield_id" ] }, + "RegisterToolGroupRequest": { + "type": "object", + "properties": { + "tool_group_id": { + "type": "string" + }, + "tool_group": { + "$ref": "#/components/schemas/ToolGroupDef" + }, + "provider_id": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "tool_group_id", + "tool_group" + ] + }, "RunEvalRequest": { "type": "object", "properties": { @@ -7868,10 +7974,6 @@ "name": "Checkpoint", "description": "Checkpoint created during training runs\n\n" }, - { - "name": "CodeInterpreterToolDefinition", - "description": "" - }, { "name": "CompletionMessage", "description": "" @@ -7926,6 +8028,10 @@ "name": "DeleteAgentsSessionRequest", "description": "" }, + { + "name": "DiscoverToolsRequest", + "description": "" + }, { "name": "EfficiencyConfig", "description": "" @@ -7956,10 +8062,6 @@ "name": "EvaluateRowsRequest", "description": "" }, - { - "name": "FunctionCallToolDefinition", - "description": "" - }, { "name": "GetAgentsSessionRequest", "description": "" @@ -8006,6 +8108,10 @@ "name": "InterleavedContentItem", "description": "" }, + { + "name": "InvokeToolRequest", + "description": "" + }, { "name": "Job", "description": "" @@ -8050,6 +8156,10 @@ "name": "LoraFinetuningConfig", "description": "" }, + { + "name": "MCPToolGroupDef", + "description": "A tool group that is defined by in a model context protocol server. Refer to https://modelcontextprotocol.io/docs/concepts/tools for more information.\n\n" + }, { "name": "Memory" }, @@ -8064,10 +8174,6 @@ "name": "MemoryRetrievalStep", "description": "" }, - { - "name": "MemoryToolDefinition", - "description": "" - }, { "name": "Message", "description": "" @@ -8107,10 +8213,6 @@ "name": "ParamType", "description": "" }, - { - "name": "PhotogenToolDefinition", - "description": "" - }, { "name": "PostTraining (Coming Soon)" }, @@ -8190,18 +8292,14 @@ "name": "RegisterShieldRequest", "description": "" }, + { + "name": "RegisterToolGroupRequest", + "description": "" + }, { "name": "ResponseFormat", "description": "" }, - { - "name": "RestAPIExecutionConfig", - "description": "" - }, - { - "name": "RestAPIMethod", - "description": "" - }, { "name": "RouteInfo", "description": "" @@ -8267,10 +8365,6 @@ "name": "ScoringResult", "description": "" }, - { - "name": "SearchToolDefinition", - "description": "" - }, { "name": "Session", "description": "A single session of an interaction with an Agentic System.\n\n" @@ -8344,6 +8438,10 @@ "name": "TokenLogProbs", "description": "" }, + { + "name": "Tool", + "description": "" + }, { "name": "ToolCall", "description": "" @@ -8360,6 +8458,10 @@ "name": "ToolChoice", "description": "" }, + { + "name": "ToolDef", + "description": "" + }, { "name": "ToolDefinition", "description": "" @@ -8368,10 +8470,29 @@ "name": "ToolExecutionStep", "description": "" }, + { + "name": "ToolGroup", + "description": "" + }, + { + "name": "ToolGroupDef", + "description": "" + }, + { + "name": "ToolGroups" + }, + { + "name": "ToolInvocationResult", + "description": "" + }, { "name": "ToolParamDefinition", "description": "" }, + { + "name": "ToolParameter", + "description": "" + }, { "name": "ToolPromptFormat", "description": "This Enum refers to the prompt format for calling custom / zero shot tools\n\n`json` --\n Refers to the json format for calling tools.\n The json format takes the form like\n {\n \"type\": \"function\",\n \"function\" : {\n \"name\": \"function_name\",\n \"description\": \"function_description\",\n \"parameters\": {...}\n }\n }\n\n`function_tag` --\n This is an example of how you could define\n your own user defined format for making tool calls.\n The function_tag format looks like this,\n (parameters)\n\nThe detailed prompts for each of these formats are added to llama cli\n\n" @@ -8384,6 +8505,9 @@ "name": "ToolResponseMessage", "description": "" }, + { + "name": "ToolRuntime" + }, { "name": "Trace", "description": "" @@ -8412,10 +8536,18 @@ "name": "UnregisterModelRequest", "description": "" }, + { + "name": "UnregisterToolGroupRequest", + "description": "" + }, { "name": "UnstructuredLogEvent", "description": "" }, + { + "name": "UserDefinedToolGroupDef", + "description": "" + }, { "name": "UserMessage", "description": "" @@ -8462,7 +8594,9 @@ "ScoringFunctions", "Shields", "SyntheticDataGeneration (Coming Soon)", - "Telemetry" + "Telemetry", + "ToolGroups", + "ToolRuntime" ] }, { @@ -8498,7 +8632,6 @@ "ChatCompletionResponseEventType", "ChatCompletionResponseStreamChunk", "Checkpoint", - "CodeInterpreterToolDefinition", "CompletionMessage", "CompletionRequest", "CompletionResponse", @@ -8511,13 +8644,13 @@ "Dataset", "DeleteAgentsRequest", "DeleteAgentsSessionRequest", + "DiscoverToolsRequest", "EfficiencyConfig", "EmbeddingsRequest", "EmbeddingsResponse", "EvalTask", "EvaluateResponse", "EvaluateRowsRequest", - "FunctionCallToolDefinition", "GetAgentsSessionRequest", "GetSpanTreeRequest", "GraphMemoryBank", @@ -8528,6 +8661,7 @@ "InsertDocumentsRequest", "InterleavedContent", "InterleavedContentItem", + "InvokeToolRequest", "Job", "JobCancelRequest", "JobStatus", @@ -8539,9 +8673,9 @@ "LogEventRequest", "LogSeverity", "LoraFinetuningConfig", + "MCPToolGroupDef", "MemoryBankDocument", "MemoryRetrievalStep", - "MemoryToolDefinition", "Message", "MetricEvent", "Model", @@ -8551,7 +8685,6 @@ "OptimizerType", "PaginatedRowsResult", "ParamType", - "PhotogenToolDefinition", "PostTrainingJob", "PostTrainingJobArtifactsResponse", "PostTrainingJobStatusResponse", @@ -8571,9 +8704,8 @@ "RegisterModelRequest", "RegisterScoringFunctionRequest", "RegisterShieldRequest", + "RegisterToolGroupRequest", "ResponseFormat", - "RestAPIExecutionConfig", - "RestAPIMethod", "RouteInfo", "RunEvalRequest", "RunShieldRequest", @@ -8588,7 +8720,6 @@ "ScoreResponse", "ScoringFn", "ScoringResult", - "SearchToolDefinition", "Session", "Shield", "ShieldCallStep", @@ -8605,13 +8736,19 @@ "SystemMessage", "TextContentItem", "TokenLogProbs", + "Tool", "ToolCall", "ToolCallDelta", "ToolCallParseStatus", "ToolChoice", + "ToolDef", "ToolDefinition", "ToolExecutionStep", + "ToolGroup", + "ToolGroupDef", + "ToolInvocationResult", "ToolParamDefinition", + "ToolParameter", "ToolPromptFormat", "ToolResponse", "ToolResponseMessage", @@ -8622,7 +8759,9 @@ "UnregisterDatasetRequest", "UnregisterMemoryBankRequest", "UnregisterModelRequest", + "UnregisterToolGroupRequest", "UnstructuredLogEvent", + "UserDefinedToolGroupDef", "UserMessage", "VectorMemoryBank", "VectorMemoryBankParams", diff --git a/docs/resources/llama-stack-spec.yaml b/docs/resources/llama-stack-spec.yaml index 8eca40cb7..5da647b54 100644 --- a/docs/resources/llama-stack-spec.yaml +++ b/docs/resources/llama-stack-spec.yaml @@ -17,6 +17,10 @@ components: AgentConfig: additionalProperties: false properties: + available_tools: + items: + type: string + type: array enable_session_persistence: type: boolean input_shields: @@ -34,6 +38,10 @@ components: items: type: string type: array + preprocessing_tools: + items: + type: string + type: array sampling_params: $ref: '#/components/schemas/SamplingParams' tool_choice: @@ -42,16 +50,6 @@ components: tool_prompt_format: $ref: '#/components/schemas/ToolPromptFormat' default: json - tools: - items: - oneOf: - - $ref: '#/components/schemas/SearchToolDefinition' - - $ref: '#/components/schemas/WolframAlphaToolDefinition' - - $ref: '#/components/schemas/PhotogenToolDefinition' - - $ref: '#/components/schemas/CodeInterpreterToolDefinition' - - $ref: '#/components/schemas/FunctionCallToolDefinition' - - $ref: '#/components/schemas/MemoryToolDefinition' - type: array required: - max_infer_iters - model @@ -490,30 +488,6 @@ components: type: object Checkpoint: description: Checkpoint created during training runs - CodeInterpreterToolDefinition: - additionalProperties: false - properties: - enable_inline_code_execution: - default: true - type: boolean - input_shields: - items: - type: string - type: array - output_shields: - items: - type: string - type: array - remote_execution: - $ref: '#/components/schemas/RestAPIExecutionConfig' - type: - const: code_interpreter - default: code_interpreter - type: string - required: - - type - - enable_inline_code_execution - type: object CompletionMessage: additionalProperties: false properties: @@ -729,6 +703,14 @@ components: - agent_id - session_id type: object + DiscoverToolsRequest: + additionalProperties: false + properties: + tool_group: + $ref: '#/components/schemas/ToolGroupDef' + required: + - tool_group + type: object EfficiencyConfig: additionalProperties: false properties: @@ -862,37 +844,6 @@ components: - scoring_functions - task_config type: object - FunctionCallToolDefinition: - additionalProperties: false - properties: - description: - type: string - function_name: - type: string - input_shields: - items: - type: string - type: array - output_shields: - items: - type: string - type: array - parameters: - additionalProperties: - $ref: '#/components/schemas/ToolParamDefinition' - type: object - remote_execution: - $ref: '#/components/schemas/RestAPIExecutionConfig' - type: - const: function_call - default: function_call - type: string - required: - - type - - function_name - - description - - parameters - type: object GetAgentsSessionRequest: additionalProperties: false properties: @@ -1017,6 +968,25 @@ components: oneOf: - $ref: '#/components/schemas/ImageContentItem' - $ref: '#/components/schemas/TextContentItem' + InvokeToolRequest: + additionalProperties: false + properties: + args: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + tool_name: + type: string + required: + - tool_name + - args + type: object Job: additionalProperties: false properties: @@ -1190,6 +1160,21 @@ components: - rank - alpha type: object + MCPToolGroupDef: + additionalProperties: false + properties: + endpoint: + $ref: '#/components/schemas/URL' + type: + const: model_context_protocol + default: model_context_protocol + type: string + required: + - type + - endpoint + title: A tool group that is defined by in a model context protocol server. Refer + to https://modelcontextprotocol.io/docs/concepts/tools for more information. + type: object MemoryBankDocument: additionalProperties: false properties: @@ -1250,135 +1235,6 @@ components: - memory_bank_ids - inserted_context type: object - MemoryToolDefinition: - additionalProperties: false - properties: - input_shields: - items: - type: string - type: array - max_chunks: - default: 10 - type: integer - max_tokens_in_context: - default: 4096 - type: integer - memory_bank_configs: - items: - oneOf: - - additionalProperties: false - properties: - bank_id: - type: string - type: - const: vector - default: vector - type: string - required: - - bank_id - - type - type: object - - additionalProperties: false - properties: - bank_id: - type: string - keys: - items: - type: string - type: array - type: - const: keyvalue - default: keyvalue - type: string - required: - - bank_id - - type - - keys - type: object - - additionalProperties: false - properties: - bank_id: - type: string - type: - const: keyword - default: keyword - type: string - required: - - bank_id - - type - type: object - - additionalProperties: false - properties: - bank_id: - type: string - entities: - items: - type: string - type: array - type: - const: graph - default: graph - type: string - required: - - bank_id - - type - - entities - type: object - type: array - output_shields: - items: - type: string - type: array - query_generator_config: - oneOf: - - additionalProperties: false - properties: - sep: - default: ' ' - type: string - type: - const: default - default: default - type: string - required: - - type - - sep - type: object - - additionalProperties: false - properties: - model: - type: string - template: - type: string - type: - const: llm - default: llm - type: string - required: - - type - - model - - template - type: object - - additionalProperties: false - properties: - type: - const: custom - default: custom - type: string - required: - - type - type: object - type: - const: memory - default: memory - type: string - required: - - type - - memory_bank_configs - - query_generator_config - - max_tokens_in_context - - max_chunks - type: object Message: oneOf: - $ref: '#/components/schemas/UserMessage' @@ -1621,26 +1477,6 @@ components: required: - type type: object - PhotogenToolDefinition: - additionalProperties: false - properties: - input_shields: - items: - type: string - type: array - output_shields: - items: - type: string - type: array - remote_execution: - $ref: '#/components/schemas/RestAPIExecutionConfig' - type: - const: photogen - default: photogen - type: string - required: - - type - type: object PostTrainingJob: additionalProperties: false properties: @@ -2039,6 +1875,19 @@ components: required: - shield_id type: object + RegisterToolGroupRequest: + additionalProperties: false + properties: + provider_id: + type: string + tool_group: + $ref: '#/components/schemas/ToolGroupDef' + tool_group_id: + type: string + required: + - tool_group_id + - tool_group + type: object ResponseFormat: oneOf: - additionalProperties: false @@ -2081,54 +1930,6 @@ components: - type - bnf type: object - RestAPIExecutionConfig: - additionalProperties: false - properties: - body: - additionalProperties: - oneOf: - - type: 'null' - - type: boolean - - type: number - - type: string - - type: array - - type: object - type: object - headers: - additionalProperties: - oneOf: - - type: 'null' - - type: boolean - - type: number - - type: string - - type: array - - type: object - type: object - method: - $ref: '#/components/schemas/RestAPIMethod' - params: - additionalProperties: - oneOf: - - type: 'null' - - type: boolean - - type: number - - type: string - - type: array - - type: object - type: object - url: - $ref: '#/components/schemas/URL' - required: - - url - - method - type: object - RestAPIMethod: - enum: - - GET - - POST - - PUT - - DELETE - type: string RouteInfo: additionalProperties: false properties: @@ -2399,37 +2200,6 @@ components: - score_rows - aggregated_results type: object - SearchToolDefinition: - additionalProperties: false - properties: - api_key: - type: string - engine: - default: brave - enum: - - bing - - brave - - tavily - type: string - input_shields: - items: - type: string - type: array - output_shields: - items: - type: string - type: array - remote_execution: - $ref: '#/components/schemas/RestAPIExecutionConfig' - type: - const: brave_search - default: brave_search - type: string - required: - - type - - api_key - - engine - type: object Session: additionalProperties: false properties: @@ -2784,6 +2554,48 @@ components: required: - logprobs_by_token type: object + Tool: + additionalProperties: false + properties: + description: + type: string + identifier: + type: string + metadata: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + parameters: + items: + $ref: '#/components/schemas/ToolParameter' + type: array + provider_id: + type: string + provider_resource_id: + type: string + tool_group: + type: string + tool_prompt_format: + $ref: '#/components/schemas/ToolPromptFormat' + default: json + type: + const: tool + default: tool + type: string + required: + - identifier + - provider_resource_id + - type + - tool_group + - description + - parameters + type: object ToolCall: additionalProperties: false properties: @@ -2848,6 +2660,36 @@ components: - auto - required type: string + ToolDef: + additionalProperties: false + properties: + description: + type: string + metadata: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + name: + type: string + parameters: + items: + $ref: '#/components/schemas/ToolParameter' + type: array + tool_prompt_format: + $ref: '#/components/schemas/ToolPromptFormat' + default: json + required: + - name + - description + - parameters + - metadata + type: object ToolDefinition: additionalProperties: false properties: @@ -2896,6 +2738,41 @@ components: - tool_calls - tool_responses type: object + ToolGroup: + additionalProperties: false + properties: + identifier: + type: string + provider_id: + type: string + provider_resource_id: + type: string + type: + const: tool_group + default: tool_group + type: string + required: + - identifier + - provider_resource_id + - provider_id + - type + type: object + ToolGroupDef: + oneOf: + - $ref: '#/components/schemas/MCPToolGroupDef' + - $ref: '#/components/schemas/UserDefinedToolGroupDef' + ToolInvocationResult: + additionalProperties: false + properties: + content: + $ref: '#/components/schemas/InterleavedContent' + error_code: + type: integer + error_message: + type: string + required: + - content + type: object ToolParamDefinition: additionalProperties: false properties: @@ -2917,6 +2794,31 @@ components: required: - param_type type: object + ToolParameter: + additionalProperties: false + properties: + default: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + description: + type: string + name: + type: string + parameter_type: + type: string + required: + type: boolean + required: + - name + - parameter_type + - description + - required + type: object ToolPromptFormat: description: "`json` --\n Refers to the json format for calling tools.\n\ \ The json format takes the form like\n {\n \"type\": \"function\"\ @@ -3091,6 +2993,14 @@ components: required: - model_id type: object + UnregisterToolGroupRequest: + additionalProperties: false + properties: + tool_group_id: + type: string + required: + - tool_group_id + type: object UnstructuredLogEvent: additionalProperties: false properties: @@ -3127,6 +3037,21 @@ components: - message - severity type: object + UserDefinedToolGroupDef: + additionalProperties: false + properties: + tools: + items: + $ref: '#/components/schemas/ToolDef' + type: array + type: + const: user_defined + default: user_defined + type: string + required: + - type + - tools + type: object UserMessage: additionalProperties: false properties: @@ -3209,29 +3134,6 @@ components: - warn - error type: string - WolframAlphaToolDefinition: - additionalProperties: false - properties: - api_key: - type: string - input_shields: - items: - type: string - type: array - output_shields: - items: - type: string - type: array - remote_execution: - $ref: '#/components/schemas/RestAPIExecutionConfig' - type: - const: wolfram_alpha - default: wolfram_alpha - type: string - required: - - type - - api_key - type: object info: description: "This is the specification of the Llama Stack that provides\n \ \ a set of endpoints and their corresponding interfaces that are tailored\ @@ -4869,9 +4771,6 @@ tags: ' name: Checkpoint -- description: - name: CodeInterpreterToolDefinition - description: name: CompletionMessage @@ -4913,6 +4812,9 @@ tags: - description: name: DeleteAgentsSessionRequest +- description: + name: DiscoverToolsRequest - description: name: EfficiencyConfig @@ -4932,9 +4834,6 @@ tags: - description: name: EvaluateRowsRequest -- description: - name: FunctionCallToolDefinition - description: name: GetAgentsSessionRequest @@ -4965,6 +4864,9 @@ tags: - description: name: InterleavedContentItem +- description: + name: InvokeToolRequest - description: name: Job - description: name: LoraFinetuningConfig +- description: 'A tool group that is defined by in a model context protocol server. + Refer to https://modelcontextprotocol.io/docs/concepts/tools for more information. + + + ' + name: MCPToolGroupDef - name: Memory - description: @@ -5003,9 +4911,6 @@ tags: - description: name: MemoryRetrievalStep -- description: - name: MemoryToolDefinition - description: name: Message - description: @@ -5027,9 +4932,6 @@ tags: name: PaginatedRowsResult - description: name: ParamType -- description: - name: PhotogenToolDefinition - name: PostTraining (Coming Soon) - description: @@ -5092,13 +4994,11 @@ tags: - description: name: RegisterShieldRequest +- description: + name: RegisterToolGroupRequest - description: name: ResponseFormat -- description: - name: RestAPIExecutionConfig -- description: - name: RestAPIMethod - description: name: RouteInfo - description: @@ -5137,9 +5037,6 @@ tags: - name: ScoringFunctions - description: name: ScoringResult -- description: - name: SearchToolDefinition - description: 'A single session of an interaction with an Agentic System. @@ -5191,6 +5088,8 @@ tags: name: TextContentItem - description: name: TokenLogProbs +- description: + name: Tool - description: name: ToolCall - description: @@ -5200,14 +5099,26 @@ tags: name: ToolCallParseStatus - description: name: ToolChoice +- description: + name: ToolDef - description: name: ToolDefinition - description: name: ToolExecutionStep +- description: + name: ToolGroup +- description: + name: ToolGroupDef +- name: ToolGroups +- description: + name: ToolInvocationResult - description: name: ToolParamDefinition +- description: + name: ToolParameter - description: "This Enum refers to the prompt format for calling custom / zero shot\ \ tools\n\n`json` --\n Refers to the json format for calling tools.\n The\ \ json format takes the form like\n {\n \"type\": \"function\",\n \ @@ -5224,6 +5135,7 @@ tags: - description: name: ToolResponseMessage +- name: ToolRuntime - description: name: Trace - description: @@ -5244,9 +5156,15 @@ tags: - description: name: UnregisterModelRequest +- description: + name: UnregisterToolGroupRequest - description: name: UnstructuredLogEvent +- description: + name: UserDefinedToolGroupDef - description: name: UserMessage - description: name: ViolationLevel -- description: - name: WolframAlphaToolDefinition x-tagGroups: - name: Operations tags: @@ -5283,6 +5198,8 @@ x-tagGroups: - Shields - SyntheticDataGeneration (Coming Soon) - Telemetry + - ToolGroups + - ToolRuntime - name: Types tags: - AgentCandidate @@ -5315,7 +5232,6 @@ x-tagGroups: - ChatCompletionResponseEventType - ChatCompletionResponseStreamChunk - Checkpoint - - CodeInterpreterToolDefinition - CompletionMessage - CompletionRequest - CompletionResponse @@ -5328,13 +5244,13 @@ x-tagGroups: - Dataset - DeleteAgentsRequest - DeleteAgentsSessionRequest + - DiscoverToolsRequest - EfficiencyConfig - EmbeddingsRequest - EmbeddingsResponse - EvalTask - EvaluateResponse - EvaluateRowsRequest - - FunctionCallToolDefinition - GetAgentsSessionRequest - GetSpanTreeRequest - GraphMemoryBank @@ -5345,6 +5261,7 @@ x-tagGroups: - InsertDocumentsRequest - InterleavedContent - InterleavedContentItem + - InvokeToolRequest - Job - JobCancelRequest - JobStatus @@ -5356,9 +5273,9 @@ x-tagGroups: - LogEventRequest - LogSeverity - LoraFinetuningConfig + - MCPToolGroupDef - MemoryBankDocument - MemoryRetrievalStep - - MemoryToolDefinition - Message - MetricEvent - Model @@ -5368,7 +5285,6 @@ x-tagGroups: - OptimizerType - PaginatedRowsResult - ParamType - - PhotogenToolDefinition - PostTrainingJob - PostTrainingJobArtifactsResponse - PostTrainingJobStatusResponse @@ -5388,9 +5304,8 @@ x-tagGroups: - RegisterModelRequest - RegisterScoringFunctionRequest - RegisterShieldRequest + - RegisterToolGroupRequest - ResponseFormat - - RestAPIExecutionConfig - - RestAPIMethod - RouteInfo - RunEvalRequest - RunShieldRequest @@ -5405,7 +5320,6 @@ x-tagGroups: - ScoreResponse - ScoringFn - ScoringResult - - SearchToolDefinition - Session - Shield - ShieldCallStep @@ -5422,13 +5336,19 @@ x-tagGroups: - SystemMessage - TextContentItem - TokenLogProbs + - Tool - ToolCall - ToolCallDelta - ToolCallParseStatus - ToolChoice + - ToolDef - ToolDefinition - ToolExecutionStep + - ToolGroup + - ToolGroupDef + - ToolInvocationResult - ToolParamDefinition + - ToolParameter - ToolPromptFormat - ToolResponse - ToolResponseMessage @@ -5439,10 +5359,11 @@ x-tagGroups: - UnregisterDatasetRequest - UnregisterMemoryBankRequest - UnregisterModelRequest + - UnregisterToolGroupRequest - UnstructuredLogEvent + - UserDefinedToolGroupDef - UserMessage - VectorMemoryBank - VectorMemoryBankParams - VersionInfo - ViolationLevel - - WolframAlphaToolDefinition diff --git a/llama_stack/apis/tools/tools.py b/llama_stack/apis/tools/tools.py index 15d59ca8f..e3c2ca52c 100644 --- a/llama_stack/apis/tools/tools.py +++ b/llama_stack/apis/tools/tools.py @@ -74,12 +74,14 @@ ToolGroupDef = register_schema( ) +@json_schema_type class ToolGroupInput(BaseModel): tool_group_id: str tool_group: ToolGroupDef provider_id: Optional[str] = None +@json_schema_type class ToolGroup(Resource): type: Literal[ResourceType.tool_group.value] = ResourceType.tool_group.value diff --git a/llama_stack/distribution/stack.py b/llama_stack/distribution/stack.py index 9d12303c9..ae96744c6 100644 --- a/llama_stack/distribution/stack.py +++ b/llama_stack/distribution/stack.py @@ -33,6 +33,7 @@ from llama_stack.apis.scoring_functions import ScoringFunctions from llama_stack.apis.shields import Shields from llama_stack.apis.synthetic_data_generation import SyntheticDataGeneration from llama_stack.apis.telemetry import Telemetry +from llama_stack.apis.tools import ToolGroups, ToolRuntime from llama_stack.distribution.datatypes import StackRunConfig from llama_stack.distribution.distribution import get_provider_registry from llama_stack.distribution.resolver import ProviderRegistry, resolve_impls @@ -63,6 +64,8 @@ class LlamaStack( Models, Shields, Inspect, + ToolGroups, + ToolRuntime, ): pass diff --git a/tests/client-sdk/agents/test_agents.py b/tests/client-sdk/agents/test_agents.py index 85a197e36..a7b08239b 100644 --- a/tests/client-sdk/agents/test_agents.py +++ b/tests/client-sdk/agents/test_agents.py @@ -4,78 +4,12 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -import json -from typing import Dict, List from uuid import uuid4 import pytest -from llama_stack.providers.tests.env import get_env_or_fail - from llama_stack_client.lib.agents.agent import Agent - -from llama_stack_client.lib.agents.custom_tool import CustomTool from llama_stack_client.lib.agents.event_logger import EventLogger -from llama_stack_client.types import CompletionMessage, ToolResponseMessage from llama_stack_client.types.agent_create_params import AgentConfig -from llama_stack_client.types.tool_param_definition_param import ( - ToolParamDefinitionParam, -) - - -class TestCustomTool(CustomTool): - """Tool to give boiling point of a liquid - Returns the correct value for water in Celcius and Fahrenheit - and returns -1 for other liquids - - """ - - def run(self, messages: List[CompletionMessage]) -> List[ToolResponseMessage]: - assert len(messages) == 1, "Expected single message" - - message = messages[0] - - tool_call = message.tool_calls[0] - - try: - response = self.run_impl(**tool_call.arguments) - response_str = json.dumps(response, ensure_ascii=False) - except Exception as e: - response_str = f"Error when running tool: {e}" - - message = ToolResponseMessage( - call_id=tool_call.call_id, - tool_name=tool_call.tool_name, - content=response_str, - role="ipython", - ) - return [message] - - def get_name(self) -> str: - return "get_boiling_point" - - def get_description(self) -> str: - return "Get the boiling point of a imaginary liquids (eg. polyjuice)" - - def get_params_definition(self) -> Dict[str, ToolParamDefinitionParam]: - return { - "liquid_name": ToolParamDefinitionParam( - param_type="string", description="The name of the liquid", required=True - ), - "celcius": ToolParamDefinitionParam( - param_type="boolean", - description="Whether to return the boiling point in Celcius", - required=False, - ), - } - - def run_impl(self, liquid_name: str, celcius: bool = True) -> int: - if liquid_name.lower() == "polyjuice": - if celcius: - return -100 - else: - return -212 - else: - return -1 @pytest.fixture(scope="session") @@ -151,12 +85,8 @@ def test_agent_simple(llama_stack_client, agent_config): def test_builtin_tool_brave_search(llama_stack_client, agent_config): agent_config = { **agent_config, - "tools": [ - { - "type": "brave_search", - "engine": "brave", - "api_key": get_env_or_fail("BRAVE_SEARCH_API_KEY"), - } + "available_tools": [ + "brave_search", ], } print(f"Agent Config: {agent_config}") @@ -167,7 +97,7 @@ def test_builtin_tool_brave_search(llama_stack_client, agent_config): messages=[ { "role": "user", - "content": "Search the web and tell me who the 44th president of the United States was. Please use tools", + "content": "Search the web and tell me who the current CEO of Meta is.", } ], session_id=session_id, @@ -178,92 +108,5 @@ def test_builtin_tool_brave_search(llama_stack_client, agent_config): assert "tool_execution>" in logs_str assert "Tool:brave_search Response:" in logs_str - assert "obama" in logs_str.lower() - if len(agent_config["input_shields"]) > 0: - assert "No Violation" in logs_str - - -def test_builtin_tool_code_execution(llama_stack_client, agent_config): - agent_config = { - **agent_config, - "tools": [ - { - "type": "code_interpreter", - } - ], - } - agent = Agent(llama_stack_client, agent_config) - session_id = agent.create_session(f"test-session-{uuid4()}") - - response = agent.create_turn( - messages=[ - { - "role": "user", - "content": "Write code to answer the question: What is the 100th prime number?", - }, - ], - session_id=session_id, - ) - logs = [str(log) for log in EventLogger().log(response) if log is not None] - logs_str = "".join(logs) - - if "Tool:code_interpreter Response" not in logs_str: - assert len(logs_str) > 0 - pytest.skip("code_interpreter not called by model") - - assert "Tool:code_interpreter Response" in logs_str - if "No such file or directory: 'bwrap'" in logs_str: - assert "prime" in logs_str - pytest.skip("`bwrap` is not available on this platform") - else: - assert "541" in logs_str - - -def test_custom_tool(llama_stack_client, agent_config): - agent_config = { - **agent_config, - "model": "meta-llama/Llama-3.2-3B-Instruct", - "tools": [ - { - "type": "brave_search", - "engine": "brave", - "api_key": get_env_or_fail("BRAVE_SEARCH_API_KEY"), - }, - { - "function_name": "get_boiling_point", - "description": "Get the boiling point of a imaginary liquids (eg. polyjuice)", - "parameters": { - "liquid_name": { - "param_type": "str", - "description": "The name of the liquid", - "required": True, - }, - "celcius": { - "param_type": "boolean", - "description": "Whether to return the boiling point in Celcius", - "required": False, - }, - }, - "type": "function_call", - }, - ], - "tool_prompt_format": "python_list", - } - - agent = Agent(llama_stack_client, agent_config, custom_tools=(TestCustomTool(),)) - session_id = agent.create_session(f"test-session-{uuid4()}") - - response = agent.create_turn( - messages=[ - { - "role": "user", - "content": "What is the boiling point of polyjuice?", - }, - ], - session_id=session_id, - ) - - logs = [str(log) for log in EventLogger().log(response) if log is not None] - logs_str = "".join(logs) - assert "-100" in logs_str - assert "CustomTool" in logs_str + assert "mark zuckerberg" in logs_str.lower() + assert "No Violation" in logs_str