fix: add traces for tool calls and mcp tool listing (#3722)

# What does this PR do?
Adds traces around tool execution and mcp tool listing for better
observability.

Closes #3108 

## Test Plan
Manually examined traces in jaeger to verify the added information was
available.

Signed-off-by: Gordon Sim <gsim@redhat.com>

llama_stack/providers/inline/agents/meta_reference/responses/streaming.py

@@ -46,6 +46,7 @@ from llama_stack.apis.inference import (
     OpenAIMessageParam,
 )
 from llama_stack.log import get_logger
+from llama_stack.providers.utils.telemetry import tracing
 
 from .types import ChatCompletionContext, ChatCompletionResult
 from .utils import convert_chat_choice_to_response_message, is_function_tool_call
@@ -595,6 +596,14 @@ class StreamingResponseOrchestrator:
                     never_allowed = mcp_tool.allowed_tools.never
 
             # Call list_mcp_tools
+            tool_defs = None
+            list_id = f"mcp_list_{uuid.uuid4()}"
+            attributes = {
+                "server_label": mcp_tool.server_label,
+                "server_url": mcp_tool.server_url,
+                "mcp_list_tools_id": list_id,
+            }
+            async with tracing.span("list_mcp_tools", attributes):
                 tool_defs = await list_mcp_tools(
                     endpoint=mcp_tool.server_url,
                     headers=mcp_tool.headers or {},
@@ -602,7 +611,7 @@ class StreamingResponseOrchestrator:
 
             # Create the MCP list tools message
             mcp_list_message = OpenAIResponseOutputMessageMCPListTools(
-                id=f"mcp_list_{uuid.uuid4()}",
+                id=list_id,
                 server_label=mcp_tool.server_label,
                 tools=[],
             )

llama_stack/providers/inline/agents/meta_reference/responses/tool_executor.py

@@ -35,6 +35,7 @@ from llama_stack.apis.inference import (
 from llama_stack.apis.tools import ToolGroups, ToolInvocationResult, ToolRuntime
 from llama_stack.apis.vector_io import VectorIO
 from llama_stack.log import get_logger
+from llama_stack.providers.utils.telemetry import tracing
 
 from .types import ChatCompletionContext, ToolExecutionResult
 
@@ -251,6 +252,12 @@ class ToolExecutor:
                 from llama_stack.providers.utils.tools.mcp import invoke_mcp_tool
 
                 mcp_tool = mcp_tool_to_server[function_name]
+                attributes = {
+                    "server_label": mcp_tool.server_label,
+                    "server_url": mcp_tool.server_url,
+                    "tool_name": function_name,
+                }
+                async with tracing.span("invoke_mcp_tool", attributes):
                     result = await invoke_mcp_tool(
                         endpoint=mcp_tool.server_url,
                         headers=mcp_tool.headers or {},
@@ -266,11 +273,16 @@ class ToolExecutor:
                     # Use vector_stores.search API instead of knowledge_search tool
                     # to support filters and ranking_options
                     query = tool_kwargs.get("query", "")
+                    async with tracing.span("knowledge_search", {}):
                         result = await self._execute_knowledge_search_via_vector_store(
                             query=query,
                             response_file_search_tool=response_file_search_tool,
                         )
             else:
+                attributes = {
+                    "tool_name": function_name,
+                }
+                async with tracing.span("invoke_tool", attributes):
                     result = await self.tool_runtime_api.invoke_tool(
                         tool_name=function_name,
                         kwargs=tool_kwargs,