diff --git a/llama_stack/apis/agents/agents.py b/llama_stack/apis/agents/agents.py
index da07d596a..5cc9ce242 100644
--- a/llama_stack/apis/agents/agents.py
+++ b/llama_stack/apis/agents/agents.py
@@ -51,6 +51,7 @@ class SearchToolDefinition(ToolDefinitionCommon):
     # NOTE: brave_search is just a placeholder since model always uses
     # brave_search as tool call name
     type: Literal[AgentTool.brave_search.value] = AgentTool.brave_search.value
+    api_key: str
     engine: SearchEngineType = SearchEngineType.brave
     remote_execution: Optional[RestAPIExecutionConfig] = None
 
@@ -58,6 +59,7 @@ class SearchToolDefinition(ToolDefinitionCommon):
 @json_schema_type
 class WolframAlphaToolDefinition(ToolDefinitionCommon):
     type: Literal[AgentTool.wolfram_alpha.value] = AgentTool.wolfram_alpha.value
+    api_key: str
     remote_execution: Optional[RestAPIExecutionConfig] = None
 
 
diff --git a/llama_stack/apis/agents/client.py b/llama_stack/apis/agents/client.py
index 8071440bc..ccca6a391 100644
--- a/llama_stack/apis/agents/client.py
+++ b/llama_stack/apis/agents/client.py
@@ -6,11 +6,12 @@
 
 import asyncio
 import json
+import os
 from typing import AsyncGenerator
 
 import fire
-
 import httpx
+from dotenv import load_dotenv
 
 from pydantic import BaseModel
 from termcolor import cprint
@@ -22,6 +23,9 @@ from .agents import *  # noqa: F403
 from .event_logger import EventLogger
 
 
+load_dotenv()
+
+
 async def get_client_impl(config: RemoteProviderConfig, _deps):
     return AgentsClient(config.url)
 
@@ -129,8 +133,11 @@ async def run_main(host: str, port: int):
     api = AgentsClient(f"http://{host}:{port}")
 
     tool_definitions = [
-        SearchToolDefinition(engine=SearchEngineType.bing),
-        WolframAlphaToolDefinition(),
+        SearchToolDefinition(
+            engine=SearchEngineType.bing,
+            api_key=os.getenv("BING_SEARCH_API_KEY"),
+        ),
+        WolframAlphaToolDefinition(api_key=os.getenv("WOLFRAM_ALPHA_API_KEY")),
         CodeInterpreterToolDefinition(),
     ]
     tool_definitions += [
diff --git a/llama_stack/providers/impls/meta_reference/agents/agents.py b/llama_stack/providers/impls/meta_reference/agents/agents.py
index 262afc611..022c8c3d1 100644
--- a/llama_stack/providers/impls/meta_reference/agents/agents.py
+++ b/llama_stack/providers/impls/meta_reference/agents/agents.py
@@ -58,21 +58,9 @@ class MetaReferenceAgentsImpl(Agents):
         builtin_tools = []
         for tool_defn in agent_config.tools:
             if isinstance(tool_defn, WolframAlphaToolDefinition):
-                key = self.config.wolfram_api_key
-                if not key:
-                    raise ValueError("Wolfram API key not defined in config")
-                tool = WolframAlphaTool(key)
+                tool = WolframAlphaTool(tool_defn.api_key)
             elif isinstance(tool_defn, SearchToolDefinition):
-                key = None
-                if tool_defn.engine == SearchEngineType.brave:
-                    key = self.config.brave_search_api_key
-                elif tool_defn.engine == SearchEngineType.bing:
-                    key = self.config.bing_search_api_key
-                if not key:
-                    raise ValueError(
-                        "Search (Brave or Bing) API key not defined in config"
-                    )
-                tool = SearchTool(tool_defn.engine, key)
+                tool = SearchTool(tool_defn.engine, tool_defn.api_key)
             elif isinstance(tool_defn, CodeInterpreterToolDefinition):
                 tool = CodeInterpreterTool()
             elif isinstance(tool_defn, PhotogenToolDefinition):
diff --git a/llama_stack/providers/impls/meta_reference/agents/config.py b/llama_stack/providers/impls/meta_reference/agents/config.py
index f1a92f2e7..17beb348e 100644
--- a/llama_stack/providers/impls/meta_reference/agents/config.py
+++ b/llama_stack/providers/impls/meta_reference/agents/config.py
@@ -4,12 +4,7 @@
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
 
-from typing import Optional
-
 from pydantic import BaseModel
 
 
-class MetaReferenceImplConfig(BaseModel):
-    brave_search_api_key: Optional[str] = None
-    bing_search_api_key: Optional[str] = None
-    wolfram_api_key: Optional[str] = None
+class MetaReferenceImplConfig(BaseModel): ...
diff --git a/requirements.txt b/requirements.txt
index 45ca7ed06..dfb37c994 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,6 +3,7 @@ fire
 httpx
 huggingface-hub
 llama-models>=0.0.16
+python-dotenv
 pydantic
 requests
 termcolor