phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-05 18:22:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

skip code interp

Browse source
This commit is contained in:
Xi Yan 2025-03-28 12:50:32 -07:00
parent 876693e710
commit c43d8bbec7
1 changed files with 22 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

79
tests/integration/agents/test_agents.py
View file

@ -8,13 +8,15 @@ from typing import Any, Dict
from uuid import uuid4 from uuid import uuid4
import pytest import pytest
from llama_stack_client import Agent, AgentEventLogger, Document
from llama_stack_client.types.shared_params.agent_config import AgentConfig, ToolConfig
from llama_stack.apis.agents.agents import ( from llama_stack.apis.agents.agents import (
AgentConfig as Server__AgentConfig, AgentConfig as Server__AgentConfig,
)
from llama_stack.apis.agents.agents import (
ToolChoice, ToolChoice,
) )
from llama_stack_client import Agent, AgentEventLogger, Document
from llama_stack_client.types.shared_params.agent_config import AgentConfig, ToolConfig
def get_boiling_point(liquid_name: str, celcius: bool = True) -> int: def get_boiling_point(liquid_name: str, celcius: bool = True) -> int:
@ -34,9 +36,7 @@ def get_boiling_point(liquid_name: str, celcius: bool = True) -> int:
return -1 return -1
def get_boiling_point_with_metadata( def get_boiling_point_with_metadata(liquid_name: str, celcius: bool = True) -> Dict[str, Any]:
liquid_name: str, celcius: bool = True
) -> Dict[str, Any]:
""" """
Returns the boiling point of a liquid in Celcius or Fahrenheit Returns the boiling point of a liquid in Celcius or Fahrenheit
@ -56,10 +56,7 @@ def get_boiling_point_with_metadata(
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
def agent_config(llama_stack_client_with_mocked_inference, text_model_id): def agent_config(llama_stack_client_with_mocked_inference, text_model_id):
available_shields = [ available_shields = [shield.identifier for shield in llama_stack_client_with_mocked_inference.shields.list()]
shield.identifier
for shield in llama_stack_client_with_mocked_inference.shields.list()
]
available_shields = available_shields[:1] available_shields = available_shields[:1]
agent_config = dict( agent_config = dict(
model=text_model_id, model=text_model_id,
@ -112,9 +109,7 @@ def test_agent_simple(llama_stack_client_with_mocked_inference, agent_config):
session_id=session_id, session_id=session_id,
) )
logs = [ logs = [str(log) for log in AgentEventLogger().log(bomb_response) if log is not None]
str(log) for log in AgentEventLogger().log(bomb_response) if log is not None
]
logs_str = "".join(logs) logs_str = "".join(logs)
assert "I can't" in logs_str assert "I can't" in logs_str
@ -175,9 +170,7 @@ def test_tool_config(llama_stack_client_with_mocked_inference, agent_config):
Server__AgentConfig(**agent_config) Server__AgentConfig(**agent_config)
def test_builtin_tool_web_search( def test_builtin_tool_web_search(llama_stack_client_with_mocked_inference, agent_config):
llama_stack_client_with_mocked_inference, agent_config
):
agent_config = { agent_config = {
**agent_config, **agent_config,
"instructions": "You are a helpful assistant that can use web search to answer questions.", "instructions": "You are a helpful assistant that can use web search to answer questions.",
@ -208,9 +201,7 @@ def test_builtin_tool_web_search(
assert found_tool_execution assert found_tool_execution
def test_builtin_tool_code_execution( def test_builtin_tool_code_execution(llama_stack_client_with_mocked_inference, agent_config):
llama_stack_client_with_mocked_inference, agent_config
):
agent_config = { agent_config = {
**agent_config, **agent_config,
"tools": [ "tools": [
@ -239,9 +230,7 @@ def test_builtin_tool_code_execution(
# This test must be run in an environment where `bwrap` is available. If you are running against a # This test must be run in an environment where `bwrap` is available. If you are running against a
# server, this means the _server_ must have `bwrap` available. If you are using library client, then # server, this means the _server_ must have `bwrap` available. If you are using library client, then
# you must have `bwrap` available in test's environment. # you must have `bwrap` available in test's environment.
def test_code_interpreter_for_attachments( def test_code_interpreter_for_attachments(llama_stack_client_with_mocked_inference, agent_config):
llama_stack_client_with_mocked_inference, agent_config
):
agent_config = { agent_config = {
**agent_config, **agent_config,
"tools": [ "tools": [
@ -303,9 +292,7 @@ def test_custom_tool(llama_stack_client_with_mocked_inference, agent_config):
assert "get_boiling_point" in logs_str assert "get_boiling_point" in logs_str
def test_custom_tool_infinite_loop( def test_custom_tool_infinite_loop(llama_stack_client_with_mocked_inference, agent_config):
llama_stack_client_with_mocked_inference, agent_config
):
client_tool = get_boiling_point client_tool = get_boiling_point
agent_config = { agent_config = {
**agent_config, **agent_config,
@ -328,9 +315,7 @@ def test_custom_tool_infinite_loop(
stream=False, stream=False,
) )
num_tool_calls = sum( num_tool_calls = sum([1 if step.step_type == "tool_execution" else 0 for step in response.steps])
[1 if step.step_type == "tool_execution" else 0 for step in response.steps]
)
assert num_tool_calls <= 5 assert num_tool_calls <= 5
@ -342,25 +327,18 @@ def test_tool_choice_required(llama_stack_client_with_mocked_inference, agent_co
def test_tool_choice_none(llama_stack_client_with_mocked_inference, agent_config): def test_tool_choice_none(llama_stack_client_with_mocked_inference, agent_config):
tool_execution_steps = run_agent_with_tool_choice( tool_execution_steps = run_agent_with_tool_choice(llama_stack_client_with_mocked_inference, agent_config, "none")
llama_stack_client_with_mocked_inference, agent_config, "none"
)
assert len(tool_execution_steps) == 0 assert len(tool_execution_steps) == 0
def test_tool_choice_get_boiling_point( def test_tool_choice_get_boiling_point(llama_stack_client_with_mocked_inference, agent_config):
llama_stack_client_with_mocked_inference, agent_config
):
if "llama" not in agent_config["model"].lower(): if "llama" not in agent_config["model"].lower():
pytest.xfail("NotImplemented for non-llama models") pytest.xfail("NotImplemented for non-llama models")
tool_execution_steps = run_agent_with_tool_choice( tool_execution_steps = run_agent_with_tool_choice(
llama_stack_client_with_mocked_inference, agent_config, "get_boiling_point" llama_stack_client_with_mocked_inference, agent_config, "get_boiling_point"
) )
assert ( assert len(tool_execution_steps) >= 1 and tool_execution_steps[0].tool_calls[0].tool_name == "get_boiling_point"
len(tool_execution_steps) >= 1
and tool_execution_steps[0].tool_calls[0].tool_name == "get_boiling_point"
)
def run_agent_with_tool_choice(client, agent_config, tool_choice): def run_agent_with_tool_choice(client, agent_config, tool_choice):
@ -390,12 +368,8 @@ def run_agent_with_tool_choice(client, agent_config, tool_choice):
return [step for step in response.steps if step.step_type == "tool_execution"] return [step for step in response.steps if step.step_type == "tool_execution"]
@pytest.mark.parametrize( @pytest.mark.parametrize("rag_tool_name", ["builtin::rag/knowledge_search", "builtin::rag"])
"rag_tool_name", ["builtin::rag/knowledge_search", "builtin::rag"] def test_rag_agent(llama_stack_client_with_mocked_inference, agent_config, rag_tool_name):
)
def test_rag_agent(
llama_stack_client_with_mocked_inference, agent_config, rag_tool_name
):
urls = ["chat.rst", "llama3.rst", "memory_optimizations.rst", "lora_finetune.rst"] urls = ["chat.rst", "llama3.rst", "memory_optimizations.rst", "lora_finetune.rst"]
documents = [ documents = [
Document( Document(
@ -444,22 +418,17 @@ def test_rag_agent(
stream=False, stream=False,
) )
# rag is called # rag is called
tool_execution_step = next( tool_execution_step = next(step for step in response.steps if step.step_type == "tool_execution")
step for step in response.steps if step.step_type == "tool_execution"
)
assert tool_execution_step.tool_calls[0].tool_name == "knowledge_search" assert tool_execution_step.tool_calls[0].tool_name == "knowledge_search"
# document ids are present in metadata # document ids are present in metadata
assert all( assert all(
doc_id.startswith("num-") doc_id.startswith("num-") for doc_id in tool_execution_step.tool_responses[0].metadata["document_ids"]
for doc_id in tool_execution_step.tool_responses[0].metadata["document_ids"]
) )
if expected_kw: if expected_kw:
assert expected_kw in response.output_message.content.lower() assert expected_kw in response.output_message.content.lower()
def test_rag_agent_with_attachments( def test_rag_agent_with_attachments(llama_stack_client_with_mocked_inference, agent_config):
llama_stack_client_with_mocked_inference, agent_config
):
urls = ["chat.rst", "llama3.rst", "memory_optimizations.rst", "lora_finetune.rst"] urls = ["chat.rst", "llama3.rst", "memory_optimizations.rst", "lora_finetune.rst"]
documents = [ documents = [
Document( Document(
@ -574,9 +543,7 @@ def test_rag_and_code_agent(llama_stack_client_with_mocked_inference, agent_conf
documents=docs, documents=docs,
stream=False, stream=False,
) )
tool_execution_step = next( tool_execution_step = next(step for step in response.steps if step.step_type == "tool_execution")
step for step in response.steps if step.step_type == "tool_execution"
)
assert tool_execution_step.tool_calls[0].tool_name == tool_name assert tool_execution_step.tool_calls[0].tool_name == tool_name
if expected_kw: if expected_kw:
assert expected_kw in response.output_message.content.lower() assert expected_kw in response.output_message.content.lower()
@ -586,9 +553,7 @@ def test_rag_and_code_agent(llama_stack_client_with_mocked_inference, agent_conf
"client_tools", "client_tools",
[(get_boiling_point, False), (get_boiling_point_with_metadata, True)], [(get_boiling_point, False), (get_boiling_point_with_metadata, True)],
) )
def test_create_turn_response( def test_create_turn_response(llama_stack_client_with_mocked_inference, agent_config, client_tools):
llama_stack_client_with_mocked_inference, agent_config, client_tools
):
client_tool, expects_metadata = client_tools client_tool, expects_metadata = client_tools
agent_config = { agent_config = {
**agent_config, **agent_config,