From 5e635d2e643576869e594578f5e531b77a27c234 Mon Sep 17 00:00:00 2001
From: Xi Yan <xiyan@meta.com>
Date: Tue, 26 Nov 2024 13:04:33 -0800
Subject: [PATCH] fix eval test

---
 llama_stack/distribution/request_headers.py  |  2 +-
 llama_stack/providers/tests/eval/conftest.py | 16 ++++++++++++++++
 llama_stack/providers/tests/eval/fixtures.py | 20 ++++++++++++++++++--
 3 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/llama_stack/distribution/request_headers.py b/llama_stack/distribution/request_headers.py
index 27ef3046a..41952edfd 100644
--- a/llama_stack/distribution/request_headers.py
+++ b/llama_stack/distribution/request_headers.py
@@ -35,7 +35,7 @@ class NeedsRequestProviderData:
             provider_data = validator(**val)
             return provider_data
         except Exception as e:
-            log.error("Error parsing provider data", e)
+            log.error(f"Error parsing provider data: {e}")
 
 
 def set_request_provider_data(headers: Dict[str, str]):
diff --git a/llama_stack/providers/tests/eval/conftest.py b/llama_stack/providers/tests/eval/conftest.py
index 171fae51a..b310439ce 100644
--- a/llama_stack/providers/tests/eval/conftest.py
+++ b/llama_stack/providers/tests/eval/conftest.py
@@ -6,10 +6,14 @@
 
 import pytest
 
+from ..agents.fixtures import AGENTS_FIXTURES
+
 from ..conftest import get_provider_fixture_overrides
 
 from ..datasetio.fixtures import DATASETIO_FIXTURES
 from ..inference.fixtures import INFERENCE_FIXTURES
+from ..memory.fixtures import MEMORY_FIXTURES
+from ..safety.fixtures import SAFETY_FIXTURES
 from ..scoring.fixtures import SCORING_FIXTURES
 from .fixtures import EVAL_FIXTURES
 
@@ -20,6 +24,9 @@ DEFAULT_PROVIDER_COMBINATIONS = [
             "scoring": "basic",
             "datasetio": "localfs",
             "inference": "fireworks",
+            "agents": "meta_reference",
+            "safety": "llama_guard",
+            "memory": "faiss",
         },
         id="meta_reference_eval_fireworks_inference",
         marks=pytest.mark.meta_reference_eval_fireworks_inference,
@@ -30,6 +37,9 @@ DEFAULT_PROVIDER_COMBINATIONS = [
             "scoring": "basic",
             "datasetio": "localfs",
             "inference": "together",
+            "agents": "meta_reference",
+            "safety": "llama_guard",
+            "memory": "faiss",
         },
         id="meta_reference_eval_together_inference",
         marks=pytest.mark.meta_reference_eval_together_inference,
@@ -40,6 +50,9 @@ DEFAULT_PROVIDER_COMBINATIONS = [
             "scoring": "basic",
             "datasetio": "huggingface",
             "inference": "together",
+            "agents": "meta_reference",
+            "safety": "llama_guard",
+            "memory": "faiss",
         },
         id="meta_reference_eval_together_inference_huggingface_datasetio",
         marks=pytest.mark.meta_reference_eval_together_inference_huggingface_datasetio,
@@ -75,6 +88,9 @@ def pytest_generate_tests(metafunc):
             "scoring": SCORING_FIXTURES,
             "datasetio": DATASETIO_FIXTURES,
             "inference": INFERENCE_FIXTURES,
+            "agents": AGENTS_FIXTURES,
+            "safety": SAFETY_FIXTURES,
+            "memory": MEMORY_FIXTURES,
         }
         combinations = (
             get_provider_fixture_overrides(metafunc.config, available_fixtures)
diff --git a/llama_stack/providers/tests/eval/fixtures.py b/llama_stack/providers/tests/eval/fixtures.py
index a6b404d0c..50dc9c16e 100644
--- a/llama_stack/providers/tests/eval/fixtures.py
+++ b/llama_stack/providers/tests/eval/fixtures.py
@@ -40,14 +40,30 @@ async def eval_stack(request):
 
     providers = {}
     provider_data = {}
-    for key in ["datasetio", "eval", "scoring", "inference"]:
+    for key in [
+        "datasetio",
+        "eval",
+        "scoring",
+        "inference",
+        "agents",
+        "safety",
+        "memory",
+    ]:
         fixture = request.getfixturevalue(f"{key}_{fixture_dict[key]}")
         providers[key] = fixture.providers
         if fixture.provider_data:
             provider_data.update(fixture.provider_data)
 
     test_stack = await construct_stack_for_test(
-        [Api.eval, Api.datasetio, Api.inference, Api.scoring],
+        [
+            Api.eval,
+            Api.datasetio,
+            Api.inference,
+            Api.scoring,
+            Api.agents,
+            Api.safety,
+            Api.memory,
+        ],
         providers,
         provider_data,
     )