test full generation + eval

llama_stack/providers/tests/datasetio/test_datasetio.py

@@ -70,6 +70,7 @@ async def register_dataset(
     if for_generation:
         dataset_schema = {
             "expected_answer": StringType(),
+            "input_query": StringType(),
             "chat_completion_input": ChatCompletionInputType(),
         }
     else:

llama_stack/providers/tests/eval/provider_config_example.yaml

@@ -16,3 +16,7 @@ providers:
       provider_type: remote::tgi
       config:
         url: http://127.0.0.1:5009
+    - provider_id: test-tgi-2
+      provider_type: remote::tgi
+      config:
+        url: http://127.0.0.1:5010

llama_stack/providers/tests/eval/test_eval.py

@@ -65,7 +65,10 @@ async def test_eval(eval_settings):
             model="Llama3.2-1B-Instruct",
             sampling_params=SamplingParams(),
         ),
-        scoring_functions=["subset_of"],
+        scoring_functions=[
+            "meta-reference::subset_of",
+            "meta-reference::llm_as_judge_8b_correctness",
+        ],
     )
     assert response.job_id == "0"
     job_status = await eval_impl.job_status(response.job_id)
@@ -74,6 +77,8 @@ async def test_eval(eval_settings):
 
     eval_response = await eval_impl.job_result(response.job_id)
 
+    print(eval_response)
     assert eval_response is not None
     assert len(eval_response.generations) == 5
-    assert "subset_of" in eval_response.scores
+    assert "meta-reference::subset_of" in eval_response.scores
+    assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores