phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-03 09:21:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

able to extract context during eval pipeline

Browse source
This commit is contained in:
Xi Yan 2024-12-19 19:49:23 -08:00
parent 40b8ec3185
commit f117a58a8b
1 changed files with 15 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

20
llama_stack/providers/inline/eval/meta_reference/eval.py
View file

@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional
from tqdm import tqdm from tqdm import tqdm
from llama_stack.apis.agents import Agents from llama_stack.apis.agents import Agents, StepType
from llama_stack.apis.datasetio import DatasetIO from llama_stack.apis.datasetio import DatasetIO
from llama_stack.apis.datasets import Datasets from llama_stack.apis.datasets import Datasets
from llama_stack.apis.eval_tasks import EvalTask from llama_stack.apis.eval_tasks import EvalTask
@ -135,11 +135,21 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate, DataSchemaValidatorM
) )
] ]
final_event = turn_response[-1].event.payload final_event = turn_response[-1].event.payload
generations.append(
{ # check if there's a memory retrieval step and extract the context
ColumnName.generated_answer.value: final_event.turn.output_message.content memory_rag_context = None
} for step in final_event.turn.steps:
if step.step_type == StepType.memory_retrieval.value:
memory_rag_context = " ".join(x.text for x in step.inserted_context)
agent_generation = {}
agent_generation[ColumnName.generated_answer.value] = (
final_event.turn.output_message.content
) )
if memory_rag_context:
agent_generation[ColumnName.context.value] = memory_rag_context
generations.append(agent_generation)
return generations return generations