phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-15 23:59:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

llm judge llamastack scorer

Browse source
This commit is contained in:
Xi Yan 2024-10-15 13:25:46 -07:00
parent 0c4ed66ecc
commit fa68809a2e
10 changed files with 199 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

1
llama_stack/distribution/registry/generator_processors/__init__.py
View file

@ -13,6 +13,7 @@ GeneratorProcessorRegistry = Registry[BaseGeneratorProcessor]()
PROCESSOR_REGISTRY = {
"mmlu": MMLUProcessor,
"judge": JudgeProcessor,
}
for k, v in PROCESSOR_REGISTRY.items():

2
llama_stack/distribution/registry/scorers/__init__.py
View file

@ -7,6 +7,7 @@
from llama_stack.apis.evals import * # noqa: F403
from llama_stack.providers.impls.meta_reference.evals.scorer.basic_scorers import * # noqa: F403
from llama_stack.providers.impls.meta_reference.evals.scorer.braintrust_scorer import * # noqa: F403
from llama_stack.providers.impls.meta_reference.evals.scorer.llm_judge_scorer import * # noqa: F403
from ..registry import Registry
@ -16,6 +17,7 @@ ScorerRegistry = Registry[BaseScorer]()
SCORER_REGISTRY = {
"accuracy": AccuracyScorer,
"random": RandomScorer,
"llamastack-llm-judge": LlamaStackLLMJudgeScorer,
"braintrust::factuality": BraintrustFactualityScorer,
"braintrust::answer-correctness": BraintrustAnswerCorrectnessScorer,
}