phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-16 06:53:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

remove prints, cleanup braintrust in this branch

Browse source
This commit is contained in:
Xi Yan 2024-10-25 17:39:05 -07:00
parent 9b410a87bf
commit 91e5ad18b0
6 changed files with 45 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

6
llama_stack/providers/impls/meta_reference/scoring/scoring.py
View file

@ -57,6 +57,7 @@ class MetaReferenceScoringImpl(Scoring, ScoringFunctionsProtocolPrivate):
await impl.initialize()
for fn_defs in impl.get_supported_scoring_fn_defs():
self.scoring_fn_id_impls[fn_defs.identifier] = impl
self.llm_as_judge_fn = impl
async def shutdown(self) -> None: ...
@ -68,9 +69,8 @@ class MetaReferenceScoringImpl(Scoring, ScoringFunctionsProtocolPrivate):
]
async def register_scoring_function(self, function_def: ScoringFnDef) -> None:
raise NotImplementedError(
"Dynamically registering scoring functions is not supported"
)
self.llm_as_judge_fn.register_scoring_fn_def(function_def)
self.scoring_fn_id_impls[function_def.identifier] = self.llm_as_judge_fn
async def validate_scoring_input_dataset_schema(self, dataset_id: str) -> None:
dataset_def = await self.datasets_api.get_dataset(dataset_identifier=dataset_id)

2
llama_stack/providers/impls/meta_reference/scoring/scoring_fn/llm_as_judge_scoring_fn.py
View file

@ -43,7 +43,7 @@ class LlmAsJudgeScoringFn(BaseScoringFn):
), "LLM Judge prompt_template not found."
assert (
fn_def.context.judge_score_regex is not None
), "LLM Judge prompt_template not found."
), "LLM Judge judge_score_regex not found."
input_query = input_row["input_query"]
expected_answer = input_row["expected_answer"]

18
llama_stack/providers/impls/third_party/scoring/braintrust/__init__.py vendored
View file

@ -1,18 +0,0 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
from typing import Any
from .config import BraintrustScoringConfig
async def get_provider_impl(config: BraintrustScoringConfig, _deps) -> Any:
pass
# from .braintrust import VLLMInferenceImpl
# impl = VLLMInferenceImpl(config)
# await impl.initialize()
# return impl

9
llama_stack/providers/impls/third_party/scoring/braintrust/config.py vendored
View file

@ -1,9 +0,0 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
from llama_stack.apis.eval import * # noqa: F401, F403
class BraintrustScoringConfig(BaseModel): ...