forked from phoenix-oss/llama-stack-mirror
8b655e3cd2
# What does this PR do? - Update `/eval-tasks` to `/benchmarks` - ⚠️ Remove differentiation between `app` v.s. `benchmark` eval task config. Now we only have `BenchmarkConfig`. The overloaded `benchmark` is confusing and do not add any value. Backward compatibility is being kept as the "type" is not being used anywhere. [//]: # (If resolving an issue, uncomment and update the line below) [//]: # (Closes #[issue-number]) ## Test Plan - This change is backward compatible - Run notebook test with ``` pytest -v -s --nbval-lax ./docs/getting_started.ipynb pytest -v -s --nbval-lax ./docs/notebooks/Llama_Stack_Benchmark_Evals.ipynb ``` <img width="846" alt="image" src="https://github.com/user-attachments/assets/d2fc06a7-593a-444f-bc1f-10ab9b0c843d" /> [//]: # (## Documentation) [//]: # (- [ ] Added a Changelog entry if the change is significant) --------- Signed-off-by: Ihar Hrachyshka <ihar.hrachyshka@gmail.com> Signed-off-by: Ben Browning <bbrownin@redhat.com> Signed-off-by: Sébastien Han <seb@redhat.com> Signed-off-by: reidliu <reid201711@gmail.com> Co-authored-by: Ihar Hrachyshka <ihar.hrachyshka@gmail.com> Co-authored-by: Ben Browning <ben324@gmail.com> Co-authored-by: Sébastien Han <seb@redhat.com> Co-authored-by: Reid <61492567+reidliu41@users.noreply.github.com> Co-authored-by: reidliu <reid201711@gmail.com> Co-authored-by: Yuan Tang <terrytangyuan@gmail.com>
73 lines
2.1 KiB
Python
73 lines
2.1 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the terms described in the LICENSE file in
|
|
# the root directory of this source tree.
|
|
|
|
from typing import Any, Dict
|
|
|
|
from llama_stack.distribution.datatypes import RoutedProtocol
|
|
from llama_stack.distribution.store import DistributionRegistry
|
|
from llama_stack.providers.datatypes import Api, RoutingTable
|
|
|
|
from .routing_tables import (
|
|
BenchmarksRoutingTable,
|
|
DatasetsRoutingTable,
|
|
ModelsRoutingTable,
|
|
ScoringFunctionsRoutingTable,
|
|
ShieldsRoutingTable,
|
|
ToolGroupsRoutingTable,
|
|
VectorDBsRoutingTable,
|
|
)
|
|
|
|
|
|
async def get_routing_table_impl(
|
|
api: Api,
|
|
impls_by_provider_id: Dict[str, RoutedProtocol],
|
|
_deps,
|
|
dist_registry: DistributionRegistry,
|
|
) -> Any:
|
|
api_to_tables = {
|
|
"vector_dbs": VectorDBsRoutingTable,
|
|
"models": ModelsRoutingTable,
|
|
"shields": ShieldsRoutingTable,
|
|
"datasets": DatasetsRoutingTable,
|
|
"scoring_functions": ScoringFunctionsRoutingTable,
|
|
"benchmarks": BenchmarksRoutingTable,
|
|
"tool_groups": ToolGroupsRoutingTable,
|
|
}
|
|
|
|
if api.value not in api_to_tables:
|
|
raise ValueError(f"API {api.value} not found in router map")
|
|
|
|
impl = api_to_tables[api.value](impls_by_provider_id, dist_registry)
|
|
await impl.initialize()
|
|
return impl
|
|
|
|
|
|
async def get_auto_router_impl(api: Api, routing_table: RoutingTable, _deps) -> Any:
|
|
from .routers import (
|
|
DatasetIORouter,
|
|
EvalRouter,
|
|
InferenceRouter,
|
|
SafetyRouter,
|
|
ScoringRouter,
|
|
ToolRuntimeRouter,
|
|
VectorIORouter,
|
|
)
|
|
|
|
api_to_routers = {
|
|
"vector_io": VectorIORouter,
|
|
"inference": InferenceRouter,
|
|
"safety": SafetyRouter,
|
|
"datasetio": DatasetIORouter,
|
|
"scoring": ScoringRouter,
|
|
"eval": EvalRouter,
|
|
"tool_runtime": ToolRuntimeRouter,
|
|
}
|
|
if api.value not in api_to_routers:
|
|
raise ValueError(f"API {api.value} not found in router map")
|
|
|
|
impl = api_to_routers[api.value](routing_table)
|
|
await impl.initialize()
|
|
return impl
|