forked from phoenix-oss/llama-stack-mirror
abdf7cddf3
* wip * dataset validation * test_scoring * cleanup * clean up test * comments * error checking * dataset client * test client: * datasetio client * clean up * basic scoring function works * scorer wip * equality scorer * score batch impl * score batch * update scoring test * refactor * validate scorer input * address comments * evals with generation * add all rows scores to ScoringResult * minor typing * bugfix * scoring function def rename * rebase name * refactor * address comments * Update iOS inference instructions for new quantization * Small updates to quantization config * Fix score threshold in faiss * Bump version to 0.0.45 * Handle both ipv6 and ipv4 interfaces together * update manifest for build templates * Update getting_started.md * chatcompletion & completion input type validation * inclusion->subsetof * error checking * scoring_function -> scoring_fn rename, scorer -> scoring_fn rename * address comments * [Evals API][5/n] fixes to generate openapi spec (#323) * generate openapi * typing comment, dataset -> dataset_id * remove custom type * sample eval run.yaml --------- Co-authored-by: Dalton Flanagan <6599399+dltn@users.noreply.github.com> Co-authored-by: Ashwin Bharambe <ashwin.bharambe@gmail.com>
79 lines
2.5 KiB
Python
79 lines
2.5 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the terms described in the LICENSE file in
|
|
# the root directory of this source tree.
|
|
import pytest
|
|
import pytest_asyncio
|
|
|
|
from llama_stack.apis.common.type_system import * # noqa: F403
|
|
from llama_stack.apis.datasetio import * # noqa: F403
|
|
from llama_stack.apis.eval.eval import ModelCandidate
|
|
from llama_stack.distribution.datatypes import * # noqa: F403
|
|
|
|
from llama_models.llama3.api import SamplingParams
|
|
|
|
from llama_stack.providers.tests.datasetio.test_datasetio import register_dataset
|
|
from llama_stack.providers.tests.resolver import resolve_impls_for_test
|
|
|
|
# How to run this test:
|
|
#
|
|
# 1. Ensure you have a conda with the right dependencies installed. This is a bit tricky
|
|
# since it depends on the provider you are testing. On top of that you need
|
|
# `pytest` and `pytest-asyncio` installed.
|
|
#
|
|
# 2. Copy and modify the provider_config_example.yaml depending on the provider you are testing.
|
|
#
|
|
# 3. Run:
|
|
#
|
|
# ```bash
|
|
# PROVIDER_ID=<your_provider> \
|
|
# PROVIDER_CONFIG=provider_config.yaml \
|
|
# pytest -s llama_stack/providers/tests/eval/test_eval.py \
|
|
# --tb=short --disable-warnings
|
|
# ```
|
|
|
|
|
|
@pytest_asyncio.fixture(scope="session")
|
|
async def eval_settings():
|
|
impls = await resolve_impls_for_test(
|
|
Api.eval, deps=[Api.datasetio, Api.scoring, Api.inference]
|
|
)
|
|
return {
|
|
"eval_impl": impls[Api.eval],
|
|
"scoring_impl": impls[Api.scoring],
|
|
"datasets_impl": impls[Api.datasets],
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_eval(eval_settings):
|
|
datasets_impl = eval_settings["datasets_impl"]
|
|
await register_dataset(
|
|
datasets_impl,
|
|
for_generation=True,
|
|
dataset_id="test_dataset_for_eval",
|
|
)
|
|
|
|
response = await datasets_impl.list_datasets()
|
|
assert len(response) == 1
|
|
|
|
eval_impl = eval_settings["eval_impl"]
|
|
response = await eval_impl.evaluate_batch(
|
|
dataset_id=response[0].identifier,
|
|
candidate=ModelCandidate(
|
|
model="Llama3.2-1B-Instruct",
|
|
sampling_params=SamplingParams(),
|
|
),
|
|
scoring_functions=["subset_of"],
|
|
)
|
|
assert response.job_id == "0"
|
|
job_status = await eval_impl.job_status(response.job_id)
|
|
|
|
assert job_status and job_status.value == "completed"
|
|
|
|
eval_response = await eval_impl.job_result(response.job_id)
|
|
|
|
assert eval_response is not None
|
|
assert len(eval_response.generations) == 5
|
|
assert "subset_of" in eval_response.scores
|