diff --git a/llama_stack/apis/evals/client.py b/llama_stack/apis/evals/client.py index 4acbff5f6..aa617a5f1 100644 --- a/llama_stack/apis/evals/client.py +++ b/llama_stack/apis/evals/client.py @@ -44,7 +44,7 @@ async def run_main(host: str, port: int): # CustomDataset response = await client.run_evals( - "Llama3.1-8B-Instruct", + "Llama3.2-1B-Instruct", "mmlu-simple-eval-en", "mmlu", ) diff --git a/llama_stack/providers/impls/meta_reference/evals/evals.py b/llama_stack/providers/impls/meta_reference/evals/evals.py index 82c2a542b..3ceb51aab 100644 --- a/llama_stack/providers/impls/meta_reference/evals/evals.py +++ b/llama_stack/providers/impls/meta_reference/evals/evals.py @@ -14,10 +14,6 @@ from llama_stack.providers.impls.meta_reference.evals.datas.dataset_registry imp get_dataset, ) -# from llama_stack.providers.impls.meta_reference.evals.tasks.task_registry import ( -# get_task, -# ) - from .config import MetaReferenceEvalsImplConfig @@ -45,7 +41,9 @@ class MetaReferenceEvalsImpl(Evals): # TODO: replace w/ batch inference & async return eval job generation_outputs = [] + print("generation start") for msg in x1[:5]: + print("generation for msg: ", msg) response = self.inference_api.chat_completion( model=model, messages=[msg], diff --git a/tests/examples/local-run.yaml b/tests/examples/local-run.yaml index f7e3b1655..b64bbea24 100644 --- a/tests/examples/local-run.yaml +++ b/tests/examples/local-run.yaml @@ -39,18 +39,18 @@ api_providers: config: {} routing_table: inference: - # - provider_type: meta-reference - # config: - # model: Llama3.2-1B-Instruct - # quantization: null - # torch_seed: null - # max_seq_len: 4096 - # max_batch_size: 1 - # routing_key: Llama3.2-1B-Instruct - - provider_type: remote::tgi + - provider_type: meta-reference config: - url: http://127.0.0.1:5009 - routing_key: Llama3.1-8B-Instruct + model: Llama3.2-1B-Instruct + quantization: null + torch_seed: null + max_seq_len: 4096 + max_batch_size: 1 + routing_key: Llama3.2-1B-Instruct + # - provider_type: remote::tgi + # config: + # url: http://127.0.0.1:5009 + # routing_key: Llama3.1-8B-Instruct safety: - provider_type: meta-reference config: