diff --git a/llama_stack/providers/impls/third_party/evals/eleuther/eleuther.py b/llama_stack/providers/impls/third_party/evals/eleuther/eleuther.py index 102d15020..706e46c61 100644 --- a/llama_stack/providers/impls/third_party/evals/eleuther/eleuther.py +++ b/llama_stack/providers/impls/third_party/evals/eleuther/eleuther.py @@ -13,12 +13,13 @@ import threading from pathlib import Path import lm_eval +import tqdm from lm_eval.api.model import LM from lm_eval.evaluator import evaluate, get_task_list from lm_eval.tasks import get_task_dict, TaskManager from termcolor import cprint -from .config import EleutherEvalsImplConfig # noqa +from .config import EleutherEvalsImplConfig # https://stackoverflow.com/questions/74703727/how-to-call-async-function-from-sync-funcion-and-get-result-while-a-loop-is-alr @@ -99,7 +100,7 @@ class EleutherEvalsWrapper(LM): res = [] if not _thr.is_alive(): _thr.start() - for req in requests: + for req in tqdm.tqdm(requests): chat_completion_coro_fn = self.inference_api.chat_completion( model=self.model, messages=[ @@ -138,6 +139,7 @@ class EleutherEvalsAdapter(Evals): eluther_wrapper = EleutherEvalsWrapper(self.inference_api, model) current_dir = Path(os.path.dirname(os.path.abspath(__file__))) + # custom registry of harness tasks task_manager = TaskManager( include_path=str(current_dir / "tasks"), ) diff --git a/tests/examples/local-run.yaml b/tests/examples/local-run.yaml index 71f745a0c..219f5ef4c 100644 --- a/tests/examples/local-run.yaml +++ b/tests/examples/local-run.yaml @@ -48,7 +48,7 @@ providers: persistence_store: namespace: null type: sqlite - db_path: /home/xiyan/.llama/runtime/kvstore.db + db_path: ~/.llama/runtime/kvstore.db telemetry: - provider_id: meta-reference provider_type: meta-reference diff --git a/tests/examples/tgi-run.yaml b/tests/examples/tgi-run.yaml new file mode 100644 index 000000000..c61e5696c --- /dev/null +++ b/tests/examples/tgi-run.yaml @@ -0,0 +1,67 @@ +version: '2' +built_at: '2024-10-08T17:40:45.325529' +image_name: local +docker_image: null +conda_env: local +apis: +- shields +- agents +- models +- memory +- memory_banks +- inference +- safety +- evals +providers: + evals: + - provider_id: eleuther + provider_type: eleuther + config: {} + inference: + - provider_id: remote::tgi + provider_type: remote::tgi + config: + url: http://127.0.0.1:5009 + safety: + - provider_id: meta-reference + provider_type: meta-reference + config: + llama_guard_shield: + model: Llama-Guard-3-1B + excluded_categories: [] + disable_input_check: false + disable_output_check: false + prompt_guard_shield: + model: Prompt-Guard-86M + memory: + - provider_id: meta-reference + provider_type: meta-reference + config: {} + agents: + - provider_id: meta-reference + provider_type: meta-reference + config: + persistence_store: + namespace: null + type: sqlite + db_path: ~/.llama/runtime/kvstore.db + telemetry: + - provider_id: meta-reference + provider_type: meta-reference + config: {} +models: +- identifier: Llama3.1-8B-Instruct + llama_model: Llama3.1-8B-Instruct + provider_id: remote::tgi +shields: +- identifier: llama_guard + type: llama_guard + provider_id: meta-reference + params: {} +memory_banks: +- identifier: vector + provider_id: meta-reference + type: vector + embedding_model: all-MiniLM-L6-v2 + chunk_size_in_tokens: 512 + overlap_size_in_tokens: null