refactor

llama_stack/providers/impls/third_party/evals/eleuther/eleuther.py

@@ -13,12 +13,13 @@ import threading
 from pathlib import Path
 
 import lm_eval
+import tqdm
 from lm_eval.api.model import LM
 from lm_eval.evaluator import evaluate, get_task_list
 from lm_eval.tasks import get_task_dict, TaskManager
 from termcolor import cprint
 
-from .config import EleutherEvalsImplConfig  # noqa
+from .config import EleutherEvalsImplConfig
 
 
 # https://stackoverflow.com/questions/74703727/how-to-call-async-function-from-sync-funcion-and-get-result-while-a-loop-is-alr
@@ -99,7 +100,7 @@ class EleutherEvalsWrapper(LM):
         res = []
         if not _thr.is_alive():
             _thr.start()
-        for req in requests:
+        for req in tqdm.tqdm(requests):
             chat_completion_coro_fn = self.inference_api.chat_completion(
                 model=self.model,
                 messages=[
@@ -138,6 +139,7 @@ class EleutherEvalsAdapter(Evals):
         eluther_wrapper = EleutherEvalsWrapper(self.inference_api, model)
         current_dir = Path(os.path.dirname(os.path.abspath(__file__)))
 
+        # custom registry of harness tasks
         task_manager = TaskManager(
             include_path=str(current_dir / "tasks"),
         )

tests/examples/local-run.yaml

@@ -48,7 +48,7 @@ providers:
       persistence_store:
         namespace: null
         type: sqlite
-        db_path: /home/xiyan/.llama/runtime/kvstore.db
+        db_path: ~/.llama/runtime/kvstore.db
   telemetry:
   - provider_id: meta-reference
     provider_type: meta-reference

tests/examples/tgi-run.yaml

@@ -0,0 +1,67 @@
+version: '2'
+built_at: '2024-10-08T17:40:45.325529'
+image_name: local
+docker_image: null
+conda_env: local
+apis:
+- shields
+- agents
+- models
+- memory
+- memory_banks
+- inference
+- safety
+- evals
+providers:
+  evals:
+  - provider_id: eleuther
+    provider_type: eleuther
+    config: {}
+  inference:
+  - provider_id: remote::tgi
+    provider_type: remote::tgi
+    config:
+      url: http://127.0.0.1:5009
+  safety:
+  - provider_id: meta-reference
+    provider_type: meta-reference
+    config:
+      llama_guard_shield:
+        model: Llama-Guard-3-1B
+        excluded_categories: []
+        disable_input_check: false
+        disable_output_check: false
+      prompt_guard_shield:
+        model: Prompt-Guard-86M
+  memory:
+  - provider_id: meta-reference
+    provider_type: meta-reference
+    config: {}
+  agents:
+  - provider_id: meta-reference
+    provider_type: meta-reference
+    config:
+      persistence_store:
+        namespace: null
+        type: sqlite
+        db_path: ~/.llama/runtime/kvstore.db
+  telemetry:
+  - provider_id: meta-reference
+    provider_type: meta-reference
+    config: {}
+models:
+- identifier: Llama3.1-8B-Instruct
+  llama_model: Llama3.1-8B-Instruct
+  provider_id: remote::tgi
+shields:
+- identifier: llama_guard
+  type: llama_guard
+  provider_id: meta-reference
+  params: {}
+memory_banks:
+- identifier: vector
+  provider_id: meta-reference
+  type: vector
+  embedding_model: all-MiniLM-L6-v2
+  chunk_size_in_tokens: 512
+  overlap_size_in_tokens: null