phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-13 04:22:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor

Browse source
This commit is contained in:
Xi Yan 2024-10-09 14:47:36 -07:00
parent adb768f827
commit 58992dacc2
3 changed files with 72 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
llama_stack/providers/impls/third_party/evals/eleuther/eleuther.py vendored
View file

@ -13,12 +13,13 @@ import threading
from pathlib import Path
import lm_eval
import tqdm
from lm_eval.api.model import LM
from lm_eval.evaluator import evaluate, get_task_list
from lm_eval.tasks import get_task_dict, TaskManager
from termcolor import cprint
from .config import EleutherEvalsImplConfig # noqa
from .config import EleutherEvalsImplConfig
# https://stackoverflow.com/questions/74703727/how-to-call-async-function-from-sync-funcion-and-get-result-while-a-loop-is-alr
@ -99,7 +100,7 @@ class EleutherEvalsWrapper(LM):
res = []
if not _thr.is_alive():
_thr.start()
for req in requests:
for req in tqdm.tqdm(requests):
chat_completion_coro_fn = self.inference_api.chat_completion(
model=self.model,
messages=[
@ -138,6 +139,7 @@ class EleutherEvalsAdapter(Evals):
eluther_wrapper = EleutherEvalsWrapper(self.inference_api, model)
current_dir = Path(os.path.dirname(os.path.abspath(__file__)))
# custom registry of harness tasks
task_manager = TaskManager(
include_path=str(current_dir / "tasks"),
)