mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-12-08 19:10:56 +00:00
120 lines
3.5 KiB
Python
120 lines
3.5 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the terms described in the LICENSE file in
|
|
# the root directory of this source tree.
|
|
|
|
from llama_stack.apis.inference import * # noqa: F403
|
|
from llama_stack.apis.evals import * # noqa: F403
|
|
import random
|
|
|
|
import lm_eval
|
|
from lm_eval.api.model import LM
|
|
from lm_eval.evaluator import evaluate, get_task_list
|
|
from lm_eval.tasks import get_task_dict, TaskManager
|
|
|
|
from .config import EleutherEvalsImplConfig # noqa
|
|
|
|
|
|
class EleutherEvalsWrapper(LM):
|
|
def __init__(
|
|
self,
|
|
inference_api: Inference,
|
|
**kwargs,
|
|
):
|
|
super().__init__(**kwargs)
|
|
self.inference_api = inference_api
|
|
self.tokenizer = None
|
|
self.tokenized_requests = False
|
|
self.kwargs = kwargs
|
|
|
|
@property
|
|
def eot_token_id(self):
|
|
raise NotImplementedError("Not implemented")
|
|
|
|
@property
|
|
def max_length(self) -> int:
|
|
return NotImplementedError("Not implemented")
|
|
|
|
@property
|
|
def max_gen_toks(self) -> int:
|
|
return NotImplementedError("Not implemented")
|
|
|
|
@property
|
|
def batch_size(self):
|
|
# Isn't used because we override _loglikelihood_tokens
|
|
raise NotImplementedError("No support for logits.")
|
|
|
|
@property
|
|
def device(self):
|
|
# Isn't used because we override _loglikelihood_tokens
|
|
raise NotImplementedError("No support for logits.")
|
|
|
|
@property
|
|
def world_size(self):
|
|
return 1
|
|
|
|
def tok_encode(self, string: str) -> List[int]:
|
|
return NotImplementedError("Not implemented")
|
|
|
|
def tok_decode(self, tokens: List[int]) -> str:
|
|
return NotImplementedError("Not implemented")
|
|
|
|
def _loglikelihood_tokens(self, requests, disable_tqdm: bool = False):
|
|
raise NotImplementedError("No support for logits.")
|
|
|
|
def _model_call(self, inps):
|
|
# Isn't used because we override _loglikelihood_tokens
|
|
raise NotImplementedError()
|
|
|
|
def _model_generate(self, context, max_length, eos_token_id):
|
|
# Isn't used because we override generate_until
|
|
raise NotImplementedError()
|
|
|
|
def loglikelihood(self, requests, disable_tqdm: bool = False):
|
|
# TODO: implement inference completion with loglikelihood
|
|
res = []
|
|
for req in requests:
|
|
res.append((-random.random(), False))
|
|
|
|
return res
|
|
|
|
def loglikelihood_rolling(self, requests, disable_tqdm: bool = False):
|
|
raise NotImplementedError("No support for logits.")
|
|
|
|
def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]:
|
|
return NotImplementedError("Not implemented")
|
|
|
|
|
|
class EleutherEvalsAdapter(Evals):
|
|
def __init__(self, config: EleutherEvalsImplConfig, inference_api: Inference):
|
|
self.inference_api = inference_api
|
|
self.eluther_wrapper = EleutherEvalsWrapper(inference_api)
|
|
|
|
async def initialize(self) -> None:
|
|
pass
|
|
|
|
async def shutdown(self) -> None:
|
|
pass
|
|
|
|
async def run_evals(
|
|
self,
|
|
model: str,
|
|
dataset: str,
|
|
task: str,
|
|
) -> EvaluateResponse:
|
|
task_manager = TaskManager()
|
|
task_dict = get_task_dict(task, task_manager)
|
|
task_types = set([t.task.OUTPUT_TYPE for t in get_task_list(task_dict)])
|
|
|
|
output = evaluate(
|
|
self.eluther_wrapper,
|
|
task_dict,
|
|
limit=2,
|
|
)
|
|
formatted_output = lm_eval.utils.make_table(output)
|
|
return EvaluateResponse(
|
|
metrics={
|
|
"metrics_table": formatted_output,
|
|
},
|
|
)
|