phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-03 09:21:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: Fix paths in Eval helper functions; update ModelCandidate to support Evals that use chat datasets

Browse source
This commit is contained in:
Jash Gulabrai 2025-04-09 15:48:46 -04:00
parent e273a68567
commit 7840a53a12
4 changed files with 46 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

34
docs/_static/llama-stack-spec.html vendored
View file

@ -6347,7 +6347,36 @@
"default": "model"
},
"model": {
"type": "string",
"oneOf": [
{
"type": "string"
},
{
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
}
}
],
"description": "The model ID to evaluate."
},
"sampling_params": {
@ -6362,8 +6391,7 @@
"additionalProperties": false,
"required": [
"type",
"model",
"sampling_params"
"model"
],
"title": "ModelCandidate",
"description": "A model candidate for evaluation."

13
docs/_static/llama-stack-spec.yaml vendored
View file

@ -4468,7 +4468,17 @@ components:
const: model
default: model
model:
type: string
oneOf:
- type: string
- type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: The model ID to evaluate.
sampling_params:
$ref: '#/components/schemas/SamplingParams'
@ -4482,7 +4492,6 @@ components:
required:
- type
- model
- sampling_params
title: ModelCandidate
description: A model candidate for evaluation.
RegexParserScoringFnParams:

4
llama_stack/apis/eval/eval.py
View file

@ -27,8 +27,8 @@ class ModelCandidate(BaseModel):
"""
type: Literal["model"] = "model"
model: str
sampling_params: SamplingParams
model: Union[str, Dict[str, Any]]
sampling_params: Optional[SamplingParams] = Field(default_factory=SamplingParams)
system_message: Optional[SystemMessage] = None

4
llama_stack/providers/remote/eval/nvidia/eval.py
View file

@ -48,13 +48,13 @@ class NVIDIAEvalImpl(
async def _evaluator_get(self, path):
"""Helper for making GET requests to the evaluator service."""
response = requests.get(url=f"{self.config.evaluator_service_url}/{path}")
response = requests.get(url=f"{self.config.evaluator_service_url}{path}")
response.raise_for_status()
return response.json()
async def _evaluator_post(self, path, data):
"""Helper for making POST requests to the evaluator service."""
response = requests.post(url=f"{self.config.evaluator_service_url}/{path}", json=data)
response = requests.post(url=f"{self.config.evaluator_service_url}{path}", json=data)
response.raise_for_status()
return response.json()