fix: Fix paths in Eval helper functions; update ModelCandidate to support Evals that use chat datasets

This commit is contained in:
Jash Gulabrai 2025-04-09 15:48:46 -04:00
parent e273a68567
commit 7840a53a12
4 changed files with 46 additions and 9 deletions

View file

@ -6347,7 +6347,36 @@
"default": "model" "default": "model"
}, },
"model": { "model": {
"type": "string", "oneOf": [
{
"type": "string"
},
{
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
}
}
],
"description": "The model ID to evaluate." "description": "The model ID to evaluate."
}, },
"sampling_params": { "sampling_params": {
@ -6362,8 +6391,7 @@
"additionalProperties": false, "additionalProperties": false,
"required": [ "required": [
"type", "type",
"model", "model"
"sampling_params"
], ],
"title": "ModelCandidate", "title": "ModelCandidate",
"description": "A model candidate for evaluation." "description": "A model candidate for evaluation."

View file

@ -4468,7 +4468,17 @@ components:
const: model const: model
default: model default: model
model: model:
type: string oneOf:
- type: string
- type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: The model ID to evaluate. description: The model ID to evaluate.
sampling_params: sampling_params:
$ref: '#/components/schemas/SamplingParams' $ref: '#/components/schemas/SamplingParams'
@ -4482,7 +4492,6 @@ components:
required: required:
- type - type
- model - model
- sampling_params
title: ModelCandidate title: ModelCandidate
description: A model candidate for evaluation. description: A model candidate for evaluation.
RegexParserScoringFnParams: RegexParserScoringFnParams:

View file

@ -27,8 +27,8 @@ class ModelCandidate(BaseModel):
""" """
type: Literal["model"] = "model" type: Literal["model"] = "model"
model: str model: Union[str, Dict[str, Any]]
sampling_params: SamplingParams sampling_params: Optional[SamplingParams] = Field(default_factory=SamplingParams)
system_message: Optional[SystemMessage] = None system_message: Optional[SystemMessage] = None

View file

@ -48,13 +48,13 @@ class NVIDIAEvalImpl(
async def _evaluator_get(self, path): async def _evaluator_get(self, path):
"""Helper for making GET requests to the evaluator service.""" """Helper for making GET requests to the evaluator service."""
response = requests.get(url=f"{self.config.evaluator_service_url}/{path}") response = requests.get(url=f"{self.config.evaluator_service_url}{path}")
response.raise_for_status() response.raise_for_status()
return response.json() return response.json()
async def _evaluator_post(self, path, data): async def _evaluator_post(self, path, data):
"""Helper for making POST requests to the evaluator service.""" """Helper for making POST requests to the evaluator service."""
response = requests.post(url=f"{self.config.evaluator_service_url}/{path}", json=data) response = requests.post(url=f"{self.config.evaluator_service_url}{path}", json=data)
response.raise_for_status() response.raise_for_status()
return response.json() return response.json()