diff --git a/docs/_static/llama-stack-spec.html b/docs/_static/llama-stack-spec.html index 567110829..6b5793119 100644 --- a/docs/_static/llama-stack-spec.html +++ b/docs/_static/llama-stack-spec.html @@ -6347,7 +6347,36 @@ "default": "model" }, "model": { - "type": "string", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + } + ], "description": "The model ID to evaluate." }, "sampling_params": { @@ -6362,8 +6391,7 @@ "additionalProperties": false, "required": [ "type", - "model", - "sampling_params" + "model" ], "title": "ModelCandidate", "description": "A model candidate for evaluation." diff --git a/docs/_static/llama-stack-spec.yaml b/docs/_static/llama-stack-spec.yaml index 1dfd17f55..8c5746900 100644 --- a/docs/_static/llama-stack-spec.yaml +++ b/docs/_static/llama-stack-spec.yaml @@ -4468,7 +4468,17 @@ components: const: model default: model model: - type: string + oneOf: + - type: string + - type: object + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object description: The model ID to evaluate. sampling_params: $ref: '#/components/schemas/SamplingParams' @@ -4482,7 +4492,6 @@ components: required: - type - model - - sampling_params title: ModelCandidate description: A model candidate for evaluation. RegexParserScoringFnParams: diff --git a/llama_stack/apis/eval/eval.py b/llama_stack/apis/eval/eval.py index 0e5959c37..f9bb9a171 100644 --- a/llama_stack/apis/eval/eval.py +++ b/llama_stack/apis/eval/eval.py @@ -27,8 +27,8 @@ class ModelCandidate(BaseModel): """ type: Literal["model"] = "model" - model: str - sampling_params: SamplingParams + model: Union[str, Dict[str, Any]] + sampling_params: Optional[SamplingParams] = Field(default_factory=SamplingParams) system_message: Optional[SystemMessage] = None diff --git a/llama_stack/providers/remote/eval/nvidia/eval.py b/llama_stack/providers/remote/eval/nvidia/eval.py index 2ef46251e..ca8464f51 100644 --- a/llama_stack/providers/remote/eval/nvidia/eval.py +++ b/llama_stack/providers/remote/eval/nvidia/eval.py @@ -48,13 +48,13 @@ class NVIDIAEvalImpl( async def _evaluator_get(self, path): """Helper for making GET requests to the evaluator service.""" - response = requests.get(url=f"{self.config.evaluator_service_url}/{path}") + response = requests.get(url=f"{self.config.evaluator_service_url}{path}") response.raise_for_status() return response.json() async def _evaluator_post(self, path, data): """Helper for making POST requests to the evaluator service.""" - response = requests.post(url=f"{self.config.evaluator_service_url}/{path}", json=data) + response = requests.post(url=f"{self.config.evaluator_service_url}{path}", json=data) response.raise_for_status() return response.json()