phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-03 17:29:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Address PR comments

Browse source
This commit is contained in:
Jash Gulabrai 2025-04-15 18:04:19 -04:00
parent b8d2e33955
commit 334cd8b34a
8 changed files with 12 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

3
docs/_static/llama-stack-spec.html vendored
View file

@ -6471,7 +6471,8 @@
"additionalProperties": false, "additionalProperties": false,
"required": [ "required": [
"type", "type",
"model" "model",
"sampling_params"
], ],
"title": "ModelCandidate", "title": "ModelCandidate",
"description": "A model candidate for evaluation." "description": "A model candidate for evaluation."

1
docs/_static/llama-stack-spec.yaml vendored
View file

@ -4553,6 +4553,7 @@ components:
required: required:
- type - type
- model - model
- sampling_params
title: ModelCandidate title: ModelCandidate
description: A model candidate for evaluation. description: A model candidate for evaluation.
RegexParserScoringFnParams: RegexParserScoringFnParams:

2
llama_stack/apis/eval/eval.py
View file

@ -28,7 +28,7 @@ class ModelCandidate(BaseModel):
type: Literal["model"] = "model" type: Literal["model"] = "model"
model: str model: str
sampling_params: Optional[SamplingParams] = Field(default_factory=SamplingParams) sampling_params: SamplingParams
system_message: Optional[SystemMessage] = None system_message: Optional[SystemMessage] = None

6
llama_stack/providers/remote/eval/nvidia/config.py
View file

@ -14,10 +14,10 @@ class NVIDIAEvalConfig(BaseModel):
Configuration for the NVIDIA NeMo Evaluator microservice endpoint. Configuration for the NVIDIA NeMo Evaluator microservice endpoint.
Attributes: Attributes:
evaluator_service_url (str): A base url for accessing the NVIDIA evaluation endpoint, e.g. http://localhost:8000. evaluator_url (str): A base url for accessing the NVIDIA evaluation endpoint, e.g. http://localhost:8000.
""" """
evaluator_service_url: str = Field( evaluator_url: str = Field(
default_factory=lambda: os.getenv("NVIDIA_EVALUATOR_URL", "http://0.0.0.0:7331"), default_factory=lambda: os.getenv("NVIDIA_EVALUATOR_URL", "http://0.0.0.0:7331"),
description="The url for accessing the evaluator service", description="The url for accessing the evaluator service",
) )
@ -25,5 +25,5 @@ class NVIDIAEvalConfig(BaseModel):
@classmethod @classmethod
def sample_run_config(cls, **kwargs) -> Dict[str, Any]: def sample_run_config(cls, **kwargs) -> Dict[str, Any]:
return { return {
"evaluator_service_url": "${env.NVIDIA_EVALUATOR_URL:http://localhost:7331}", "evaluator_url": "${env.NVIDIA_EVALUATOR_URL:http://localhost:7331}",
} }

4
llama_stack/providers/remote/eval/nvidia/eval.py
View file

@ -53,13 +53,13 @@ class NVIDIAEvalImpl(
async def _evaluator_get(self, path): async def _evaluator_get(self, path):
"""Helper for making GET requests to the evaluator service.""" """Helper for making GET requests to the evaluator service."""
response = requests.get(url=f"{self.config.evaluator_service_url}{path}") response = requests.get(url=f"{self.config.evaluator_url}{path}")
response.raise_for_status() response.raise_for_status()
return response.json() return response.json()
async def _evaluator_post(self, path, data): async def _evaluator_post(self, path, data):
"""Helper for making POST requests to the evaluator service.""" """Helper for making POST requests to the evaluator service."""
response = requests.post(url=f"{self.config.evaluator_service_url}{path}", json=data) response = requests.post(url=f"{self.config.evaluator_url}{path}", json=data)
response.raise_for_status() response.raise_for_status()
return response.json() return response.json()

2
llama_stack/templates/nvidia/run-with-safety.yaml
View file

@ -56,7 +56,7 @@ providers:
- provider_id: nvidia - provider_id: nvidia
provider_type: remote::nvidia provider_type: remote::nvidia
config: config:
evaluator_service_url: ${env.NVIDIA_EVALUATOR_URL:http://localhost:7331} evaluator_url: ${env.NVIDIA_EVALUATOR_URL:http://localhost:7331}
post_training: post_training:
- provider_id: nvidia - provider_id: nvidia
provider_type: remote::nvidia provider_type: remote::nvidia

2
llama_stack/templates/nvidia/run.yaml
View file

@ -51,7 +51,7 @@ providers:
- provider_id: nvidia - provider_id: nvidia
provider_type: remote::nvidia provider_type: remote::nvidia
config: config:
evaluator_service_url: ${env.NVIDIA_EVALUATOR_URL:http://localhost:7331} evaluator_url: ${env.NVIDIA_EVALUATOR_URL:http://localhost:7331}
post_training: post_training:
- provider_id: nvidia - provider_id: nvidia
provider_type: remote::nvidia provider_type: remote::nvidia

5
tests/unit/providers/nvidia/test_eval.py
View file

@ -33,7 +33,7 @@ class TestNVIDIAEvalImpl(unittest.TestCase):
self.agents_api = MagicMock() self.agents_api = MagicMock()
self.config = NVIDIAEvalConfig( self.config = NVIDIAEvalConfig(
evaluator_service_url=os.environ["NVIDIA_EVALUATOR_URL"], evaluator_url=os.environ["NVIDIA_EVALUATOR_URL"],
) )
self.eval_impl = NVIDIAEvalImpl( self.eval_impl = NVIDIAEvalImpl(
@ -56,9 +56,6 @@ class TestNVIDIAEvalImpl(unittest.TestCase):
self.mock_evaluator_get = self.evaluator_get_patcher.start() self.mock_evaluator_get = self.evaluator_get_patcher.start()
self.mock_evaluator_post = self.evaluator_post_patcher.start() self.mock_evaluator_post = self.evaluator_post_patcher.start()
# Set up async test helper
# self.run_async = self._create_async_helper()
def tearDown(self): def tearDown(self):
"""Clean up after each test.""" """Clean up after each test."""
self.evaluator_get_patcher.stop() self.evaluator_get_patcher.stop()