phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-03 09:21:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: Update NVIDIA Eval README

Browse source
This commit is contained in:
Jash Gulabrai 2025-04-07 10:30:08 -04:00
parent 4317a0ddcc
commit f939117dbf
1 changed files with 39 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

72
llama_stack/providers/remote/eval/nvidia/README.md
View file

@ -18,9 +18,7 @@ POST /eval/benchmarks
"dataset_id": "",
"scoring_functions": [],
"metadata": {
"config": {
"type": "mmlu"
}
"type": "mmlu"
}
}
```
@ -36,31 +34,29 @@ POST /eval/benchmarks
"dataset_id": "",
"scoring_functions": [],
"metadata": {
"config": {
"type": "custom",
"params": {
"parallelism": 8
},
"tasks": {
"qa": {
"type": "completion",
"params": {
"template": {
"prompt": "{{prompt}}",
"max_tokens": 200
}
},
"dataset": {
"files_url": "hf://datasets/default/sample-basic-test/testing/testing.jsonl"
},
"metrics": {
"bleu": {
"type": "bleu",
"params": {
"references": [
"{{ideal_response}}"
]
}
"type": "custom",
"params": {
"parallelism": 8
},
"tasks": {
"qa": {
"type": "completion",
"params": {
"template": {
"prompt": "{{prompt}}",
"max_tokens": 200
}
},
"dataset": {
"files_url": "hf://datasets/default/sample-basic-test/testing/testing.jsonl"
},
"metrics": {
"bleu": {
"type": "bleu",
"params": {
"references": [
"{{ideal_response}}"
]
}
}
}
@ -78,12 +74,16 @@ POST /eval/benchmarks/{benchmark_id}/jobs
```json
{
"benchmark_id": "my-custom-benchmark",
"task_config": {
"benchmark_config": {
"eval_candidate": {
"type": "model",
"model": "meta/llama-3.1-8b-instruct"
"model": "meta/llama-3.1-8b-instruct",
"sampling_params": {
"max_tokens": 100,
"temperature": 0.7
}
},
"scoring_params": []
"scoring_params": {}
}
}
```
@ -91,7 +91,8 @@ POST /eval/benchmarks/{benchmark_id}/jobs
Response example:
```json
{
"job_id": "1234"
"job_id": "1234",
"status": "in_progress"
}
```
@ -100,9 +101,14 @@ Response example:
GET /eval/benchmarks/{benchmark_id}/jobs/{job_id}
```
### Example for cancelling a job
```
POST /eval/benchmarks/{benchmark_id}/jobs/{job_id}/cancel
```
### Example for getting the results
```
GET /eval/benchmarks/{benchmark_id}/result
GET /eval/benchmarks/{benchmark_id}/results
```
```json
{