mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-12-28 20:52:00 +00:00
cc77f79f55
# What does this PR do? This PR adds support for NVIDIA's NeMo Evaluator API to the Llama Stack eval module. The integration enables users to evaluate models via the Llama Stack interface. ## Test Plan [Describe the tests you ran to verify your changes with result summaries. *Provide clear instructions so the plan can be easily re-executed.*] 1. Added unit tests and successfully ran from root of project: `./scripts/unit-tests.sh tests/unit/providers/nvidia/test_eval.py` ``` tests/unit/providers/nvidia/test_eval.py::TestNVIDIAEvalImpl::test_job_cancel PASSED tests/unit/providers/nvidia/test_eval.py::TestNVIDIAEvalImpl::test_job_result PASSED tests/unit/providers/nvidia/test_eval.py::TestNVIDIAEvalImpl::test_job_status PASSED tests/unit/providers/nvidia/test_eval.py::TestNVIDIAEvalImpl::test_register_benchmark PASSED tests/unit/providers/nvidia/test_eval.py::TestNVIDIAEvalImpl::test_run_eval PASSED ``` 2. Verified I could build the Llama Stack image: `LLAMA_STACK_DIR=$(pwd) llama stack build --template nvidia --image-type venv` Documentation added to `llama_stack/providers/remote/eval/nvidia/README.md` --------- Co-authored-by: Jash Gulabrai <jgulabrai@nvidia.com>
759 lines
13 KiB
JSON
759 lines
13 KiB
JSON
{
|
|
"bedrock": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"boto3",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"langdetect",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn"
|
|
],
|
|
"cerebras": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"cerebras_cloud_sdk",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"langdetect",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch torchvision --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"ci-tests": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"fastapi",
|
|
"fire",
|
|
"fireworks-ai",
|
|
"httpx",
|
|
"langdetect",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"sqlite-vec",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch torchvision --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"dell": [
|
|
"aiohttp",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"huggingface_hub",
|
|
"langdetect",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch torchvision --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"dev": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"fastapi",
|
|
"fire",
|
|
"fireworks-ai",
|
|
"httpx",
|
|
"langdetect",
|
|
"litellm",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"sqlite-vec",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch torchvision --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"fireworks": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"fireworks-ai",
|
|
"httpx",
|
|
"langdetect",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch torchvision --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"groq": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"datasets",
|
|
"emoji",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"langdetect",
|
|
"litellm",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn"
|
|
],
|
|
"hf-endpoint": [
|
|
"aiohttp",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"huggingface_hub",
|
|
"langdetect",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn"
|
|
],
|
|
"hf-serverless": [
|
|
"aiohttp",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"huggingface_hub",
|
|
"langdetect",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch torchvision --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"meta-reference-gpu": [
|
|
"accelerate",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"fairscale",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fbgemm-gpu-genai==1.1.2",
|
|
"fire",
|
|
"httpx",
|
|
"langdetect",
|
|
"lm-format-enforcer",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentence-transformers",
|
|
"sentencepiece",
|
|
"torch",
|
|
"torchao==0.8.0",
|
|
"torchvision",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn",
|
|
"zmq"
|
|
],
|
|
"nvidia": [
|
|
"aiohttp",
|
|
"aiosqlite",
|
|
"blobfile",
|
|
"chardet",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn"
|
|
],
|
|
"ollama": [
|
|
"aiohttp",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"langdetect",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"ollama",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn"
|
|
],
|
|
"open-benchmark": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"langdetect",
|
|
"litellm",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"sqlite-vec",
|
|
"together",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn"
|
|
],
|
|
"passthrough": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"langdetect",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch torchvision --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"remote-vllm": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"langdetect",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch torchvision --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"sambanova": [
|
|
"aiosqlite",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn"
|
|
],
|
|
"tgi": [
|
|
"aiohttp",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"huggingface_hub",
|
|
"langdetect",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch torchvision --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"together": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"langdetect",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"together",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch torchvision --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"verification": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"langdetect",
|
|
"litellm",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"sqlite-vec",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch torchvision --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"vllm-gpu": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"emoji",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"langdetect",
|
|
"matplotlib",
|
|
"mcp",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pymongo",
|
|
"pypdf",
|
|
"pythainlp",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"tree_sitter",
|
|
"uvicorn",
|
|
"vllm",
|
|
"sentence-transformers --no-deps",
|
|
"torch torchvision --index-url https://download.pytorch.org/whl/cpu"
|
|
]
|
|
}
|