mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-16 06:53:47 +00:00
b76bef169c
# What does this PR do? - fixes to nvidia inference provider to account for strategy update - update nvidia templates ## Test Plan ``` llama stack run ./llama_stack/templates/nvidia/run.yaml --port 5000 LLAMA_STACK_BASE_URL="http://localhost:5000" pytest -v tests/client-sdk/inference/test_inference.py --html=report.html --self-contained-html ``` <img width="1288" alt="image" src="https://github.com/user-attachments/assets/d20f9aea-525e-47de-a5be-586e022e0d55" /> **NOTE** - vision inference broken - tool calling broken - /completion broken cc @mattf @cdgamarose-nv for improving NVIDIA inference adapter ## Sources Please link relevant resources if necessary. ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Ran pre-commit to handle lint / formatting issues. - [ ] Read the [contributor guideline](https://github.com/meta-llama/llama-stack/blob/main/CONTRIBUTING.md), Pull Request section? - [ ] Updated relevant documentation. - [ ] Wrote necessary unit or integration tests.
440 lines
8.1 KiB
JSON
440 lines
8.1 KiB
JSON
{
|
|
"hf-serverless": [
|
|
"aiohttp",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"huggingface_hub",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"together": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"together",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"vllm-gpu": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"vllm",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"remote-vllm": [
|
|
"aiosqlite",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"fireworks": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"fireworks-ai",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"tgi": [
|
|
"aiohttp",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"huggingface_hub",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"bedrock": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"boto3",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"meta-reference-gpu": [
|
|
"accelerate",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"fairscale",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"lm-format-enforcer",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentence-transformers",
|
|
"sentencepiece",
|
|
"torch",
|
|
"torchvision",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"zmq",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"nvidia": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"meta-reference-quantized-gpu": [
|
|
"accelerate",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"fairscale",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fbgemm-gpu",
|
|
"fire",
|
|
"httpx",
|
|
"lm-format-enforcer",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentence-transformers",
|
|
"sentencepiece",
|
|
"torch",
|
|
"torchao==0.5.0",
|
|
"torchvision",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"zmq",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"cerebras": [
|
|
"aiosqlite",
|
|
"blobfile",
|
|
"cerebras_cloud_sdk",
|
|
"chardet",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"ollama": [
|
|
"aiohttp",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"ollama",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"hf-endpoint": [
|
|
"aiohttp",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"huggingface_hub",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"opentelemetry-exporter-otlp-proto-http",
|
|
"opentelemetry-sdk",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"requests",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
]
|
|
}
|