mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-12-22 17:56:15 +00:00
7301403ce3
# What does this PR do? - add /eval, /scoring, /datasetio API providers to distribution templates - regenerate build.yaml / run.yaml files - fix `template.py` to take in list of providers instead of only first one - override memory provider as faiss default for all distro (as only 1 memory provider is needed to start basic flow, chromadb/pgvector need additional setup step). ``` python llama_stack/scripts/distro_codegen.py ``` - updated README to start UI via conda builds. ## Test Plan ``` python llama_stack/scripts/distro_codegen.py ``` - Use newly generated `run.yaml` to start server ``` llama stack run ./llama_stack/templates/together/run.yaml ``` <img width="1191" alt="image" src="https://github.com/user-attachments/assets/62f7d179-0cd0-427c-b6e8-e087d4648f09"> #### Registration ``` ❯ llama-stack-client datasets register \ --dataset-id "mmlu" \ --provider-id "huggingface" \ --url "https://huggingface.co/datasets/llamastack/evals" \ --metadata '{"path": "llamastack/evals", "name": "evals__mmlu__details", "split": "train"}' \ --schema '{"input_query": {"type": "string"}, "expected_answer": {"type": "string", "chat_completion_input": {"type": "string"}}}' ❯ llama-stack-client datasets list ┏━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ identifier ┃ provider_id ┃ metadata ┃ type ┃ ┡━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━┩ │ mmlu │ huggingface │ {'path': 'llamastack/evals', 'name': │ dataset │ │ │ │ 'evals__mmlu__details', 'split': │ │ │ │ │ 'train'} │ │ └────────────┴─────────────┴─────────────────────────────────────────┴─────────┘ ``` ``` ❯ llama-stack-client datasets register \ --dataset-id "simpleqa" \ --provider-id "huggingface" \ --url "https://huggingface.co/datasets/llamastack/evals" \ --metadata '{"path": "llamastack/evals", "name": "evals__simpleqa", "split": "train"}' \ --schema '{"input_query": {"type": "string"}, "expected_answer": {"type": "string", "chat_completion_input": {"type": "string"}}}' ❯ llama-stack-client datasets list ┏━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ identifier ┃ provider_id ┃ metadata ┃ type ┃ ┡━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━┩ │ mmlu │ huggingface │ {'path': 'llamastack/evals', 'name': 'evals__mmlu__details', │ dataset │ │ │ │ 'split': 'train'} │ │ │ simpleqa │ huggingface │ {'path': 'llamastack/evals', 'name': 'evals__simpleqa', │ dataset │ │ │ │ 'split': 'train'} │ │ └────────────┴─────────────┴───────────────────────────────────────────────────────────────┴─────────┘ ``` ``` ❯ llama-stack-client eval_tasks register \ > --eval-task-id meta-reference-mmlu \ > --provider-id meta-reference \ > --dataset-id mmlu \ > --scoring-functions basic::regex_parser_multiple_choice_answer ❯ llama-stack-client eval_tasks register \ --eval-task-id meta-reference-simpleqa \ --provider-id meta-reference \ --dataset-id simpleqa \ --scoring-functions llm-as-judge::405b-simpleqa ❯ llama-stack-client eval_tasks list ┏━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┓ ┃ dataset_id ┃ identifier ┃ metadata ┃ provider_id ┃ provider_resour… ┃ scoring_functio… ┃ type ┃ ┡━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━┩ │ mmlu │ meta-reference-… │ {} │ meta-reference │ meta-reference-… │ ['basic::regex_… │ eval_task │ │ simpleqa │ meta-reference-… │ {} │ meta-reference │ meta-reference-… │ ['llm-as-judge:… │ eval_task │ └────────────┴──────────────────┴──────────┴────────────────┴──────────────────┴──────────────────┴───────────┘ ``` #### Test with UI ``` streamlit run app.py ``` ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Ran pre-commit to handle lint / formatting issues. - [ ] Read the [contributor guideline](https://github.com/meta-llama/llama-stack/blob/main/CONTRIBUTING.md), Pull Request section? - [ ] Updated relevant documentation. - [ ] Wrote necessary unit or integration tests.
371 lines
6.5 KiB
JSON
371 lines
6.5 KiB
JSON
{
|
|
"hf-serverless": [
|
|
"aiohttp",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"huggingface_hub",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"together": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"together",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"vllm-gpu": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"vllm",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"remote-vllm": [
|
|
"aiosqlite",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"fireworks": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"fireworks-ai",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"tgi": [
|
|
"aiohttp",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"huggingface_hub",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"bedrock": [
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"boto3",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"meta-reference-gpu": [
|
|
"accelerate",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"fairscale",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"lm-format-enforcer",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"torch",
|
|
"torchvision",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"zmq",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"meta-reference-quantized-gpu": [
|
|
"accelerate",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"fairscale",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fbgemm-gpu",
|
|
"fire",
|
|
"httpx",
|
|
"lm-format-enforcer",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"torch",
|
|
"torchao==0.5.0",
|
|
"torchvision",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"zmq",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"cerebras": [
|
|
"aiosqlite",
|
|
"blobfile",
|
|
"cerebras_cloud_sdk",
|
|
"chardet",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"ollama": [
|
|
"aiohttp",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"ollama",
|
|
"openai",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
],
|
|
"hf-endpoint": [
|
|
"aiohttp",
|
|
"aiosqlite",
|
|
"autoevals",
|
|
"blobfile",
|
|
"chardet",
|
|
"chromadb-client",
|
|
"datasets",
|
|
"faiss-cpu",
|
|
"fastapi",
|
|
"fire",
|
|
"httpx",
|
|
"huggingface_hub",
|
|
"matplotlib",
|
|
"nltk",
|
|
"numpy",
|
|
"openai",
|
|
"pandas",
|
|
"pillow",
|
|
"psycopg2-binary",
|
|
"pypdf",
|
|
"redis",
|
|
"scikit-learn",
|
|
"scipy",
|
|
"sentencepiece",
|
|
"tqdm",
|
|
"transformers",
|
|
"uvicorn",
|
|
"sentence-transformers --no-deps",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu"
|
|
]
|
|
}
|