mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-04 04:04:14 +00:00
bcc7f2c7d0
# What does this PR do? ## Test Plan ``` cd /docs/source/distributions/k8s-benchmark # start mock server python openai-mock-server.py --port 8000 # start stack server uv run --with llama-stack python -m llama_stack.core.server.server docs/source/distributions/k8s-benchmark/stack_run_config.yaml # run benchmark script uv run python3 benchmark.py --duration 30 --concurrent 50 --base-url=http://localhost:8321/v1/openai/v1 --model=vllm-inference/meta-llama/Llama-3.2-3B-Instruct ``` Before: ============================================================ BENCHMARK RESULTS ============================================================ Total time: 30.00s Concurrent users: 50 Total requests: 1267 Successful requests: 1267 Failed requests: 0 Success rate: 100.0% Requests per second: 42.23 After: ============================================================ BENCHMARK RESULTS ============================================================ Total time: 30.00s Concurrent users: 50 Total requests: 1449 Successful requests: 1449 Failed requests: 0 Success rate: 100.0% Requests per second: 48.30
116 lines
3.3 KiB
YAML
116 lines
3.3 KiB
YAML
version: '2'
|
|
image_name: kubernetes-benchmark-demo
|
|
apis:
|
|
- agents
|
|
- inference
|
|
- safety
|
|
- telemetry
|
|
- tool_runtime
|
|
- vector_io
|
|
providers:
|
|
inference:
|
|
- provider_id: vllm-inference
|
|
provider_type: remote::vllm
|
|
config:
|
|
url: ${env.VLLM_URL:=http://localhost:8000/v1}
|
|
max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
|
|
api_token: ${env.VLLM_API_TOKEN:=fake}
|
|
tls_verify: ${env.VLLM_TLS_VERIFY:=true}
|
|
- provider_id: sentence-transformers
|
|
provider_type: inline::sentence-transformers
|
|
config: {}
|
|
vector_io:
|
|
- provider_id: ${env.ENABLE_CHROMADB:+chromadb}
|
|
provider_type: remote::chromadb
|
|
config:
|
|
url: ${env.CHROMADB_URL:=}
|
|
kvstore:
|
|
type: postgres
|
|
host: ${env.POSTGRES_HOST:=localhost}
|
|
port: ${env.POSTGRES_PORT:=5432}
|
|
db: ${env.POSTGRES_DB:=llamastack}
|
|
user: ${env.POSTGRES_USER:=llamastack}
|
|
password: ${env.POSTGRES_PASSWORD:=llamastack}
|
|
safety:
|
|
- provider_id: llama-guard
|
|
provider_type: inline::llama-guard
|
|
config:
|
|
excluded_categories: []
|
|
agents:
|
|
- provider_id: meta-reference
|
|
provider_type: inline::meta-reference
|
|
config:
|
|
persistence_store:
|
|
type: postgres
|
|
host: ${env.POSTGRES_HOST:=localhost}
|
|
port: ${env.POSTGRES_PORT:=5432}
|
|
db: ${env.POSTGRES_DB:=llamastack}
|
|
user: ${env.POSTGRES_USER:=llamastack}
|
|
password: ${env.POSTGRES_PASSWORD:=llamastack}
|
|
responses_store:
|
|
type: postgres
|
|
host: ${env.POSTGRES_HOST:=localhost}
|
|
port: ${env.POSTGRES_PORT:=5432}
|
|
db: ${env.POSTGRES_DB:=llamastack}
|
|
user: ${env.POSTGRES_USER:=llamastack}
|
|
password: ${env.POSTGRES_PASSWORD:=llamastack}
|
|
telemetry:
|
|
- provider_id: meta-reference
|
|
provider_type: inline::meta-reference
|
|
config:
|
|
service_name: "${env.OTEL_SERVICE_NAME:=\u200B}"
|
|
sinks: ${env.TELEMETRY_SINKS:=console}
|
|
tool_runtime:
|
|
- provider_id: brave-search
|
|
provider_type: remote::brave-search
|
|
config:
|
|
api_key: ${env.BRAVE_SEARCH_API_KEY:+}
|
|
max_results: 3
|
|
- provider_id: tavily-search
|
|
provider_type: remote::tavily-search
|
|
config:
|
|
api_key: ${env.TAVILY_SEARCH_API_KEY:+}
|
|
max_results: 3
|
|
- provider_id: rag-runtime
|
|
provider_type: inline::rag-runtime
|
|
config: {}
|
|
- provider_id: model-context-protocol
|
|
provider_type: remote::model-context-protocol
|
|
config: {}
|
|
metadata_store:
|
|
type: postgres
|
|
host: ${env.POSTGRES_HOST:=localhost}
|
|
port: ${env.POSTGRES_PORT:=5432}
|
|
db: ${env.POSTGRES_DB:=llamastack}
|
|
user: ${env.POSTGRES_USER:=llamastack}
|
|
password: ${env.POSTGRES_PASSWORD:=llamastack}
|
|
table_name: llamastack_kvstore
|
|
inference_store:
|
|
type: postgres
|
|
host: ${env.POSTGRES_HOST:=localhost}
|
|
port: ${env.POSTGRES_PORT:=5432}
|
|
db: ${env.POSTGRES_DB:=llamastack}
|
|
user: ${env.POSTGRES_USER:=llamastack}
|
|
password: ${env.POSTGRES_PASSWORD:=llamastack}
|
|
models:
|
|
- metadata:
|
|
embedding_dimension: 384
|
|
model_id: all-MiniLM-L6-v2
|
|
provider_id: sentence-transformers
|
|
model_type: embedding
|
|
- model_id: ${env.INFERENCE_MODEL}
|
|
provider_id: vllm-inference
|
|
model_type: llm
|
|
shields:
|
|
- shield_id: ${env.SAFETY_MODEL:=meta-llama/Llama-Guard-3-1B}
|
|
vector_dbs: []
|
|
datasets: []
|
|
scoring_fns: []
|
|
benchmarks: []
|
|
tool_groups:
|
|
- toolgroup_id: builtin::websearch
|
|
provider_id: tavily-search
|
|
- toolgroup_id: builtin::rag
|
|
provider_id: rag-runtime
|
|
server:
|
|
port: 8323
|