phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-02 08:44:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

first draft

Browse source
This commit is contained in:
Kai Wu 2025-07-25 10:41:06 -07:00
parent 025163d8e6
commit e614241876
9 changed files with 64 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

8
docs/source/distributions/k8s/vllm-k8s.yaml.template
View file

@ -25,14 +25,16 @@ spec:
app.kubernetes.io/name: vllm
workload-type: inference
spec:
nodeSelector:
eks.amazonaws.com/nodegroup: gpu
# Removed nodeSelector for GPU nodes as they don't appear to exist in the cluster
# If you have GPU nodes with a different label, you can uncomment and modify this section
# nodeSelector:
# <your-gpu-node-label-key>: <your-gpu-node-label-value>
containers:
- name: vllm
image: vllm/vllm-openai:latest
command: ["/bin/sh", "-c"]
args:
- "vllm serve ${INFERENCE_MODEL} --dtype float16 --enforce-eager --max-model-len 4096 --gpu-memory-utilization 0.6 --enable-auto-tool-choice --tool-call-parser llama4_pythonic"
- "vllm serve ${INFERENCE_MODEL} --enforce-eager --max-model-len 8192 --gpu-memory-utilization 0.7 --enable-auto-tool-choice --tool-call-parser llama3_json --max-num-seqs 4"
env:
- name: INFERENCE_MODEL
value: "${INFERENCE_MODEL}"