phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-17 20:02:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

benchmark, local test, ttft, duration

Browse source 
# What does this PR do?


## Test Plan
# What does this PR do?


## Test Plan
# What does this PR do?


## Test Plan
# What does this PR do?


## Test Plan
This commit is contained in:
Eric Huang 2025-08-15 10:54:20 -07:00
parent f66ae3b3b1
commit 244ff9efbd
13 changed files with 633 additions and 328 deletions
Download patch file Download diff file Expand all files Collapse all files

6
docs/source/distributions/k8s-benchmark/openai-mock-server.py
View file

@ -23,7 +23,7 @@ app = Flask(__name__)
# Models from environment variables
def get_models():
models_str = os.getenv("MOCK_MODELS", "mock-inference")
models_str = os.getenv("MOCK_MODELS", "meta-llama/Llama-3.2-3B-Instruct")
model_ids = [m.strip() for m in models_str.split(",") if m.strip()]
return {
@ -49,13 +49,13 @@ def generate_random_text(length=50):
]
return " ".join(random.choices(words, k=length))
@app.route('/models', methods=['GET'])
@app.route('/v1/models', methods=['GET'])
def list_models():
models = get_models()
print(f"[MOCK] Returning models: {[m['id'] for m in models['data']]}")
return jsonify(models)
@app.route('/chat/completions', methods=['POST'])
@app.route('/v1/chat/completions', methods=['POST'])
def chat_completions():
"""Return OpenAI-formatted chat completion responses."""
data = request.get_json()