phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-17 23:17:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chore: update the vLLM inference impl to use OpenAIMixin for openai-compat functions

Browse source 
inference recordings from Qwen3-0.6B and vLLM 0.8.3 -
```
docker run --gpus all -v ~/.cache/huggingface:/root/.cache/huggingface -p 8000:8000 --ipc=host \
    vllm/vllm-openai:latest \
    --model Qwen/Qwen3-0.6B --enable-auto-tool-choice --tool-call-parser hermes
```

test with -

```
./scripts/integration-tests.sh --stack-config server:ci-tests --setup vllm --subdirs inference
```
This commit is contained in:
Matthew Farrellee 2025-09-10 10:10:10 -04:00
parent c86e45496e
commit c2a9c65fff
33 changed files with 51813 additions and 203 deletions
Download patch file Download diff file Expand all files Collapse all files

71
tests/integration/recordings/responses/934eabb2c7bd.json Normal file
View file

@ -0,0 +1,71 @@
{
"request": {
"method": "POST",
"url": "http://localhost:8000/v1/v1/completions",
"headers": {},
"body": {
"model": "Qwen/Qwen3-0.6B",
"prompt": "<|begin_of_text|>Michael Jordan was born in 1963. He played basketball for the Chicago Bulls. He retired in 2003.Please respond in JSON format with the schema: {\"properties\": {\"name\": {\"title\": \"Name\", \"type\": \"string\"}, \"year_born\": {\"title\": \"Year Born\", \"type\": \"string\"}, \"year_retired\": {\"title\": \"Year Retired\", \"type\": \"string\"}}, \"required\": [\"name\", \"year_born\", \"year_retired\"], \"title\": \"AnswerFormat\", \"type\": \"object\"}",
"extra_body": {
"guided_json": {
"properties": {
"name": {
"title": "Name",
"type": "string"
},
"year_born": {
"title": "Year Born",
"type": "string"
},
"year_retired": {
"title": "Year Retired",
"type": "string"
}
},
"required": [
"name",
"year_born",
"year_retired"
],
"title": "AnswerFormat",
"type": "object"
}
},
"stream": false,
"temperature": 0.0,
"max_tokens": 50
},
"endpoint": "/v1/completions",
"model": "Qwen/Qwen3-0.6B"
},
"response": {
"body": {
"__type__": "openai.types.completion.Completion",
"__data__": {
"id": "cmpl-d0abbd0dc22c41a091a2a72ce913cbed",
"choices": [
{
"finish_reason": "stop",
"index": 0,
"logprobs": null,
"text": "{ \n \"name\": \"Michael Jordan\",\n \"year_born\": \"1963\",\n \"year_retired\": \"2003\"\n}",
"stop_reason": null,
"prompt_logprobs": null
}
],
"created": 1757524134,
"model": "Qwen/Qwen3-0.6B",
"object": "text_completion",
"system_fingerprint": null,
"usage": {
"completion_tokens": 36,
"prompt_tokens": 128,
"total_tokens": 164,
"completion_tokens_details": null,
"prompt_tokens_details": null
}
}
},
"is_streaming": false
}
}