mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-15 14:43:48 +00:00
chore: update the vLLM inference impl to use OpenAIMixin for openai-compat functions
inference recordings from Qwen3-0.6B and vLLM 0.8.3 - ``` docker run --gpus all -v ~/.cache/huggingface:/root/.cache/huggingface -p 8000:8000 --ipc=host \ vllm/vllm-openai:latest \ --model Qwen/Qwen3-0.6B --enable-auto-tool-choice --tool-call-parser hermes ``` test with - ``` ./scripts/integration-tests.sh --stack-config server:ci-tests --setup vllm --subdirs inference ```
This commit is contained in:
parent
c86e45496e
commit
c2a9c65fff
33 changed files with 51813 additions and 203 deletions
85
tests/integration/recordings/responses/fded9afd616c.json
Normal file
85
tests/integration/recordings/responses/fded9afd616c.json
Normal file
|
@ -0,0 +1,85 @@
|
|||
{
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "http://localhost:8000/v1/v1/completions",
|
||||
"headers": {},
|
||||
"body": {
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"prompt": "Hello, world!",
|
||||
"stream": false,
|
||||
"extra_body": {
|
||||
"prompt_logprobs": 1
|
||||
}
|
||||
},
|
||||
"endpoint": "/v1/completions",
|
||||
"model": "Qwen/Qwen3-0.6B"
|
||||
},
|
||||
"response": {
|
||||
"body": {
|
||||
"__type__": "openai.types.completion.Completion",
|
||||
"__data__": {
|
||||
"id": "cmpl-df4fc1ac600a43778b6ad1e3781a4129",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "length",
|
||||
"index": 0,
|
||||
"logprobs": null,
|
||||
"text": " I'm a 14-year-old girl with a passion for science and tech",
|
||||
"stop_reason": null,
|
||||
"prompt_logprobs": [
|
||||
null,
|
||||
{
|
||||
"11": {
|
||||
"logprob": -8.8668212890625,
|
||||
"rank": 1031,
|
||||
"decoded_token": ","
|
||||
},
|
||||
"21806": {
|
||||
"logprob": -4.8355712890625,
|
||||
"rank": 1,
|
||||
"decoded_token": "\u0120Answer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"1879": {
|
||||
"logprob": -4.7821879386901855,
|
||||
"rank": 10,
|
||||
"decoded_token": "\u0120world"
|
||||
},
|
||||
"358": {
|
||||
"logprob": -0.9071879386901855,
|
||||
"rank": 1,
|
||||
"decoded_token": "\u0120I"
|
||||
}
|
||||
},
|
||||
{
|
||||
"0": {
|
||||
"logprob": -1.6575366258621216,
|
||||
"rank": 2,
|
||||
"decoded_token": "!"
|
||||
},
|
||||
"13": {
|
||||
"logprob": -1.4700366258621216,
|
||||
"rank": 1,
|
||||
"decoded_token": "."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"created": 1757524113,
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"object": "text_completion",
|
||||
"system_fingerprint": null,
|
||||
"usage": {
|
||||
"completion_tokens": 16,
|
||||
"prompt_tokens": 4,
|
||||
"total_tokens": 20,
|
||||
"completion_tokens_details": null,
|
||||
"prompt_tokens_details": null
|
||||
}
|
||||
}
|
||||
},
|
||||
"is_streaming": false
|
||||
}
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue