mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-15 22:47:59 +00:00
chore: update the vLLM inference impl to use OpenAIMixin for openai-compat functions
inference recordings from Qwen3-0.6B and vLLM 0.8.3 - ``` docker run --gpus all -v ~/.cache/huggingface:/root/.cache/huggingface -p 8000:8000 --ipc=host \ vllm/vllm-openai:latest \ --model Qwen/Qwen3-0.6B --enable-auto-tool-choice --tool-call-parser hermes ``` test with - ``` ./scripts/integration-tests.sh --stack-config server:ci-tests --setup vllm --subdirs inference ```
This commit is contained in:
parent
c86e45496e
commit
c2a9c65fff
33 changed files with 51813 additions and 203 deletions
86
tests/integration/recordings/responses/6cf57d8dfcfc.json
Normal file
86
tests/integration/recordings/responses/6cf57d8dfcfc.json
Normal file
|
@ -0,0 +1,86 @@
|
|||
{
|
||||
"request": {
|
||||
"method": "POST",
|
||||
"url": "http://localhost:8000/v1/v1/completions",
|
||||
"headers": {},
|
||||
"body": {
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"prompt": "<|begin_of_text|>Complete the sentence: Micheael Jordan is born in ",
|
||||
"logprobs": 1,
|
||||
"stream": false,
|
||||
"temperature": 0.0,
|
||||
"max_tokens": 5
|
||||
},
|
||||
"endpoint": "/v1/completions",
|
||||
"model": "Qwen/Qwen3-0.6B"
|
||||
},
|
||||
"response": {
|
||||
"body": {
|
||||
"__type__": "openai.types.completion.Completion",
|
||||
"__data__": {
|
||||
"id": "cmpl-b114bec84b5a430ba94520ac9e7a6d8d",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "length",
|
||||
"index": 0,
|
||||
"logprobs": {
|
||||
"text_offset": [
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
4
|
||||
],
|
||||
"token_logprobs": [
|
||||
-0.03187580779194832,
|
||||
-0.014971747994422913,
|
||||
-0.3393988609313965,
|
||||
-1.2200194597244263,
|
||||
-0.762129545211792
|
||||
],
|
||||
"tokens": [
|
||||
"1",
|
||||
"9",
|
||||
"5",
|
||||
"8",
|
||||
","
|
||||
],
|
||||
"top_logprobs": [
|
||||
{
|
||||
"1": -0.03187580779194832
|
||||
},
|
||||
{
|
||||
"9": -0.014971747994422913
|
||||
},
|
||||
{
|
||||
"5": -0.3393988609313965
|
||||
},
|
||||
{
|
||||
"8": -1.2200194597244263
|
||||
},
|
||||
{
|
||||
",": -0.762129545211792
|
||||
}
|
||||
]
|
||||
},
|
||||
"text": "1958,",
|
||||
"stop_reason": null,
|
||||
"prompt_logprobs": null
|
||||
}
|
||||
],
|
||||
"created": 1757524133,
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"object": "text_completion",
|
||||
"system_fingerprint": null,
|
||||
"usage": {
|
||||
"completion_tokens": 5,
|
||||
"prompt_tokens": 18,
|
||||
"total_tokens": 23,
|
||||
"completion_tokens_details": null,
|
||||
"prompt_tokens_details": null
|
||||
}
|
||||
}
|
||||
},
|
||||
"is_streaming": false
|
||||
}
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue