phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-12 04:50:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Sync with latest vLLM APIs

Browse source
This commit is contained in:
Fred Reiss 2025-02-10 17:09:19 -08:00 • committed by Ashwin Bharambe
parent c08ca19d79
commit 7d062550c9
1 changed files with 11 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

20
llama_stack/providers/inline/inference/vllm/vllm.py
View file

@ -31,7 +31,7 @@ from llama_models.llama3.api.tokenizer import Tokenizer
from vllm.engine.arg_utils import AsyncEngineArgs from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.async_llm_engine import AsyncLLMEngine from vllm.engine.async_llm_engine import AsyncLLMEngine
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
from vllm.entrypoints.openai.serving_engine import BaseModelPath from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
from llama_stack.apis.common.content_types import ( from llama_stack.apis.common.content_types import (
InterleavedContent, InterleavedContent,
@ -323,15 +323,17 @@ class VLLMInferenceImpl(Inference, ModelsProtocolPrivate):
self.chat = OpenAIServingChat( self.chat = OpenAIServingChat(
engine_client=self.engine, engine_client=self.engine,
model_config=model_config, model_config=model_config,
base_model_paths=[ models=OpenAIServingModels(
# The layer below us will only see resolved model IDs engine_client=self.engine,
BaseModelPath(resolved_model_id, resolved_model_id) model_config=model_config,
], base_model_paths=[
# The layer below us will only see resolved model IDs
BaseModelPath(resolved_model_id, resolved_model_id)
],
),
response_role="assistant", response_role="assistant",
lora_modules=None, request_logger=None, # Use default logging
prompt_adapters=None, chat_template=None, # Use default template from model checkpoint
request_logger=None,
chat_template=None,
enable_auto_tools=True, enable_auto_tools=True,
tool_parser=tool_parser, tool_parser=tool_parser,
chat_template_content_format="auto", chat_template_content_format="auto",