From 6ec9eabbeb76fd0938c47aa72753b87950e611b8 Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Thu, 19 Dec 2024 11:35:25 -0800 Subject: [PATCH] Redo code change after merge --- llama_stack/providers/inline/inference/vllm/vllm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llama_stack/providers/inline/inference/vllm/vllm.py b/llama_stack/providers/inline/inference/vllm/vllm.py index ac38e13b5..12c6c0370 100644 --- a/llama_stack/providers/inline/inference/vllm/vllm.py +++ b/llama_stack/providers/inline/inference/vllm/vllm.py @@ -169,7 +169,8 @@ class VLLMInferenceImpl(Inference, ModelsProtocolPrivate): log.info("Sampling params: %s", sampling_params) request_id = _random_uuid() - prompt = await chat_completion_request_to_prompt(request, self.formatter) + prompt = await chat_completion_request_to_prompt(request, self.config.model, + self.formatter) vllm_sampling_params = self._sampling_params(request.sampling_params) results_generator = self.engine.generate( prompt, vllm_sampling_params, request_id