phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-17 18:22:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

indicate to mypy that InferenceProvider.batch_completion/batch_chat_completion is concrete

Browse source
This commit is contained in:
Matthew Farrellee 2025-08-22 15:22:20 -04:00
parent 2ee898cc4c
commit 5f0d3d473e
5 changed files with 2 additions and 89 deletions
Download patch file Download diff file Expand all files Collapse all files

22
llama_stack/providers/remote/inference/vllm/vllm.py
View file

@ -711,25 +711,3 @@ class VLLMInferenceAdapter(Inference, ModelsProtocolPrivate):
user=user,
)
return await self.client.chat.completions.create(**params) # type: ignore
async def batch_completion(
self,
model_id: str,
content_batch: list[InterleavedContent],
sampling_params: SamplingParams | None = None,
response_format: ResponseFormat | None = None,
logprobs: LogProbConfig | None = None,
):
raise NotImplementedError("Batch completion is not supported for Ollama")
async def batch_chat_completion(
self,
model_id: str,
messages_batch: list[list[Message]],
sampling_params: SamplingParams | None = None,
tools: list[ToolDefinition] | None = None,
tool_config: ToolConfig | None = None,
response_format: ResponseFormat | None = None,
logprobs: LogProbConfig | None = None,
):
raise NotImplementedError("Batch chat completion is not supported for vLLM")