diff --git a/llama_stack/providers/inline/inference/meta_reference/inference.py b/llama_stack/providers/inline/inference/meta_reference/inference.py index 2fdc8f2d5..1e668b183 100644 --- a/llama_stack/providers/inline/inference/meta_reference/inference.py +++ b/llama_stack/providers/inline/inference/meta_reference/inference.py @@ -46,7 +46,7 @@ class MetaReferenceInferenceImpl(Inference, ModelsProtocolPrivate): self.generator = Llama.build(self.config) async def register_model(self, model: Model) -> None: - if model.identifier != self.model.descriptor(): + if model.provider_resource_id != self.model.descriptor(): raise ValueError( f"Model mismatch: {model.identifier} != {self.model.descriptor()}" ) @@ -68,7 +68,7 @@ class MetaReferenceInferenceImpl(Inference, ModelsProtocolPrivate): async def completion( self, - model: str, + model_id: str, content: InterleavedTextMedia, sampling_params: Optional[SamplingParams] = SamplingParams(), response_format: Optional[ResponseFormat] = None, @@ -79,7 +79,7 @@ class MetaReferenceInferenceImpl(Inference, ModelsProtocolPrivate): assert logprobs.top_k == 1, f"Unexpected top_k={logprobs.top_k}" request = CompletionRequest( - model=model, + model=model_id, content=content, sampling_params=sampling_params, response_format=response_format, @@ -186,7 +186,7 @@ class MetaReferenceInferenceImpl(Inference, ModelsProtocolPrivate): async def chat_completion( self, - model: str, + model_id: str, messages: List[Message], sampling_params: Optional[SamplingParams] = SamplingParams(), response_format: Optional[ResponseFormat] = None, @@ -201,7 +201,7 @@ class MetaReferenceInferenceImpl(Inference, ModelsProtocolPrivate): # wrapper request to make it easier to pass around (internal only, not exposed to API) request = ChatCompletionRequest( - model=model, + model=model_id, messages=messages, sampling_params=sampling_params, tools=tools or [], @@ -386,7 +386,7 @@ class MetaReferenceInferenceImpl(Inference, ModelsProtocolPrivate): async def embeddings( self, - model: str, + model_id: str, contents: List[InterleavedTextMedia], ) -> EmbeddingsResponse: raise NotImplementedError() diff --git a/llama_stack/providers/inline/inference/vllm/vllm.py b/llama_stack/providers/inline/inference/vllm/vllm.py index 3b1a0dd50..8869cc07f 100644 --- a/llama_stack/providers/inline/inference/vllm/vllm.py +++ b/llama_stack/providers/inline/inference/vllm/vllm.py @@ -110,7 +110,7 @@ class VLLMInferenceImpl(Inference, ModelsProtocolPrivate): async def completion( self, - model: str, + model_id: str, content: InterleavedTextMedia, sampling_params: Optional[SamplingParams] = SamplingParams(), response_format: Optional[ResponseFormat] = None, @@ -120,7 +120,7 @@ class VLLMInferenceImpl(Inference, ModelsProtocolPrivate): log.info("vLLM completion") messages = [UserMessage(content=content)] return self.chat_completion( - model=model, + model=model_id, messages=messages, sampling_params=sampling_params, stream=stream, @@ -129,7 +129,7 @@ class VLLMInferenceImpl(Inference, ModelsProtocolPrivate): async def chat_completion( self, - model: str, + model_id: str, messages: List[Message], sampling_params: Optional[SamplingParams] = SamplingParams(), tools: Optional[List[ToolDefinition]] = None, @@ -144,7 +144,7 @@ class VLLMInferenceImpl(Inference, ModelsProtocolPrivate): assert self.engine is not None request = ChatCompletionRequest( - model=model, + model=model_id, messages=messages, sampling_params=sampling_params, tools=tools or [], @@ -215,7 +215,7 @@ class VLLMInferenceImpl(Inference, ModelsProtocolPrivate): yield chunk async def embeddings( - self, model: str, contents: list[InterleavedTextMedia] + self, model_id: str, contents: list[InterleavedTextMedia] ) -> EmbeddingsResponse: log.info("vLLM embeddings") # TODO diff --git a/llama_stack/providers/remote/inference/ollama/ollama.py b/llama_stack/providers/remote/inference/ollama/ollama.py index 938d05c08..f5750e0cf 100644 --- a/llama_stack/providers/remote/inference/ollama/ollama.py +++ b/llama_stack/providers/remote/inference/ollama/ollama.py @@ -66,8 +66,10 @@ class OllamaInferenceAdapter(Inference, ModelsProtocolPrivate): pass async def register_model(self, model: Model) -> None: - if model.identifier not in OLLAMA_SUPPORTED_MODELS: - raise ValueError(f"Model {model.identifier} is not supported by Ollama") + if model.provider_resource_id not in OLLAMA_SUPPORTED_MODELS: + raise ValueError( + f"Model {model.provider_resource_id} is not supported by Ollama" + ) async def list_models(self) -> List[Model]: ollama_to_llama = {v: k for k, v in OLLAMA_SUPPORTED_MODELS.items()} @@ -94,7 +96,7 @@ class OllamaInferenceAdapter(Inference, ModelsProtocolPrivate): async def completion( self, - model: str, + model_id: str, content: InterleavedTextMedia, sampling_params: Optional[SamplingParams] = SamplingParams(), response_format: Optional[ResponseFormat] = None, @@ -102,7 +104,7 @@ class OllamaInferenceAdapter(Inference, ModelsProtocolPrivate): logprobs: Optional[LogProbConfig] = None, ) -> AsyncGenerator: request = CompletionRequest( - model=model, + model=model_id, content=content, sampling_params=sampling_params, stream=stream, @@ -148,7 +150,7 @@ class OllamaInferenceAdapter(Inference, ModelsProtocolPrivate): async def chat_completion( self, - model: str, + model_id: str, messages: List[Message], sampling_params: Optional[SamplingParams] = SamplingParams(), response_format: Optional[ResponseFormat] = None, @@ -159,7 +161,7 @@ class OllamaInferenceAdapter(Inference, ModelsProtocolPrivate): logprobs: Optional[LogProbConfig] = None, ) -> AsyncGenerator: request = ChatCompletionRequest( - model=model, + model=model_id, messages=messages, sampling_params=sampling_params, tools=tools or [], @@ -271,7 +273,7 @@ class OllamaInferenceAdapter(Inference, ModelsProtocolPrivate): async def embeddings( self, - model: str, + model_id: str, contents: List[InterleavedTextMedia], ) -> EmbeddingsResponse: raise NotImplementedError() diff --git a/llama_stack/providers/remote/inference/vllm/vllm.py b/llama_stack/providers/remote/inference/vllm/vllm.py index bd7f5073c..3a8b8c326 100644 --- a/llama_stack/providers/remote/inference/vllm/vllm.py +++ b/llama_stack/providers/remote/inference/vllm/vllm.py @@ -45,8 +45,15 @@ class VLLMInferenceAdapter(Inference, ModelsProtocolPrivate): self.client = OpenAI(base_url=self.config.url, api_key=self.config.api_token) async def register_model(self, model: Model) -> None: - for running_model in self.client.models.list(): - repo = running_model.id + pass + + async def shutdown(self) -> None: + pass + + async def list_models(self) -> List[Model]: + models = [] + for model in self.client.models.list(): + repo = model.id if repo not in self.huggingface_repo_to_llama_model_id: print(f"Unknown model served by vllm: {repo}") continue @@ -67,7 +74,7 @@ class VLLMInferenceAdapter(Inference, ModelsProtocolPrivate): async def completion( self, - model: str, + model_id: str, content: InterleavedTextMedia, sampling_params: Optional[SamplingParams] = SamplingParams(), response_format: Optional[ResponseFormat] = None, @@ -78,7 +85,7 @@ class VLLMInferenceAdapter(Inference, ModelsProtocolPrivate): async def chat_completion( self, - model: str, + model_id: str, messages: List[Message], sampling_params: Optional[SamplingParams] = SamplingParams(), response_format: Optional[ResponseFormat] = None, @@ -89,7 +96,7 @@ class VLLMInferenceAdapter(Inference, ModelsProtocolPrivate): logprobs: Optional[LogProbConfig] = None, ) -> AsyncGenerator: request = ChatCompletionRequest( - model=model, + model=model_id, messages=messages, sampling_params=sampling_params, tools=tools or [], @@ -173,7 +180,7 @@ class VLLMInferenceAdapter(Inference, ModelsProtocolPrivate): async def embeddings( self, - model: str, + model_id: str, contents: List[InterleavedTextMedia], ) -> EmbeddingsResponse: raise NotImplementedError() diff --git a/llama_stack/providers/tests/inference/fixtures.py b/llama_stack/providers/tests/inference/fixtures.py index 2a4eba3ad..59bd492b9 100644 --- a/llama_stack/providers/tests/inference/fixtures.py +++ b/llama_stack/providers/tests/inference/fixtures.py @@ -49,7 +49,7 @@ def inference_meta_reference(inference_model) -> ProviderFixture: providers=[ Provider( provider_id=f"meta-reference-{i}", - provider_type="meta-reference", + provider_type="inline::meta-reference", config=MetaReferenceInferenceConfig( model=m, max_seq_len=4096,