phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-01 16:24:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

return model in vllm

Browse source
This commit is contained in:
Dinesh Yeduguru 2024-11-13 12:20:08 -08:00
parent e272f8aa62
commit 3b68e6cbbe
1 changed files with 3 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

4
llama_stack/providers/remote/inference/vllm/vllm.py
View file

@ -131,7 +131,8 @@ class VLLMInferenceAdapter(Inference, ModelRegistryHelper, ModelsProtocolPrivate
): ):
yield chunk yield chunk
async def register_model(self, model: Model) -> None: async def register_model(self, model: Model) -> Model:
print(f"model: {model}")
model = await super().register_model(model) model = await super().register_model(model)
res = self.client.models.list() res = self.client.models.list()
available_models = [m.id for m in res] available_models = [m.id for m in res]
@ -139,6 +140,7 @@ class VLLMInferenceAdapter(Inference, ModelRegistryHelper, ModelsProtocolPrivate
raise ValueError( raise ValueError(
f"Model {model.provider_resource_id} is not being served by vLLM" f"Model {model.provider_resource_id} is not being served by vLLM"
) )
return model
async def _get_params( async def _get_params(
self, request: Union[ChatCompletionRequest, CompletionRequest] self, request: Union[ChatCompletionRequest, CompletionRequest]