phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-03 19:57:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge 2b54b57145 into 188a56af5c

Browse source
This commit is contained in:
Akram Ben Aissi 2025-10-03 19:32:38 +00:00 • committed by GitHub
commit d178e29753
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 13 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

24
llama_stack/providers/remote/inference/vllm/vllm.py
View file

@ -9,7 +9,6 @@ from typing import Any
from urllib.parse import urljoin from urllib.parse import urljoin
import httpx import httpx
from openai import APIConnectionError
from openai.types.chat.chat_completion_chunk import ( from openai.types.chat.chat_completion_chunk import (
ChatCompletionChunk as OpenAIChatCompletionChunk, ChatCompletionChunk as OpenAIChatCompletionChunk,
) )
@ -339,16 +338,19 @@ class VLLMInferenceAdapter(OpenAIMixin, LiteLLMOpenAIMixin, Inference, ModelsPro
pass # Ignore statically unknown model, will check live listing pass # Ignore statically unknown model, will check live listing
try: try:
res = self.client.models.list() res = self.client.models.list()
except APIConnectionError as e: available_models = [m.id async for m in res]
raise ValueError( if model.provider_resource_id not in available_models:
f"Failed to connect to vLLM at {self.config.url}. Please check if vLLM is running and accessible at that URL." raise ValueError(
) from e f"Model {model.provider_resource_id} is not being served by vLLM. "
available_models = [m.id async for m in res] f"Available models: {', '.join(available_models)}"
if model.provider_resource_id not in available_models: )
raise ValueError( except Exception as e:
f"Model {model.provider_resource_id} is not being served by vLLM. " if self.config.refresh_models:
f"Available models: {', '.join(available_models)}" raise ValueError(f"Model verification failed: {e}") from e
) # if refresh_models is false, gracefully continue without verification
log.warning(f"Model verification failed for model {model.model_id} with error {e}")
log.warning("Continuing without live check (refresh_models=false).")
return model return model
async def _get_params(self, request: ChatCompletionRequest) -> dict: async def _get_params(self, request: ChatCompletionRequest) -> dict: