phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-03 12:52:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

update the code with aysnc iterator as suggested by Ben

Browse source
This commit is contained in:
Sumit Jaiswal 2025-06-02 23:49:08 +05:30
parent b413c7562b
commit 3840ef7a98
No known key found for this signature in database
GPG key ID: A4604B39D64D6AEC
3 changed files with 22 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

2
llama_stack/distribution/routers/inference.py
View file

@ -602,7 +602,7 @@ class InferenceRouter(Inference):
async def health(self) -> dict[str, HealthResponse]:
health_statuses = {}
timeout = 0.5
timeout = 1 # increasing the timeout to 1 second for health checks
for provider_id, impl in self.routing_table.impls_by_provider_id.items():
try:
# check if the provider has a health method

6
llama_stack/providers/remote/inference/vllm/vllm.py
View file

@ -313,10 +313,10 @@ class VLLMInferenceAdapter(Inference, ModelsProtocolPrivate):
"""
try:
client = self._create_client() if self.client is None else self.client
client.models.list() # Ensure the client is initialized
_ = [m async for m in client.models.list()] # Ensure the client is initialized
return HealthResponse(status=HealthStatus.OK)
except Exception as ex:
return HealthResponse(status=HealthStatus.ERROR, message=f"Health check failed: {str(ex)}")
except Exception as e:
return HealthResponse(status=HealthStatus.ERROR, message=f"Health check failed: {str(e)}")
async def _get_model(self, model_id: str) -> Model:
if not self.model_store: