Fix Dell distro and TGI inference

This commit is contained in:
Connor Hack 2025-08-08 12:29:33 -04:00
parent 9b70bb9d4b
commit ad66968909
2 changed files with 4 additions and 1 deletions

View file

@ -309,7 +309,7 @@ class TGIAdapter(_HfAdapter):
raise ValueError("You must provide a URL in run.yaml (or via the TGI_URL environment variable) to use TGI.")
log.info(f"Initializing TGI client with url={config.url}")
self.client = AsyncInferenceClient(
model=config.url,
model=config.url,provider="hf-inference"
)
endpoint_info = await self.client.get_endpoint_info()
self.max_tokens = endpoint_info["max_total_tokens"]