phoenix-oss/llama-stack
1
0
Fork 0
forked from phoenix-oss/llama-stack-mirror
Code Pull requests Releases Packages 2 Activity Actions

Add logs (prints :/) to dump out what URL vllm / tgi is connecting to

Browse source
This commit is contained in:
Ashwin Bharambe 2024-11-19 15:50:26 -08:00
parent 887ccc2143
commit 7bfcfe80b5
2 changed files with 2 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
llama_stack/providers/remote/inference/tgi/tgi.py
View file

@ -264,6 +264,7 @@ class _HfAdapter(Inference, ModelsProtocolPrivate):
class TGIAdapter(_HfAdapter):
async def initialize(self, config: TGIImplConfig) -> None:
print(f"Initializing TGI client with url={config.url}")
self.client = AsyncInferenceClient(model=config.url, token=config.api_token)
endpoint_info = await self.client.get_endpoint_info()
self.max_tokens = endpoint_info["max_total_tokens"]