phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-29 15:23:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

error handling

Browse source
This commit is contained in:
Xi Yan 2024-10-07 17:40:43 -07:00
parent 8a67e7a2bd
commit f1d31fe9b5
2 changed files with 6 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
llama_stack/distribution/routers/routers.py
View file

@ -101,7 +101,7 @@ class InferenceRouter(Inference):
tool_choice: Optional[ToolChoice] = ToolChoice.auto,
tool_prompt_format: Optional[ToolPromptFormat] = ToolPromptFormat.json,
stream: Optional[bool] = False,
logprobs: Optional[LogProbConfig] = False,
logprobs: Optional[LogProbConfig] = None,
) -> AsyncGenerator:
params = dict(
model=model,
@ -125,7 +125,7 @@ class InferenceRouter(Inference):
content: InterleavedTextMedia,
sampling_params: Optional[SamplingParams] = SamplingParams(),
stream: Optional[bool] = False,
logprobs: Optional[LogProbConfig] = False,
logprobs: Optional[LogProbConfig] = None,
) -> Union[CompletionResponse, CompletionResponseStreamChunk]:
return await self.routing_table.get_provider_impl(model).completion(
model=model,

4
llama_stack/providers/impls/meta_reference/inference/inference.py
View file

@ -135,6 +135,10 @@ class MetaReferenceInferenceImpl(Inference, RoutableProvider):
assert (
len(token_result.logprobs) == 1
), "Expected logprob to contain 1 result for the current token"
assert (
logprobs.top_k == 1
), "Only top_k=1 is supported for LogProbConfig"
logprobs.append(
TokenLogProbs(
logprobs_by_token={