phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-07 02:58:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add comment

Browse source
This commit is contained in:
Sixian Yi 2025-01-26 22:49:12 -08:00
parent 667c71f4e7
commit 0052089ab8
1 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
llama_stack/providers/utils/inference/openai_compat.py
View file

@ -124,7 +124,9 @@ def convert_openai_completion_logprobs(
if hasattr(logprobs, "top_logprobs"): if hasattr(logprobs, "top_logprobs"):
return [TokenLogProbs(logprobs_by_token=x) for x in logprobs.top_logprobs] return [TokenLogProbs(logprobs_by_token=x) for x in logprobs.top_logprobs]
# Together supports logprobs (top_k=1) but not top_logprobs (top_k>1). # Together supports logprobs with top_k=1 only. This means for each token position,
# they return only the logprobs for the selected token (vs. the top n most likely tokens).
# Here we construct the response by matching the selected token with the logprobs.
if logprobs.tokens and logprobs.token_logprobs: if logprobs.tokens and logprobs.token_logprobs:
return [ return [
TokenLogProbs(logprobs_by_token={token: token_lp}) TokenLogProbs(logprobs_by_token={token: token_lp})
@ -139,9 +141,9 @@ def convert_openai_completion_logprobs_stream(
if logprobs is None: if logprobs is None:
return None return None
if isinstance(logprobs, float): if isinstance(logprobs, float):
# Adapt response from Together CompletionChoicesChunk
return [TokenLogProbs(logprobs_by_token={text: logprobs})] return [TokenLogProbs(logprobs_by_token={text: logprobs})]
if hasattr(logprobs, "top_logprobs"): if hasattr(logprobs, "top_logprobs"):
# Adapt response from Together CompletionChoicesChunk
return [TokenLogProbs(logprobs_by_token=x) for x in logprobs.top_logprobs] return [TokenLogProbs(logprobs_by_token=x) for x in logprobs.top_logprobs]
return None return None