phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

(fix) hf don't fail when logprob is None

Browse source
This commit is contained in:
ishaan-jaff 2023-11-06 14:22:09 -08:00
parent 8d9e2574cf
commit 3c67de7f04
1 changed files with 5 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

3
litellm/llms/huggingface_restapi.py
View file

@ -270,6 +270,7 @@ def completion(
headers=headers,
data=json.dumps(data)
)
print(response.text)
## Some servers might return streaming responses even though stream was not set to true. (e.g. Baseten)
is_streamed = False
@ -330,6 +331,7 @@ def completion(
model_response.choices[0].finish_reason = completion_response[0]["details"]["finish_reason"]
sum_logprob = 0
for token in completion_response[0]["details"]["tokens"]:
if token["logprob"] != None:
sum_logprob += token["logprob"]
model_response["choices"][0]["message"]._logprob = sum_logprob
if "best_of" in optional_params and optional_params["best_of"] > 1:
@ -338,6 +340,7 @@ def completion(
for idx, item in enumerate(completion_response[0]["details"]["best_of_sequences"]):
sum_logprob = 0
for token in item["tokens"]:
if token["logprob"] != None:
sum_logprob += token["logprob"]
if len(item["generated_text"]) > 0:
message_obj = Message(content=output_parser(item["generated_text"]), logprobs=sum_logprob)