Add ToolPromptFormat to ChatFormat.encode_message so that tools are encoded properly

This commit is contained in:
Hardik Shah 2024-08-26 17:03:34 -07:00
parent decbbc127b
commit 69d9655ecd
4 changed files with 21 additions and 5 deletions

View file

@ -104,6 +104,7 @@ class MetaReferenceInferenceImpl(Inference):
top_p=request.sampling_params.top_p,
max_gen_len=request.sampling_params.max_tokens,
logprobs=request.logprobs,
tool_prompt_format=request.tool_prompt_format,
):
buffer += token_result.text
tokens.append(token_result.token)