From 70a7e4d51e3341942699bc6d027d0346bc53952b Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Fri, 11 Apr 2025 20:30:44 -0700 Subject: [PATCH] fix: unhide python_start, python_end --- llama_stack/models/llama/llama4/tokenizer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llama_stack/models/llama/llama4/tokenizer.py b/llama_stack/models/llama/llama4/tokenizer.py index 8eabc3205..0d2cc7ce5 100644 --- a/llama_stack/models/llama/llama4/tokenizer.py +++ b/llama_stack/models/llama/llama4/tokenizer.py @@ -56,8 +56,8 @@ LLAMA4_TEXT_POST_TRAIN_SPECIAL_TOKENS = [ "<|text_post_train_reserved_special_token_3|>", "<|text_post_train_reserved_special_token_4|>", "<|text_post_train_reserved_special_token_5|>", - "<|text_post_train_reserved_special_token_6|>", - "<|text_post_train_reserved_special_token_7|>", + "<|python_start|>", + "<|python_end|>", "<|finetune_right_pad|>", ] + get_reserved_special_tokens( "text_post_train", 61, 8