diff --git a/model/minimind_tokenizer/tokenizer_config.json b/model/minimind_tokenizer/tokenizer_config.json index 5f3fa2b..066d177 100644 --- a/model/minimind_tokenizer/tokenizer_config.json +++ b/model/minimind_tokenizer/tokenizer_config.json @@ -40,5 +40,5 @@ "tokenizer_class": "PreTrainedTokenizerFast", "unk_token": "", "use_default_system_prompt": false, - "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'user\\n' + content + '\\nassistant\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '' + '\\n' }}{% endif %}{% endfor %}" + "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ 'system\n你是一个乐于助人的助手\n' }}{% endif %}{{'' + message['role'] + '\n' + message['content'] + '' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ 'assistant\n' }}{% endif %}" } \ No newline at end of file diff --git a/train_tokenizer.py b/train_tokenizer.py index 06716d1..630efc4 100644 --- a/train_tokenizer.py +++ b/train_tokenizer.py @@ -104,7 +104,7 @@ def train_tokenizer(): "tokenizer_class": "PreTrainedTokenizerFast", "unk_token": "", "use_default_system_prompt": False, - "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'user\\n' + content + '\\nassistant\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '' + '\\n' }}{% endif %}{% endfor %}" + "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ 'system\n你是一个乐于助人的助手\n' }}{% endif %}{{'' + message['role'] + '\n' + message['content'] + '' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ 'assistant\n' }}{% endif %}" } # 保存配置文件 @@ -122,14 +122,23 @@ def eval_tokenizer(): messages = [ {"role": "system", "content": "你是一个优秀的聊天机器人,总是给我正确的回应!"}, - {"role": "user", "content": '你来自哪里?'}, - {"role": "assistant", "content": '我来自地球'} + {"role": "user", "content": '你来自哪里?'} ] new_prompt = tokenizer.apply_chat_template( messages, tokenize=False ) + add_generation_prompt_messages = [ + {"role": "user", "content": '你来自哪里?'} + ] + add_generation_prompt_new_prompt = tokenizer.apply_chat_template( + add_generation_prompt_messages, + tokenize=False, + add_generation_prompt=True + ) print(new_prompt) + print("----------") + print(add_generation_prompt_new_prompt) # 获取实际词汇表长度(包括特殊符号) actual_vocab_size = len(tokenizer)