update 0-eval_pretrain.py

2024-09-14 14:14:12 +08:00 · 2024-09-14 14:14:12 +08:00 · a8405b08a2
parent b00a3b5ff7
commit a8405b08a2
1 changed files with 1 additions and 2 deletions
--- a/0-eval_pretrain.py
+++ b/0-eval_pretrain.py
@ -16,8 +16,7 @@ def count_parameters(model):


 def init_model(lm_config):
-    tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer',
-                                              trust_remote_code=True, use_fast=False)
+    tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer')
    model_from = 1  # 1从权重，2用transformers

    if model_from == 1: