github
/
minimind
mirror of https://github.com/jingyaogong/minimind.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:daiweidaichenzi/minimind

pull/791/head
baihaochen 2026-05-14 14:29:06 +08:00
parent 8baa9c5f71 de683eafae
commit 09e491f940
3 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
trainer/train_pretrain.py
View File

@ -87,7 +87,7 @@ if __name__ == "__main__":
parser.add_argument("--save_dir", type=str, default="../out", help="模型保存目录") parser.add_argument("--save_dir", type=str, default="../out", help="模型保存目录")
parser.add_argument('--save_weight', default='pretrain', type=str, help="保存权重的前缀名") parser.add_argument('--save_weight', default='pretrain', type=str, help="保存权重的前缀名")
parser.add_argument("--epochs", type=int, default=2, help="训练轮数") parser.add_argument("--epochs", type=int, default=2, help="训练轮数")
parser.add_argument("--batch_size", type=int, default=32, help="batch size") parser.add_argument("--batch_size", type=int, default=128, help="batch size")
parser.add_argument("--learning_rate", type=float, default=5e-4, help="初始学习率") parser.add_argument("--learning_rate", type=float, default=5e-4, help="初始学习率")
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备") parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备")
parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型") parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型")

3
trainer/train_tokenizer.py
View File

@ -22,6 +22,8 @@ def get_texts(data_path):
continue continue
def train_tokenizer(data_path, tokenizer_dir, vocab_size, special_tokens_num=SPECIAL_TOKENS_NUM): def train_tokenizer(data_path, tokenizer_dir, vocab_size, special_tokens_num=SPECIAL_TOKENS_NUM):
#tokenizer处理文本过程
#原始文本->normalizer->pre_tokenizer->model->post_processor
tokenizer = Tokenizer(models.BPE()) tokenizer = Tokenizer(models.BPE())
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
@ -48,6 +50,7 @@ def train_tokenizer(data_path, tokenizer_dir, vocab_size, special_tokens_num=SPE
special_tokens=all_special_tokens special_tokens=all_special_tokens
) )
texts = get_texts(data_path) texts = get_texts(data_path)
tokenizer.train_from_iterator(texts, trainer=trainer) tokenizer.train_from_iterator(texts, trainer=trainer)
tokenizer.decoder = decoders.ByteLevel() tokenizer.decoder = decoders.ByteLevel()
tokenizer.add_special_tokens(special_tokens_list) tokenizer.add_special_tokens(special_tokens_list)