merge local changes

dataset/lm_dataset.py

@@ -81,9 +81,9 @@ class SFTDataset(Dataset):
         tools = None
         for message in conversations:
             message = dict(message)
-            # 如果消息是system角色，且带了tools字段
+            #如果消息是system角色，且带了tools字段
             if message.get("role") == "system" and message.get("tools"):
-                # json.loads把json格式字符串解析为python对象
+                #json.loads把json格式字符串解析为python对象
                 tools = json.loads(message["tools"]) if isinstance(message["tools"], str) else message["tools"]
             if message.get("tool_calls") and isinstance(message["tool_calls"], str):
                 message["tool_calls"] = json.loads(message["tool_calls"])

trainer/train_pretrain.py

@@ -87,7 +87,7 @@ if __name__ == "__main__":
     parser.add_argument("--save_dir", type=str, default="../out", help="模型保存目录")
     parser.add_argument('--save_weight', default='pretrain', type=str, help="保存权重的前缀名")
     parser.add_argument("--epochs", type=int, default=2, help="训练轮数")
-    parser.add_argument("--batch_size", type=int, default=32, help="batch size")
+    parser.add_argument("--batch_size", type=int, default=128, help="batch size")
     parser.add_argument("--learning_rate", type=float, default=5e-4, help="初始学习率")
     parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备")
     parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型")

trainer/train_tokenizer.py

@@ -22,6 +22,8 @@ def get_texts(data_path):
                 continue
 
 def train_tokenizer(data_path, tokenizer_dir, vocab_size, special_tokens_num=SPECIAL_TOKENS_NUM):
+    #tokenizer处理文本过程
+    #原始文本->normalizer->pre_tokenizer->model->post_processor
     tokenizer = Tokenizer(models.BPE())
     tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
     
@@ -48,6 +50,7 @@ def train_tokenizer(data_path, tokenizer_dir, vocab_size, special_tokens_num=SPE
         special_tokens=all_special_tokens
     )
     texts = get_texts(data_path)
+   
     tokenizer.train_from_iterator(texts, trainer=trainer)
     tokenizer.decoder = decoders.ByteLevel()
     tokenizer.add_special_tokens(special_tokens_list)