update

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // 使用 IntelliSense 了解相关属性。 
+    // 悬停以查看现有属性的描述。
+    // 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: Current File with Arguments",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "args": "${command:pickArgs}"
+        }
+    ]
+}

eval_model.py

@@ -39,13 +39,10 @@ def get_prompt_datas(args):
     if args.model_mode == 0:
         # pretrain模型的接龙能力（无法对话）
         prompt_datas = [
-            '马克思主义基本原理',
+            '凤清儿自然',
-            '人类大脑的主要功能',
+            '唐火儿鼓起的小腹',
-            '万有引力原理是',
+            '马上黯淡下去唐火儿便绝望地'
-            '世界上最高的山峰是',
+
-            '二氧化碳在空气中',
-            '地球上最大的动物有',
-            '杭州市的美食有'
         ]
     else:
         if args.lora_name == 'None':
@@ -116,7 +113,7 @@ def main():
     # 模型未经过外推微调时，在更长的上下文的chat_template时难免出现性能的明显退化，因此需要注意此处设置
     parser.add_argument('--history_cnt', default=0, type=int)
     parser.add_argument('--load', default=0, type=int, help="0: 原生torch权重，1: transformers加载")
-    parser.add_argument('--model_mode', default=1, type=int,
+    parser.add_argument('--model_mode', default=0, type=int,
                         help="0: 预训练模型，1: SFT-Chat模型，2: RLHF-Chat模型，3: Reason模型，4: RLAIF-Chat模型")
     args = parser.parse_args()
 

file_2025-06-28_23-53-55_099857.log

@@ -0,0 +1 @@
+2025-06-28 23:53:56.752 | INFO     | __main__:Logger:28 - LLM可训练总参数量：25.830 百万

file_2025-06-28_23-54-45_149311.log

@@ -0,0 +1 @@
+2025-06-28 23:54:45.767 | INFO     | __main__:Logger:28 - LLM可训练总参数量：25.830 百万

process.ipynb

@@ -0,0 +1,320 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>text</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>&lt;|im_start|&gt;鉴别一组中文文章的风格和特点，例如官方、口语、文言等。需要提供样例文...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>&lt;|im_start|&gt;根据输入的内容，编写一个类别标签。\\n这是一篇介绍如何阅读心电图的文...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>&lt;|im_start|&gt;客户要求看一份报告，但他没有说明他需要哪份报告。他是指哪份报告？这个...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>&lt;|im_start|&gt;请从这首诗选出三句话，重新组成另一首新的诗歌。秋天美景尽收眼底，枫叶...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>&lt;|im_start|&gt;引述一段启发人心的一句话，用于鼓励团队士气。“团队的力量不在于每个人...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                text\n",
+       "0  <|im_start|>鉴别一组中文文章的风格和特点，例如官方、口语、文言等。需要提供样例文...\n",
+       "1  <|im_start|>根据输入的内容，编写一个类别标签。\\n这是一篇介绍如何阅读心电图的文...\n",
+       "2  <|im_start|>客户要求看一份报告，但他没有说明他需要哪份报告。他是指哪份报告？这个...\n",
+       "3  <|im_start|>请从这首诗选出三句话，重新组成另一首新的诗歌。秋天美景尽收眼底，枫叶...\n",
+       "4  <|im_start|>引述一段启发人心的一句话，用于鼓励团队士气。“团队的力量不在于每个人..."
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df=pd.read_json(\"dataset/pretrain_hq.jsonl\",lines=True)\n",
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'<|im_start|>鉴别一组中文文章的风格和特点，例如官方、口语、文言等。需要提供样例文章才能准确鉴别不同的风格和特点。<|im_end|> <|im_start|>好的，现在帮我查一下今天的天气怎么样?今天的天气依据地区而异。请问你需要我帮你查询哪个地区的天气呢？<|im_end|> <|im_start|>打开闹钟功能，定一个明天早上七点的闹钟。好的，我已经帮您打开闹钟功能，闹钟将在明天早上七点准时响起。<|im_end|> <|im_start|>为以下场景写一句话描述：一个孤独的老人坐在公园长椅上看着远处。一位孤独的老人坐在公园长椅上凝视远方。<|im_end|> <|im_start|>非常感谢你的回答。请告诉我，这些数据是关于什么主题的？这些数据是关于不同年龄段的男女人口比例分布的。<|im_end|> <|im_start|>帮我想一个有趣的标题。这个挺有趣的：\"如何成为一名成功的魔术师\" 调皮的标题往往会吸引读者的注意力。<|im_end|> <|im_start|>回答一个问题，地球的半径是多少？地球的平均半径约为6371公里，这是地球自赤道到两极的距离的平均值。<|im_end|> <|im_start|>识别文本中的语气，并将其分类为喜悦、悲伤、惊异等。\\n文本：“今天是我的生日！”这个文本的语气是喜悦。<|im_end|>'"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.iloc[0]['text']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'已处理并保存 252 条数据'"
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def process_and_save_dataset(dataset_list, max_length=512, output_file=\"dataset/pretrain_hq.jsonl\"):\n",
+    "    import json\n",
+    "    \n",
+    "    combined_text = \"\"\n",
+    "    result_list = []\n",
+    "    \n",
+    "    for data in dataset_list:\n",
+    "        # 添加特殊Token\n",
+    "        processed_text = \"<|im_start|>\" + data.strip() + \"<|im_end|>\"\n",
+    "        \n",
+    "        # 如果当前文本加上新样本会超过最大长度\n",
+    "        if len(combined_text + \" \" + processed_text) > max_length and combined_text:\n",
+    "            # 保存当前拼接的文本\n",
+    "            result_list.append({\"text\": combined_text+\"<|endoftext|>\"*10})\n",
+    "            # 重置拼接文本\n",
+    "            combined_text = processed_text\n",
+    "        else:\n",
+    "            # 添加空格分隔符（如果combined_text不为空）\n",
+    "            if combined_text:\n",
+    "                combined_text += \" \" + processed_text\n",
+    "            else:\n",
+    "                combined_text = processed_text\n",
+    "    \n",
+    "    # 处理最后剩余的文本\n",
+    "    if combined_text:\n",
+    "        result_list.append({\"text\": combined_text + \"<|endoftext|>\"*10})\n",
+    "    \n",
+    "    # 以最小IO方式保存到jsonl文件\n",
+    "    with open(output_file, 'a', encoding='utf-8') as f:\n",
+    "        for item in result_list:\n",
+    "            f.write(json.dumps(item, ensure_ascii=False) + '\\n')\n",
+    "    \n",
+    "    return f\"已处理并保存 {len(result_list)} 条数据\"\n",
+    "\n",
+    "# 使用示例\n",
+    "raw=\"\"\n",
+    "for file in os.listdir():\n",
+    "    if file.endswith(\".txt\"):\n",
+    "        with open(file, \"r\", encoding=\"utf-8\") as f:\n",
+    "            raw += f.read()\n",
+    "\n",
+    "cleaned_raw = raw.replace(\"\"\",\"\").replace(\"\"\",\"\").replace(\"'\",\"\").replace(\"'\",\"\").replace(\"，\",\"\").replace(\"\\n\",\"\")\n",
+    "dataset = cleaned_raw.split(\"。\")\n",
+    "\n",
+    "# 处理并保存数据\n",
+    "process_and_save_dataset(dataset[1:], max_length=512)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## remove"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'已从 dataset/pretrain_hq.jsonl 中删除最后 362 条数据，剩余 1413259 条'"
+      ]
+     },
+     "execution_count": 27,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def remove_last_n_entries(jsonl_file, n=362):\n",
+    "    import json\n",
+    "    \n",
+    "    # 读取所有行\n",
+    "    with open(jsonl_file, 'r', encoding='utf-8') as f:\n",
+    "        lines = f.readlines()\n",
+    "    \n",
+    "    # 计算要保留的行数\n",
+    "    keep_count = max(0, len(lines) - n)\n",
+    "    remaining_lines = lines[:keep_count]\n",
+    "    \n",
+    "    # 重写文件\n",
+    "    with open(jsonl_file, 'w', encoding='utf-8') as f:\n",
+    "        f.writelines(remaining_lines)\n",
+    "    \n",
+    "    return f\"已从 {jsonl_file} 中删除最后 {n} 条数据，剩余 {keep_count} 条\"\n",
+    "\n",
+    "# 使用示例\n",
+    "remove_last_n_entries(\"dataset/pretrain_hq.jsonl\", 362)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 添加结束符"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "已处理 1413511 条数据，修改了 1413511 条数据添加结束标记\n"
+     ]
+    }
+   ],
+   "source": [
+    "def add_endoftext_to_jsonl(jsonl_file, batch_size=1000):\n",
+    "    import json\n",
+    "    import os\n",
+    "    import tempfile\n",
+    "    \n",
+    "    # 创建临时文件\n",
+    "    temp_file = tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', delete=False)\n",
+    "    temp_filename = temp_file.name\n",
+    "    \n",
+    "    endoftext_str = \"<|endoftext|>\" * 10  # 重复150次\n",
+    "    \n",
+    "    # 记录处理的行数和修改的行数\n",
+    "    processed = 0\n",
+    "    modified = 0\n",
+    "    \n",
+    "    try:\n",
+    "        # 逐批次处理文件\n",
+    "        with open(jsonl_file, 'r', encoding='utf-8') as in_file:\n",
+    "            batch = []\n",
+    "            for line in in_file:\n",
+    "                processed += 1\n",
+    "                try:\n",
+    "                    data = json.loads(line.strip())\n",
+    "                    # 检查是否以<|endoftext|>结尾\n",
+    "                    if \"text\" in data and not data[\"text\"].endswith(\"<|endoftext|>\"):\n",
+    "                        data[\"text\"] += endoftext_str\n",
+    "                        modified += 1\n",
+    "                    batch.append(json.dumps(data, ensure_ascii=False) + '\\n')\n",
+    "                    \n",
+    "                    # 批处理写入\n",
+    "                    if len(batch) >= batch_size:\n",
+    "                        temp_file.writelines(batch)\n",
+    "                        batch = []\n",
+    "                except json.JSONDecodeError:\n",
+    "                    # 处理无效JSON的情况\n",
+    "                    temp_file.write(line)\n",
+    "            \n",
+    "            # 写入最后一批\n",
+    "            if batch:\n",
+    "                temp_file.writelines(batch)\n",
+    "        \n",
+    "        # 关闭临时文件\n",
+    "        temp_file.close()\n",
+    "        \n",
+    "        # 替换原始文件\n",
+    "        os.replace(temp_filename, jsonl_file)\n",
+    "        \n",
+    "        return f\"已处理 {processed} 条数据，修改了 {modified} 条数据添加结束标记\"\n",
+    "    \n",
+    "    except Exception as e:\n",
+    "        # 发生错误时删除临时文件\n",
+    "        if os.path.exists(temp_filename):\n",
+    "            os.unlink(temp_filename)\n",
+    "        return f\"处理出错: {str(e)}\"\n",
+    "\n",
+    "# 使用示例\n",
+    "file_path = \"dataset/pretrain_hq.jsonl\"\n",
+    "result = add_endoftext_to_jsonl(file_path)\n",
+    "print(result)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "minimind",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

trainer/train_pretrain.py

@@ -1,7 +1,10 @@
 import os
 import sys
 __package__ = "trainer"
-sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+# 获取脚本所在目录的绝对路径
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+PROJECT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, '..'))
+sys.path.append(PROJECT_ROOT)
 
 import argparse
 import time
@@ -16,13 +19,13 @@ from contextlib import nullcontext
 from transformers import AutoTokenizer
 from model.model_minimind import MiniMindConfig, MiniMindForCausalLM
 from dataset.lm_dataset import PretrainDataset
-
+from loguru import logger
 warnings.filterwarnings('ignore')
-
+logger.add("file_{time}.log")
 
 def Logger(content):
     if not ddp or dist.get_rank() == 0:
-        print(content)
+        logger.info(content)
 
 
 def get_lr(current_step, total_steps, lr):
@@ -73,7 +76,8 @@ def train_epoch(epoch, wandb):
                     loss.item() * args.accumulation_steps,
                     optimizer.param_groups[-1]['lr'],
                     spend_time / (step + 1) * iter_per_epoch // 60 - spend_time // 60))
-
+            decoded_text = tokenizer.decode(X[0].tolist())
+            Logger(f"原始文本：{decoded_text}")
             if (wandb is not None) and (not ddp or dist.get_rank() == 0):
                 wandb.log({"loss": loss.item() * args.accumulation_steps,
                            "lr": optimizer.param_groups[-1]['lr'],
@@ -95,7 +99,9 @@ def train_epoch(epoch, wandb):
 
 
 def init_model(lm_config):
-    tokenizer = AutoTokenizer.from_pretrained('../model/')
+    # 使用基于项目根目录的路径
+    tokenizer_path = os.path.join(PROJECT_ROOT, 'model')
+    tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
     model = MiniMindForCausalLM(lm_config).to(args.device)
     Logger(f'LLM可训练总参数量：{sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e6:.3f} 百万')
     return model, tokenizer
@@ -116,7 +122,8 @@ def init_distributed_mode():
 # torchrun --nproc_per_node 2 1-pretrain.py
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="MiniMind Pretraining")
-    parser.add_argument("--out_dir", type=str, default="../out")
+    # 使用基于项目根目录的路径
+    parser.add_argument("--out_dir", type=str, default=os.path.join(PROJECT_ROOT, "out"))
     # 若要以最快速度实现zero则epochs设置为1轮；否则应当利用有限的数据训练2~6个epochs。
     parser.add_argument("--epochs", type=int, default=1)
     parser.add_argument("--batch_size", type=int, default=32)
@@ -137,7 +144,8 @@ if __name__ == "__main__":
     parser.add_argument('--num_hidden_layers', default=8, type=int)
     parser.add_argument('--max_seq_len', default=512, type=int)
     parser.add_argument('--use_moe', default=False, type=bool)
-    parser.add_argument("--data_path", type=str, default="../dataset/pretrain_hq.jsonl")
+    # 使用基于项目根目录的路径
+    parser.add_argument("--data_path", type=str, default=os.path.join(PROJECT_ROOT, "dataset/pretrain_hq.jsonl"))
     args = parser.parse_args()
 
     lm_config = MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=args.use_moe)