Merge 7227c6a21c into dddedc6881
100apps
GitHub
commit
05ae7c646b
|
|
@ -3,4 +3,12 @@ __pycache__/
|
|||
.DS_Store
|
||||
out
|
||||
website/
|
||||
docs-minimind/
|
||||
docs-minimind/
|
||||
minimind-3
|
||||
dataset/**
|
||||
!dataset/lm_dataset.py
|
||||
!dataset/__init__.py
|
||||
!dataset/dataset.md
|
||||
checkpoints/
|
||||
.venv/
|
||||
.uv-cache/
|
||||
|
|
@ -0,0 +1 @@
|
|||
3.10
|
||||
|
|
@ -0,0 +1,227 @@
|
|||
{
|
||||
// 使用 IntelliSense 了解可用属性。
|
||||
// 悬停以查看现有属性的相关描述。
|
||||
// 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
//
|
||||
// 使用说明：
|
||||
// 1. 先执行 `uv sync` 安装依赖
|
||||
// 2. 在 VS Code 中按 Ctrl+Shift+P → "Python: Select Interpreter"
|
||||
// 选择 `./.venv/Scripts/python.exe` (Windows) 或 `./.venv/bin/python` (Linux/Mac)
|
||||
// 3. 按 F5 或 Ctrl+Shift+D 选择对应配置启动调试
|
||||
//
|
||||
// 注：对应 README.md 中所有 `uv run python` / `uv run streamlit` 的单卡命令
|
||||
// 多卡训练 (`torchrun`) 建议在终端直接执行，不在此列出
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
// ==================== 1. 推理与评测 ====================
|
||||
{
|
||||
"name": "推理: eval_llm (transformers格式)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/eval_llm.py",
|
||||
"cwd": "${workspaceFolder}",
|
||||
"console": "integratedTerminal",
|
||||
"args": ["--load_from", "./minimind-3"]
|
||||
},
|
||||
{
|
||||
"name": "推理: eval_llm (pytorch格式)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/eval_llm.py",
|
||||
"cwd": "${workspaceFolder}",
|
||||
"console": "integratedTerminal",
|
||||
"args": ["--load_from", "./model", "--weight", "full_sft"]
|
||||
},
|
||||
{
|
||||
"name": "推理: eval_llm (open_thinking)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/eval_llm.py",
|
||||
"cwd": "${workspaceFolder}",
|
||||
"console": "integratedTerminal",
|
||||
"args": ["--load_from", "./minimind-3", "--open_thinking", "1"]
|
||||
},
|
||||
{
|
||||
"name": "推理: eval_llm (RoPE外推)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/eval_llm.py",
|
||||
"cwd": "${workspaceFolder}",
|
||||
"console": "integratedTerminal",
|
||||
"args": ["--weight", "full_sft", "--inference_rope_scaling"]
|
||||
},
|
||||
{
|
||||
"name": "评测: eval_toolcall (full_sft)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/scripts/eval_toolcall.py",
|
||||
"cwd": "${workspaceFolder}",
|
||||
"console": "integratedTerminal",
|
||||
"args": ["--weight", "full_sft"]
|
||||
},
|
||||
{
|
||||
"name": "评测: eval_toolcall (agent)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/scripts/eval_toolcall.py",
|
||||
"cwd": "${workspaceFolder}",
|
||||
"console": "integratedTerminal",
|
||||
"args": ["--weight", "agent"]
|
||||
},
|
||||
|
||||
// ==================== 2. WebUI 与服务 ====================
|
||||
{
|
||||
"name": "服务: WebUI (streamlit)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"module": "streamlit",
|
||||
"cwd": "${workspaceFolder}/scripts",
|
||||
"console": "integratedTerminal",
|
||||
"args": ["run", "web_demo.py"]
|
||||
},
|
||||
{
|
||||
"name": "服务: OpenAI API Server",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/scripts/serve_openai_api.py",
|
||||
"cwd": "${workspaceFolder}/scripts",
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "服务: Chat API 测试",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/scripts/chat_api.py",
|
||||
"cwd": "${workspaceFolder}/scripts",
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
|
||||
// ==================== 3. 预训练与微调 ====================
|
||||
{
|
||||
"name": "训练: Pretrain (预训练)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/trainer/train_pretrain.py",
|
||||
"cwd": "${workspaceFolder}/trainer",
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "训练: Pretrain (断点续训)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/trainer/train_pretrain.py",
|
||||
"cwd": "${workspaceFolder}/trainer",
|
||||
"console": "integratedTerminal",
|
||||
"args": ["--from_resume", "1"]
|
||||
},
|
||||
{
|
||||
"name": "训练: Full SFT (全参数微调)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/trainer/train_full_sft.py",
|
||||
"cwd": "${workspaceFolder}/trainer",
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "训练: Full SFT (断点续训)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/trainer/train_full_sft.py",
|
||||
"cwd": "${workspaceFolder}/trainer",
|
||||
"console": "integratedTerminal",
|
||||
"args": ["--from_resume", "1"]
|
||||
},
|
||||
{
|
||||
"name": "训练: LoRA",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/trainer/train_lora.py",
|
||||
"cwd": "${workspaceFolder}/trainer",
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "训练: Distillation (知识蒸馏)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/trainer/train_distillation.py",
|
||||
"cwd": "${workspaceFolder}/trainer",
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "训练: Tokenizer (分词器训练)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/trainer/train_tokenizer.py",
|
||||
"cwd": "${workspaceFolder}/trainer",
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
|
||||
// ==================== 4. 强化学习 ====================
|
||||
{
|
||||
"name": "RL: DPO",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/trainer/train_dpo.py",
|
||||
"cwd": "${workspaceFolder}/trainer",
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "RL: PPO",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/trainer/train_ppo.py",
|
||||
"cwd": "${workspaceFolder}/trainer",
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "RL: GRPO",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/trainer/train_grpo.py",
|
||||
"cwd": "${workspaceFolder}/trainer",
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "RL: CISPO (GRPO变体)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/trainer/train_grpo.py",
|
||||
"cwd": "${workspaceFolder}/trainer",
|
||||
"console": "integratedTerminal",
|
||||
"args": ["--loss_type", "cispo"]
|
||||
},
|
||||
{
|
||||
"name": "RL: Agentic RL",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/trainer/train_agent.py",
|
||||
"cwd": "${workspaceFolder}/trainer",
|
||||
"console": "integratedTerminal"
|
||||
},
|
||||
{
|
||||
"name": "RL: Agentic RL (SGLang)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/trainer/train_agent.py",
|
||||
"cwd": "${workspaceFolder}/trainer",
|
||||
"console": "integratedTerminal",
|
||||
"args": [
|
||||
"--rollout_engine", "sglang",
|
||||
"--sglang_base_url", "http://localhost:8998",
|
||||
"--sglang_shared_path", "./ckpt_mm",
|
||||
"--data_path", "../dataset/agent_rl_math.jsonl",
|
||||
"--use_wandb"
|
||||
]
|
||||
},
|
||||
|
||||
// ==================== 5. 工具与转换 ====================
|
||||
{
|
||||
"name": "工具: 模型转换",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/scripts/convert_model.py",
|
||||
"cwd": "${workspaceFolder}/scripts",
|
||||
"console": "integratedTerminal"
|
||||
}
|
||||
]
|
||||
}
|
||||
42
README.md
42
README.md
|
|
@ -220,7 +220,7 @@ minimind2系列旧模型均经过权重映射+（微调训练）QKVO线性层校
|
|||
* Ubuntu==20.04
|
||||
* CUDA==12.2
|
||||
* Python==3.10.16
|
||||
* [requirements.txt](./requirements.txt)
|
||||
* [uv](https://docs.astral.sh/uv/) + [pyproject.toml](./pyproject.toml)
|
||||
|
||||
</details>
|
||||
|
||||
|
|
@ -229,7 +229,8 @@ minimind2系列旧模型均经过权重映射+（微调训练）QKVO线性层校
|
|||
```bash
|
||||
# 克隆仓库、安装依赖
|
||||
git clone --depth 1 https://github.com/jingyaogong/minimind
|
||||
cd minimind && pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple
|
||||
cd minimind
|
||||
uv sync
|
||||
```
|
||||
|
||||
## Ⅰ 🚀 模型推理
|
||||
|
|
@ -239,7 +240,7 @@ cd minimind && pip install -r requirements.txt -i https://mirrors.aliyun.com/pyp
|
|||
在项目根目录：
|
||||
```bash
|
||||
# 方式1
|
||||
modelscope download --model gongjy/minimind-3 --local_dir ./minimind-3
|
||||
uv run modelscope download --model gongjy/minimind-3 --local_dir ./minimind-3
|
||||
# 方式2
|
||||
git clone https://huggingface.co/jingyaogong/minimind-3
|
||||
```
|
||||
|
|
@ -248,17 +249,17 @@ git clone https://huggingface.co/jingyaogong/minimind-3
|
|||
|
||||
```bash
|
||||
# 方式1：使用 Transformers 格式模型
|
||||
python eval_llm.py --load_from ./minimind-3
|
||||
uv run python eval_llm.py --load_from ./minimind-3
|
||||
# 方式2：基于 PyTorch 模型（确保./out目录下有对应权重）
|
||||
python eval_llm.py --load_from ./model --weight full_sft
|
||||
uv run python eval_llm.py --load_from ./model --weight full_sft
|
||||
```
|
||||
|
||||
### 3'（可选）WebUI
|
||||
|
||||
```bash
|
||||
# 可能需要`python>=3.10`，安装 `pip install streamlit`
|
||||
# 使用 `uv sync` 安装依赖后即可运行
|
||||
# ⚠️ 须先将 transformers 格式模型文件夹复制到 ./scripts/ 目录下（例如：cp -r minimind-3 ./scripts/minimind-3），web_demo 脚本会自动扫描该目录下包含权重文件的子文件夹，如不存在则报错
|
||||
cd scripts && streamlit run web_demo.py
|
||||
cd scripts && uv run streamlit run web_demo.py
|
||||
```
|
||||
|
||||
### 4'（可选）第三方推理框架
|
||||
|
|
@ -282,7 +283,7 @@ print(torch.cuda.is_available())
|
|||
|
||||
若你计划使用 CUDA 训练，建议先确认当前环境是否已正确识别 GPU。
|
||||
若 `cuda` 不可用，也仍可根据自身设备选择 `CPU` 或 `MPS` 运行，但训练速度与兼容性会有非常大的差异。
|
||||
如需安装或更换 PyTorch 版本，可参考 [torch_stable](https://download.pytorch.org/whl/torch_stable.html) 与[链接](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=%E5%AE%89%E8%A3%85torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187)
|
||||
默认 `pyproject.toml` 使用 PyTorch 官方 `cu128` wheel，可覆盖 Windows 11 + RTX 50 系显卡的常见 CUDA 运行环境；如需切换 CPU 或其它 CUDA 版本，请调整 `pyproject.toml` 中的 `pytorch-cu128` 索引与 `torch` / `torchvision` 版本后重新执行 `uv sync`。
|
||||
|
||||
</details>
|
||||
|
||||
|
|
@ -291,7 +292,11 @@ print(torch.cuda.is_available())
|
|||
从下文提供的[数据集下载链接](https://www.modelscope.cn/datasets/gongjy/minimind_dataset/files) 下载所需数据文件，并放入 `./dataset` 目录
|
||||
|
||||
> 当前默认仅需下载 `pretrain_t2t_mini.jsonl` 与 `sft_t2t_mini.jsonl`，即可较快复现 `MiniMind Zero` 对话模型。
|
||||
如有更多需求，下文提供多种搭配方案，可根据自身任务目标与 GPU 资源灵活选择。
|
||||
|
||||
```bash
|
||||
// （可选）如有更多需求，下文提供多种搭配方案，可根据自身任务目标与 GPU 资源灵活选择。
|
||||
uv run modelscope download --dataset gongjy/minimind_dataset --local_dir ./dataset
|
||||
```
|
||||
|
||||
### 2' 开始训练
|
||||
|
||||
|
|
@ -301,8 +306,8 @@ print(torch.cuda.is_available())
|
|||
所有训练脚本均支持检查点保存。添加 `--from_resume 1` 参数后，即可自动检测并恢复训练进度：
|
||||
|
||||
```bash
|
||||
python train_pretrain.py --from_resume 1
|
||||
python train_full_sft.py --from_resume 1
|
||||
uv run python train_pretrain.py --from_resume 1
|
||||
uv run python train_full_sft.py --from_resume 1
|
||||
# ...
|
||||
```
|
||||
|
||||
|
|
@ -319,7 +324,7 @@ python train_full_sft.py --from_resume 1
|
|||
#### 2.1 预训练（必须）
|
||||
|
||||
```bash
|
||||
cd trainer && python train_pretrain.py
|
||||
cd trainer && uv run python train_pretrain.py
|
||||
```
|
||||
|
||||
> 训练后，将得到 `out/pretrain_*.pth` 作为输出权重（其中 `*` 为模型 dimension，默认为 `768`）
|
||||
|
|
@ -327,17 +332,22 @@ cd trainer && python train_pretrain.py
|
|||
#### 2.2 指令微调（必须）
|
||||
|
||||
```bash
|
||||
cd trainer && python train_full_sft.py
|
||||
cd trainer && uv run python train_full_sft.py
|
||||
```
|
||||
|
||||
> 训练后，将得到 `out/full_sft_*.pth` 作为输出权重（其中 `full` 表示全参数微调）
|
||||
|
||||
#### 2.3 测试已训练模型（可选）
|
||||
|
||||
确保待测试的模型 `*.pth` 文件位于 `./out/` 目录下；也可直接前往[此处](https://www.modelscope.cn/models/gongjy/minimind-3-pytorch/files)下载我已训练好的 `*.pth` 权重。
|
||||
确保待测试的模型 `*.pth` 文件位于 `./out/` 目录下；也可直接前往[此处](https://www.modelscope.cn/models/gongjy/minimind-3-pytorch/files)下载我已训练好的 `*.pth` 权重
|
||||
|
||||
```bash
|
||||
python eval_llm.py --weight full_sft
|
||||
// （可选） 下载已经训练好的模型
|
||||
uv run modelscope download --model gongjy/minimind-3-pytorch --local_dir ./out
|
||||
```
|
||||
|
||||
```bash
|
||||
uv run python eval_llm.py --weight full_sft
|
||||
```
|
||||
|
||||
> `--weight` 用于指定权重名称前缀，例如 `pretrain`、`full_sft` 等；更多参数可直接参考 `eval_llm.py`
|
||||
|
|
@ -350,7 +360,7 @@ python eval_llm.py --weight full_sft
|
|||
2、若你的设备有 `N (N > 1)` 张显卡，可通过以下方式启动单机 `N` 卡训练（DDP，也支持扩展到多机多卡）：
|
||||
|
||||
```bash
|
||||
torchrun --nproc_per_node N train_xxx.py
|
||||
uv run torchrun --nproc_per_node N train_xxx.py
|
||||
```
|
||||
|
||||
3、可根据需要开启 wandb 记录训练过程。
|
||||
|
|
|
|||
31
README_en.md
31
README_en.md
|
|
@ -219,7 +219,7 @@ After this update, maintenance for the entire `minimind-v1` series will be disco
|
|||
* Ubuntu==20.04
|
||||
* CUDA==12.2
|
||||
* Python==3.10.16
|
||||
* [requirements.txt](./requirements.txt)
|
||||
* [uv](https://docs.astral.sh/uv/) + [pyproject.toml](./pyproject.toml)
|
||||
|
||||
</details>
|
||||
|
||||
|
|
@ -228,7 +228,8 @@ After this update, maintenance for the entire `minimind-v1` series will be disco
|
|||
```bash
|
||||
# Clone repository and install dependencies
|
||||
git clone --depth 1 https://github.com/jingyaogong/minimind
|
||||
cd minimind && pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple
|
||||
cd minimind
|
||||
uv sync
|
||||
```
|
||||
|
||||
## Ⅰ 🚀 Model Inference
|
||||
|
|
@ -238,7 +239,7 @@ cd minimind && pip install -r requirements.txt -i https://mirrors.aliyun.com/pyp
|
|||
In the project root directory:
|
||||
```bash
|
||||
# Method 1
|
||||
modelscope download --model gongjy/minimind-3 --local_dir ./minimind-3
|
||||
uv run modelscope download --model gongjy/minimind-3 --local_dir ./minimind-3
|
||||
# Method 2
|
||||
git clone https://huggingface.co/jingyaogong/minimind-3
|
||||
```
|
||||
|
|
@ -247,17 +248,17 @@ git clone https://huggingface.co/jingyaogong/minimind-3
|
|||
|
||||
```bash
|
||||
# Method 1: Using Transformers format model
|
||||
python eval_llm.py --load_from ./minimind-3
|
||||
uv run python eval_llm.py --load_from ./minimind-3
|
||||
# Method 2: Based on PyTorch model (ensure corresponding weights are in the ./out directory)
|
||||
python eval_llm.py --load_from ./model --weight full_sft
|
||||
uv run python eval_llm.py --load_from ./model --weight full_sft
|
||||
```
|
||||
|
||||
### 3' (Optional) WebUI
|
||||
|
||||
```bash
|
||||
# May need `python>=3.10`, install `pip install streamlit`
|
||||
# Run this after installing dependencies with `uv sync`
|
||||
# ⚠️ You must first copy the transformers-format model folder into ./scripts/ (e.g.: cp -r minimind-3 ./scripts/minimind-3). The web_demo script will auto-scan subdirectories containing weight files; it will throw an error if none are found.
|
||||
cd scripts && streamlit run web_demo.py
|
||||
cd scripts && uv run streamlit run web_demo.py
|
||||
```
|
||||
|
||||
### 4' (Optional) Third-party Inference Frameworks
|
||||
|
|
@ -281,7 +282,7 @@ print(torch.cuda.is_available())
|
|||
|
||||
If you plan to use CUDA for training, it is recommended to first confirm whether the current environment has correctly recognized the GPU.
|
||||
If `cuda` is not available, you can still choose `CPU` or `MPS` to run based on your device, but training speed and compatibility will differ significantly.
|
||||
If you need to install or switch PyTorch versions, refer to [torch_stable](https://download.pytorch.org/whl/torch_stable.html) and [this link](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=%E5%AE%89%E8%A3%85torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187)
|
||||
By default, `pyproject.toml` uses the official PyTorch `cu128` wheels, which cover common CUDA runtime setups for Windows 11 + RTX 50-series GPUs. To switch to CPU or another CUDA version, adjust the `pytorch-cu128` index and the `torch` / `torchvision` versions in `pyproject.toml`, then run `uv sync` again.
|
||||
|
||||
</details>
|
||||
|
||||
|
|
@ -300,8 +301,8 @@ For other use cases, several data combinations are provided below and can be sel
|
|||
All training scripts support checkpoint saving. After adding the `--from_resume 1` parameter, the script will automatically detect and resume training progress:
|
||||
|
||||
```bash
|
||||
python train_pretrain.py --from_resume 1
|
||||
python train_full_sft.py --from_resume 1
|
||||
uv run python train_pretrain.py --from_resume 1
|
||||
uv run python train_full_sft.py --from_resume 1
|
||||
# ...
|
||||
```
|
||||
|
||||
|
|
@ -318,7 +319,7 @@ python train_full_sft.py --from_resume 1
|
|||
#### 2.1 Pretraining (Required)
|
||||
|
||||
```bash
|
||||
cd trainer && python train_pretrain.py
|
||||
cd trainer && uv run python train_pretrain.py
|
||||
```
|
||||
|
||||
> After training, `out/pretrain_*.pth` will be produced as output weights (where `*` is the model dimension, default `768`)
|
||||
|
|
@ -326,7 +327,7 @@ cd trainer && python train_pretrain.py
|
|||
#### 2.2 Instruction Fine-tuning (Required)
|
||||
|
||||
```bash
|
||||
cd trainer && python train_full_sft.py
|
||||
cd trainer && uv run python train_full_sft.py
|
||||
```
|
||||
|
||||
> After training, `out/full_sft_*.pth` will be produced as output weights (where `full` indicates full-parameter fine-tuning)
|
||||
|
|
@ -336,7 +337,7 @@ cd trainer && python train_full_sft.py
|
|||
Ensure the model `*.pth` files to be tested are located in the `./out/` directory; you can also go directly to [here](https://www.modelscope.cn/models/gongjy/minimind-3-pytorch/files) to download my pre-trained `*.pth` weights.
|
||||
|
||||
```bash
|
||||
python eval_llm.py --weight full_sft
|
||||
uv run python eval_llm.py --weight full_sft
|
||||
```
|
||||
|
||||
> `--weight` is used to specify the weight name prefix, such as `pretrain`, `full_sft`, etc.; for more parameters, refer directly to `eval_llm.py`
|
||||
|
|
@ -349,7 +350,7 @@ python eval_llm.py --weight full_sft
|
|||
2. If your device has `N (N > 1)` GPUs, you can start single-machine `N`-GPU training as follows (DDP, also supports extension to multi-machine multi-GPU):
|
||||
|
||||
```bash
|
||||
torchrun --nproc_per_node N train_xxx.py
|
||||
uv run torchrun --nproc_per_node N train_xxx.py
|
||||
```
|
||||
|
||||
3. You can enable wandb to record the training process as needed.
|
||||
|
|
@ -1989,4 +1990,4 @@ If `MiniMind` has been helpful to your research or work, feel free to cite:
|
|||
|
||||
# ⚖️ License
|
||||
|
||||
This project is open-sourced under the [Apache License 2.0](LICENSE).
|
||||
This project is open-sourced under the [Apache License 2.0](LICENSE).
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ def main():
|
|||
parser.add_argument('--open_thinking', default=0, type=int, help="是否开启自适应思考（0=否，1=是）")
|
||||
parser.add_argument('--historys', default=0, type=int, help="携带历史对话轮数（需为偶数，0表示不携带历史）")
|
||||
parser.add_argument('--show_speed', default=1, type=int, help="显示decode速度（tokens/s）")
|
||||
parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu', type=str, help="运行设备")
|
||||
parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'mps' if torch.backends.mps.is_available() and torch.backends.mps.is_built() else 'cpu', type=str, help="运行设备")
|
||||
args = parser.parse_args()
|
||||
|
||||
prompts = [
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
[project]
|
||||
name = "minimind"
|
||||
version = "0.1.0"
|
||||
description = "MiniMind training and inference scripts."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10,<3.11"
|
||||
dependencies = [
|
||||
"datasets==3.6.0",
|
||||
"datasketch==1.6.4",
|
||||
"einops==0.8.1",
|
||||
"fastapi==0.115.14",
|
||||
"flask==3.0.3",
|
||||
"flask-cors==4.0.0",
|
||||
"jieba==0.42.1",
|
||||
"jinja2==3.1.2",
|
||||
"jsonlines==4.0.0",
|
||||
"marshmallow==3.22.0",
|
||||
"modelscope>=1.36.3",
|
||||
"ngrok==1.4.0",
|
||||
"nltk==3.8",
|
||||
"numpy==1.26.4",
|
||||
"openai==1.59.6",
|
||||
"psutil==5.9.8",
|
||||
"pydantic==2.11.5",
|
||||
"requests==2.32.5",
|
||||
"rich==13.7.1",
|
||||
"scikit-learn==1.5.1",
|
||||
"sentence-transformers==2.3.1",
|
||||
"setuptools>=82.0.1",
|
||||
"simhash==2.1.2",
|
||||
"streamlit==1.50.0",
|
||||
"swanlab==0.7.11",
|
||||
"tiktoken==0.10.0",
|
||||
"tokenizers==0.22.1",
|
||||
"torch==2.10.0",
|
||||
"torchvision==0.25.0",
|
||||
"transformers==4.57.6",
|
||||
"trl==0.13.0",
|
||||
"ujson==5.1.0",
|
||||
"uvicorn==0.35.0",
|
||||
"wandb==0.18.3",
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
package = false
|
||||
|
||||
[tool.uv.sources]
|
||||
torch = [
|
||||
{ index = "pytorch-cu128", marker = "sys_platform == 'win32' or sys_platform == 'linux'" },
|
||||
]
|
||||
torchvision = [
|
||||
{ index = "pytorch-cu128", marker = "sys_platform == 'win32' or sys_platform == 'linux'" },
|
||||
]
|
||||
|
||||
[[tool.uv.index]]
|
||||
url = "https://pypi.tuna.tsinghua.edu.cn/simple"
|
||||
default = true
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "pytorch-cu128"
|
||||
url = "https://download.pytorch.org/whl/cu128"
|
||||
explicit = true
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
datasets==3.6.0
|
||||
datasketch==1.6.4
|
||||
Flask==3.0.3
|
||||
Flask_Cors==4.0.0
|
||||
jieba==0.42.1
|
||||
jsonlines==4.0.0
|
||||
marshmallow==3.22.0
|
||||
# matplotlib==3.10.0
|
||||
ngrok==1.4.0
|
||||
nltk==3.8
|
||||
numpy==1.26.4
|
||||
openai==1.59.6
|
||||
# peft==0.7.1
|
||||
psutil==5.9.8
|
||||
pydantic==2.11.5
|
||||
rich==13.7.1
|
||||
scikit_learn==1.5.1
|
||||
sentence_transformers==2.3.1
|
||||
simhash==2.1.2
|
||||
tiktoken==0.10.0
|
||||
transformers==4.57.6
|
||||
jinja2==3.1.2
|
||||
jsonlines==4.0.0
|
||||
trl==0.13.0
|
||||
ujson==5.1.0
|
||||
wandb==0.18.3
|
||||
streamlit==1.50.0
|
||||
einops==0.8.1
|
||||
swanlab==0.7.11
|
||||
modelscope==1.30.0
|
||||
# torch==2.6.0
|
||||
# torchvision==0.21.0
|
||||
|
|
@ -54,15 +54,27 @@ TEST_CASES = [
|
|||
]
|
||||
|
||||
|
||||
def _resolve_path(path):
|
||||
if os.path.isabs(path):
|
||||
return path
|
||||
abs_path = os.path.abspath(path)
|
||||
if os.path.isdir(abs_path):
|
||||
return abs_path
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
return os.path.abspath(os.path.join(script_dir, path))
|
||||
|
||||
|
||||
def init_model(args):
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.load_from)
|
||||
load_path = _resolve_path(args.load_from)
|
||||
tokenizer = AutoTokenizer.from_pretrained(load_path)
|
||||
if 'model' in args.load_from:
|
||||
model = MiniMindForCausalLM(MiniMindConfig(hidden_size=args.hidden_size, num_hidden_layers=args.num_hidden_layers, use_moe=bool(args.use_moe)))
|
||||
moe_suffix = '_moe' if args.use_moe else ''
|
||||
ckp = f'./{args.save_dir}/{args.weight}_{args.hidden_size}{moe_suffix}.pth'
|
||||
save_dir = _resolve_path(args.save_dir)
|
||||
ckp = os.path.join(save_dir, f'{args.weight}_{args.hidden_size}{moe_suffix}.pth')
|
||||
model.load_state_dict(torch.load(ckp, map_location=args.device), strict=True)
|
||||
else:
|
||||
model = AutoModelForCausalLM.from_pretrained(args.load_from, trust_remote_code=True)
|
||||
model = AutoModelForCausalLM.from_pretrained(load_path, trust_remote_code=True)
|
||||
get_model_params(model, model.config)
|
||||
return model.half().eval().to(args.device), tokenizer
|
||||
|
||||
|
|
@ -212,7 +224,7 @@ def main():
|
|||
parser.add_argument('--temperature', default=0.9, type=float, help="生成温度，控制随机性（0-1，越大越随机）")
|
||||
parser.add_argument('--top_p', default=0.9, type=float, help="nucleus采样阈值（0-1）")
|
||||
parser.add_argument('--show_speed', default=0, type=int, help="显示decode速度（tokens/s）")
|
||||
parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu', type=str, help="运行设备")
|
||||
parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'mps' if torch.backends.mps.is_available() and torch.backends.mps.is_built() else 'cpu', type=str, help="运行设备")
|
||||
parser.add_argument('--api_base_url', default="http://localhost:11434/v1", type=str, help="OpenAI兼容接口的base_url")
|
||||
parser.add_argument('--api_key', default='sk-123', type=str, help="OpenAI兼容接口的api_key")
|
||||
parser.add_argument('--api_model', default='jingyaogong/minimind-3:latest', type=str, help="API请求时使用的模型名称")
|
||||
|
|
|
|||
|
|
@ -238,7 +238,7 @@ if __name__ == "__main__":
|
|||
parser.add_argument('--max_seq_len', default=8192, type=int, help="最大序列长度")
|
||||
parser.add_argument('--use_moe', default=0, type=int, choices=[0, 1], help="是否使用MoE架构（0=否，1=是）")
|
||||
parser.add_argument('--inference_rope_scaling', default=False, action='store_true', help="启用RoPE位置编码外推（4倍，仅解决位置编码问题）")
|
||||
parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu', type=str, help="运行设备")
|
||||
parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'mps' if torch.backends.mps.is_available() and torch.backends.mps.is_built() else 'cpu', type=str, help="运行设备")
|
||||
args = parser.parse_args()
|
||||
device = args.device
|
||||
model, tokenizer = init_model(args)
|
||||
|
|
|
|||
|
|
@ -377,7 +377,7 @@ if __name__ == "__main__":
|
|||
parser.add_argument("--epochs", type=int, default=1, help="训练轮数")
|
||||
parser.add_argument("--batch_size", type=int, default=2, help="批次大小")
|
||||
parser.add_argument("--learning_rate", type=float, default=3e-7, help="学习率")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() and torch.backends.mps.is_built() else "cpu", help="训练设备")
|
||||
parser.add_argument("--dtype", type=str, default="bfloat16", help="数据类型 bfloat16/float16")
|
||||
parser.add_argument("--num_workers", type=int, default=8, help="数据加载线程数")
|
||||
parser.add_argument("--accumulation_steps", type=int, default=1, help="梯度累积步数")
|
||||
|
|
|
|||
|
|
@ -150,7 +150,7 @@ if __name__ == "__main__":
|
|||
parser.add_argument("--epochs", type=int, default=6, help="训练轮数")
|
||||
parser.add_argument("--batch_size", type=int, default=32, help="batch size")
|
||||
parser.add_argument("--learning_rate", type=float, default=5e-6, help="初始学习率")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() and torch.backends.mps.is_built() else "cpu", help="训练设备")
|
||||
parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型")
|
||||
parser.add_argument("--num_workers", type=int, default=8, help="数据加载线程数")
|
||||
parser.add_argument("--accumulation_steps", type=int, default=1, help="梯度累积步数")
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ if __name__ == "__main__":
|
|||
parser.add_argument("--epochs", type=int, default=1, help="训练轮数")
|
||||
parser.add_argument("--batch_size", type=int, default=4, help="batch size")
|
||||
parser.add_argument("--learning_rate", type=float, default=4e-8, help="初始学习率（建议<=5e-8避免遗忘）")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() and torch.backends.mps.is_built() else "cpu", help="训练设备")
|
||||
parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型")
|
||||
parser.add_argument("--num_workers", type=int, default=8, help="数据加载线程数")
|
||||
parser.add_argument("--accumulation_steps", type=int, default=1, help="梯度累积步数")
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ if __name__ == "__main__":
|
|||
parser.add_argument("--epochs", type=int, default=2, help="训练轮数")
|
||||
parser.add_argument("--batch_size", type=int, default=16, help="batch size")
|
||||
parser.add_argument("--learning_rate", type=float, default=1e-5, help="初始学习率")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() and torch.backends.mps.is_built() else "cpu", help="训练设备")
|
||||
parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型")
|
||||
parser.add_argument("--num_workers", type=int, default=8, help="数据加载线程数")
|
||||
parser.add_argument("--accumulation_steps", type=int, default=1, help="梯度累积步数")
|
||||
|
|
|
|||
|
|
@ -209,7 +209,7 @@ if __name__ == "__main__":
|
|||
parser.add_argument("--epochs", type=int, default=1, help="训练轮数")
|
||||
parser.add_argument("--batch_size", type=int, default=2, help="batch size")
|
||||
parser.add_argument("--learning_rate", type=float, default=3e-7, help="初始学习率")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() and torch.backends.mps.is_built() else "cpu", help="训练设备")
|
||||
parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型")
|
||||
parser.add_argument("--num_workers", type=int, default=8, help="数据加载线程数")
|
||||
parser.add_argument("--accumulation_steps", type=int, default=1, help="梯度累积步数")
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ if __name__ == "__main__":
|
|||
parser.add_argument("--epochs", type=int, default=10, help="训练轮数")
|
||||
parser.add_argument("--batch_size", type=int, default=32, help="batch size")
|
||||
parser.add_argument("--learning_rate", type=float, default=1e-4, help="初始学习率")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() and torch.backends.mps.is_built() else "cpu", help="训练设备")
|
||||
parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型")
|
||||
parser.add_argument("--num_workers", type=int, default=8, help="数据加载线程数")
|
||||
parser.add_argument("--accumulation_steps", type=int, default=1, help="梯度累积步数")
|
||||
|
|
|
|||
|
|
@ -300,7 +300,7 @@ if __name__ == "__main__":
|
|||
parser.add_argument("--batch_size", type=int, default=2, help="batch size")
|
||||
parser.add_argument("--learning_rate", type=float, default=3e-7, help="Actor学习率")
|
||||
parser.add_argument("--critic_learning_rate", type=float, default=5e-7, help="Critic学习率")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() and torch.backends.mps.is_built() else "cpu", help="训练设备")
|
||||
parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型")
|
||||
parser.add_argument("--num_workers", type=int, default=8, help="数据加载线程数")
|
||||
parser.add_argument("--accumulation_steps", type=int, default=1, help="梯度累积步数")
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ if __name__ == "__main__":
|
|||
parser.add_argument("--epochs", type=int, default=2, help="训练轮数")
|
||||
parser.add_argument("--batch_size", type=int, default=32, help="batch size")
|
||||
parser.add_argument("--learning_rate", type=float, default=5e-4, help="初始学习率")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", help="训练设备")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() and torch.backends.mps.is_built() else "cpu", help="训练设备")
|
||||
parser.add_argument("--dtype", type=str, default="bfloat16", help="混合精度类型")
|
||||
parser.add_argument("--num_workers", type=int, default=8, help="数据加载线程数")
|
||||
parser.add_argument("--accumulation_steps", type=int, default=8, help="梯度累积步数")
|
||||
|
|
|
|||
Loading…
Reference in New Issue