diff --git a/2-sft_vlm.py b/2-sft_vlm.py deleted file mode 100644 index 877b76a..0000000 --- a/2-sft_vlm.py +++ /dev/null @@ -1,264 +0,0 @@ -import os -import platform -import argparse -import time -import math -import warnings -import json - -import pandas as pd -import torch -import torch.nn.functional as F -import torch.distributed as dist -from contextlib import nullcontext - -from torch import optim -from torch.nn.parallel import DistributedDataParallel -from torch.utils.data import DataLoader, DistributedSampler -from transformers import AutoTokenizer, AutoModel -from model.model import Transformer -from model.LMConfig import LMConfig -from model.dataset import SFTDataset, SFTDataset_multi -from model.vision_utils import get_vision_model, get_img_embedding - -warnings.filterwarnings('ignore') - - -def count_parameters(model): - return sum(p.numel() for p in model.parameters() if p.requires_grad) - - -def Logger(content): - if not ddp or dist.get_rank() == 0: - print(content) - - -def get_lr(it, all): - warmup_iters = args.warmup_iters - lr_decay_iters = all - min_lr = args.learning_rate / 10 - - if it < warmup_iters: - return args.learning_rate * it / warmup_iters - if it > lr_decay_iters: - return min_lr - decay_ratio = (it - warmup_iters) / (lr_decay_iters - warmup_iters) - assert 0 <= decay_ratio <= 1 - coeff = 0.5 * (1.0 + math.cos(math.pi * decay_ratio)) - return min_lr + coeff * (args.learning_rate - min_lr) - - -def train_epoch(epoch, wandb): - start_time = time.time() - for step, (X, Y, loss_mask, image_process) in enumerate(train_loader): - X = X.to(args.device) - Y = Y.to(args.device) - loss_mask = loss_mask.to(args.device) - image_process = image_process.to(args.device) - image_encoders = get_img_embedding(image_process, vision_model) - lr = get_lr(epoch * iter_per_epoch + step, args.epochs * iter_per_epoch) - for param_group in optimizer.param_groups: - param_group['lr'] = lr - - with ctx: - logits = model(X, Y, image_encoders=image_encoders).logits - loss = F.cross_entropy(logits.view(-1, logits.size(-1)), Y.view(-1), ignore_index=0, reduction='none') - loss_mask = loss_mask.view(-1) - loss = torch.sum(loss * loss_mask) / loss_mask.sum() - - scaler.scale(loss).backward() - - if (step + 1) % args.accumulation_steps == 0: - scaler.unscale_(optimizer) - torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) - - scaler.step(optimizer) - scaler.update() - - optimizer.zero_grad(set_to_none=True) - - if step % args.log_interval == 0: - spend_time = time.time() - start_time - Logger( - 'Epoch:[{}/{}]({}/{}) loss:{:.3f} lr:{:.7f} epoch_Time:{}min:'.format( - epoch, - args.epochs, - step, - iter_per_epoch, - loss.item(), - optimizer.param_groups[-1]['lr'], - spend_time / (step + 1) * iter_per_epoch // 60 - spend_time // 60)) - - if (wandb is not None) and (not ddp or dist.get_rank() == 0): - wandb.log({"loss": loss, - "lr": optimizer.param_groups[-1]['lr'], - "epoch_Time": spend_time / (step + 1) * iter_per_epoch // 60 - spend_time // 60}) - - if (step + 1) % args.save_interval == 0 and (not ddp or dist.get_rank() == 0): - model.eval() - moe_path = '_moe' if lm_config.use_moe else '' - if args.multi: # 多图训练权重保存 - ckp = f'{args.save_dir}/{lm_config.dim}{moe_path}_vlm_sft_multi.pth' - else: # 单图训练权重保存 - ckp = f'{args.save_dir}/{lm_config.dim}{moe_path}_vlm_sft.pth' - - if isinstance(model, torch.nn.parallel.DistributedDataParallel): - state_dict = model.module.state_dict() - else: - state_dict = model.state_dict() - - torch.save(state_dict, ckp) - model.train() - if args.save_last and (not ddp or dist.get_rank() == 0) and args.multi: - model.eval() - moe_path = '_moe' if lm_config.use_moe else '' - ckp = f'{args.save_dir}/{lm_config.dim}{moe_path}_vlm_sft_multi.pth' - - if isinstance(model, torch.nn.parallel.DistributedDataParallel): - state_dict = model.module.state_dict() - else: - state_dict = model.state_dict() - - torch.save(state_dict, ckp) - model.train() - - -def init_model(lm_config): - tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer') - moe_path = '_moe' if lm_config.use_moe else '' - - # 多图推理建议在单图推理之后 - if args.multi: - ckp = f'./out/{lm_config.dim}{moe_path}_vlm_sft.pth' - else: - ckp = f'./out/{lm_config.dim}{moe_path}_vlm_pretrain.pth' - - model = Transformer(lm_config) - state_dict = torch.load(ckp, map_location=args.device) - - # 处理不需要的前缀 - unwanted_prefix = '_orig_mod.' - for k, v in list(state_dict.items()): - if k.startswith(unwanted_prefix): - state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k) - - # 加载到模型中 - model.load_state_dict(state_dict, strict=False) - model = model.to(args.device) - - print(f'模型可学习参数: {count_parameters(model) / 1e6} 百万 = {count_parameters(model) / 1e9} B (Billion)') - - (vision_model, preprocess) = get_vision_model(args.visual_encoder) - vision_model = vision_model.to(args.device) - return model, tokenizer, (vision_model, preprocess) - - -def init_distributed_mode(): - if not ddp: return - global ddp_local_rank, DEVICE - - dist.init_process_group(backend="nccl") - ddp_rank = int(os.environ["RANK"]) - ddp_local_rank = int(os.environ["LOCAL_RANK"]) - ddp_world_size = int(os.environ["WORLD_SIZE"]) - DEVICE = f"cuda:{ddp_local_rank}" - torch.cuda.set_device(DEVICE) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="MiniMind-V-SFT") - parser.add_argument("--out_dir", type=str, default="out", help="Output directory") - parser.add_argument("--epochs", type=int, default=19, help="Number of epochs") - parser.add_argument("--batch_size", type=int, default=32, help="Batch size") - parser.add_argument("--learning_rate", type=float, default=1e-6, help="Learning rate") - parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", - help="Device to use") - parser.add_argument("--dtype", type=str, default="bfloat16", help="Data type") - parser.add_argument("--use_wandb", default=False, action="store_true", help="Use Weights & Biases") - parser.add_argument("--wandb_project", type=str, default="MiniMind-V", help="Weights & Biases project name") - parser.add_argument("--num_workers", type=int, default=8, help="Number of workers for data loading") - parser.add_argument("--data_path", type=str, default="./dataset/LLaVA-Instruct/llava_instruct_230k.json", - help="Path to training data") - parser.add_argument("--data_path_multi", type=str, default="./dataset/sft_multi_images/output.json", - help="Path to multi images training data") - parser.add_argument("--ddp", action="store_true", help="Use DistributedDataParallel") - parser.add_argument("--accumulation_steps", type=int, default=1, help="Gradient accumulation steps") - parser.add_argument("--grad_clip", type=float, default=1.0, help="Gradient clipping threshold") - parser.add_argument("--warmup_iters", type=int, default=0, help="Number of warmup iterations") - parser.add_argument("--log_interval", type=int, default=100, help="Logging interval") - parser.add_argument("--save_interval", type=int, default=100, help="Model saving interval") - parser.add_argument('--local_rank', type=int, default=-1, help='local rank for distributed training') - parser.add_argument('--multi', type=bool, default=False, help='multi-images training') - parser.add_argument('--save_last', type=bool, default=True, help='save last step model') - parser.add_argument('--visual_encoder', type=str, default="clip", help='type of visual endcoder') - - args = parser.parse_args() - - if args.visual_encoder == "clip": - lm_config = LMConfig() - else: - lm_config = LMConfig(image_special_token='<' * 98 + '>' * 98, image_ids=[30] * 98 + [32] * 98) - - max_seq_len = lm_config.max_seq_len - args.save_dir = os.path.join(args.out_dir) - os.makedirs(args.save_dir, exist_ok=True) - os.makedirs(args.out_dir, exist_ok=True) - tokens_per_iter = args.batch_size * max_seq_len - torch.manual_seed(1337) - device_type = "cuda" if "cuda" in args.device else "cpu" - - args.wandb_run_name = f"MiniMind-V-SFT-Epoch-{args.epochs}-BatchSize-{args.batch_size}-LearningRate-{args.learning_rate}" - - ctx = nullcontext() if device_type == "cpu" else torch.cuda.amp.autocast() - ddp = int(os.environ.get("RANK", -1)) != -1 # is this a ddp run? - ddp_local_rank, DEVICE = 0, "cuda:0" - if ddp: - init_distributed_mode() - args.device = torch.device(DEVICE) - - if args.use_wandb and (not ddp or ddp_local_rank == 0): - import wandb - - wandb.init(project=args.wandb_project, name=args.wandb_run_name) - else: - wandb = None - - model, tokenizer, (vision_model, preprocess) = init_model(lm_config) - - use_version = 0 - if args.multi: - print("进行多图训练，建议在指令微调后进行...") - train_ds = SFTDataset_multi(args.data_path_multi, tokenizer, vision_model=(vision_model, preprocess), - image_special_token=lm_config.image_special_token, - max_length=max_seq_len) - else: - train_ds = SFTDataset(args.data_path, tokenizer, vision_model=(vision_model, preprocess), - image_special_token=lm_config.image_special_token, - max_length=max_seq_len) - train_sampler = DistributedSampler(train_ds) if ddp else None - train_loader = DataLoader( - train_ds, - batch_size=args.batch_size, - pin_memory=True, - drop_last=False, - shuffle=False, - num_workers=args.num_workers, - sampler=train_sampler - ) - - scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype in ['float16', 'bfloat16'])) - optimizer = optim.Adam(model.parameters(), lr=args.learning_rate) - - if False and not lm_config.use_moe and platform.system() != 'Windows' and float( - torch.__version__.split('.')[0]) >= 2: - Logger("compiling the model... (takes a ~minute)") - unoptimized_model = model - model = torch.compile(model) - - if ddp: - model._ddp_params_and_buffers_to_ignore = {"pos_cis"} - model = DistributedDataParallel(model, device_ids=[ddp_local_rank]) - - iter_per_epoch = len(train_loader) - for epoch in range(args.epochs): - train_epoch(epoch, wandb) diff --git a/3-eval_chat.py b/3-eval_chat.py deleted file mode 100644 index 52cb898..0000000 --- a/3-eval_chat.py +++ /dev/null @@ -1,149 +0,0 @@ -import os -import random -import numpy as np -import torch -import warnings -from PIL import Image -from transformers import AutoTokenizer, AutoModelForCausalLM -from model.model import Transformer -from model.LMConfig import LMConfig -from model.vision_utils import get_vision_model, get_img_process, get_img_embedding - -warnings.filterwarnings('ignore') - - -def count_parameters(model): - return sum(p.numel() for p in model.parameters() if p.requires_grad) - - -def init_model(lm_config, device, multi): - tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer') - model_from = 1 # 1从权重，2用transformers - - if model_from == 1: - moe_path = '_moe' if lm_config.use_moe else '' - if multi: - ckp = f'./out/{lm_config.dim}{moe_path}_vlm_sft_multi.pth' - else: - ckp = f'./out/{lm_config.dim}{moe_path}_vlm_sft.pth' - model = Transformer(lm_config) - state_dict = torch.load(ckp, map_location=device) - - # 处理不需要的前缀 - unwanted_prefix = '_orig_mod.' - for k, v in list(state_dict.items()): - if k.startswith(unwanted_prefix): - state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k) - model.load_state_dict(state_dict, strict=False) - else: - model = AutoModelForCausalLM.from_pretrained('minimind-v-v1-small', trust_remote_code=True) - - model = model.to(device) - print(f'模型参数: {count_parameters(model) / 1e6} 百万 = {count_parameters(model) / 1e9} B (Billion)') - - vision_model, preprocess = get_vision_model(encoder_type="clip") - vision_model = vision_model.to(device) - return model, tokenizer, vision_model, preprocess - - -def setup_seed(seed): - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - torch.cuda.manual_seed(seed) - torch.cuda.manual_seed_all(seed) - torch.backends.cudnn.deterministic = True - torch.backends.cudnn.benchmark = False - - -if __name__ == "__main__": - # -------------------------- 基本参数设置 ----------------------------------- - multi = False # 设置multi参数，控制单图或多图推理 - out_dir = 'out' - temperature = 0.5 - top_k = 8 - setup_seed(1337) - device = 'cuda:0' if torch.cuda.is_available() else 'cpu' - dtype = 'bfloat16' - max_seq_len = 1024 - encoder_type = "clip" - # lm_config = LMConfig() - if encoder_type == "clip": - lm_config = LMConfig() - else: - lm_config = LMConfig(image_special_token='<' * 98 + '>' * 98, image_ids=[30] * 98 + [32] * 98) - lm_config.max_seq_len = max_seq_len - model, tokenizer, vision_model, preprocess = init_model(lm_config, device, multi) - model.eval() - - # -------------------------- 问题和目录设置 ----------------------------------- - if multi: - image_dir = './dataset/eval_multi_images/bird/' - prompt = f"{lm_config.image_special_token}\n{lm_config.image_special_token}\nName all the differences between these two birds." - else: - image_dir = './dataset/eval_images/' - prompt = lm_config.image_special_token + '\n这个图片描述的是什么内容？' - - image_files = sorted(os.listdir(image_dir)) - - # -------------------------- 推理逻辑 ----------------------------------- - if multi: - # 多图推理：所有图像编码一次性推理 - image_encoders = [] - for image_file in image_files: - image_path = os.path.join(image_dir, image_file) - image = Image.open(image_path).convert('RGB') - image_process = get_img_process(image, preprocess).to(vision_model.device) - image_encoder = get_img_embedding(image_process, vision_model).unsqueeze(0) - image_encoders.append(image_encoder) - print(f'[Image]: {image_file}') - image_encoders = torch.cat(image_encoders, dim=0).unsqueeze(0) - - messages = [{"role": "user", "content": prompt}] - new_prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)[ - -(max_seq_len - 1):] - x = tokenizer(new_prompt).data['input_ids'] - x = torch.tensor(x, dtype=torch.long, device=device)[None, ...] - - with torch.no_grad(): - res_y = model.generate(x, tokenizer.eos_token_id, max_new_tokens=max_seq_len, temperature=temperature, - top_k=top_k, stream=True, image_encoders=image_encoders) - print('[A]: ', end='') - history_idx = 0 - for y in res_y: - answer = tokenizer.decode(y[0].tolist()) - if answer and answer[-1] == '�': - y = next(res_y) - continue - print(answer[history_idx:], end='', flush=True) - history_idx = len(answer) - print('\n') - - else: - # 单图推理：对每张图像单独推理 - for image_file in image_files: - image_path = os.path.join(image_dir, image_file) - image = Image.open(image_path).convert('RGB') - image_process = get_img_process(image, preprocess).to(vision_model.device) - image_encoder = get_img_embedding(image_process, vision_model).unsqueeze(0) - - messages = [{"role": "user", "content": prompt}] - new_prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)[ - -(max_seq_len - 1):] - x = tokenizer(new_prompt).data['input_ids'] - x = torch.tensor(x, dtype=torch.long, device=device)[None, ...] - - print(f'[Image]: {image_file}') - with torch.no_grad(): - res_y = model.generate(x, tokenizer.eos_token_id, max_new_tokens=max_seq_len, temperature=temperature, - top_k=top_k, stream=True, image_encoders=image_encoder) - print('[A]: ', end='') - history_idx = 0 - for y in res_y: - answer = tokenizer.decode(y[0].tolist()) - if answer and answer[-1] == '�': - y = next(res_y) - continue - print(answer[history_idx:], end='', flush=True) - history_idx = len(answer) - print('\n') \ No newline at end of file diff --git a/README.md b/README.md index 6e5c0b8..b71fa2d 100644 --- a/README.md +++ b/README.md @@ -26,195 +26,224 @@ -* 本开源项目旨在从0开始，最快3小时训练一个小参数量的，具备视觉模态能力的语言模型**MiniMind-V** -* **MiniMind-V**同样极其轻量，最小版本体积仅为 GPT3 的约 $\frac{1}{7000}$，力求做到个人GPU也可快速推理甚至训练。 -* 这不仅是一个开源模型的实现，也是入门视觉语言模型（VLM）的教程。 -* 希望此项目能为研究者提供一个抛砖引玉的入门示例，帮助大家快速上手并对VLM领域产生更多的探索与创新。 +* 此项目旨在从0开始，仅用3块钱成本 + 2小时！即可训练出26M参数的超小多模态视觉语言模型**MiniMind-V**。 +* **MiniMind-V**最小版本体积仅为 GPT3 的约 $\frac{1}{7000}$，力求做到个人GPU也可快速推理甚至训练。 +* **MiniMind-V**是[MiniMind](https://github.com/jingyaogong/minimind)纯语言模型的视觉能力额外拓展。 +* 项目同时包含了VLM大模型的极简结构、数据集清洗、预训练(Pretrain)、监督微调(SFT)等全过程代码。 +* 这不仅是一个开源VLM模型的最小实现，也是入门视觉语言模型的简明教程。 +* 希望此项目能为所有人提供一个抛砖引玉的示例，一起感受创造的乐趣！推动更广泛AI社区的进步！ - > 为防止误读，「从0开始」特指基于纯语言模型MiniMind（这是一个完全从0训练的类GPT模型）做进一步的，从0到1的视觉能力拓展。 - > 若需详细了解后者，请参考孪生项目[MiniMind](https://github.com/jingyaogong/minimind)。 +> 为防止误解，“2小时” 基于NVIDIA 3090硬件设备（单卡）测试`1 epoch`，“3块钱” 指GPU服务器租用成本。 - > 为防止误读，「最快3小时」是指您需要具备＞本人硬件配置的机器，具体规格的详细信息将在下文提供。 -![](./images/modelscope-demo.gif)
-Demo已部署至ModelScope创空间，可以在此网站上体验： +![minimind2-v](./images/minimind2-v.gif) -[ModelScope在线体验](https://modelscope.cn/studios/gongjy/minimind-v) +[🔗🤖在线体验](https://www.modelscope.cn/studios/gongjy/MiniMind-V) | [🔗🎞️视频介绍](https://www.bilibili.com/video/BV1Sh1vYBEzY) -[🎉🎉BiliBili项目视频🎉🎉](https://www.bilibili.com/video/BV1Sh1vYBEzY)
# 📌 Introduction -视觉语言模型（VLM）如 GPT-4V、Qwen-VL、LlaVA 等，虽然在效果上令人惊艳， -但这些动辄 100 亿级参数的庞大模型，往往需要极高的硬件配置。 -对于个人设备来说，不仅显存远远不足以支持训练，甚至连推理都十分困难。 -我们通过阅读论文或公众号讲解来了解略显新颖的 VLM， -往往只能一知半解云里雾里。 -而我们真正需要了解的是： -构建多模态大模型是否真的如想象中那样复杂？它的代码实现到底如何？ -训练过程究竟难不难？如果我只有一张 2080Ti 显卡，能否从0开始进行训练？ - -通过 **MiniMind-V**，本项目希望回答这些问题， -帮助研究者在有限的硬件条件下理解视觉语言模型的核心原理。 +“用乐高拼出一架飞机，远比坐在头等舱里飞行更让人兴奋！” +构建VLM范式的多模态大模型是否真的如想象中那样复杂？它的代码实现到底如何？ +训练过程究竟难不难？那么现在，探索它们的答案，一起感受创造的乐趣吧！ > [!TIP] -> （截至2024-10-04）MiniMind-V 系列已完成了 2 个型号模型的预训练，最小仅需27M（0.027B），即可具备识图和对话的能力！ +> （截至2025-02-20）MiniMind-V 系列已完成了以下型号模型训练，最小仅需26M (0.026B)，即可具备识图和对话的能力！ -| 模型 (大小) | tokenizer长度 | 推理占用 | release | 主观评分（/100） | -|---------------------------|-------------|--------|------------|------------| -| minimind-v-v1-small (27M) | 6400 | 0.6 GB | 2024.10.04 | 50' | -| minimind-v-v1 (109M) | 6400 | 1.1 GB | 2024.10.04 | 60' | - -> 该分析在具有Torch 2.1.2、CUDA 12.2和Flash Attention 2的2×RTX 3090 GPU上进行。 +| 模型 (大小) | 推理占用 | release | +|---------------------------|--------|------------| +| MiniMind2-V (104M) | 0.6 GB | 2025.02.20 | +| MiniMind2-Small-V (26M) | 1.1 GB | 2025.02.20 | +| minimind-v-v1-small (27M) | 0.6 GB | 2024.10.04 | +| minimind-v-v1 (109M) | 1.1 GB | 2024.10.04 | ### 👉**最近更新**
- 2024-10-05 (newest 🎉) + 2025-02-20 (newest 🎉) + +- MiniMind2-V伴随MiniMind2同步更新 +- 大幅减少所有冗余代码，规范代码格式 +- 大幅精简模型冗余结构 +- 更新数据集格式，拓展新的SFT数据集 +- 比前代VLM更优秀的效果！ + +
+ +
+ 2024-10-05 - MiniMind-V如期而至，首次开源
-# 📌 Environment +# 📌 快速开始 -仅是我个人的软硬件环境配置，自行酌情变动： +
+分享本人的软硬件配置（仅供参考） -```bash -CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz -内存：128 GB -显卡：NVIDIA GeForce RTX 3090(24GB) * 2 -环境：python 3.9 + Torch 2.1.2 + DDP单机多卡训练 -``` - -* Ubuntu == 20.04 -* Python == 3.9 -* Pytorch == 2.1.2 -* CUDA == 12.2 +* CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz +* RAM: 128 GB +* GPU: NVIDIA GeForce RTX 3090(24GB) * 8 +* Ubuntu==20.04 +* CUDA==12.2 +* Python==3.10.16 * [requirements.txt](./requirements.txt) -# 📌 Quick Start Test +
-> BTW: 如果没有git-lfs，请先安装 `sudo apt-get update`, `sudo apt-get install git-lfs` +### 第0步 -* 0.克隆项目 - ```bash - git clone https://github.com/jingyaogong/minimind-v - cd minimind-v - ``` +```bash +# 克隆代码仓库 +git clone https://github.com/jingyaogong/minimind-v +``` -* 1.环境安装 - ```bash - pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple - ``` +```bash +# 下载clip模型到 ./model/vision_model 目录下 +git clone https://huggingface.co/openai/clip-vit-base-patch16 +# or +git clone https://www.modelscope.cn/models/openai-mirror/clip-vit-base-patch16 +``` - ```text - # 测试torch是否可用cuda - import torch - print(torch.cuda.is_available()) - ``` +## Ⅰ 测试已有模型效果 - > 如果不可用，请自行去[torch_stable](https://download.pytorch.org/whl/torch_stable.html) - 下载whl文件安装。参考[链接](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=%E5%AE%89%E8%A3%85torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187) +### 1.环境准备 + +```bash +pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple +``` + +### 2.下载模型 + +```bash +git clone https://huggingface.co/jingyaogong/MiniMind2-V +``` + +### 3.命令行问答 + +```bash +# load=0: load from pytorch model, load=1: load from transformers-hf model +python eval_vlm.py --load 1 +``` + +### 4.或启动WebUI + +```bash +python web_demo_vlm.py +``` + +## Ⅱ 从0开始自己训练 + +### 1.环境准备 + +```bash +pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple +``` + +
+注：提前测试Torch是否可用cuda + +```bash +import torch +print(torch.cuda.is_available()) +``` + +如果不可用，请自行去[torch_stable](https://download.pytorch.org/whl/torch_stable.html) +下载whl文件安装。参考[链接](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=%E5%AE%89%E8%A3%85torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187) + +
+ +### 2.数据下载 + +从下文提供的[数据集下载链接](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset) +下载需要的数据文件（创建`./dataset`目录）并放到`./dataset`下。 + +`*.jsonl`为问答数据集，`*images`为配套的图片数据，下载完成后需要解压图像数据。 + +
+注：数据集须知 + +请预留~5GB空间存放数据集，若无多余空间存放pretrain数据， +可尝试跳过pretrain训练步骤直接进行sft训练。 + +
+ +### 3.开始训练 + +**3.1 预训练（学图像描述）** + +```bash +python train_pretrain_vlm.py --epochs 4 +``` + +> 执行预训练，得到 `pretrain_vlm_*.pth` 作为预训练的输出权重（其中*为模型的dimension，默认为512） -* 2.下载预训练的模型权重文件到项目根目录 `minimind-v-v1` - ```bash - git clone https://huggingface.co/jingyaogong/minimind-v-v1 - ``` +**3.2 监督微调（学看图对话方式）** -* 3.下载预训练的`clip-vit-base-patch32` 模型，在 `model/clip_model` 目录下： - ```bash - cd model/clip_model - git clone https://hf-mirror.com/openai/clip-vit-base-patch32 - ``` +```bash +python train_sft_vlm.py --epochs 4 +``` -* 4.启动聊天网页测试对话 - ```bash - python web_server.py - ``` +> 执行监督微调，得到 `sft_vlm_*.pth` 作为指令微调的输出权重 -![web_server](images/web_server.gif) +
+注：训练须知 -# 📌 Quick Start Train +所有训练过程默认每隔100步保存1次参数到文件`./out/***.pth`（每次会覆盖掉旧权重文件）。 -> BTW: 如果没有git-lfs，请先安装 `sudo apt-get update`, `sudo apt-get install git-lfs` +
-* 0.克隆项目代码 - ```text - git clone https://github.com/jingyaogong/minimind-v & cd minimind-v - ``` -* 1.环境安装 - ```bash - pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple - ``` +--- - ```text - # 测试torch是否可用cuda - import torch - print(torch.cuda.is_available()) - ``` +### 4.测试模型效果 - > 如果不可用，请自行去[torch_stable](https://download.pytorch.org/whl/torch_stable.html) - 下载whl文件安装。参考[链接](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=%E5%AE%89%E8%A3%85torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187) +确保需要测试的模型`*.pth`文件位于`./out/`目录下。 +也可以直接去[此处](https://huggingface.co/jingyaogong/MiniMind2-V-PyTorch)下载使用我训练的`*.pth`文件。 -* 2.下载 `clip-vit-base-patch32` 模型，放到 `./model/clip_model` 目录下： - ```bash - cd ./model/clip_model & git clone https://hf-mirror.com/openai/clip-vit-base-patch32 - ``` +```bash +python eval_vlm.py --model_mode 1 # 默认为0：测试pretrain模型效果，设置为1：测试sft模型效果 +``` -* 3.如果需要自己训练 +--- - * 3.1 下载数据集 ([百度网盘](https://pan.baidu.com/s/1Nz36OBBvVBGEx-PwIb7ofg?pwd=6666) - or [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset))文件到`./dataset` - 目录下，并解压`pretrain_images.zip` 和 `sft_images.zip` - * 3.2 在`./model/LMConfig.py` 中调整model的参数配置 - > 这里仅需调整dim和n_layers参数，分别是`(512+8)`或`(768+16)`，对应于`minimind-v-v1-small`和`minimind-v-v1` - * 3.3 下载minimind语言模型的预训练权重([百度网盘](https://pan.baidu.com/s/1LE1SPoPYGS7VNtT1tpf7DA?pwd=6666) - or [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset/tree/main/out))，放到`./out/` - 目录下，命名为`*_llm.pth` - * 3.4 `python 1-pretrain_vlm.py` 执行预训练，得到 `*_vlm_pretrain.pth` 作为预训练的输出权重 - * 3.5 `python 2-sft_vlm.py` 执行指令微调，得到 `*_vlm_sft.pth` 作为指令微调的输出权重 - * 3.6 `python 2-sft_vlm.py --multi True` 在指令微调的基础上执行多图指令微调（效果仅参考），得到 `*_vlm_sft_multi.pth` 作为多图指令微调的输出权重，512+8 模型显存占用约为8198M +> [!TIP] +> 训练脚本均为Pytorch原生框架，均支持多卡加速，假设你的设备有N (N＞1) 张显卡： -* 4.测试自己训练的模型推理效果 - * 确保需要使用的，训练完成的参数权重`*.pth`文件位于`./out/`目录下 - * 也可以直接下载使用我已经训练好的 `*.pth` [权重文件](https://pan.baidu.com/s/1LE1SPoPYGS7VNtT1tpf7DA?pwd=6666)使用 - ```text - minimind-v/out - ├── 512_llm.pth - ├── 512_vlm_pretrain.pth - ├── 512_vlm_sft.pth - ├── 768_llm.pth - ├── 768_vlm_pretrain.pth - ├── 768_vlm_sft.pth - ``` - * `python 3-eval_chat.py`测试模型的对话效果，其中测试图片在`./dataset/eval_images`下，可自行更换 - ![eval_chat](images/3-eval_chat.png) - * `python 3-eval_chat.py`调整[multi](./3-eval_chat.py#L61)变量，可以测试模型的多图对话效果，其中测试图片在`./dataset/eval_multi_images`下，可自行更换（多图数据集规模相对较小且为英文对话，数据集仅包含两图对比的场景，因此微调效果有限）。 +单机N卡启动训练方式 (DDP, 支持多机多卡集群) -🍭 【Tip】预训练和全参指令微调pretrain和sft均支持多卡加速 +```bash +torchrun --nproc_per_node N train_xxx.py +``` -* 单机N卡启动训练(DDP) - ```bash - torchrun --nproc_per_node N 1-pretrain_vlm.py - # and - torchrun --nproc_per_node N 2-sft_vlm.py - ``` +
+注：其它须知 -* 记录训练过程 - ```bash - torchrun --nproc_per_node N 1-pretrain_vlm.py --use_wandb - # and - python 1-pretrain_vlm.py --use_wandb - ``` - 通过添加`--use_wandb`参数，可以记录训练过程，训练完成后，可以在wandb网站上查看训练过程。通过修改`wandb_project` - 和`wandb_run_name`参数，可以指定项目名称和运行名称。 +单机N卡启动训练 (DeepSpeed) + +```bash +deepspeed --master_port 29500 --num_gpus=N train_xxx.py +``` + +可根据需要开启wandb记录训练过程 + +```bash +# 需要登录: wandb login +torchrun --nproc_per_node N train_xxx.py --use_wandb +# and +python train_xxx.py --use_wandb +``` + +通过添加`--use_wandb`参数，可以记录训练过程，训练完成后，可以在wandb网站上查看训练过程。通过修改`wandb_project` +和`wandb_run_name`参数，可以指定项目名称和运行名称。 + +
# 📌 VLM Detail @@ -222,70 +251,63 @@ MiniMind-V (VLM)的基座语言模型MiniMind (LLM)来自孪生项目[minimind]( 具体的模型结构、训练细节、原理、测试效果等均可移步[minimind](https://github.com/jingyaogong/minimind)项目查阅。 此处为减少冗余，省略讨论LLM的相关部分，默认您已对MiniMind (LLM)的细节有基本的了解。 -> PS: 即使您不希望了解MiniMind (LLM)的细节，也可直接参考Quick Test和Quick Start中快速测试或训练MiniMind-V， -> 这并不受太大影响。 +> 即使您不太了解LLM的细节，也可参考“快速开始”流程训练一个MiniMind-V， +> 这并不受到影响，仓库致力于最低成本的开箱即用！ -MiniMind-V的结构几乎不变，仅增加Visual Encoder和特征投影两个子模块，增加模态混合分支，以支持多种模态信息的输入： +MiniMind-V的结构仅增加Visual Encoder和特征投影两个子模块，增加模态混合分支，以支持多种模态信息的输入： ![LLM-structure](./images/VLM-structure.png) ![LLM-structure](./images/VLM-structure-moe.png) -此时，不妨思考2个很有意思的问题：什么叫做**L**arge **L**anguage **M**odel(LLM)？什么叫做多模态模型？ -* [这篇文章](https://www.jiqizhixin.com/articles/2024-09-15-3)完美吐露了本人的想法，即LLM这个命名很不准确！ +
+ 【重要】一些有趣的思考 - > 大语言模型（LLM）名字虽然带有语言二字，但它们其实与语言关系不大，这只是历史问题，更确切的名字应该是自回归 Transformer - 或者其他。 - LLM 更多是一种统计建模的通用技术，它们主要通过自回归 Transformer 来模拟 token 流，而这些 token - 可以代表文本、图片、音频、动作选择、甚至是分子等任何东西。 - 因此，只要能将问题转化为模拟一系列离散 token 的流程，理论上都可以应用 LLM 来解决。 - 实际上，随着大型语言模型技术栈的日益成熟，我们可能会看到越来越多的问题被纳入这种建模范式。也就是说，问题固定在使用 LLM - 进行『下一个 token 的预测』，只是每个领域中 token 的用途和含义有所不同。 +此处不妨展开想一想两个问题： -* [李玺老师](https://person.zju.edu.cn/xilics#694283)的公开讲话同样佐证了类似观点（原话大意如下）： +* 什么叫做**L**arge **L**anguage **M**odel (LLM)？ +* 什么叫做多模态模型？ - > 文本、视频、语音、动作等在人类看来属于「多模态」信号，但所谓的「模态」其实只是人类在信息存储方式上的一种分类概念。 - 就像`.txt`和`.png`文件，虽然在视觉呈现和高级表现形式上有所不同，但它们本质上并没有根本区别。 - 之所以出现「多模态」这个概念，仅仅是因为人类在不同的感知层面上对这些信号的分类需求。 - 然而，对于机器来说，无论信号来自何种「模态」，最终它们都只是以一串二进制的「单模态」数字序列来呈现。 - 机器并不会区分这些信号的模态来源，而只是处理和分析这些序列背后所承载的信息内容。 +[这篇文章](https://www.jiqizhixin.com/articles/2024-09-15-3)完美吻合本人的想法： +大语言模型（LLM）名字虽然带有语言二字，但它们其实与语言关系不大，这只是历史问题，更确切的名字应该是自回归 Transformer +或者其他。LLM 更多是一种统计建模的通用技术，它们主要通过自回归 Transformer 来模拟 token 流，而这些 token +可以代表文本、图片、音频、动作选择、甚至是分子等任何东西。 +因此，只要能将问题转化为模拟一系列离散 token 的流程，理论上都可以应用 LLM 来解决。 +实际上，随着大型语言模型技术栈的日益成熟，我们可能会看到越来越多的问题被纳入这种建模范式。也就是说，问题固定在使用 LLM +进行『下一个 token 的预测』，只是每个领域中 token 的用途和含义有所不同。 ---- +[ZJU-LiXi老师](https://person.zju.edu.cn/xilics#694283)同样谈及过类似观点（原话大意如下）： +文本、视频、语音、动作等在人类看来属于「多模态」信号，但所谓的「模态」其实只是人类在信息存储方式上的一种分类概念。 +就像`.txt`和`.png`文件，虽然在视觉呈现和高级表现形式上有所不同，但它们本质上并没有根本区别。 +之所以出现「多模态」这个概念，仅仅是因为人类在不同的感知层面上对这些信号的分类需求。 +然而，对于机器来说，无论信号来自何种「模态」，最终它们都只是以一串二进制的「单模态」数字序列来呈现。 +机器并不会区分这些信号的模态来源，而只是处理和分析这些序列背后所承载的信息内容。 -私以为，**G**enerative **P**retrained **T**ransformer (GPT) 比 **L**arge **L**anguage **M**odel (LLM)更为贴切， +个人认为**G**enerative **P**retrained **T**ransformer (GPT) 比 **L**arge **L**anguage **M**odel (LLM)更为贴切， 因此本人表达上更习惯用"GPT"去代表LLM/VLM/类GPT架构的系列模型，而非为了蹭OpenAI的热度。 ---- - 至此，我们可以用一句话总结GPT的所作所为： + GPT模型根据现有token预测输出下一个下下一个下下下一个token ...，直到模型输出结束符；此处的"token"其实并不需要一定是文本！ ---- +```text +> 对于LLM模型，如果需要理解"图片"，我们只要把"图片"作为对一种特殊的从来没见过的"外国语言"，通过"外语词典"翻译后即可作为特殊的语言输入LLM +> 对于LLM模型，如果需要理解"音频"，我们只要把"音频"作为对一种特殊的从来没见过的"外国语言"，通过"外语词典"翻译后即可作为特殊的语言输入LLM +> ... +``` -* 对于LLM模型，如果需要理解"图片"，我们只要把"图片"作为对一种特殊的从来没见过的"外国语言"，通过"外语词典"翻译后即可作为特殊的语言输入LLM -* 对于LLM模型，如果需要理解"音频"，我们只要把"音频"作为对一种特殊的从来没见过的"外国语言"，通过"外语词典"翻译后即可作为特殊的语言输入LLM -* ... - ---- - -**所以，为了得到MiniMind-V，我们只需要完成2件事即可：** +**为了得到MiniMind-V，我们只需要完成这2件事即可：** 1. 借助擅长翻译图片的 **"外语词典"** ，把图片从 **"外国语言"** 翻译为模型便于理解的 **"LLM语言"** 2. 训练微调LLM，使其和 **"外语词典"** 度过磨合期，从而更好的理解图片 ---- - -"外语词典"一般称之为Visual Encoder模型。 +"外语词典" 称之为Visual Encoder模型。 和LlaVA、Qwen-VL等视觉语言模型类似，MiniMind-V同样选用开源Clip系列模型作为Visual Encoder。 -具体使用[clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32)， -一种基于 ViT-B/32 架构的经典Visual Encoder用于描述图像文本信息。 -输入的图像尺寸为224x224，因为划分的Patch是32×32，所以会产生7*7+1(cls_token)=50个token作为encoder编码层的输入， +具体使用[clip-vit-base-patch16](https://huggingface.co/openai/clip-vit-base-patch16)， +一种基于 ViT-B/16 架构的经典Visual Encoder用于描述图像文本信息。 +输入的图像尺寸为224x224，因为划分的Patch是16×16，所以会产生16*16=196个token作为encoder编码层的输入， 最终产生1×768维的嵌入向量用于和文本对计算误差。 我们并不需要最终嵌入表示，因此只取encoder层的输出，也就是VIT核心主干的输出特征即可。 -在代码中对应[./model/vision_utils.py](./model/vision_utils.py)的get_img_embedding中的hook函数。 -它拿到前一层维度50×768大小的特征，我们把它作为50个visual token输入MiniMind-V。 -也有clip-vit-large-patch14这种更大，图像理解能力更强的Clip模型， -但是单图片会产生257个token，对于minimind这种量级模型，图片token的上下文占比太长，反倒不利于训练。 - +它拿到前一层维度196×768大小的特征，我们把它作为196个visual token输入MiniMind-V。 与LLM的结合在获取图像encoder特征后，一方面需要把768维度的visual token对齐到LLM的文本token， 另一方面，要将图像特征映射到与文本embedding相同的空间，即文本token和原生的视觉token需要磨合并不能直接地一视同仁， 可以称之为跨模态的特征对齐。 @@ -295,6 +317,9 @@ GPT模型根据现有token预测输出下一个下下一个下下下一个token 至此，MiniMind-V的内部结构变化已经呈现完毕。 +
+ + --- 下面，我们简单讨论MiniMind-V的外部输入输出的变化。 @@ -307,46 +332,45 @@ VLM的输入依然是一段文本，其中包含特殊的占位符。 \n这个图像中有什么内容？ ``` -minimind-v使用50个字符组成的 `<<<...>>>` 占位符代替图像， -之所以是50个字符，前面有所提及： -任何图像都被clip模型encoder为50×768维的token。 -因此minimind-v的prompt： +在`minimind-v`中，使用196个字符组成的 `@@@...@@@` +占位符代替图像，之所以是196个字符，前面有所提及： +任何图像都被clip模型encoder为196×768维的token， +因此`minimind-v`的prompt为： ```text -<<<<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>>>>\n这个图片描述的是什么内容？ +@@@......@@@\n这个图片描述的是什么内容？ ``` 计算完embedding和projection，并对图像部分token替换后整个计算过程到输出则和LLM部分没有任何区别。 ![input](./images/minimind-v-input.png) -多图实现方法就是通过注入多个\图像占位符进行实现，不需要修改任何框架。 - -> ps: 唯一值得注意的点是，如果在训练过程中存在不同conversations插入图片数量不同的情况，需要利用空特征将较短的特征进行填充（对应[dataset的第267行](./model/dataset.py#L267)），以保证能够在同样大小下被dataloader读取。 - -> pps: 在prompt中不需要如此做，仍旧是根据插入图像的数量来进行占位符的注入。因此，最终输入给LLM的input feature不会受填充特征的影响。 - +一次性多图的实现方法就是通过注入多个``图像占位符进行实现，不需要修改任何框架。
- 实现视频理解能力的思考 + 视频理解的拓展思路 + +write by [@xinyanghuang7](https://github.com/xinyanghuang7) 对于多模态大模型的视频理解能力，一个可行的思路是参考现有MiniCPM-V 2.6 进行视频理解的Python示例。 主要思想是通过提取视频关键帧，而后进行多图推理。 因此，如果希望在MiniMind-V中添加视频理解能力，可以在现有多图训练的基础上，参考此python脚本中对于关键帧的提取方法，而后加大训练文件中支持图片的数量。 所支持的MAX_NUM_FRAMES越多，所消耗的显存越大。 -```python +```text import torch from PIL import Image from transformers import AutoModel, AutoTokenizer -from decord import VideoReader, cpu # pip install decord +from decord import VideoReader, cpu # pip install decord model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True, - attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager + attn_implementation='sdpa', + torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager model = model.eval().cuda() tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True) -MAX_NUM_FRAMES=64 # if cuda OOM set a smaller number +MAX_NUM_FRAMES = 64 # if cuda OOM set a smaller number + def encode_video(video_path): def uniform_sample(l, n): @@ -364,17 +388,18 @@ def encode_video(video_path): print('num frames:', len(frames)) return frames -video_path="video_test.mp4" + +video_path = "video_test.mp4" frames = encode_video(video_path) question = "Describe the video" msgs = [ - {'role': 'user', 'content': frames + [question]}, + {'role': 'user', 'content': frames + [question]}, ] # Set decode params for video params = {} params["use_image_id"] = False -params["max_slice_nums"] = 2 # 如果cuda OOM且视频分辨率大于448*448可设为1 +params["max_slice_nums"] = 2 # 如果cuda OOM且视频分辨率大于448*448可设为1 answer = model.chat( image=None, @@ -384,226 +409,208 @@ answer = model.chat( ) print(answer) ``` +
-至此，MiniMind-V的所有细节已经呈现完毕。 - -MiniMind-V的实现未参考 **任何** 第三方代码，完全基于MiniMind尽可能做最小改动产生，故代码实现和LlaVA等模型必然存在很大区别。 -MiniMind-V与MiniMind的代码核心改动不超过100行，上手难度低。 +至此，`MiniMind-V`的所有细节已经呈现完毕。 +`MiniMind-V`的模型子类完全继承自`MiniMind`， +仅基于后者做**最小**变更而产生， +其核心算法改动`< 50行`，迁移难度极低。 +因此可能和`LlAVA`等模型细节可能存在区别，但思路完全统一。 # 📌 Experiment -## 数据集 +## Ⅰ 数据集 来源：[Chinese-LLaVA-Vision](https://huggingface.co/datasets/LinkSoul/Chinese-LLaVA-Vision-Instructions) -包含约60万张预训练图像和<10万张指令微调图像，来自CC-3M和COCO 2014，问答内容经过翻译，对中文支持更友好。并进一步经过resize和整理压缩。 +包含约57万张预训练图像，来自CC-3M和COCO 2014； +[llava-en-zh-300k](https://huggingface.co/datasets/BUAADreamer/llava-en-zh-300k) +包含300k条指令微调数据和15万张图像。 +问答内容经过翻译， +对中文支持更友好，进一步经过整理并`resize`。 -预训练数据集格式： +(pretrain_vlm_data.jsonl) 预训练数据集格式： -```json +```json lines { - "id": "GCC_train_000644518", - "image": "GCC_train_000644518.jpg", "conversations": [ { - "from": "human", - "value": "写一篇简短但有益的图片摘要.\n" + "role": "user", + "content": "提供给定图像的简要描述。\n" }, { - "from": "gpt", - "value": "在黑色背景的金属锅中加入盐水,慢动作fps" + "role": "assistant", + "content": "橄榄油是自由使用的健康成分。" } - ] + ], + "image": "GCC_train_002582585.jpg" } ``` -指令微调数据集格式： +(sft_vlm_data.jsonl) 单图指令微调数据集格式： -```json +```json lines { - "id": "000000334872", - "image": "000000334872.jpg", "conversations": [ { - "from": "human", - "value": "\n照片中的人们在下山滑雪还是越野滑雪?" + "role": "user", + "content": "闹钟的位置对睡眠质量有什么影响？" }, { - "from": "gpt", - "value": "照片中的人们在森林里越野滑雪,因为他们在一条小径上而不是在陡坡上滑雪。" + "role": "assistant", + "content": "把数字闹钟放在床头柜..." } - ] + ], + "image": "train-00000-of-00001_image_0_0.jpg" } ``` -多图指令微调数据集格式： +(sft_vlm_data_multi.jsonl) 多图指令微调数据集格式： -```json +```json lines { - "id": "000000000000", - "image": "0.jpg, 1.jpg", - "conversations": [ - { - "from": "human", - "value": "context: Source Image: Target Image: Instruction: What is the correct image edit instruction that can transfrom the source image to target image?" - }, - { - "from": "gpt", - "value": "take the people out of the back in the photo. Remove the two people behind the woman in the white dress and the man in the blue suit. remove people behind the couple in the centre" - } - ] - } + "conversations": [ + { + "role": "user", + "content": "context: Source Image: Target Image: Instruction: What is the correct image edit instruction that can transfrom the source image to target image?" + }, + { + "role": "assistant", + "content": "take the people out of the back in the photo. Remove the two people behind the woman in the white dress and the man in the blue suit. remove people behind the couple in the centre" + } + ], + "image": "0.jpg, 1.jpg" +} ``` -```json -{ - "id": "000000013633", - "image": "27266.jpg, 27267.jpg", - "conversations": [ - { - "from": "human", - "value": "\n\nName all the differences between these two birds." - }, - { - "from": "gpt", - "value": "animal1 is brown with white tuft while animal2 is white with dark brown wings" - } - ] - } -``` +
+ 数据说明 -注： -+ 对于指令微调，仅保留了一轮对话，训练单轮对话模型，防止小模型性能被长文本拉低。 -+ 多图数据集规模相对较小且为英文对话，数据集仅包含两图对比的场景，因此微调效果有限，这里只提供一种参考思路。 +* 多图数据集规模相对较小且为英文对话，数据集仅包含两图对比的场景，因此微调效果有限，这里只提供一种参考思路。 +* `jsonl`均为文本指令，`images.zip`均为配套的图像数据（下载后需要解压） -最终的数据集下载地址：[百度网盘](https://pan.baidu.com/s/1Nz36OBBvVBGEx-PwIb7ofg?pwd=6666) | [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset) +
-多图对话数据集：[HuggingFace](https://hf-mirror.com/datasets/xinyanghuang/minimind-v_multi_image/tree/main) +数据集下载地址：([ModelScope](https://www.modelscope.cn/datasets/gongjy/minimind-v_dataset) | [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset)) -## 训练 +## Ⅱ 训练 + +> train_pretrain_vlm 预训练从595K条数据集中学习图片的通用知识，比如鹿是鹿，狗是狗。 -指令微调从230K条真实对话数据集中学习对图片提问的真实问答格式。 +> train_sft_vlm -多图微调提供两个数据集，图像转换数据集和鸟类对比数据集，长度分别为3.5k和13.6k的真实问答格式。 +指令微调从300K条真实对话数据集中学习对图片提问的真实问答格式，更符合与人类的交流习惯。 -`1-pretrain_vlm.py` 执行预训练，得到 `*_vlm_pretrain.pth` 作为预训练的输出权重。 +> train_sft_vlm -`2-sft_vlm.py` 执行指令微调，得到 `*_vlm_sft.pth` 作为指令微调的输出权重。 +多图微调提供demo：鸟类对比数据集，长度为13.6k的真实问答格式。 -训练时均冻结visual encoder也就是clip模型，只微调Projection和LLM两部分。 +训练时均冻结visual encoder也就是clip模型梯度， +只训练Projection和LLM两部分。 +预训练中，只设置Projection和LLM的最后一层参数可学习。 +指令微调中，设置Projection和LLM的全部参数可学习。 -> Pretrain 512+8 模型 (训练时间和Loss参考图) -![input](./images/1-pretrain-512.png) -> Pretrain 768+16 模型 (训练时间和Loss参考图) -![input](./images/1-pretrain-768.png) -> SFT 512+8 模型 (训练时间和Loss参考图) -![input](./images/2-sft-512.png) -> SFT 768+16 模型 (训练时间和Loss参考图) -![input](./images/2-sft-768.png) +> 训练时间和Loss走势（仅供参考） -## 训练完成的模型权重 +Pretrain [512+8] & [768+16] +![input](./images/pretrain_loss.png) -(`.pth`权重文件) 下载地址：[百度网盘](https://pan.baidu.com/s/1a7_C7HdCMfnG2Dia3q85FQ?pwd=6666) +SFT [512+8] & [768+16] +![input](./images/sft_loss.png) -(`transformers`模型文件) -下载地址：[HuggingFace](https://huggingface.co/collections/jingyaogong/minimind-v-67000833fb60b3a2e1f3597d) +## Ⅲ 模型权重 -> 注：HuggingFace版本均为指令微调后的MiniMind-V模型 +(原生PyTorch`*.pth`权重文件) 下载地址： +([ModelScope](https://www.modelscope.cn/models/gongjy/MiniMind2-V-PyTorch) | [HuggingFace](https://huggingface.co/jingyaogong/MiniMind2-V-PyTorch)) -| Model Name | params | Config | file_name | -|---------------------|--------|-----------------------------|-----------------------------------------------------| -| minimind-v-v1-small | 27M | d_model=512
n_layers=8 | 预训练：512_vllm_pretrain.pth
指令微调：512_vllm_sft.pth | -| minimind-v-v1 | 109M | d_model=768
n_layers=16 | 预训练：768_vllm_pretrain.pth
指令微调：768_vllm_sft.pth | +(`Transformers`格式模型) +下载地址： +([ModelScope](https://www.modelscope.cn/profile/gongjy) | [HuggingFace](https://huggingface.co/collections/jingyaogong/minimind-v-67000833fb60b3a2e1f3597d)) + +> 注：Transformers版本均为单图指令微调后的`MiniMind-V`模型 # 📌 Test ### 效果测试 #### 单图对话 + - - - - + + - - - - - + + + - - - - - + + + - - - - - + + + - - - - - + + + - - - - - + + + - - - - - + + + - - - - - + + + - - - - - - - - - - - - - - - - - - - + + +
图片512_pretrain512_sft768_pretrain768_sftMiniMind2-VMiniMind2-V-Small
a-girl.png头发和化妆,我喜欢她的自然头发!这个图片描绘了一个年轻的女人,她穿着一套西装,戴着一条领带,这表明她可能正在参加一个特别的时装活动或庆祝活动。人为出演员的冒险片。这个图片描绘了一个女人的肖像，她穿着一件粉红色的裙子。 + city-traffic +                              + 图中是一个繁忙的城市街道，一条长长的街道两旁都是高楼大厦。这条街上挤满了汽车、卡车和公共汽车，还有许多其他车辆在路上行驶。在街道上，可以看到许多汽车，有的在高速行驶，而其他的则停在街道一侧。此外还有一辆公交车也停在街道的右侧。街道上可以看到交通灯，表明这是一个繁忙的城市环境。图中是一个繁忙的城市景象，有几辆汽车和一辆卡车行驶在城市街道上。可以看到许多交通信号灯，其中一些位于街道左侧，另一些则在右侧。可以看到有几个人在街上行走，其中一些人站得离街道更近一些，而另一些则距离较远。还有一个停车标志位于画面的左侧，暗示着城市环境。可以看到街道上有两辆汽车，一辆在右边，另一辆在左边，还有一辆在左边。这幅图像捕捉到了都市环境中典型的一天。
a-girl.png水中的化石, 一个由环绕的环形化石团组成的静止线.图片显示一只大型的 octopus, 一个大型的 octopus, 可能是一个潜在的海洋生物, 它在水面上, 或在海洋中 。海星和触角。图片显示了海星在海滩上，包括海星，以及一个水下物体。 + astronaut +                              + 图片显示了一个宇航员的宇航员身穿宇航服，坐在一架大型航天飞机上。他们似乎正在进行一次宇航员登机或下机的旅程。在宇航员的身后，有一个火箭发射架，可能是用来支撑宇航员在旅程中的任务。此外，还有一架飞机停在机库附近，进一步表明这是一次航空展。在飞机的周围，还有一些人，但他们看起来离飞机很近。可以看到一个人站在飞机附近，可能正在观察或等待航天飞机准备起飞。场景中，一名士兵戴着头盔站在一架大型飞机上。这架飞机似乎是一架军用军用飞机，似乎正准备登上一架飞机。另一个人则站在前面，可能正在观察飞行过程。在飞机周围，有几个人，其中一些站在左侧，另一些则站在右侧。他们似乎正在观看飞行员的表现。此外，还有一辆卡车停在靠近左侧的位置，可能是为了更具体地观察飞行过程。
a-girl.png在野外,在山谷里。图片中的植物和一只灰熊坐在草地上。一只灰熊的近景图片显示一只灰熊站在一片开放的草地上，周围有树木和草丛，还有一只背包放在上面。 + dog-woman-sea +                              + 图片中，一个女人坐在沙滩上，手里拿着一只白色的狗。她看起来像是个女人，坐在沙地上，看着她。一只狗也坐在她旁边，看起来很放松和舒适。海滩上散布着其他沙滩游客，有些人坐着，而另一些人则坐在更远的地方。背景中可以看到一艘船，这表明这是一个受欢迎的海滩旅游目的地。两个人坐在海滩上，一边懒洋洋地躺在沙滩上，另一边则坐着。他们似乎正在享受海边时光。海滩上有几把椅子，其中一把靠近沙滩的左侧，另一把在中间。此外，还有一只狗躺在沙地上，为这个场景增添了一种放松的气氛。
a-girl.png一群游客观看了这部电影。这个图片描绘了一群海鸥在水面飞翔,在水面上。海鸥的出现表明,它们正在寻找食物。海鸥在水面上筑巢,可能是为了保护自己免受潜在的危险,如海鸥的尖锐牙齿和爬行动物。一群海豚或绵羊在一天的航行中乘船捕鱼这个图片显示一群人在海豚和海豚附近的大群中游泳。 + rainbow-falls +                              + 照片捕捉到一幅美丽如画的大自然场景，背景是高山峦崖。在水边，一座巨大的喷泉横跨着水面，吸引着许多游客。水面上有几个人，他们或站或坐在喷泉周围，或站或坐。有些人可以看到他们在水中行走，而其他人则站在水边。总体而言，这幅画描绘的是一个美丽而宁静的环境，在那里人们可以欣赏到如画般的美景。在一个美丽的蓝色天空下，一座巨大而巨大的白色瀑布上方悬挂着一只巨大的湿流水。这只瀑布位于一座山上，为整个场景增添了一种迷人而又宁静的气氛。在这幅图像的背景中，可以看到几艘船，其中一些靠近水边，其他的则离得较远。这些船只似乎正在为风景或户外活动做准备。
a-girl.png一个女孩和她的朋友坐在一张长凳上,穿着长长的白色长袍。这个场景描绘了一个充满活力的年轻女孩,她们穿着一件黑色和白色的服装,在一群人中间站着,他们都穿着黑色和白色的服装,这表明他们的服装是生动的、优雅的,在他们身边。在场景中,有两个女孩在背后,一个女人在背后,另一个女人站着,他们都穿着黑色的服装。这表明他们正在享受他们的服装和服装,可能正在参加一个特别的节日或庆祝活动。女孩们在城市的街道上。这个图片描绘了一个穿着传统服装的男人和女人，站在他们旁边，他们正在一起度过一个家庭时光。在整个场景中，可以看到一个小男孩和一个女孩，他们穿着牛仔帽，这表明他们正在参加一个家庭聚会，这可能是一次聚会或庆祝，或者他们可能正在讨论一个有趣的活动或活动。 + elderly-reading +                              + 图中，一个男人坐在公园的长椅上，旁边是一把绿色椅子。他身边有一本打开的书，上面写着"读书"一句话，暗示他可能正在阅读。公园里有一张长椅和一张公园长椅，为周围的环境增添了几分生气。在公园的周围，有几辆汽车和一辆卡车，表明这是一个公共区域。此外，还可以看到一个人站在公园的不同位置上，可能是等着上路或过马路。一个穿着短裤的老人坐在公园长椅上，周围是树木。他似乎正在读一本书，可能是在读书。背景中有一座长凳，为这个场景提供了充足的座位。在背景中，可以看到一把椅子和一张餐桌，这说明这个场景可能是在一个户外座位区，那里有椅子供人们坐下来放松。
a-girl.png这张照片中有几只鹿。这个图片记录了一只白尾鹿, 它坐在草地上, 用它的照片来捕捉一只红鹿.这只动物看起来好像准备躲在树后面,他看上去很威严,因为他无法控制自己。这个图片描绘了一只母鹿和一只鹿，这只母鹿在树林中站着，一只羊和一只鹿。 + panda-grassland +                              + 图中，一只白色的棕熊坐在草地上，旁边是一只长着棕色斑点的大熊。这只熊看起来很害羞或顽皮，因为它正躺在草地上休息，看上去很放松。在这幅图像中，一只棕色的熊正在草地上漫步。这只熊被放置在草地上，占据了画面的大部分空间。它似乎正在自然环境中行走，可能是在草地上。在背景中，有几棵树，为画面增添了自然元素。一只鸟在场景的中间附近飞翔，为画面增添了生气勃勃的气氛。
a-girl.png这个花束的花期几乎没有进数。图片显示一只红色和黄色的花朵, 它们被称为“花瓶”。花头的贴近。图片显示了红色的花朵，周围有几个玫瑰花。 + bicycle-flowers +                              + 图片展示了一个漂亮的花瓶，里面插满了五颜六色的鲜花和花束。这些花束散落在整个花瓶中，给人一种赏心悦目的感觉。花瓶里插着五颜六色鲜花，创造出一种令人赏心悦目的景象。这些鲜花被摆放在一张桌子上，很可能是为了展示它们的美丽而摆放的。场景中，一辆绿色和紫色相间的自行车停在一栋建筑旁边，它被放置在一棵大树旁。这辆自行车被摆放在附近，为这个场景增添了几分色彩。除了自行车外，还有一些其他的自行车，包括两个位于前景中的一个和位于背景中靠近中心位置的另一个。自行车的存在表明它可能是停在那里的。
a-girl.png宇航员在太空任务中与地球相姿态。这个图像描绘了一个充满活力的月球,在月球上散步。宇航员在任务期间在摇篮上休息,与他的团队在背景。这个图片描绘了一个宇航员在太空站的形象。
a-girl.png一只老虎在水里看着摄像机。图片显示一只大棕色的海豹在水里游泳,在水里休息。动物园里被囚禁的老虎图片显示一只小熊，躺在一棵树枝上。
a-girl.png这个是濒危物种。图片中,一只黑白的猫在岩石上散步。野外云层的豹在洞穴外的岩石上,在日出时该图片展示了一只小熊猫在岩石上散步的照片。 + dance +                              + 图片中的女人穿着一件白色连衣裙，脚上还系着一条黑色网球裙。她正在表演一个网球比赛，很可能是在比赛中。在背景中可以看到几把椅子，可能是为了观众或其他观众的座位安排而摆放的。此外，还有一个长凳放在场景左侧，为人们提供了一个休息的地方。一名身穿白色衣服的女子站在舞台上，手里拿着一只手拿着白色飞盘。她似乎正在参加一个舞台舞会或比赛。场景中还有其他几个人，其中一个站在舞台左侧，另一个站在右侧，第三个人则站在场地右侧。舞台上有几个观众，有的站着，有的坐着，还有一些站着。这看起来像是一场欢乐的节日或活动。
@@ -616,66 +623,35 @@ MiniMind-V与MiniMind的代码核心改动不超过100行，上手难度低。图片1 图片2 512_sft_multi + 768_sft_multi - a-bird.png - a-bird.png - animal1 has a brown and black head with a black and white striped head . animal2 has a black head with a white stripe on its wings . + a-bird.png + a-bird.png + 这幅图像显示了一种鸟簸戮的场景：一个女人站在红绿相间的红绿相间的紫色鸟簸戴在她身上。女人站在红色的鸟簸戴在她身上，而她的翻领上的那只红鸟则站在她身后。 + 这两只鸟在同一片树林中飞翔，有的位于画面中心，而另一些则较小，形成了鲜明对比。这种鸟类的出现突出了其飞行能力和适应性，因为它们能够在树林中快速迅速移动。此外，两只鸟的位置不同，一个在图像的左边，另一个在右边，这表明它们在同一片树林中移动得很近。这种鸟类的自然行为也有助于区分这两种鸟类物种。 -### 启动推理 +### 效果小结： -```bash -python web_server.py +视觉信号对于LLM视作一种特殊的外语， +因此“学习外语”的能力高低， +很大程度上取决于LLM的能力。 +LLM性能越强，对应的VLM必然越强，此时效果增益会很明显。 + +#### 未来值得改进的方面： + +```text +> 更简单的Projection的跨模态特征对齐方式，相较于Cross-Attention可能处于劣势。 +> Clip模型可以尝试更大性能更强的large系列，用更具细粒度的token表征图像特征，目前仍粗糙。 +> 分辨率不高，理论上只有224×224（minimind-v数据集为节省空间，仅设定为128×128）。 +> ... ``` -![web_server1](./images/web_server1.png) -![web_server2](./images/web_server2.png) - -### 效果总结 - ---- -根据提供的表格数据，四个模型的表现可以总结如下： - -1. **512_pretrain**: - - **描述简略且不准确**：大部分描述无法清晰表达图像内容，常常给出一些不相关的叙述。例如，在海星的图像中描述为“水中的化石”，与实际内容偏差较大。 - - **缺乏细节**：大多数情况下，只给出简单的、模糊的描述，无法深入解释图像的细节或背景。例如，对于老虎的图像，仅说“在水里看着摄像机”。 - -2. **512_sft**: - - **描述更具体**：相比512_pretrain，512_sft在描述图像内容时更加详细，并尝试捕捉场景的具体元素。比如描述女子图像时，提到了“西装”和“领带”，细节较为清晰。 - - **偶尔出错或冗余**：部分描述显得过于复杂甚至与图片无关，如描述海豚图像时，提到海鸥、筑巢等不相关的内容。 - -3. **768_pretrain**: - - **信息不连贯**：该模型的表现较为散乱，描述经常模糊且不完整。例如，在描述女子图像时，只提到“人为出演员的冒险片”，没有清楚地解释图像内容。 - - **部分准确，但总体信息量少**：一些描述虽然与图像相关，但非常简短。例如，海星的描述只有“海星和触角”，无法提供完整的画面感。 - -4. **768_sft**: - - **描述全面且具体**：该模型的描述是四个模型中最详细和精确的。比如，描述熊的图像时提到了“站在一片开放的草地上，周围有树木和草丛，还有一只背包”，能够准确捕捉到多个图像元素。 - - **具备更强的理解力**：该模型能够识别图像的场景和背景，提供合理的解释和推测。例如，描述“家庭聚会”或“庆祝活动”，这些解释让图像更具上下文联系。 - -### GPT-4o总结： - -- **512_pretrain**的表现最差，描述简单且不准确。 -- **512_sft**的描述详细度有所提升，但偶尔出现不相关信息。 -- **768_pretrain**信息连贯性差，但在某些方面能提供基本描述。 -- **768_sft**表现最佳，能够给出详细、准确的描述，并且能够很好地推测图像的上下文。 - -### 个人小结： - -- 视觉信号对于LLM是一种特殊的外语，因此“学习外语”的能力高低，很大程度上取决于LLM的能力。 -- LLM性能越强，对应的VLM必然越强，且效果增益很明显。 -- 值得改进的方面： - - 更简单的Projection的跨模态特征对齐方式，相较于Cross-Attention会带来更大的性能损失。 - - Clip模型可以尝试更大性能更强的large系列，用更具细粒度的token表征图像特征，目前仍非常粗糙。 - - 分辨率不高，理论上只有224×224（minimind-v数据集为节省空间，仅设定为128×128）。 - - ... - ---- - # 📌 Acknowledge > [!TIP] diff --git a/README_en.md b/README_en.md index 9761a72..73a3d81 100644 --- a/README_en.md +++ b/README_en.md @@ -27,315 +27,386 @@ +* This project aims to train a super-small multimodal vision-language model, **MiniMind-V**, with just a cost of 3 RMB + and 2 hours of work, starting from scratch! +* The smallest version of **MiniMind-V** is only about $\frac{1}{7000}$ the size of GPT-3, designed to enable fast + inference and even training on personal GPUs. +* **MiniMind-V** is an extension of the visual capabilities of the [MiniMind](https://github.com/jingyaogong/minimind) + pure language model. +* The project includes full code for the minimalist structure of large VLM models, dataset cleaning, pretraining, and + supervised fine-tuning (SFT). +* This is not only the smallest implementation of an open-source VLM model but also a concise tutorial for beginners in + vision-language models. +* The hope is that this project can provide a useful example to inspire others and share the joy of creation, helping to + drive progress in the wider AI community! -* This open-source project aims to train a small-parameter visual modal capable language model, **MiniMind-V**, from scratch within as fast as 3 hours. -* **MiniMind-V** is extremely lightweight, with the smallest version being about $\frac{1}{7000}$ the size of GPT-3, aiming to be quickly inferable and trainable even on personal GPUs. -* This is not only an implementation of an open-source model but also a tutorial for getting started with Visual Language Models (VLMs). -* We hope this project can provide researchers with a starting example, helping everyone to get up to speed and generate more exploration and innovation in the VLM field. - - > To avoid misunderstanding, "from scratch" specifically refers to further developing the pure language model MiniMind (which is a fully from-scratch trained GPT-like model) with visual capabilities. - > For more details on the latter, please refer to the twin project [MiniMind](https://github.com/jingyaogong/minimind). - - > To avoid misunderstanding, "as fast as 3 hours" means you need to have a machine with a hardware configuration higher than mine. The detailed specifications will be provided below. - - - -![modelscope](./images/modelscope-demo.gif) +> To avoid misunderstandings, the "2 hours" is based on testing (`1 epoch`) with an NVIDIA 3090 hardware device (single GPU), and +> the "3 RMB" refers to GPU server rental costs.
-The demo has been deployed to ModelScope's creative space, where you can experience it on this website: +![minimind2-v](./images/minimind2-v.gif) -[🔗ModelScope Online Experience🔗](https://modelscope.cn/studios/gongjy/minimind-v) - -[🎉🎉BiliBili Video🎉🎉](https://www.bilibili.com/video/BV1Sh1vYBEzY) +[🔗🤖 Online Experience](https://www.modelscope.cn/studios/gongjy/MiniMind-V) | [🔗🎞️ Video Introduction](https://www.bilibili.com/video/BV1Sh1vYBEzY)
# 📌 Introduction -Visual Language Models (VLMs) like GPT-4V, Qwen-VL, LlaVA, etc., although impressive in performance, often require extremely high hardware configurations. -For personal devices, not only is the GPU memory far from sufficient to support training, but even inference can be very difficult. -We learn about the somewhat novel VLMs through reading papers or public account explanations, but often end up with a vague understanding. -What we really need to know is: -Is multimodal large models really as complex as imagined? What is their code implementation like? -Is the training process really that difficult? Can I start training from scratch with just one 2080Ti GPU? - -Through **MiniMind-V**, this project hopes to answer these questions and help researchers understand the core principles of visual language models under limited hardware conditions. +“Building a plane with Legos is much more exciting than flying in first class!” +Is it really as complex as imagined to build a VLM-based multimodal large model? How is the code implementation done? +Is the training process difficult? Now, let's explore the answers and feel the joy of creation together! > [!TIP] -> (As of 2024-10-04) The MiniMind-V series has completed pre-training of 2 model versions, requiring as little as 27M (0.027B) to have image recognition and dialogue capabilities! +> (As of 2025-02-20) The MiniMind-V series has completed the training of the following model versions, with the smallest +> requiring only 26M (0.026B) parameters, capable of both image recognition and conversation! -| Model (Size) | Tokenizer Length | Inference Usage | Release | Subjective Rating (/100) | -| --------------------------- | ------------- | -------- | ------------ | ------------ | -| minimind-v-v1-small (27M) | 6400 | 0.6 GB | 2024.10.04 | 50' | -| minimind-v-v1 (109M) | 6400 | 1.1 GB | 2024.10.04 | 60' | - -> This analysis was conducted on 2×RTX 3090 GPUs with Torch 2.1.2, CUDA 12.2, and Flash Attention 2. +| Model (Size) | Inference Memory | Release | +|---------------------------|------------------|------------| +| MiniMind2-V (104M) | 0.6 GB | 2025.02.20 | +| MiniMind2-Small-V (26M) | 1.1 GB | 2025.02.20 | +| minimind-v-v1-small (27M) | 0.6 GB | 2024.10.04 | +| minimind-v-v1 (109M) | 1.1 GB | 2024.10.04 | ### 👉**Recent Updates**
- 2024-10-05 (newest 🎉) + 2025-02-20 (newest 🎉) -- MiniMind-V arrives as scheduled, first open-source release +- MiniMind2-V updated alongside MiniMind2 +- Significant reduction of all redundant code, standardized code format +- Major simplification of the model's redundant structure +- Updated dataset format, expanded with new SFT datasets +- Better performance than the previous VLM version!
-# 📌 Environment +
+ 2024-10-05 -This is my personal software and hardware configuration; adjust as necessary: +- MiniMind-V released on schedule, first open-source release -```bash -CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz -Memory: 128 GB -GPU: NVIDIA GeForce RTX 3090(24GB) * 2 -Environment: python 3.9 + Torch 2.1.2 + DDP single-machine multi-GPU training -``` +
-* Ubuntu == 20.04 -* Python == 3.9 -* Pytorch == 2.1.2 -* CUDA == 12.2 +# 📌 Quick Start + +
+Sharing my hardware and software configuration (for reference only) + +* CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz +* RAM: 128 GB +* GPU: NVIDIA GeForce RTX 3090(24GB) * 8 +* Ubuntu==20.04 +* CUDA==12.2 +* Python==3.10.16 * [requirements.txt](./requirements.txt) -# 📌 Quick Start Test +
-> BTW: If you don't have Git LFS installed, please install it first with `sudo apt-get update`, `sudo apt-get install git-lfs`. +### Step 0 +```bash +# Clone the code repository +git clone https://github.com/jingyaogong/minimind-v +``` -* 0. Clone the project - ```bash - git clone https://github.com/jingyaogong/minimind-v - cd minimind-v - ``` +```bash +# Download the clip model to the ./model/vision_model directory +git clone https://huggingface.co/openai/clip-vit-base-patch16 +# or +git clone https://www.modelscope.cn/models/openai-mirror/clip-vit-base-patch16 +``` -* 1. Install the environment - ```bash - pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple - ``` - ```text - # Test if torch can use CUDA - import torch - print(torch.cuda.is_available()) - ``` +## Ⅰ Test an existing model's performance - > If it is not available, please go to [torch_stable](https://download.pytorch.org/whl/torch_stable.html) - to download the whl file for installation. Refer to [this link](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=安装torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187) - - -* 2. Download the pre-trained model weights to the project root directory `minimind-v-v1` - ```bash - git clone https://huggingface.co/jingyaogong/minimind-v-v1 - ``` +### 1. Environment Preparation -* 3. Download the pre-trained `clip-vit-base-patch32` model to the `model/clip_model` directory: - ```bash - cd model/clip_model - git clone https://hf-mirror.com/openai/clip-vit-base-patch32 - ``` +```bash +pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple +``` -* 4. Start the chat web server for testing conversations - ```bash - python web_server.py - ``` +### 2. Download the model -![web_server](images/web_server.gif) +```bash +git clone https://huggingface.co/jingyaogong/MiniMind2-V +``` -# 📌 Quick Start Train +### 3. Command-line Q&A -> BTW: If you don't have Git LFS installed, please install it first with `sudo apt-get update`, `sudo apt-get install git-lfs`. +```bash +# load=0: load from pytorch model, load=1: load from transformers-hf model +python eval_vlm.py --load 1 +``` -* 0.Clone the project code - ```text - git clone https://github.com/jingyaogong/minimind-v & cd minimind-v - ``` +### 4. Or start the WebUI -* 1.Environment setup - ```bash - pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple - ``` - ```text - # Test if torch can use CUDA - import torch - print(torch.cuda.is_available()) - ``` +```bash +python web_demo_vlm.py +``` - > If it is not available, please go to [torch_stable](https://download.pytorch.org/whl/torch_stable.html) - to download the whl file for installation. Refer to [this link](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=安装torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187) - - -* 2.Download the `clip-vit-base-patch32` model and place it in the `./model/clip_model` directory: - ```bash - cd ./model/clip_model & git clone https://hf-mirror.com/openai/clip-vit-base-patch32 - ``` - -* 3.If you want to train it yourself +## Ⅱ Train from scratch - * 3.1 Download all contents of the dataset ([Baidu Netdisk](https://pan.baidu.com/s/1Nz36OBBvVBGEx-PwIb7ofg?pwd=6666) or [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset)) to the `./dataset` directory, and unzip `pretrain_images.zip` and `sft_images.zip` - * 3.2 Adjust the model parameters in `./model/LMConfig.py` - > Only need to adjust the dim and n_layers parameters, which are `(512+8)` or `(768+16)`, corresponding to `minimind-v-v1-small` and `minimind-v-v1` - * 3.3 Download the pre-trained weight file ([Baidu Netdisk](https://pan.baidu.com/s/1LE1SPoPYGS7VNtT1tpf7DA?pwd=6666) or [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset/tree/main/out)) of the MiniMind language model and place it in the `./out/` directory, named `*_llm.pth` - * 3.4 Execute `python 1-pretrain_vlm.py` for pre-training, obtaining `*_vlm_pretrain.pth` as the output weights - * 3.5 Execute `python 2-sft_vlm.py` for instruction fine-tuning, obtaining `*_vlm_sft.pth` as the output weights for fine-tuning - * 3.6 Execute `python 2-sft_vlm.py --multi True` Perform multi-graph instruction fine-tuning based on instruction fine-tuning, resulting in `*_vlm_sft_multi.pth` as the output weight for multi-graph instruction fine-tuning, with a memory usage of approximately 8198M for a 512+8 model. +### 1. Environment Preparation -* 4.Test the inference effect of the self-trained model - * Ensure that the used, completed training parameter weights `*.pth` files are located in the `./out/` directory - * You can also directly download the [completed model weight files](https://pan.baidu.com/s/1LE1SPoPYGS7VNtT1tpf7DA?pwd=6666) and use the `*.pth` weight files I have trained - ```text - minimind-v/out - ├── 512_llm.pth - ├── 512_vlm_pretrain.pth - ├── 512_vlm_sft.pth - ├── 768_llm.pth - ├── 768_vlm_pretrain.pth - ├── 768_vlm_sft.pth - ``` - * Use `python 3-eval_chat.py` to test the conversation effect of the model, where the test images are in `./dataset/eval_images`, and you can replace them as needed - [eval_chat](images/3-eval_chat.png) - * Use `python 3-eval_chat.py` Adjust the [multi](./3-eval_chat.py#L61) variable to test the model's multi-graph conversation effect. The test images are located in `./dataset/eval_multi_images`, and you can replace them as needed (The multi-graph dataset is relatively small and contains English dialogues, with the dataset only including two-image comparison scenarios, so the fine-tuning effect is limited). +```bash +pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple +``` -🍭 【Tip】Both pretraining and full-parameter instruction fine-tuning (pretrain and sft) support multi-GPU acceleration +
+Note: Test if Torch can use CUDA -* Single machine N-card training launch (DDP) - ```bash - torchrun --nproc_per_node N 1-pretrain_vlm.py - # and - torchrun --nproc_per_node N 2-sft_vlm.py - ``` +```bash +import torch +print(torch.cuda.is_available()) +``` -* Record the training process - ```bash - torchrun --nproc_per_node N 1-pretrain_vlm.py --use_wandb - # and - python 1-pretrain_vlm.py --use_wandb - ``` - By adding the `--use_wandb` parameter, you can record the training process, and after the training is complete, you can view the training process on the wandb website. You can specify the project name and run name by modifying the `wandb_project` and `wandb_run_name` parameters. +If unavailable, download the whl file from [torch_stable](https://download.pytorch.org/whl/torch_stable.html) for +installation. Refer +to [this link](https://blog.csdn.net/weixin_45456738/article/details/141029610?ops_request_misc=&request_id=&biz_id=102&utm_term=%E5%AE%89%E8%A3%85torch&utm_medium=distribute.pc_search_result.none-task-blog-2~all~sobaiduweb~default-2-141029610.nonecase&spm=1018.2226.3001.4187) +for help. + +
+ +### 2. Download Data + +Download the required dataset files from +the [dataset download link](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset), create a `./dataset` +directory, and place the files under `./dataset`. + +`*.jsonl` is the Q&A dataset, and `*images` are the accompanying image data. After downloading, decompress the image +data. + +
+Note: Dataset Details + +Please reserve about 5GB of space for the dataset. If there is insufficient space for pretrain data, you can try +skipping the pretrain training step and proceed directly to SFT training. + +
+ +### 3. Start Training + +**3.1 Pretraining (Learning image description)** + +```bash +python train_pretrain_vlm.py --epochs 4 +``` + +> Run pretraining to get `pretrain_vlm_*.pth` as the pretrained model's output weights (* represents the model +> dimension, default is 512). + +**3.2 Supervised Fine-Tuning (Learning image-caption dialogue style)** + +```bash +python train_sft_vlm.py --epochs 4 +``` + +> Perform supervised fine-tuning to get `sft_vlm_*.pth` as the output weights for the fine-tuned model. + +
+Note: Training Details + +By default, the training process saves model parameters every 100 steps to the `./out/***.pth` file (it will overwrite +previous weight files). + +
+ +--- + +### 4. Test the Model's Performance + +Ensure that the model `*.pth` file you want to test is located in the `./out/` directory. +You can also directly download the pre-trained `*.pth` file +from [here](https://huggingface.co/jingyaogong/MiniMind2-V-PyTorch). + +```bash +python eval_vlm.py --model_mode 1 # Default is 0: test pretrain model, set to 1: test sft model +``` + +--- + +> [!TIP] +> The training scripts are based on PyTorch's native framework and support multi-card acceleration. If your device has +> N (N>1) GPUs: + +Single-machine N-card training method (DDP, supports multi-machine multi-card cluster) + +```bash +torchrun --nproc_per_node N train_xxx.py +``` + +
+Note: Other Details + +Single-machine N-card training (DeepSpeed) + +```bash +deepspeed --master_port 29500 --num_gpus=N train_xxx.py +``` + +You can enable wandb logging during training: + +```bash +# You need to log in: wandb login +torchrun --nproc_per_node N train_xxx.py --use_wandb +# and +python train_xxx.py --use_wandb +``` + +By adding the `--use_wandb` parameter, you can log the training process, and after training is complete, you can view +the process on the wandb website. You can specify the project name and run name by modifying the `wandb_project` +and `wandb_run_name` parameters. + +
# 📌 VLM Detail -The base language model MiniMind (LLM) for MiniMind-V (VLM) comes from the twin project [minimind](https://github.com/jingyaogong/minimind). For specific details on the model architecture, training specifics, principles, and test results, please refer to the [minimind](https://github.com/jingyaogong/minimind) project. To avoid redundancy, we will not discuss the LLM-related parts here, assuming you have a basic understanding of MiniMind (LLM). +The base language model of MiniMind-V (VLM), MiniMind (LLM), comes from the twin +project [minimind](https://github.com/jingyaogong/minimind). For detailed information on the model structure, training +specifics, principles, and testing results, please refer to the [minimind](https://github.com/jingyaogong/minimind) +project. To reduce redundancy, the discussion on LLM-related topics is omitted here, assuming you have a basic +understanding of MiniMind (LLM). -> PS: Even if you do not wish to delve into the details of MiniMind (LLM), you can directly refer to Quick Test and Quick Start to quickly test or train MiniMind-V. This will not be significantly affected. +> Even if you are not very familiar with the details of LLMs, you can still follow the "Quick Start" guide to train a +> MiniMind-V, as it remains unaffected and the repository focuses on the lowest cost for out-of-the-box use! -The structure of MiniMind-V remains almost unchanged, with only two additional sub-modules added: Visual Encoder and feature projection, as well as a multimodal fusion branch, to support input from multiple modalities: +MiniMind-V's structure adds two submodules, a Visual Encoder and a feature projection, with a modality-mixing branch to +support inputs from multiple modalities: ![LLM-structure](./images/VLM-structure.png) ![LLM-structure](./images/VLM-structure-moe.png) -At this point, it's interesting to ponder two questions: What is a **L**arge **L**anguage **M**odel (LLM)? And what is a multimodal model? -* [This article](https://www.jiqizhixin.com/articles/2024-09-15-3) perfectly articulates my thoughts, suggesting that the term LLM is quite inaccurate! +
+ [Important] Some Interesting Thoughts - > Although Large Language Models (LLMs) carry the word "language" in their name, they are actually not very related to language; this is merely a historical issue. A more accurate name would be autoregressive Transformer or something similar. - LLMs are more of a general statistical modeling technique, primarily using autoregressive Transformers to simulate token streams, and these tokens can represent text, images, audio, action choices, or even molecules, among other things. - Therefore, theoretically, any problem that can be framed as a process of simulating a series of discrete tokens can be addressed using LLMs. - In fact, as the large language model technology stack matures, we may see an increasing number of problems being brought into this modeling paradigm. That is, the problem is fixed on using LLMs for 'predicting the next token', with the usage and meaning of tokens varying across different domains. +Let's take a moment to think about two questions: -* [Professor Li Xi](https://person.zju.edu.cn/xilics#694283) similarly corroborates my view (the exact wording is not recalled, but the gist is as follows): +* What is a **Large Language Model (LLM)**? +* What is a multimodal model? - > Text, video, speech, and actions, which appear to humans as "multimodal" signals, are essentially just a classification concept for information storage by humans. - Just like `.txt` and `.png` files, although they differ in visual presentation and higher-level representation, there is no fundamental difference at their core. - The notion of "multimodality" arises simply because of the human need to categorize these signals at different perceptual levels. - However, for machines, regardless of the "modality" of the signal, they ultimately present as a string of binary "unimodal" digital sequences. - Machines do not differentiate the modality source of these signals but rather process and analyze the information content carried by these sequences. +[This article](https://www.jiqizhixin.com/articles/2024-09-15-3) perfectly aligns with my thoughts: +Although the name "large language model" (LLM) contains the word "language," they are actually not closely related to +language; this is just a historical issue. A more accurate name would be self-regressive Transformer or something else. +LLMs are more of a general statistical modeling technology, mainly using a self-regressive Transformer to simulate token +flows. These tokens can represent text, images, audio, action choices, and even molecules—anything, really. +Therefore, as long as the problem can be converted into a process of simulating a series of discrete tokens, LLM can +theoretically solve it. In fact, with the increasing maturity of large language model technologies, we may see more and +more problems falling under this modeling paradigm. In other words, the problem is fixed in using LLM to "predict the +next token," but the role and meaning of the tokens differ in each domain. ---- +[ZJU-LiXi](https://person.zju.edu.cn/xilics#694283) has also mentioned a similar viewpoint (roughly stated below): +Text, video, audio, actions, etc., are considered "multimodal" signals in human perception, but the term "modality" is +essentially just a classification concept based on how humans store information. Just like `.txt` and `.png` files, +though they differ in visual presentation and higher-level forms, they are fundamentally the same. The concept of " +multimodal" arose simply because humans need to categorize these signals based on different sensory dimensions. +However, for machines, regardless of the signal's "modality," they are ultimately presented as a sequence of binary " +monomodal" numbers. Machines do not differentiate the origin of these signals; they just process and analyze the +information contained within these sequences. -I personally believe that **G**enerative **P**retrained **T**ransformer (GPT) is a more fitting term than **L**arge **L**anguage **M**odel (LLM), -and thus I prefer to use "GPT" to represent LLM/VLM/GPT-like architectures, rather than to piggyback on OpenAI's popularity. +Personally, I think **Generative Pretrained Transformer (GPT)** is a more fitting term than **Large Language Model (LLM) +**, and I prefer to use "GPT" to represent models in the LLM/VLM/GPT-like architecture series rather than to ride on +OpenAI's coattails. ---- +To summarize what GPTs do in one sentence: -In summary, we can encapsulate what GPT does in one sentence: -GPT models predict the next, and the next, and the next token... until the model outputs an end token; here, the "token" does not necessarily have to be text! +A GPT model predicts the next, next-next, next-next-next token, etc., based on the current token... until the model +outputs the end token; here, the "token" doesn’t necessarily have to be text! ---- +```text +> For an LLM model, if we need to understand an "image," we just treat the "image" as a special "foreign language" that has never been encountered before, and translate it into the "LLM language" via a "foreign language dictionary." +> For an LLM model, if we need to understand "audio," we just treat "audio" as a special "foreign language" that has never been encountered before, and translate it into the "LLM language" via a "foreign language dictionary." +> ... +``` -* For LLM models, if understanding "images" is required, we can treat "images" as a special kind of "foreign language" never seen before, translating them through a "foreign dictionary" into a special language input for the LLM. -* For LLM models, if understanding "audio" is required, we can treat "audio" as a special kind of "foreign language" never seen before, translating them through a "foreign dictionary" into a special language input for the LLM. -* ... +**To obtain MiniMind-V, we only need to do these 2 things:** ---- +1. Use the **"foreign language dictionary"** that is good at translating images, to translate the image from the **" + foreign language"** into a model-understandable **"LLM language."** +2. Fine-tune the LLM so that it and the **"foreign language dictionary"** go through a period of adaptation, thereby + better understanding images. -**So, to get MiniMind-V, we only need to accomplish two things:** - -1. Use a "foreign dictionary" proficient in translating images to translate the "foreign language" of images into the "LLM language" that the model can understand. -2. Fine-tune the LLM so that it goes through a period of adjustment with the "foreign dictionary," thereby better understanding images. - ---- - -The "foreign dictionary" is generally referred to as the Visual Encoder model. -Similar to visual-language models such as LlaVA and Qwen-VL, MiniMind-V also selects open-source Clip series models as the Visual Encoder. -Specifically, it uses [clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32), a classic Visual Encoder based on the ViT-B/32 architecture, for describing image-text information. -The input image size is 224x224, and since the patches are 32×32, it generates 7*7+1(cls_token)=50 tokens as input to the encoder layer, -ultimately producing a 1×768 dimensional embedding vector for calculating error with text. -We do not need the final embedding representation, so we only take the output of the encoder layer, which is the output features of the VIT backbone. -In the code, this corresponds to the hook function in [./model/vision_utils.py](./model/vision_utils.py)'s get_img_embedding. -It retrieves the 50×768 dimensional features from the previous layer, which we then input as 50 visual tokens into MiniMind-V. -There are also larger Clip models like clip-vit-large-patch14, which have a stronger image understanding capability, -but a single image would generate 257 tokens, which, for a model of MiniMind's scale, would result in too long a context of image tokens, which is not conducive to training. - -After obtaining the image encoder features, on one hand, it is necessary to align the 768-dimensional visual tokens with the text tokens of the LLM, -on the other hand, the image features must be mapped to the same space as the text embeddings, i.e., the text tokens and the native visual tokens need to be aligned and cannot be treated equally; this can be called cross-modal feature alignment. -[LlaVA-1](https://arxiv.org/pdf/2304.08485) accomplished this with a simple unbiased linear transformation, and the results were excellent; MiniMind-V does the same. +The "foreign language dictionary" is referred to as the Visual Encoder model. +Like LlaVA, Qwen-VL, and other visual language models, MiniMind-V also uses the open-source Clip series models as the +Visual Encoder. +Specifically, we use [clip-vit-base-patch16](https://huggingface.co/openai/clip-vit-base-patch16), a classic Visual +Encoder based on the ViT-B/16 architecture for describing image-text information. +The input image size is 224x224, and because the Patch size is 16×16, it generates 16*16=196 tokens as the input to the +encoder layer, which produces a 1×768 dimensional embedding vector for calculating error with the text. +We don’t need the final embedding representation, so we only take the output from the encoder layer, which is the output +feature from the core ViT backbone. +It receives the feature of size 196×768 from the previous layer, which we use as 196 visual tokens to input into +MiniMind-V. +After obtaining the image encoder features, the integration with the LLM requires aligning the 768-dimensional visual +tokens with the LLM's text tokens, and mapping the image features into the same space as text embeddings. In other +words, the image features and native visual tokens cannot be directly treated the same; they require cross-modal feature +alignment. +[LlaVA-1](https://arxiv.org/pdf/2304.08485) uses a simple unbiased linear transformation to achieve this, with great +success, and MiniMind-V does the same. ![llava-structure](./images/llava-structure.png) -With this, the internal structural changes of MiniMind-V have been presented. +With that, the internal structural changes of MiniMind-V are now fully presented. + +
+ --- -Next, we briefly discuss the changes in the external input and output of MiniMind-V. +Next, let's briefly discuss the changes in the external input and output of MiniMind-V. -The input for VLM is still a piece of text, which includes a special placeholder. -After computing the text embedding, the vector generated by the image encoder can be projected into the part of the embedding corresponding to the placeholder, replacing the original placeholder embedding. +The input to the VLM is still a segment of text containing special placeholders. +After computing the text embedding, the vector generated by the image encoder can be projected onto the corresponding +embedding part of the placeholder, replacing the original placeholder embedding. For example: ```text -\nWhat is the content of this image? +\nWhat is in this image? ``` -minimind-v uses a 50-character `<<<...>>>` placeholder to replace the image, -the reason for 50 characters was mentioned earlier: -any image is encoded by the clip model into 50×768 dimensional tokens. -Therefore, the prompt for minimind-v: +In `minimind-v`, the image is replaced by a 196-character `@@@...@@@` placeholder. The reason for using 196 characters +is explained earlier: +Any image is encoded by the Clip model as 196×768-dimensional tokens, +thus the `minimind-v` prompt becomes: ```text -<<<<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>>>>\nWhat is the description of this picture? +@@@......@@@\nWhat is this image describing? ``` -After computing the embedding and projection, and replacing the image part tokens, -the entire computation process to output is no different from the LLM part. +After calculating the embedding and projection, and replacing the image token part, the entire calculation process to +output is no different from that of the LLM part. ![input](./images/minimind-v-input.png) -The method of implementing multi-graph processing is achieved by injecting multiple `` placeholders, without the need to modify any framework. - -> ps: The only point worth noting is that if there are different numbers of images inserted in different conversations during training, you need to use empty features to pad the shorter features (corresponding to [line 267 of the dataset](./model/dataset.py#L267)) to ensure they can be read by the dataloader in the same size. - -> pps: This does not need to be done in the prompt; the placeholders are still injected based on the number of images inserted. Therefore, the input feature given to the LLM will not be affected by the padded features. +For handling multiple images at once, this can be achieved by injecting multiple `` placeholders without needing +to modify the framework at all.
- Considerations for Implementing Video Understanding Capabilities + Expansion Ideas for Video Understanding -For the video understanding capabilities of multi-modal large models, a feasible approach is to refer to the existing Python example for video understanding in MiniCPM-V 2.6. -The main idea is to extract key frames from the video and then perform multi-graph inference. -Therefore, if you want to add video understanding capabilities to MiniMind-V, you can build on the existing multi-graph training and refer to the method of extracting key frames in this Python script, then increase the number of images supported in the training files. -The more MAX_NUM_FRAMES supported, the larger the memory consumption. +written by [@xinyanghuang7](https://github.com/xinyanghuang7) -```python +For the video understanding capabilities of multimodal large models, one feasible approach is to refer to the existing +MiniCPM-V 2.6 Python example for video understanding. +The main idea is to extract key frames from the video and then perform multi-image inference. +Therefore, if you want to add video understanding capabilities to MiniMind-V, you can base it on the existing +multi-image training, refer to the key frame extraction method in this Python script, and increase the number of images +supported in the training files. +The more MAX_NUM_FRAMES supported, the more GPU memory it will consume. + +```text import torch from PIL import Image from transformers import AutoModel, AutoTokenizer -from decord import VideoReader, cpu # pip install decord +from decord import VideoReader, cpu # pip install decord model = AutoModel.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True, - attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager + attn_implementation='sdpa', + torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager model = model.eval().cuda() tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-2_6', trust_remote_code=True) -MAX_NUM_FRAMES=64 # if cuda OOM set a smaller number +MAX_NUM_FRAMES = 64 # if cuda OOM set a smaller number + def encode_video(video_path): def uniform_sample(l, n): @@ -353,17 +424,18 @@ def encode_video(video_path): print('num frames:', len(frames)) return frames -video_path="video_test.mp4" + +video_path = "video_test.mp4" frames = encode_video(video_path) question = "Describe the video" msgs = [ - {'role': 'user', 'content': frames + [question]}, + {'role': 'user', 'content': frames + [question]}, ] # Set decode params for video params = {} params["use_image_id"] = False -params["max_slice_nums"] = 2 # If cuda OOM and video resolution is greater than 448*448, set to 1 +params["max_slice_nums"] = 2 # If cuda OOM and video resolution is greater than 448*448, set to 1 answer = model.chat( image=None, @@ -373,229 +445,215 @@ answer = model.chat( ) print(answer) ``` +
-With this, all the details of MiniMind-V have been presented. - -The implementation of MiniMind-V did not reference any third-party code; it is based on MiniMind with minimal modifications, hence the code implementation is significantly different from models like LlaVA. -The core changes between MiniMind-V and MiniMind do not exceed 100 lines, making it easy to get started. +At this point, all the details of `MiniMind-V` have been presented. +The `MiniMind-V` model subclass completely inherits from `MiniMind`, +and is generated with **minimal** changes based on the latter, +with core algorithm modifications `< 50 lines`, making the migration difficulty very low. +Therefore, there may be differences with models like `LlAVA`, but the overall idea remains consistent. # 📌 Experiment -## Dataset +## Ⅰ Dataset -Source: [Chinese-LLaVA-Vision](https://huggingface.co/datasets/LinkSoul/Chinese-LLaVA-Vision-Instructions) -Contains approximately 600,000 pre-training images and <100,000 instruction fine-tuning images, derived from CC-3M and COCO 2014, with translated Q&A content for better support of the Chinese language. The dataset has been further resized, organized, and compressed. +Source: [Chinese-LLaVA-Vision](https://huggingface.co/datasets/LinkSoul/Chinese-LLaVA-Vision-Instructions) +Contains approximately 570,000 pre-trained images from CC-3M and COCO 2014; +[llava-en-zh-300k](https://huggingface.co/datasets/BUAADreamer/llava-en-zh-300k) +Contains 300k instruction fine-tuning data and 150k images. +The Q&A content has been translated, with better support for Chinese, further organized and resized. -Pre-training dataset format: +(pretrain_vlm_data.jsonl) Pre-training dataset format: -```json +```json lines { - "id": "GCC_train_000644518", - "image": "GCC_train_000644518.jpg", "conversations": [ { - "from": "human", - "value": "Write a brief but informative image summary.\n" + "role": "user", + "content": "Provide a brief description of the given image.\n" }, { - "from": "gpt", - "value": "Adding saltwater to a metal pot with a black background, in slow motion fps" + "role": "assistant", + "content": "Olive oil is a healthy ingredient for free use." } - ] + ], + "image": "GCC_train_002582585.jpg" } ``` -Instruction fine-tuning dataset format: +(sft_vlm_data.jsonl) Single image instruction fine-tuning dataset format: -```json +```json lines { - "id": "000000334872", - "image": "000000334872.jpg", "conversations": [ { - "from": "human", - "value": "\nAre people in the photo going downhill skiing or cross-country skiing?" + "role": "user", + "content": "What impact does the location of the alarm clock have on sleep quality?" }, { - "from": "gpt", - "value": "People in the photo are cross-country skiing in the forest because they are on a trail rather than on a steep slope." + "role": "assistant", + "content": "Place the digital alarm clock on the nightstand..." } - ] + ], + "image": "train-00000-of-00001_image_0_0.jpg" } ``` -Multi-graph Instruction Fine-tuning Dataset Format: +(sft_vlm_data_multi.jsonl) Multi-image instruction fine-tuning dataset format: -```json +```json lines { - "id": "000000000000", - "image": "0.jpg, 1.jpg", - "conversations": [ - { - "from": "human", - "value": "context: Source Image: Target Image: Instruction: What is the correct image edit instruction that can transform the source image to the target image?" - }, - { - "from": "gpt", - "value": "take the people out of the back in the photo. Remove the two people behind the woman in the white dress and the man in the blue suit. remove people behind the couple in the centre" - } - ] - } + "conversations": [ + { + "role": "user", + "content": "context: Source Image: Target Image: Instruction: What is the correct image edit instruction that can transform the source image to target image?" + }, + { + "role": "assistant", + "content": "take the people out of the back in the photo. Remove the two people behind the woman in the white dress and the man in the blue suit. remove people behind the couple in the center" + } + ], + "image": "0.jpg, 1.jpg" +} ``` -```json -{ - "id": "000000013633", - "image": "27266.jpg, 27267.jpg", - "conversations": [ - { - "from": "human", - "value": "\n\nName all the differences between these two birds." - }, - { - "from": "gpt", - "value": "animal1 is brown with white tuft while animal2 is white with dark brown wings" - } - ] - } -``` +
+ Data Description -Notes: -+ For instruction fine-tuning, only one round of conversation is retained, training a single-round conversation model to prevent small model performance from being dragged down by long texts. -+ The multi-graph dataset is relatively small and contains English dialogues, with the dataset only including two-image comparison scenarios, so the fine-tuning effect is limited. Here, only a reference approach is provided. +* The multi-image dataset is relatively small and contains English conversations, focusing only on scenes with two image + comparisons. Therefore, the fine-tuning effect is limited, and this is just one reference approach. -Final dataset download link: [Baidu Netdisk](https://pan.baidu.com/s/1Nz36OBBvVBGEx-PwIb7ofg?pwd=6666) | [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset) -multi_image_dataset: [HuggingFace](https://hf-mirror.com/datasets/xinyanghuang/minimind-v_multi_image/tree/main) +* `jsonl` contains textual instructions, and `images.zip` contains the corresponding image data (to be unzipped after + download). -## Training +
-The pre-training learns general knowledge about images, such as what a deer or dog is, from 595K datasets. +Dataset download +link: ([ModelScope](https://www.modelscope.cn/datasets/gongjy/minimind-v_dataset) | [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset)) -The instruction fine-tuning learns the real Q&A format for asking questions about images from 230K real dialogue datasets. +## Ⅱ Training -Two datasets are provided for multi-graph fine-tuning: the Image Transformation dataset and the Bird Comparison dataset, with lengths of 3.5k and 13.6k respectively, in real Q&A format. +> train_pretrain_vlm -`1-pretrain_vlm.py` executes pre-training, yielding `*_vlm_pretrain.pth` as the output weights of pre-training. +Pre-training learns general image knowledge from a dataset of 595K samples, such as a deer is a deer, a dog is a dog. -`2-sft_vlm.py` performs instruction fine-tuning, resulting in `*_vlm_sft.pth` as the output weights of instruction fine-tuning. +> train_sft_vlm -During training, the visual encoder, which is the CLIP model, is frozen, and only the Projection and LLM parts are fine-tuned. +Instruction fine-tuning learns the real Q&A format for image-related questions from a dataset of 300K real +conversations, which better aligns with human communication habits. -> Pretrain 512+8 Model (Training Time and Loss Reference Chart) -![input](./images/1-pretrain-512.png) -> Pretrain 768+16 Model (Training Time and Loss Reference Chart) -![input](./images/1-pretrain-768.png) -> SFT 512+8 Model (Training Time and Loss Reference Chart) -![input](./images/2-sft-512.png) -> SFT 768+16 Model (Training Time and Loss Reference Chart) -![input](./images/2-sft-768.png) +> train_sft_vlm -## Trained Model Weights +Multi-image fine-tuning provides a demo: a bird comparison dataset with 13.6k real Q&A formats. -(`.pth` weight files) Download link: [Baidu Netdisk](https://pan.baidu.com/s/1a7_C7HdCMfnG2Dia3q85FQ?pwd=6666) +During training, the visual encoder, i.e., the CLIP model's gradients, are frozen, and only the Projection and LLM parts +are trained. +In pre-training, only the last layer parameters of Projection and LLM are learnable. +In instruction fine-tuning, all parameters of Projection and LLM are learnable. -(`transformers` model files) -Download link: [HuggingFace](https://huggingface.co/collections/jingyaogong/minimind-v-67000833fb60b3a2e1f3597d) +> Training Time and Loss Trend (for reference only) -> Note: HuggingFace versions are all post-instruction fine-tuned MiniMind-V models +Pretrain [512+8] & [768+16] +![input](./images/pretrain_loss.png) -| Model Name | params | Config | file_name | -|------------------------|--------|-----------------------------|-----------------------------------------------------| -| minimind-v-v1-small | 27M | d_model=512
n_layers=8 | Pre-trained: 512_vllm_pretrain.pth
Fine-tuned: 512_vllm_sft.pth | -| minimind-v-v1 | 109M | d_model=768
n_layers=16 | Pre-trained: 768_vllm_pretrain.pth
Fine-tuned: 768_vllm_sft.pth | +SFT [512+8] & [768+16] +![input](./images/sft_loss.png) + +## Ⅲ Model Weights + +(Native PyTorch `*.pth` weight files) Download link: +([ModelScope](https://www.modelscope.cn/models/gongjy/MiniMind2-V-PyTorch) | [HuggingFace](https://huggingface.co/jingyaogong/MiniMind2-V-PyTorch)) + +(`Transformers` format models) +Download link: +([ModelScope](https://www.modelscope.cn/profile/gongjy) | [HuggingFace](https://huggingface.co/collections/jingyaogong/minimind-v-67000833fb60b3a2e1f3597d)) + +> Note: The Transformers version is the `MiniMind-V` model after single-image instruction fine-tuning # 📌 Test -### Effectiveness Testing +### Effect Test -#### Single-image conversation +#### Single Image Dialogue - - - - + + - - - - - + + + - - - - - + + + - - - - - + + + - - - - - + + + - - - - - + + + - - - - - + + + - - - - - + + + - - - - - - - - - - - - - - - - - - - + + +
Image512_pretrain512_sft768_pretrain768_sftMiniMind2-VMiniMind2-V-Small
a-girl.pngHair and makeup, I like her natural hair!This picture depicts a young woman, she is wearing a suit, and a tie, which indicates that she may be attending a special fashion event or celebration.A human-made actor's adventure movie.This picture portrays a portrait of a woman, who is wearing a pink dress. + city-traffic +                              + The image shows a busy city street with tall buildings on both sides of a long road. The street is packed with cars, trucks, and buses, along with many other vehicles driving on the road. There are many cars visible on the street, some driving at high speed, while others are parked at the side. Additionally, there is a bus parked on the right side of the street. Traffic lights are visible on the street, indicating a busy city environment.The image depicts a bustling urban scene with several cars and a truck driving down the city street. There are many traffic lights visible, some on the left side of the street, and others on the right. Several pedestrians can be seen on the street, some standing closer to the road while others are farther away. A parking sign is located on the left side of the image, hinting at the city environment. Two cars are visible on the street, one on the right and the other on the left, with another one also on the left side. This image captures a typical day in a city environment.
a-starfish.pngFossils in water, a still line composed of a ring of fossil clusters.The picture shows a large octopus, a potential marine creature, it is either on the surface of the water or in the ocean.Starfish and tentacles.The picture shows a starfish on the beach, including the starfish, and an underwater object. + astronaut +                              + The image shows an astronaut in a spacesuit sitting in a large spacecraft. They appear to be embarking on or disembarking from a space mission. Behind the astronaut, there is a rocket launch platform, likely to support the astronaut's mission during the journey. Additionally, an aircraft is parked near a hangar, further suggesting this is an aviation event. There are a few people around the aircraft, some standing close, possibly observing or waiting for the spacecraft to prepare for takeoff.The scene shows a soldier wearing a helmet standing on a large airplane. This aircraft appears to be a military one, likely preparing to board another plane. Another person stands in front, possibly observing the flight process. There are several people around the airplane, some standing on the left side, others on the right. They seem to be watching the pilot's performance. Additionally, a truck is parked near the left side, likely to observe the flight process more closely.
a-bear.pngIn the wild, in the valley.The picture features plants and a grizzly bear sitting on the grass.A close-up of a grizzly bearThe picture shows a grizzly bear standing in an open field of grass, surrounded by trees and bushes, with a backpack placed on top of it. + dog-woman-sea +                              + The image shows a woman sitting on the beach, holding a white dog in her hands. She appears to be a woman sitting on the sand, looking at her. A dog is also sitting beside her, looking relaxed and comfortable. Other beachgoers are scattered across the beach, some sitting, while others are farther away. A boat can be seen in the background, indicating that this is a popular beach destination for tourists.Two people are sitting on the beach, one lounging lazily on the sand, while the other is sitting. They seem to be enjoying their time by the sea. There are a few beach chairs, one close to the left side of the sand, another in the middle. Additionally, a dog is lying on the sand, adding to the relaxed atmosphere of the scene.
some-dolphins.pngA group of tourists watched this movie.This picture depicts a flock of seagulls flying over the water, on the water. The presence of seagulls suggests that they are looking for food. Seagulls nest on the water, possibly to protect themselves from potential dangers, such as sharp teeth and reptiles of seagulls.A group of dolphins or sheep fishing by boat during a day's voyageThis picture shows a group of people swimming among a large group of dolphins and dolphins nearby. + rainbow-falls +                              + The photo captures a beautiful natural scene with high mountains in the background. By the water, a large fountain spans across the surface, attracting many visitors. There are several people on the water's surface, some standing or sitting around the fountain, while others walk in the water. Overall, this image portrays a beautiful and peaceful environment where people can enjoy scenic views of nature.Under a beautiful blue sky, a massive white waterfall hangs above with a huge stream of wet water. This waterfall is located on a mountain, adding a charming and serene atmosphere to the whole scene. In the background of this image, several boats can be seen, some near the water's edge, others farther away. These boats seem to be preparing for scenic or outdoor activities.
three-girls.pngA girl and her friends are sitting on a bench, wearing long white robes.This scene depicts a vibrant young girl, they are dressed in black and white attire, standing in the middle of a crowd, all of whom are also dressed in black and white, indicating their outfits are lively and elegant. In the scene, there are two girls in the background, a woman behind, and another woman standing, all wearing black clothes. This suggests they are enjoying their outfits and attire, possibly attending a special festival or celebration.The girls are on the city streets.This picture portrays a man and a woman in traditional clothing, standing next to them, they are spending family time together. Throughout the scene, you can see a little boy and a girl, both wearing cowboy hats, which suggests they are participating in a family gathering, which could be a party or celebration, or they might be discussing an interesting activity or event. + elderly-reading +                              + The image shows a man sitting on a park bench next to a green chair. There is an open book beside him with the words "reading" written on it, suggesting he may be reading. The park has a bench and a park seat, adding life to the surrounding environment. There are several cars and a truck nearby, indicating this is a public space. Additionally, a person can be seen standing at different locations in the park, possibly waiting to cross the road or walk further.An elderly person wearing shorts sits on a park bench surrounded by trees. He seems to be reading a book, possibly engaged in reading. In the background, there is another bench providing ample seating for the scene. You can also see a chair and a table in the background, suggesting this may be an outdoor seating area where people can relax.
two-deer.pngThis photo contains several deer.This picture captures a white-tailed deer, sitting on the grass, using its photo to capture a red deer.The animal looks as if it is about to hide behind a tree, he looks majestic, as he cannot control himself.This picture depicts a doe and a deer, the doe is standing in the woods, a sheep, and a deer. + panda-grassland +                              + The image shows a white brown bear sitting on the grass, next to a large bear with brown spots. This bear seems shy or playful as it lies on the grass, resting and looking relaxed.In this image, a brown bear is strolling on the grass. The bear occupies much of the frame, seemingly walking in its natural environment on the grass. In the background, there are several trees, adding natural elements to the scene. A bird is flying near the middle of the scene, bringing a lively atmosphere to the image.
two-red-flowers.pngThe bouquet's flowers have barely bloomed.The picture shows red and yellow flowers, which are called "vase".The flower heads are close together.The picture shows red flowers, surrounded by several roses. + bicycle-flowers +                              + The image shows a beautiful vase filled with colorful flowers and bouquets. The bouquets are scattered throughout the vase, creating a visually pleasant scene. The colorful flowers in the vase create a delightful visual. These flowers are placed on a table, likely displayed to showcase their beauty.The scene shows a green and purple bicycle parked next to a building, placed near a large tree. This bicycle is situated nearby, adding some color to the scene. In addition to the bicycle, there are other bicycles, including one in the foreground and another near the center of the background. The presence of the bicycles suggests they may be parked there.
an-astronaut.pngAn astronaut posing with Earth during a space mission.This image depicts a dynamic moon, walking on the moon.An astronaut resting in a cradle during a mission, with his team in the background.This picture portrays an astronaut in the image of a space station.
a-tiger-in-water.pngA tiger in the water looking at the camera.The picture shows a large brown seal swimming in the water, resting in the water.A tiger caged in a zooThe picture shows a small bear, lying on a branch of a tree.
a-leopard-on-cliff.pngThis is an endangered species.In the picture, a black and white cat is walking on the rocks.A leopard in the wild on the rocks outside a cave, at sunriseThis picture shows a red panda walking on the rocks. + dance +                              + The woman in the image is wearing a white dress with a black tennis skirt. She is performing a tennis match, likely part of the competition. Several chairs are visible in the background, possibly set up for the audience or other spectators. Additionally, a bench is placed on the left side of the scene, providing a place for people to rest.A woman in white clothing stands on a stage holding a white frisbee. She seems to be participating in a stage dance or competition. Several other people are present in the scene, one standing on the left side of the stage, another on the right, and a third person standing near the right of the venue. The stage has several spectators, some standing, others sitting, with some remaining standing. This appears to be a joyful festival or event.
-#### Multi-image conversation +#### Multiple Image Dialogue (Effect is Limited) @@ -603,72 +661,40 @@ Download link: [HuggingFace](https://huggingface.co/collections/jingyaogong/mini + - - - + + + +
Image1 Image2 512_sft_multi768_sft_multi
a-bird.pnga-bird.pnganimal1 has a brown and black head with a black and white striped head . animal2 has a black head with a white stripe on its wings .a-bird.pnga-bird.pngThis image displays a bird scenario: a woman standing with a red and green mixed purple bird perched on her. The woman stands with the bird on her shoulders, while the red bird on her collar stands behind her.The two birds are flying in the same forest, some are in the center of the image, while others are smaller, creating a contrast. The birds’ presence highlights their flight ability and adaptability as they swiftly move through the woods. Additionally, the birds’ positions vary, with one on the left and the other on the right, indicating they are moving close within the same forest. Their natural behavior helps distinguish the two bird species.
+### Effect Summary: -### Start Inference +Visual signals are treated as a special foreign language by LLMs, so the "language learning" ability highly depends on +the LLM's capacity. The stronger the LLM, the more powerful the corresponding VLM, and the performance boost becomes +significant. -```bash -python web_server.py +#### Future Areas for Improvement: + +```text +> Simpler projection-based cross-modal feature alignment, which may be inferior compared to Cross-Attention. +> The Clip model could try larger, more powerful large series for finer-grained token representations of image features, as they are still coarse. +> The resolution is not high, theoretically only 224×224 (the minimind-v dataset is set to 128×128 for space saving). +> ... ``` -![web_server1](./images/web_server1.png) -![web_server2](./images/web_server2.png) - -### Summary of Effects - ---- -Based on the provided table data, the performance of the four models can be summarized as follows: - -1. **512_pretrain**: - - **Brief and inaccurate descriptions**: Most descriptions fail to clearly convey the image content, often providing unrelated narratives. For example, the starfish image is described as "fossils in water," which is far off the mark. - - **Lack of detail**: In most cases, only simple, vague descriptions are given, failing to delve into the details or context of the image. For instance, for the tiger image, it simply says "looking at the camera in the water." - -2. **512_sft**: - - **More specific descriptions**: Compared to 512_pretrain, 512_sft provides more detailed explanations and attempts to capture specific elements of the scene. For example, when describing the woman image, it mentions "suit" and "tie," giving a clearer depiction. - - **Occasional errors or redundancy**: Some descriptions are overly complex or even irrelevant to the image, such as mentioning seagulls, nesting, etc., in the dolphin image, which are not related. - -3. **768_pretrain**: - - **Incoherent information**: The performance of this model is quite scattered, with descriptions often being vague and incomplete. For example, in describing the woman image, it only mentions "a human-made actor's adventure movie," without clearly explaining the image content. - - **Partially accurate but with less overall information**: Some descriptions, although relevant to the image, are very brief. For example, the starfish description only states "starfish and tentacles," lacking a full sense of the scene. - -4. **768_sft**: - - **Comprehensive and specific descriptions**: This model's descriptions are the most detailed and precise among the four. For instance, when describing the bear image, it mentions "standing in an open field of grass, surrounded by trees and bushes, with a backpack," capturing multiple elements of the image. - - **Stronger comprehension ability**: This model can identify the scene and context of the image, providing reasonable interpretations and speculations. For example, describing a "family gathering" or "celebration" gives the image a more contextual connection. - -### GPT=4o Summary: - -- **512_pretrain** performs the worst, with simple and inaccurate descriptions. -- **512_sft** has improved detail in descriptions but occasionally includes irrelevant information. -- **768_pretrain** has poor coherence in information, yet provides basic descriptions in some aspects. -- **768_sft** performs the best, offering detailed, accurate descriptions, and is able to make good context-based inferences. - -### Personal Summary: - -- Visual signals are a special kind of foreign language for LLMs, so the ability to "learn foreign languages" largely depends on the capabilities of the LLM. -- The stronger the performance of the LLM, the stronger the corresponding VLM will be, and the performance gain will be significant. -- Areas for improvement: - - Simpler projection methods for cross-modal feature alignment result in greater performance loss compared to Cross-Attention. - - Larger and more powerful large-series Clip models can be tried, using more fine-grained token representations for image features, which are currently very rough. - - The resolution is not high, theoretically only 224×224 (minimind-v dataset is set to 128×128 to save space). - - ... - ---- - # 📌 Acknowledge > [!TIP] > If you find `MiniMind-V` helpful, please consider giving it a ⭐ on GitHub.
-> Given the limited expertise, there may be unknown issues, and we welcome everyone to discuss, correct, or submit PRs to improve the project in Issues.
+> Given the limited expertise, there may be unknown issues, and we welcome everyone to discuss, correct, or submit PRs +> to improve the project in Issues.
> Your support is the driving force behind continuous improvements to the project. Thank you! ## 🤝 [Contributors](https://github.com/jingyaogong/minimind/graphs/contributors) @@ -695,7 +721,6 @@ Based on the provided table data, the performance of the four models can be summ - ## 🫶Supporter @@ -720,7 +745,6 @@ Based on the provided table data, the performance of the four models can be summ Star History Chart - # License This repository is licensed under the [Apache-2.0 License](LICENSE). diff --git a/dataset/eval_images/一个女子.png b/dataset/eval_images/一个女子.png deleted file mode 100644 index 28832d0..0000000 Binary files a/dataset/eval_images/一个女子.png and /dev/null differ diff --git a/dataset/eval_images/一个海星.png b/dataset/eval_images/一个海星.png deleted file mode 100644 index e62392f..0000000 Binary files a/dataset/eval_images/一个海星.png and /dev/null differ diff --git a/dataset/eval_images/一个熊.png b/dataset/eval_images/一个熊.png deleted file mode 100644 index 2257324..0000000 Binary files a/dataset/eval_images/一个熊.png and /dev/null differ diff --git a/dataset/eval_images/一些海豚.png b/dataset/eval_images/一些海豚.png deleted file mode 100644 index 2710d7c..0000000 Binary files a/dataset/eval_images/一些海豚.png and /dev/null differ diff --git a/dataset/eval_images/三个女孩.png b/dataset/eval_images/三个女孩.png deleted file mode 100644 index 7817ef4..0000000 Binary files a/dataset/eval_images/三个女孩.png and /dev/null differ diff --git a/dataset/eval_images/两头鹿.png b/dataset/eval_images/两头鹿.png deleted file mode 100644 index 14bc9ba..0000000 Binary files a/dataset/eval_images/两头鹿.png and /dev/null differ diff --git a/dataset/eval_images/两朵红花.png b/dataset/eval_images/两朵红花.png deleted file mode 100644 index 738f8d3..0000000 Binary files a/dataset/eval_images/两朵红花.png and /dev/null differ diff --git a/dataset/eval_images/城市车水马龙-city-traffic.jpg b/dataset/eval_images/城市车水马龙-city-traffic.jpg new file mode 100644 index 0000000..ede43c4 Binary files /dev/null and b/dataset/eval_images/城市车水马龙-city-traffic.jpg differ diff --git a/dataset/eval_images/太空宇航员-Astronaut-Space.jpg b/dataset/eval_images/太空宇航员-Astronaut-Space.jpg new file mode 100644 index 0000000..e2cca53 Binary files /dev/null and b/dataset/eval_images/太空宇航员-Astronaut-Space.jpg differ diff --git a/dataset/eval_images/太空宇航员.png b/dataset/eval_images/太空宇航员.png deleted file mode 100644 index c1970d4..0000000 Binary files a/dataset/eval_images/太空宇航员.png and /dev/null differ diff --git a/dataset/eval_images/小狗美女海边-Dog-Woman-Sea.jpg b/dataset/eval_images/小狗美女海边-Dog-Woman-Sea.jpg new file mode 100644 index 0000000..4b9aa27 Binary files /dev/null and b/dataset/eval_images/小狗美女海边-Dog-Woman-Sea.jpg differ diff --git a/dataset/eval_images/彩虹瀑布-Rainbow-Falls.jpg b/dataset/eval_images/彩虹瀑布-Rainbow-Falls.jpg new file mode 100644 index 0000000..0ea9bf8 Binary files /dev/null and b/dataset/eval_images/彩虹瀑布-Rainbow-Falls.jpg differ diff --git a/dataset/eval_images/椅子老人看书-Chair-Elderly-Reading.jpg b/dataset/eval_images/椅子老人看书-Chair-Elderly-Reading.jpg new file mode 100644 index 0000000..e2d86e3 Binary files /dev/null and b/dataset/eval_images/椅子老人看书-Chair-Elderly-Reading.jpg differ diff --git a/dataset/eval_images/熊猫草地-Panda-Grassland.jpg b/dataset/eval_images/熊猫草地-Panda-Grassland.jpg new file mode 100644 index 0000000..9c6ba26 Binary files /dev/null and b/dataset/eval_images/熊猫草地-Panda-Grassland.jpg differ diff --git a/dataset/eval_images/老虎在水里.png b/dataset/eval_images/老虎在水里.png deleted file mode 100644 index 9f52900..0000000 Binary files a/dataset/eval_images/老虎在水里.png and /dev/null differ diff --git a/dataset/eval_images/自行车鲜花-Bicycle-Flowers.jpg b/dataset/eval_images/自行车鲜花-Bicycle-Flowers.jpg new file mode 100644 index 0000000..755da50 Binary files /dev/null and b/dataset/eval_images/自行车鲜花-Bicycle-Flowers.jpg differ diff --git a/dataset/eval_images/舞蹈-dance.jpg b/dataset/eval_images/舞蹈-dance.jpg new file mode 100644 index 0000000..9095487 Binary files /dev/null and b/dataset/eval_images/舞蹈-dance.jpg differ diff --git a/dataset/eval_images/豹子在悬崖.png b/dataset/eval_images/豹子在悬崖.png deleted file mode 100644 index 84c7ffa..0000000 Binary files a/dataset/eval_images/豹子在悬崖.png and /dev/null differ diff --git a/eval_vlm.py b/eval_vlm.py new file mode 100644 index 0000000..7e09187 --- /dev/null +++ b/eval_vlm.py @@ -0,0 +1,141 @@ +import argparse +import os +import random +import numpy as np +import torch +import warnings +from PIL import Image +from transformers import AutoTokenizer, AutoModelForCausalLM +from model.model_vlm import MiniMindVLM +from model.VLMConfig import VLMConfig +from transformers import logging as hf_logging + +hf_logging.set_verbosity_error() + +warnings.filterwarnings('ignore') + + +def count_parameters(model): + return sum(p.numel() for p in model.parameters() if p.requires_grad) + + +def init_model(lm_config, device): + tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer') + if args.load == 0: + moe_path = '_moe' if args.use_moe else '' + modes = {0: 'pretrain_vlm', 1: 'sft_vlm', 2: 'sft_vlm_multi'} + ckp = f'./{args.out_dir}/{modes[args.model_mode]}_{args.dim}{moe_path}.pth' + model = MiniMindVLM(lm_config) + state_dict = torch.load(ckp, map_location=device) + model.load_state_dict({k: v for k, v in state_dict.items() if 'mask' not in k}, strict=False) + else: + transformers_model_path = 'MiniMind2-V' + tokenizer = AutoTokenizer.from_pretrained(transformers_model_path) + model = AutoModelForCausalLM.from_pretrained(transformers_model_path, trust_remote_code=True) + + print(f'VLM参数量：{sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e6:.3f} 百万') + + vision_model, preprocess = MiniMindVLM.get_vision_model() + return model.eval().to(device), tokenizer, vision_model.eval().to(device), preprocess + + +def setup_seed(seed): + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Chat with MiniMind") + parser.add_argument('--lora_name', default='None', type=str) + parser.add_argument('--out_dir', default='out', type=str) + parser.add_argument('--temperature', default=0.65, type=float) + parser.add_argument('--top_p', default=0.85, type=float) + parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu', type=str) + # MiniMind2-Small (26M)：(dim=512, n_layers=8) + # MiniMind2 (104M)：(dim=768, n_layers=16) + parser.add_argument('--dim', default=512, type=int) + parser.add_argument('--n_layers', default=8, type=int) + parser.add_argument('--max_seq_len', default=8192, type=int) + parser.add_argument('--use_moe', default=False, type=bool) + # 默认单图推理，设置为2为多图推理 + parser.add_argument('--use_multi', default=1, type=int) + parser.add_argument('--stream', default=True, type=bool) + parser.add_argument('--load', default=0, type=int, help="0: 原生torch权重，1: transformers加载") + parser.add_argument('--model_mode', default=1, type=int, + help="0: Pretrain模型，1: SFT模型，2: SFT-多图模型 (beta拓展)") + args = parser.parse_args() + + lm_config = VLMConfig(dim=args.dim, n_layers=args.n_layers, max_seq_len=args.max_seq_len, use_moe=args.use_moe) + + model, tokenizer, vision_model, preprocess = init_model(lm_config, args.device) + + + def chat_with_vlm(prompt, pixel_tensors, image_names): + messages = [{"role": "user", "content": prompt}] + + new_prompt = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True + )[-args.max_seq_len + 1:] + + print(f'[Image]: {image_names}') + with torch.no_grad(): + x = torch.tensor(tokenizer(new_prompt)['input_ids'], device=args.device).unsqueeze(0) + outputs = model.generate( + x, + eos_token_id=tokenizer.eos_token_id, + max_new_tokens=args.max_seq_len, + temperature=args.temperature, + top_p=args.top_p, + stream=True, + pad_token_id=tokenizer.pad_token_id, + pixel_tensors=pixel_tensors + ) + print('🤖️: ', end='') + try: + if not args.stream: + print(tokenizer.decode(outputs.squeeze()[x.shape[1]:].tolist(), skip_special_tokens=True), end='') + else: + history_idx = 0 + for y in outputs: + answer = tokenizer.decode(y[0].tolist(), skip_special_tokens=True) + if (answer and answer[-1] == '�') or not answer: + continue + print(answer[history_idx:], end='', flush=True) + history_idx = len(answer) + except StopIteration: + print("No answer") + print('\n') + + + # 单图推理：每1个图像单独推理 + if args.use_multi == 1: + image_dir = './dataset/eval_images/' + prompt = f"{model.params.image_special_token}\n描述一下这个图像的内容。" + + for image_file in os.listdir(image_dir): + image = Image.open(os.path.join(image_dir, image_file)).convert('RGB') + pixel_tensors = MiniMindVLM.image2tensor(image, preprocess).to(args.device).unsqueeze(0) + chat_with_vlm(prompt, pixel_tensors, image_file) + + # 2图推理：目录下的两个图像编码，一次性推理（power by ） + if args.use_multi == 2: + args.model_mode = 2 + image_dir = './dataset/eval_multi_images/bird/' + prompt = (f"{lm_config.image_special_token}\n" + f"{lm_config.image_special_token}\n" + f"比较一下两张图像的异同点。") + pixel_tensors_multi = [] + for image_file in os.listdir(image_dir): + image = Image.open(os.path.join(image_dir, image_file)).convert('RGB') + pixel_tensors_multi.append(MiniMindVLM.image2tensor(image, preprocess)) + pixel_tensors = torch.cat(pixel_tensors_multi, dim=0).to(args.device).unsqueeze(0) + # 同样内容重复10次 + for _ in range(10): + chat_with_vlm(prompt, pixel_tensors, (', '.join(os.listdir(image_dir)))) diff --git a/export_model.py b/export_model.py deleted file mode 100644 index 11926de..0000000 --- a/export_model.py +++ /dev/null @@ -1,40 +0,0 @@ -import torch -import warnings -from transformers import AutoTokenizer, AutoModelForCausalLM -from model.LMConfig import LMConfig -from model.model import Transformer - -warnings.filterwarnings('ignore', category=UserWarning) - - -def count_parameters(model): - return sum(p.numel() for p in model.parameters() if p.requires_grad) - - -def export_transformers_model(): - tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer', - trust_remote_code=True, use_fast=False) - LMConfig.register_for_auto_class() - Transformer.register_for_auto_class("AutoModelForCausalLM") - - lm_config = LMConfig() - lm_model = Transformer(lm_config) - device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') - moe_path = '_moe' if lm_config.use_moe else '' - ckpt_path = f'./out/{lm_config.dim}{moe_path}_vlm_sft.pth' - - state_dict = torch.load(ckpt_path, map_location=device) - unwanted_prefix = '_orig_mod.' - for k, v in list(state_dict.items()): - if k.startswith(unwanted_prefix): - state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k) - lm_model.load_state_dict(state_dict, strict=False) - print(f'模型参数: {count_parameters(lm_model) / 1e6} 百万 = {count_parameters(lm_model) / 1e9} B (Billion)') - - lm_model.save_pretrained("minimind-v-v1-small", safe_serialization=False) - tokenizer.save_pretrained("minimind-v-v1-small") - - -if __name__ == '__main__': - # 1 - export_transformers_model() diff --git a/images/1-pretrain-512.png b/images/1-pretrain-512.png deleted file mode 100644 index a30c402..0000000 Binary files a/images/1-pretrain-512.png and /dev/null differ diff --git a/images/1-pretrain-768.png b/images/1-pretrain-768.png deleted file mode 100644 index 4591a60..0000000 Binary files a/images/1-pretrain-768.png and /dev/null differ diff --git a/images/2-sft-512.png b/images/2-sft-512.png deleted file mode 100644 index b6882a0..0000000 Binary files a/images/2-sft-512.png and /dev/null differ diff --git a/images/2-sft-768.png b/images/2-sft-768.png deleted file mode 100644 index 8994fa9..0000000 Binary files a/images/2-sft-768.png and /dev/null differ diff --git a/images/VLM-structure-moe.png b/images/VLM-structure-moe.png index 8c5aec1..dfa52a0 100644 Binary files a/images/VLM-structure-moe.png and b/images/VLM-structure-moe.png differ diff --git a/images/VLM-structure.png b/images/VLM-structure.png index 7677824..e8ecd87 100644 Binary files a/images/VLM-structure.png and b/images/VLM-structure.png differ diff --git a/images/minimind-v-input.png b/images/minimind-v-input.png index ac3a4cb..26386e2 100644 Binary files a/images/minimind-v-input.png and b/images/minimind-v-input.png differ diff --git a/images/minimind2-v.gif b/images/minimind2-v.gif new file mode 100644 index 0000000..c115799 Binary files /dev/null and b/images/minimind2-v.gif differ diff --git a/images/modelscope-demo.gif b/images/modelscope-demo.gif deleted file mode 100644 index b03ece4..0000000 Binary files a/images/modelscope-demo.gif and /dev/null differ diff --git a/images/pretrain_loss.png b/images/pretrain_loss.png new file mode 100644 index 0000000..c1421da Binary files /dev/null and b/images/pretrain_loss.png differ diff --git a/images/sft_loss.png b/images/sft_loss.png new file mode 100644 index 0000000..f815919 Binary files /dev/null and b/images/sft_loss.png differ diff --git a/images/web_server.gif b/images/web_server.gif deleted file mode 100644 index fc4dabd..0000000 Binary files a/images/web_server.gif and /dev/null differ diff --git a/images/web_server1.png b/images/web_server1.png deleted file mode 100644 index bbd85a0..0000000 Binary files a/images/web_server1.png and /dev/null differ diff --git a/images/web_server2.png b/images/web_server2.png deleted file mode 100644 index 65f5b2a..0000000 Binary files a/images/web_server2.png and /dev/null differ diff --git a/model/LMConfig.py b/model/LMConfig.py index 3113da1..fb0a76d 100644 --- a/model/LMConfig.py +++ b/model/LMConfig.py @@ -7,31 +7,31 @@ class LMConfig(PretrainedConfig): def __init__( self, - dim: int = 512, # 768 - n_layers: int = 8, # 16 - n_heads: int = 16, - n_kv_heads: int = 8, + dim: int = 512, + n_layers: int = 8, + n_heads: int = 8, + n_kv_heads: int = 2, vocab_size: int = 6400, hidden_dim: int = None, multiple_of: int = 64, norm_eps: float = 1e-5, - max_seq_len: int = 512, + max_seq_len: int = 8192, + rope_theta: int = 1e6, dropout: float = 0.0, flash_attn: bool = True, - image_special_token: str = '<' * 25 + '>' * 25, - image_ids: List = [30] * 25 + [32] * 25, #################################################### # Here are the specific configurations of MOE # When use_moe is false, the following is invalid #################################################### use_moe: bool = False, - num_experts_per_tok=2, - n_routed_experts=4, + #################################################### + num_experts_per_tok: int = 2, + n_routed_experts: int = 4, n_shared_experts: bool = True, - scoring_func='softmax', - aux_loss_alpha=0.01, - seq_aux=True, - norm_topk_prob=True, + scoring_func: str = 'softmax', + aux_loss_alpha: float = 0.1, + seq_aux: bool = True, + norm_topk_prob: bool = True, **kwargs, ): self.dim = dim @@ -43,10 +43,9 @@ class LMConfig(PretrainedConfig): self.multiple_of = multiple_of self.norm_eps = norm_eps self.max_seq_len = max_seq_len + self.rope_theta = rope_theta self.dropout = dropout self.flash_attn = flash_attn - self.image_special_token = image_special_token - self.image_ids = image_ids #################################################### # Here are the specific configurations of MOE # When use_moe is false, the following is invalid diff --git a/model/VLMConfig.py b/model/VLMConfig.py new file mode 100644 index 0000000..113fb3a --- /dev/null +++ b/model/VLMConfig.py @@ -0,0 +1,16 @@ +from .LMConfig import LMConfig +from typing import List + + +class VLMConfig(LMConfig): + model_type = "minimind-v" + + def __init__( + self, + image_special_token: str = '@' * 196, + image_ids: List = [34] * 196, + **kwargs, + ): + self.image_special_token = image_special_token + self.image_ids = image_ids + super().__init__(**kwargs) diff --git a/model/clip_model/README.md b/model/clip_model/README.md deleted file mode 100644 index b92a1d5..0000000 --- a/model/clip_model/README.md +++ /dev/null @@ -1,5 +0,0 @@ -* 需要把clip-vit-base-patch32模型下载到此目录下 - -```bash -git clone https://hf-mirror.com/openai/clip-vit-base-patch32 -``` \ No newline at end of file diff --git a/model/dataset.py b/model/dataset.py index d93de28..dae04a7 100644 --- a/model/dataset.py +++ b/model/dataset.py @@ -1,278 +1,86 @@ import json -import random -import re - -import pandas as pd -import numpy as np from PIL import Image from torch.utils.data import Dataset, DataLoader import torch -from model.vision_utils import get_img_process +from .model_vlm import MiniMindVLM import os os.environ["TOKENIZERS_PARALLELISM"] = "false" -class PretrainDataset(Dataset): - def __init__(self, json_path, tokenizer, vision_model=None, max_length=1024, - prompt_max_len=512, - answer_max_len=256, - image_special_token='<' * 25 + '>' * 25): +class VLMDataset(Dataset): + def __init__(self, jsonl_path, images_path, tokenizer, preprocess=None, max_length=512, + image_special_token='@' * 196): super().__init__() - with open(json_path, 'r', encoding='utf-8') as f: - self.data = json.load(f) + self.samples = self.load_data(jsonl_path) + self.images_path = images_path - self.max_length = max_length - self.prompt_max_len = prompt_max_len - self.answer_max_len = answer_max_len self.tokenizer = tokenizer - self.vision_model, self.preprocess = vision_model - self.padding = 0 - self.bos_id = self.tokenizer('assistant').data['input_ids'] - self.dataset_path = './dataset/pretrain_images/' - self.image_special_token = image_special_token + self.max_length = max_length + self.preprocess = preprocess + self.image_token = image_special_token + self.bos_id = tokenizer('assistant\n', add_special_tokens=False).input_ids + self.eos_id = tokenizer('\n', add_special_tokens=False).input_ids def __len__(self): - return len(self.data) + return len(self.samples) - def find_sublist_index(self, main_list, sub_list) -> int: - last_index = -1 - for i in range(len(main_list) - len(sub_list) + 1): - if main_list[i:i + len(sub_list)] == sub_list: - last_index = i - return last_index + def load_data(self, path): + samples = [] + with open(path, 'r', encoding='utf-8') as f: + for line_num, line in enumerate(f, 1): + data = json.loads(line.strip()) + samples.append(data) + return samples - def safe_eval(self, s): - try: - res = eval(s) - except Exception as e: - return [] - return res - - def __getitem__(self, index: int): - sample = self.data[index] - image_name = sample['image'] - conversation = sample['conversations'] - # minimind-v的image的特殊占位符，对应每张图切分成M个token，和get_img_process中的数量对应 + def _create_chat_prompt(self, conversations): messages = [] - # 遍历 conversation 列表 - for i in range(0, len(conversation), 2): - # 检查是否有配对的问题和回答 - if i + 1 < len(conversation): - q = conversation[i]['value'].replace('', self.image_special_token) - a = conversation[i + 1]['value'] - - if q and a: - messages.append({"role": "user", "content": q}) - messages.append({"role": "assistant", "content": a}) - - new_prompt = self.tokenizer.apply_chat_template( + for i, turn in enumerate(conversations): + role = 'user' if i % 2 == 0 else 'assistant' + messages.append({"role": role, "content": turn['content'].replace('', self.image_token)}) + return self.tokenizer.apply_chat_template( messages, tokenize=False, - add_generation_prompt=True + add_generation_prompt=False ) - input_id = self.tokenizer(new_prompt).data['input_ids'][:self.max_length] - question_length = self.find_sublist_index(input_id, self.bos_id) + len(self.bos_id) - padding_len = self.max_length - len(input_id) - input_id = input_id + [self.padding] * padding_len - mask_len = len(input_id) - question_length - padding_len - loss_mask = [0] * question_length + [1] * (mask_len) + [0] * padding_len - - input_id = np.array(input_id) - - X = np.array(input_id[:-1]).astype(np.int64) - Y = np.array(input_id[1:]).astype(np.int64) - loss_mask = np.array(loss_mask[1:]).astype(np.int64) - - X_tensor = torch.from_numpy(X) - Y_tensor = torch.from_numpy(Y) - loss_mask_tensor = torch.from_numpy(loss_mask) - - image = Image.open(f'{self.dataset_path}{image_name}') - image_encoders = get_img_process(image, self.preprocess) - - return X_tensor, Y_tensor, loss_mask_tensor, image_encoders - - -class SFTDataset(Dataset): - def __init__(self, json_path, tokenizer, vision_model=None, max_length=1024, - prompt_max_len=512, - answer_max_len=256, - image_special_token='<' * 25 + '>' * 25): - - super().__init__() - with open(json_path, 'r', encoding='utf-8') as f: - self.data = json.load(f) - - self.max_length = max_length - self.prompt_max_len = prompt_max_len - self.answer_max_len = answer_max_len - self.tokenizer = tokenizer - self.vision_model, self.preprocess = vision_model - self.padding = 0 - self.bos_id = self.tokenizer('assistant').data['input_ids'] - self.dataset_path = './dataset/sft_images/' - self.image_special_token = image_special_token - - def __len__(self): - return len(self.data) - - def find_sublist_index(self, main_list, sub_list) -> int: - last_index = -1 - for i in range(len(main_list) - len(sub_list) + 1): - if main_list[i:i + len(sub_list)] == sub_list: - last_index = i - return last_index - - def safe_eval(self, s): - try: - res = eval(s) - except Exception as e: - return [] - return res + def _generate_loss_mask(self, input_ids): + loss_mask = [0] * len(input_ids) + i = 0 + while i < len(input_ids): + if input_ids[i:i + len(self.bos_id)] == self.bos_id: + start = i + len(self.bos_id) + end = start + while end < len(input_ids): + if input_ids[end:end + len(self.eos_id)] == self.eos_id: + break + end += 1 + for j in range(start + 1, min(end + len(self.eos_id) + 1, self.max_length)): + loss_mask[j] = 1 + i = end + len(self.eos_id) if end < len(input_ids) else len(input_ids) + else: + i += 1 + return loss_mask def __getitem__(self, index: int): - sample = self.data[index] - image_name = 'COCO_train2014_' + sample['image'] - conversation = sample['conversations'] - # minimind-v的image的特殊占位符，对应每张图切分成M个token，和get_img_process中的数量对应 - messages = [] - # 遍历 conversation 列表 - # for i in range(0, len(conversation), 2): - for i in range(0, 1): - # 检查是否有配对的问题和回答 - if i + 1 < len(conversation): - q = conversation[i]['value'].replace('', self.image_special_token) - a = conversation[i + 1]['value'] + sample = self.samples[index] + image_paths = sample['image'] + prompt = self._create_chat_prompt(sample['conversations']) + input_ids = self.tokenizer(prompt).input_ids[:self.max_length] + input_ids += [self.tokenizer.pad_token_id] * (self.max_length - len(input_ids)) + loss_mask = self._generate_loss_mask(input_ids) - if q and a: - messages.append({"role": "user", "content": q}) - messages.append({"role": "assistant", "content": a}) + X = torch.tensor(input_ids[:-1], dtype=torch.long) + Y = torch.tensor(input_ids[1:], dtype=torch.long) + loss_mask = torch.tensor(loss_mask[1:], dtype=torch.long) - new_prompt = self.tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True - ) - input_id = self.tokenizer(new_prompt).data['input_ids'][:self.max_length] + image_tensors = [] + for image_name in image_paths.split(','): + image_name = image_name.strip() + image = Image.open(f'{self.images_path}/{image_name}') + image_tensor = MiniMindVLM.image2tensor(image, self.preprocess) + image_tensors.append(image_tensor) + image_tensors = torch.stack(image_tensors, dim=0) - question_length = self.find_sublist_index(input_id, self.bos_id) + len(self.bos_id) - padding_len = self.max_length - len(input_id) - input_id = input_id + [self.padding] * padding_len - mask_len = len(input_id) - question_length - padding_len - loss_mask = [0] * question_length + [1] * (mask_len) + [0] * padding_len - - input_id = np.array(input_id) - - X = np.array(input_id[:-1]).astype(np.int64) - Y = np.array(input_id[1:]).astype(np.int64) - loss_mask = np.array(loss_mask[1:]).astype(np.int64) - - X_tensor = torch.from_numpy(X) - Y_tensor = torch.from_numpy(Y) - loss_mask_tensor = torch.from_numpy(loss_mask) - - image = Image.open(f'{self.dataset_path}{image_name}') - image_encoders = get_img_process(image, self.preprocess) - - return X_tensor, Y_tensor, loss_mask_tensor, image_encoders - - -class SFTDataset_multi(Dataset): - def __init__(self, json_path, tokenizer, vision_model=None, max_length=1024, - prompt_max_len=512, - answer_max_len=256, - image_special_token='<' * 25 + '>' * 25): - - super().__init__() - with open(json_path, 'r', encoding='utf-8') as f: - self.data = json.load(f) - - self.max_length = max_length - self.prompt_max_len = prompt_max_len - self.answer_max_len = answer_max_len - self.tokenizer = tokenizer - self.vision_model, self.preprocess = vision_model - self.padding = 0 - self.bos_id = self.tokenizer('assistant').data['input_ids'] - self.dataset_path = './dataset/sft_multi_images/' - self.image_special_token = image_special_token - - def __len__(self): - return len(self.data) - - def find_sublist_index(self, main_list, sub_list) -> int: - last_index = -1 - for i in range(len(main_list) - len(sub_list) + 1): - if main_list[i:i + len(sub_list)] == sub_list: - last_index = i - return last_index - - def safe_eval(self, s): - try: - res = eval(s) - except Exception as e: - return [] - return res - - def __getitem__(self, index: int): - sample = self.data[index] - image_names = sample['image'].split(', ') - conversation = sample['conversations'] - - messages = [] - for i in range(0, 1): - if i + 1 < len(conversation): - q = conversation[i]['value'].replace('', self.image_special_token) - a = conversation[i + 1]['value'] - - if q and a: - messages.append({"role": "user", "content": q}) - messages.append({"role": "assistant", "content": a}) - - # print(messages) # [{'role': 'user', 'content': '<<<<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>>>>\nAre the two images below that resemble each other described by the same term? You must choose your answer from the Choice List. Choice_List: True, False.'}, {'role': 'assistant', 'content': 'False'}][{'role': 'user', 'content': '<<<<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>>>><<<<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>>>>\nDo these four pictures fall into the same category? You must choose your answer from the Choice List. Choice_List: True, False.'}, {'role': 'assistant', 'content': 'False'}] - new_prompt = self.tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True - ) - input_id = self.tokenizer(new_prompt).data['input_ids'][:self.max_length] - # print(len(input_id)) # 165 or 263 or 259 - - question_length = self.find_sublist_index(input_id, self.bos_id) + len(self.bos_id) - padding_len = self.max_length - len(input_id) - input_id = input_id + [self.padding] * padding_len - mask_len = len(input_id) - question_length - padding_len - loss_mask = [0] * question_length + [1] * (mask_len) + [0] * padding_len - - input_id = np.array(input_id) - - X = np.array(input_id[:-1]).astype(np.int64) - Y = np.array(input_id[1:]).astype(np.int64) - loss_mask = np.array(loss_mask[1:]).astype(np.int64) - - X_tensor = torch.from_numpy(X) - Y_tensor = torch.from_numpy(Y) - loss_mask_tensor = torch.from_numpy(loss_mask) - - # 读取多张图像 - image_encoders = [] - for image_name in image_names: - image = Image.open(f'{self.dataset_path}{image_name.strip()}') # 去掉可能的空格 - image_encoders.append(get_img_process(image, self.preprocess)['pixel_values']) - - # 确定目标形状 - max_images = 2 # 根据你的需求设置 一次性最大输入几张图片 - target_shape = (max_images, 3, 224, 224) - - # 创建填充张量 - padded_image_encoders = torch.zeros(target_shape, dtype=torch.float32) - - # 填充图像编码 - for i, img_enc in enumerate(image_encoders): - if i < max_images: - padded_image_encoders[i] = img_enc.squeeze(0) - - return X_tensor, Y_tensor, loss_mask_tensor, padded_image_encoders + return X, Y, loss_mask, image_tensors diff --git a/model/minimind_tokenizer/tokenizer_config.json b/model/minimind_tokenizer/tokenizer_config.json index 5f3fa2b..0b14ab9 100644 --- a/model/minimind_tokenizer/tokenizer_config.json +++ b/model/minimind_tokenizer/tokenizer_config.json @@ -1,44 +1,43 @@ { - "add_bos_token": false, - "add_eos_token": false, - "add_prefix_space": true, - "added_tokens_decoder": { - "0": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - }, - "1": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - }, - "2": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - } + "add_bos_token": false, + "add_eos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true }, - "additional_special_tokens": [], - "bos_token": "", - "clean_up_tokenization_spaces": false, - "eos_token": "", - "legacy": true, - "model_max_length": 1000000000000000019884624838656, - "pad_token": null, - "sp_model_kwargs": {}, - "spaces_between_special_tokens": false, - "tokenizer_class": "PreTrainedTokenizerFast", - "unk_token": "", - "use_default_system_prompt": false, - "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'user\\n' + content + '\\nassistant\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '' + '\\n' }}{% endif %}{% endfor %}" + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": true, + "model_max_length": 32768, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "PreTrainedTokenizerFast", + "unk_token": "", + "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{{ 'system\\n' + system_message + '\\n' }}{% else %}{{ 'system\\n你是 MiniMind，是一个有用的人工智能助手。\\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'user\\n' + content + '\\nassistant\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '' + '\\n' }}{% endif %}{% endfor %}" } \ No newline at end of file diff --git a/model/model.py b/model/model.py index 9e91abf..1dc6478 100644 --- a/model/model.py +++ b/model/model.py @@ -4,7 +4,7 @@ import inspect import time from .LMConfig import LMConfig -from typing import Any, Optional, Tuple +from typing import Any, Optional, Tuple, List import numpy as np import torch import torch.nn.functional as F @@ -19,15 +19,11 @@ class RMSNorm(torch.nn.Module): self.eps = eps self.weight = nn.Parameter(torch.ones(dim)) - def _norm(self, x): - return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) - def forward(self, x): - output = self._norm(x.float()).type_as(x) - return output * self.weight + return self.weight * (x.float() * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)).type_as(x) -def precompute_pos_cis(dim: int, end: int, theta: float = 10000.0): +def precompute_pos_cis(dim: int, end: int = int(32 * 1024), theta: float = 1e6): freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim)) t = torch.arange(end, device=freqs.device) # type: ignore freqs = torch.outer(t, freqs).float() # type: ignore @@ -76,71 +72,69 @@ class Attention(nn.Module): self.wk = nn.Linear(args.dim, self.n_kv_heads * self.head_dim, bias=False) self.wv = nn.Linear(args.dim, self.n_kv_heads * self.head_dim, bias=False) self.wo = nn.Linear(args.n_heads * self.head_dim, args.dim, bias=False) - self.k_cache, self.v_cache = None, None self.attn_dropout = nn.Dropout(args.dropout) self.resid_dropout = nn.Dropout(args.dropout) self.dropout = args.dropout self.flash = hasattr(torch.nn.functional, 'scaled_dot_product_attention') and args.flash_attn - # print("WARNING: using slow attention. Flash Attention requires PyTorch >= 2.0") mask = torch.full((1, 1, args.max_seq_len, args.max_seq_len), float("-inf")) mask = torch.triu(mask, diagonal=1) self.register_buffer("mask", mask, persistent=False) - def forward(self, x: torch.Tensor, pos_cis: torch.Tensor, kv_cache=False): - bsz, seqlen, _ = x.shape - + def forward(self, + x: torch.Tensor, + pos_cis: torch.Tensor, + past_key_value: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + use_cache=False): + bsz, seq_len, _ = x.shape xq, xk, xv = self.wq(x), self.wk(x), self.wv(x) - - xq = xq.view(bsz, seqlen, self.n_local_heads, self.head_dim) - xk = xk.view(bsz, seqlen, self.n_local_kv_heads, self.head_dim) - xv = xv.view(bsz, seqlen, self.n_local_kv_heads, self.head_dim) + xq = xq.view(bsz, seq_len, self.n_local_heads, self.head_dim) + xk = xk.view(bsz, seq_len, self.n_local_kv_heads, self.head_dim) + xv = xv.view(bsz, seq_len, self.n_local_kv_heads, self.head_dim) xq, xk = apply_rotary_emb(xq, xk, pos_cis) + # kv_cache实现 + if past_key_value is not None: + xk = torch.cat([past_key_value[0], xk], dim=1) + xv = torch.cat([past_key_value[1], xv], dim=1) + past_kv = (xk, xv) if use_cache else None - # 更高效的kv_cache实现 - if kv_cache and self.eval(): - if seqlen == 1 and all(cache is not None for cache in (self.k_cache, self.v_cache)): - xk = torch.cat((self.k_cache, xk), dim=1) - xv = torch.cat((self.v_cache, xv), dim=1) - self.k_cache, self.v_cache = xk, xv - - xk = repeat_kv(xk, self.n_rep) # (bs, seqlen, n_local_heads, head_dim) - xv = repeat_kv(xv, self.n_rep) # (bs, seqlen, n_local_heads, head_dim) - - xq = xq.transpose(1, 2) - xk = xk.transpose(1, 2) - xv = xv.transpose(1, 2) - - if self.flash and seqlen != 1: - output = torch.nn.functional.scaled_dot_product_attention(xq, xk, xv, attn_mask=None, - dropout_p=self.dropout if self.training else 0.0, - is_causal=True) + xq, xk, xv = ( + xq.transpose(1, 2), + repeat_kv(xk, self.n_rep).transpose(1, 2), + repeat_kv(xv, self.n_rep).transpose(1, 2) + ) + if self.flash and seq_len != 1: + dropout_p = self.dropout if self.training else 0.0 + output = F.scaled_dot_product_attention( + xq, xk, xv, + attn_mask=None, + dropout_p=dropout_p, + is_causal=True + ) else: - scores = torch.matmul(xq, xk.transpose(2, 3)) / math.sqrt(self.head_dim) - scores = scores + self.mask[:, :, :seqlen, :seqlen] # (bs, n_local_heads, seqlen, cache_len + seqlen) + scores = (xq @ xk.transpose(-2, -1)) / math.sqrt(self.head_dim) + scores += self.mask[:, :, :seq_len, :seq_len] scores = F.softmax(scores.float(), dim=-1).type_as(xq) scores = self.attn_dropout(scores) - output = torch.matmul(scores, xv) # (bs, n_local_heads, seqlen, head_dim) + output = scores @ xv - output = output.transpose(1, 2).contiguous().view(bsz, seqlen, -1) - - output = self.wo(output) - output = self.resid_dropout(output) - return output + output = output.transpose(1, 2).reshape(bsz, seq_len, -1) + output = self.resid_dropout(self.wo(output)) + return output, past_kv class FeedForward(nn.Module): - def __init__(self, dim: int, hidden_dim: int, multiple_of: int, dropout: float): + def __init__(self, config: LMConfig): super().__init__() - if hidden_dim is None: - hidden_dim = 4 * dim + if config.hidden_dim is None: + hidden_dim = 4 * config.dim hidden_dim = int(2 * hidden_dim / 3) - hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of) - self.w1 = nn.Linear(dim, hidden_dim, bias=False) - self.w2 = nn.Linear(hidden_dim, dim, bias=False) - self.w3 = nn.Linear(dim, hidden_dim, bias=False) - self.dropout = nn.Dropout(dropout) + config.hidden_dim = config.multiple_of * ((hidden_dim + config.multiple_of - 1) // config.multiple_of) + self.w1 = nn.Linear(config.dim, config.hidden_dim, bias=False) + self.w2 = nn.Linear(config.hidden_dim, config.dim, bias=False) + self.w3 = nn.Linear(config.dim, config.hidden_dim, bias=False) + self.dropout = nn.Dropout(config.dropout) def forward(self, x): return self.dropout(self.w2(F.silu(self.w1(x)) * self.w3(x))) @@ -168,7 +162,6 @@ class MoEGate(nn.Module): def forward(self, hidden_states): bsz, seq_len, h = hidden_states.shape - hidden_states = hidden_states.view(-1, h) logits = F.linear(hidden_states, self.weight, None) if self.scoring_func == 'softmax': @@ -200,7 +193,7 @@ class MoEGate(nn.Module): fi = ce * self.n_routed_experts aux_loss = (Pi * fi).sum() * self.alpha else: - aux_loss = None + aux_loss = 0 return topk_idx, topk_weight, aux_loss @@ -209,50 +202,35 @@ class MOEFeedForward(nn.Module): super().__init__() self.config = config self.experts = nn.ModuleList([ - FeedForward( - dim=config.dim, - hidden_dim=config.hidden_dim, - multiple_of=config.multiple_of, - dropout=config.dropout, - ) + FeedForward(config) for _ in range(config.n_routed_experts) ]) - self.gate = MoEGate(config) if config.n_shared_experts is not None: - self.shared_experts = FeedForward( - dim=config.dim, - hidden_dim=config.hidden_dim, - multiple_of=config.multiple_of, - dropout=config.dropout, - ) + self.shared_experts = FeedForward(config) def forward(self, x): identity = x orig_shape = x.shape bsz, seq_len, _ = x.shape - # 使用门控机制选择专家 topk_idx, topk_weight, aux_loss = self.gate(x) - x = x.view(-1, x.shape[-1]) flat_topk_idx = topk_idx.view(-1) - if self.training: # 训练模式下，重复输入数据 x = x.repeat_interleave(self.config.num_experts_per_tok, dim=0) y = torch.empty_like(x, dtype=torch.float16) for i, expert in enumerate(self.experts): - y[flat_topk_idx == i] = expert(x[flat_topk_idx == i]) + y[flat_topk_idx == i] = expert(x[flat_topk_idx == i]).to(y.dtype) # 确保类型一致 y = (y.view(*topk_weight.shape, -1) * topk_weight.unsqueeze(-1)).sum(dim=1) y = y.view(*orig_shape) else: # 推理模式下，只选择最优专家 y = self.moe_infer(x, flat_topk_idx, topk_weight.view(-1, 1)).view(*orig_shape) - if self.config.n_shared_experts is not None: y = y + self.shared_experts(identity) - + self.aux_loss = aux_loss return y @torch.no_grad() @@ -271,7 +249,7 @@ class MOEFeedForward(nn.Module): expert = self.experts[i] exp_token_idx = token_idxs[start_idx:end_idx] expert_tokens = x[exp_token_idx] - expert_out = expert(expert_tokens) + expert_out = expert(expert_tokens).to(expert_cache.dtype) expert_out.mul_(flat_expert_weights[idxs[start_idx:end_idx]]) # 使用 scatter_add_ 进行 sum 操作 expert_cache.scatter_add_(0, exp_token_idx.view(-1, 1).repeat(1, x.shape[-1]), expert_out) @@ -279,222 +257,120 @@ class MOEFeedForward(nn.Module): return expert_cache -class TransformerBlock(nn.Module): - def __init__(self, layer_id: int, args: LMConfig): +class MiniMindBlock(nn.Module): + def __init__(self, layer_id: int, config: LMConfig): super().__init__() - self.n_heads = args.n_heads - self.dim = args.dim - self.head_dim = args.dim // args.n_heads - self.attention = Attention(args) + self.n_heads = config.n_heads + self.dim = config.dim + self.head_dim = config.dim // config.n_heads + self.attention = Attention(config) self.layer_id = layer_id - self.attention_norm = RMSNorm(args.dim, eps=args.norm_eps) - self.ffn_norm = RMSNorm(args.dim, eps=args.norm_eps) + self.attention_norm = RMSNorm(config.dim, eps=config.norm_eps) + self.ffn_norm = RMSNorm(config.dim, eps=config.norm_eps) + self.feed_forward = FeedForward(config) if not config.use_moe else MOEFeedForward(config) - if args.use_moe: - self.feed_forward = MOEFeedForward(args) - else: - self.feed_forward = FeedForward( - dim=args.dim, - hidden_dim=args.hidden_dim, - multiple_of=args.multiple_of, - dropout=args.dropout, - ) - - def forward(self, x, pos_cis, kv_cache=False): - h = x + self.attention(self.attention_norm(x), pos_cis, kv_cache) - out = h + self.feed_forward(self.ffn_norm(h)) - return out - - -class VisionProj(nn.Module): - def __init__(self, vision_out_dim=768, lm_dim=512, image_ids=[1, 2, 3, 4]): - super().__init__() - self.vision_out_dim = vision_out_dim - self.lm_dim = lm_dim - self.image_ids = image_ids - self.vision_proj = nn.Sequential( - nn.Linear(self.vision_out_dim, self.lm_dim), + def forward(self, x, pos_cis, past_key_value=None, use_cache=False): + h_attn, past_kv = self.attention( + self.attention_norm(x), + pos_cis, + past_key_value=past_key_value, + use_cache=use_cache ) - - def forward(self, image_encoders): - vision_proj = self.vision_proj(image_encoders) - return vision_proj + h = x + h_attn + out = h + self.feed_forward(self.ffn_norm(h)) + return out, past_kv -class Transformer(PreTrainedModel): +class MiniMindLM(PreTrainedModel): config_class = LMConfig - last_loss: Optional[torch.Tensor] def __init__(self, params: LMConfig = None): - super().__init__(params) - if not params: - params = LMConfig() - self.params = params - self.vocab_size = params.vocab_size - self.n_layers = params.n_layers - # image的特殊占位符，对应每张图切分成M个token，和get_img_process中的数量对应 - self.image_ids = params.image_ids - - self.tok_embeddings = nn.Embedding(self.vocab_size, params.dim) + self.params = params or LMConfig() + super().__init__(self.params) + self.vocab_size, self.n_layers = params.vocab_size, params.n_layers + self.tok_embeddings = nn.Embedding(params.vocab_size, params.dim) self.dropout = nn.Dropout(params.dropout) - self.layers = torch.nn.ModuleList() - for layer_id in range(self.n_layers): - self.layers.append(TransformerBlock(layer_id, params)) + self.layers = nn.ModuleList([MiniMindBlock(l, params) for l in range(self.n_layers)]) self.norm = RMSNorm(params.dim, eps=params.norm_eps) - self.output = nn.Linear(params.dim, self.vocab_size, bias=False) + self.output = nn.Linear(params.dim, params.vocab_size, bias=False) self.tok_embeddings.weight = self.output.weight - pos_cis = precompute_pos_cis(self.params.dim // self.params.n_heads, self.params.max_seq_len) - self.register_buffer("pos_cis", pos_cis, persistent=False) - - self.apply(self._init_weights) - - for pn, p in self.named_parameters(): - if pn.endswith('w3.weight') or pn.endswith('wo.weight'): - torch.nn.init.normal_(p, mean=0.0, std=0.02 / math.sqrt(2 * params.n_layers)) - - self.last_loss = None + self.register_buffer("pos_cis", + precompute_pos_cis(dim=params.dim // params.n_heads, theta=params.rope_theta), + persistent=False) self.OUT = CausalLMOutputWithPast() - self._no_split_modules = [name for name, _ in self.named_modules()] - - self.vision_proj = VisionProj(768, params.dim, self.image_ids) - - def _init_weights(self, module): - if isinstance(module, nn.Linear): - torch.nn.init.normal_(module.weight, mean=0.0, std=0.02) - if module.bias is not None: - torch.nn.init.zeros_(module.bias) - elif isinstance(module, nn.Embedding): - torch.nn.init.normal_(module.weight, mean=0.0, std=0.02) - - # VLM - def count_vision_proj(self, tokens, h, image_encoders=None, seqlen=200): - # 查找token中片段的索引，为了替换做准备 - def find_indices(tokens, image_ids): - image_ids_tensor = torch.tensor(image_ids).to(tokens.device) - len_image_ids = len(image_ids) - # .generate时，在初始化后直接跳过 - if len_image_ids > tokens.size(1): - return None - - # 使用view来创建一个视图，便于处理滑动窗口 - tokens_view = tokens.unfold(1, len_image_ids, 1) # 在第二维度创建滑动窗口 - # 检查每个滑动窗口是否与image_ids_tensor相等 - matches = (tokens_view == image_ids_tensor).all(dim=2) # 对窗口中的每一行进行比较 - - # 提取匹配的索引 - indices = {} - for batch_idx in range(tokens.size(0)): - match_indices = matches[batch_idx].nonzero(as_tuple=True)[0] # 获取非零（匹配）索引 - if match_indices.numel() > 0: # 如果有匹配 - indices[batch_idx] = [(idx.item(), idx.item() + len_image_ids - 1) for idx in match_indices] - return indices if indices else None - - image_indices = find_indices(tokens, self.image_ids) # 字典形式存储索引 - - # 如果此时有图像编码 - if image_encoders is not None: - vision_proj = self.vision_proj(image_encoders) - vision_proj = vision_proj.unsqueeze(1) if len(vision_proj.shape) == 3 else vision_proj - if image_indices is not None: - # 创建一个新的张量来存储拼接后的结果 - new_h = [] - for i in range(h.size(0)): - # i即为current_batch_idx索引 - img_idx = 0 - if i in image_indices: # 直接从字典中获取 - for start_idx, end_idx in image_indices[i]: - # 插入vision_proj特征 - before = h[i][:start_idx, :] - after = h[i][end_idx + 1:, :] - # 拼接 before, vision_proj, after - h[i] = torch.cat((before, vision_proj[i][img_idx], after), dim=0)[:seqlen] - img_idx += 1 - new_h.append(h[i]) - # 将所有拼接后的结果重新堆叠起来 - new_h = torch.stack(new_h, dim=0) # torch.Size([32, 511, 512]) - return new_h - - return h - - def forward(self, tokens: Optional[torch.Tensor] = None, targets: Optional[torch.Tensor] = None, - kv_cache=False, image_encoders=None, **keyargs): - current_idx = 0 - if 'input_ids' in keyargs: - tokens = keyargs['input_ids'] - if 'attention_mask' in keyargs: - targets = keyargs['attention_mask'] - if 'current_idx' in keyargs: - current_idx = int(keyargs['current_idx']) - - _bsz, seqlen = tokens.shape - # language proj token - h = self.tok_embeddings(tokens) - # vision proj token - h = self.count_vision_proj(tokens=tokens, h=h, image_encoders=image_encoders, seqlen=seqlen) - h = self.dropout(h) - - pos_cis = self.pos_cis[current_idx:current_idx + seqlen] - for idx, layer in enumerate(self.layers): - h = layer(h, pos_cis, kv_cache) - - h = self.norm(h) - - if targets is not None: - logits = self.output(h) - self.last_loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=0, - reduction='none') - else: - logits = self.output(h[:, [-1], :]) - self.last_loss = None + def forward(self, + input_ids: Optional[torch.Tensor] = None, + past_key_values: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None, + use_cache: bool = False, + **args): + past_key_values = past_key_values or [None] * len(self.layers) + start_pos = args.get('start_pos', 0) + h = self.dropout(self.tok_embeddings(input_ids)) + pos_cis = self.pos_cis[start_pos:start_pos + input_ids.size(1)] + past_kvs = [] + for l, layer in enumerate(self.layers): + h, past_kv = layer( + h, pos_cis, + past_key_value=past_key_values[l], + use_cache=use_cache + ) + past_kvs.append(past_kv) + logits = self.output(self.norm(h)) + aux_loss = sum(l.feed_forward.aux_loss for l in self.layers if isinstance(l.feed_forward, MOEFeedForward)) self.OUT.__setitem__('logits', logits) - self.OUT.__setitem__('loss', self.last_loss) + self.OUT.__setitem__('aux_loss', aux_loss) + self.OUT.__setitem__('past_key_values', past_kvs) return self.OUT @torch.inference_mode() - def generate(self, idx, eos, max_new_tokens, temperature=0.7, top_k=8, stream=True, rp=1., kv_cache=True, - image_encoders=None): - # rp: repetition_penalty - index = idx.shape[1] - init_inference = True - while idx.shape[1] < max_new_tokens - 1: - if init_inference or not kv_cache: - inference_res, init_inference = self(idx, kv_cache=kv_cache, image_encoders=image_encoders), False + def generate(self, input_ids, eos_token_id=2, max_new_tokens=1024, temperature=0.75, top_p=0.90, + stream=False, rp=1., use_cache=True, pad_token_id=0, **args): + # 流式生成 + if stream: + return self._stream(input_ids, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache, **args) + + # 直接生成 + generated = [] + for i in range(input_ids.size(0)): + non_pad = input_ids[i][input_ids[i] != pad_token_id].unsqueeze(0) + out = self._stream(non_pad, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache, **args) + tokens_list = [tokens[:, -1:] for tokens in out] + gen = torch.cat(tokens_list, dim=-1) if tokens_list else non_pad + full_sequence = torch.cat([non_pad, gen], dim=-1) + generated.append(full_sequence) + max_length = max(seq.size(1) for seq in generated) + generated = [ + torch.cat( + [seq, torch.full((1, max_length - seq.size(1)), pad_token_id, dtype=seq.dtype, device=seq.device)], + dim=-1) + for seq in generated + ] + return torch.cat(generated, dim=0) + + def _stream(self, input_ids, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache, **args): + start, first_seq, past_kvs = input_ids.shape[1], True, None + while input_ids.shape[1] < max_new_tokens - 1: + if first_seq or not use_cache: + out, first_seq = self(input_ids, past_key_values=past_kvs, use_cache=use_cache, **args), False else: - inference_res = self(idx[:, -1:], kv_cache=kv_cache, current_idx=idx.shape[1] - 1) - - logits = inference_res.logits - logits = logits[:, -1, :] - - for token in set(idx.tolist()[0]): - logits[:, token] /= rp - - if temperature == 0.0: - _, idx_next = torch.topk(logits, k=1, dim=-1) - else: - logits = logits / temperature - if top_k is not None: - v, _ = torch.topk(logits, min(top_k, logits.size(-1))) - logits[logits < v[:, [-1]]] = -float('Inf') - - probs = F.softmax(logits, dim=-1) - idx_next = torch.multinomial(probs, num_samples=1, generator=None) - - if idx_next == eos: + out = self(input_ids[:, -1:], past_key_values=past_kvs, use_cache=use_cache, + start_pos=input_ids.shape[1] - 1, **args) + logits, past_kvs = out.logits[:, -1, :], out.past_key_values + logits[:, list(set(input_ids.tolist()[0]))] /= rp + logits /= (temperature + 1e-9) + if top_p is not None and top_p < 1.0: + sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1) + sorted_probs = F.softmax(sorted_logits, dim=-1) + cumulative_probs = torch.cumsum(sorted_probs, dim=-1) + sorted_indices_to_remove = cumulative_probs > top_p + sorted_indices_to_remove[:, 1:] = sorted_indices_to_remove[:, :-1].clone() + sorted_indices_to_remove[:, 0] = False + indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove) + logits[indices_to_remove] = -float('Inf') + input_ids_next = torch.multinomial(F.softmax(logits, dim=-1), num_samples=1) + input_ids = torch.cat((input_ids, input_ids_next), dim=1) + yield input_ids[:, start:] + if input_ids_next.item() == eos_token_id: break - - idx = torch.cat((idx, idx_next), dim=1) - if stream: - yield idx[:, index:] - - if not stream: - yield idx[:, index:] - - @torch.inference_mode() - def eval_answer(self, idx): - idx_cond = idx if idx.size(1) <= self.params.max_seq_len else idx[:, -self.params.max_seq_len:] - inference_res = self(idx_cond) - logits = inference_res.logits - logits = logits[:, -1, :] - return logits diff --git a/model/model_vlm.py b/model/model_vlm.py new file mode 100644 index 0000000..9a5f8f3 --- /dev/null +++ b/model/model_vlm.py @@ -0,0 +1,130 @@ +from .VLMConfig import VLMConfig +from .model import * +from typing import Optional, Tuple, List +from torch import nn +import warnings +from transformers import CLIPProcessor, CLIPModel +import torch + +warnings.filterwarnings('ignore') + + +class VisionProj(nn.Module): + def __init__(self, ve_dim=768, lm_dim=512): + super().__init__() + self.ve_dim = ve_dim + self.lm_dim = lm_dim + self.vision_proj = nn.Sequential( + nn.Linear(self.ve_dim, self.lm_dim) + ) + + def forward(self, image_encoders): + vision_proj = self.vision_proj(image_encoders) + return vision_proj + + +# 继承自语言模型 +class MiniMindVLM(MiniMindLM): + config_class = VLMConfig + + def __init__(self, params: VLMConfig = None): + super().__init__(params) + if not params: params = VLMConfig() + self.params = params + self.vision_encoder, self.processor = self.__class__.get_vision_model() + self.vision_proj = VisionProj(lm_dim=params.dim) + + @staticmethod + def get_vision_model(model_path="./model/vision_model/clip-vit-base-patch16"): + model = CLIPModel.from_pretrained(model_path) + processor = CLIPProcessor.from_pretrained(model_path) + # 冻结 vision_encoder 的所有参数 + for param in model.parameters(): + param.requires_grad = False + return model.eval(), processor + + @staticmethod + def image2tensor(image, processor): + if image.mode in ['RGBA', 'LA']: image = image.convert('RGB') + inputs = processor(images=image, return_tensors="pt")['pixel_values'] + return inputs + + @staticmethod + def get_image_embeddings(image_tensors, vision_model): + with torch.no_grad(): + outputs = vision_model.vision_model(pixel_values=image_tensors) + img_embedding = outputs.last_hidden_state[:, 1:, :].squeeze() + return img_embedding + + def count_vision_proj(self, tokens, h, vision_tensors=None, seqlen=512): + def find_indices(tokens, image_ids): + image_ids_tensor = torch.tensor(image_ids).to(tokens.device) + len_image_ids = len(image_ids) + if len_image_ids > tokens.size(1): + return None + tokens_view = tokens.unfold(1, len_image_ids, 1) + matches = (tokens_view == image_ids_tensor).all(dim=2) + return { + batch_idx: [(idx.item(), idx.item() + len_image_ids - 1) for idx in + matches[batch_idx].nonzero(as_tuple=True)[0]] + for batch_idx in range(tokens.size(0)) if matches[batch_idx].any() + } or None + + image_indices = find_indices(tokens, self.params.image_ids) + if vision_tensors is not None and image_indices: + vision_proj = self.vision_proj(vision_tensors) + if len(vision_proj.shape) == 3: + vision_proj = vision_proj.unsqueeze(1) + new_h = [] + for i in range(h.size(0)): + if i in image_indices: + h_i = h[i] + img_idx = 0 + for start_idx, end_idx in image_indices[i]: + if img_idx < vision_proj.size(1): + h_i = torch.cat((h_i[:start_idx], vision_proj[i][img_idx], h_i[end_idx + 1:]), dim=0)[ + :seqlen] + img_idx += 1 + new_h.append(h_i) + else: + new_h.append(h[i]) + return torch.stack(new_h, dim=0) + return h + + def forward(self, + input_ids: Optional[torch.Tensor] = None, + past_key_values: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None, + use_cache: bool = False, + **args): + start_pos = args.get('start_pos', 0) + pixel_tensors = args.get('pixel_tensors', None) + h = self.tok_embeddings(input_ids) + + if pixel_tensors is not None and start_pos == 0: + if len(pixel_tensors.shape) == 6: + pixel_tensors = pixel_tensors.squeeze(2) + bs, num, c, im_h, im_w = pixel_tensors.shape + stack_dim = 1 if bs > 1 else 0 + vision_tensors = torch.stack([ + MiniMindVLM.get_image_embeddings(pixel_tensors[:, i, :, :, :], self.vision_encoder) + for i in range(num) + ], dim=stack_dim) + h = self.count_vision_proj(tokens=input_ids, h=h, vision_tensors=vision_tensors, seqlen=input_ids.shape[1]) + + pos_cis = self.pos_cis[start_pos:start_pos + input_ids.shape[1]] + past_kvs = [] + for l, layer in enumerate(self.layers): + h, past_kv = layer( + h, pos_cis, + past_key_value=past_key_values[l] if past_key_values else None, + use_cache=use_cache + ) + past_kvs.append(past_kv) + + logits = self.output(self.norm(h)) + aux_loss = sum(l.feed_forward.aux_loss for l in self.layers if isinstance(l.feed_forward, MOEFeedForward)) + + self.OUT.__setitem__('logits', logits) + self.OUT.__setitem__('aux_loss', aux_loss) + self.OUT.__setitem__('past_key_values', past_kvs) + return self.OUT diff --git a/model/vision_model/README.md b/model/vision_model/README.md new file mode 100644 index 0000000..31c8c59 --- /dev/null +++ b/model/vision_model/README.md @@ -0,0 +1,11 @@ +需要把`clip-vit-base-patch16`模型下载到此目录下 + +```text +git clone https://huggingface.co/openai/clip-vit-base-patch16 +``` + +or + +```text +git clone https://www.modelscope.cn/openai-mirror/clip-vit-base-patch16 +``` \ No newline at end of file diff --git a/model/vision_utils.py b/model/vision_utils.py deleted file mode 100644 index adc6a2d..0000000 --- a/model/vision_utils.py +++ /dev/null @@ -1,72 +0,0 @@ -import warnings -from transformers import CLIPProcessor, CLIPModel, SiglipProcessor, SiglipModel -from PIL import Image -import requests -import torch -import transformers - -warnings.filterwarnings('ignore') - - -def get_vision_model(encoder_type): - # 加载预训练的CLIP模型和处理器 - if encoder_type == "clip": - model_path = "./model/clip_model/clip-vit-base-patch32" - model = CLIPModel.from_pretrained(model_path) - processor = CLIPProcessor.from_pretrained(model_path) - else: - model_path = "./model/siglip_model/siglip-vit-base-patch16" - model = SiglipModel.from_pretrained(model_path) - processor = SiglipProcessor.from_pretrained(model_path) - return (model, processor) - - -def get_img_process(image, processor): - # 将图像调整为224*224大小 - image = image.resize((224, 224)) - if image.mode in ['RGBA', 'LA']: # 处理有透明通道的图像 - image = image.convert('RGB') - # 使用CLIPProcessor处理每个patch - # inputs = processor(images=image, return_tensors="pt", clean_up_tokenization_spaces=False) - inputs = processor(images=image, return_tensors="pt") - return inputs - - -def get_img_embedding(batch_encoding, vision_model): - embeddings = [] - - def hook_fn(module, input, output): - # 将特征添加到 embeddings 列表中 - embeddings.append(output.last_hidden_state) - - # 从 BatchEncoding 中提取图像张量 - if (isinstance(batch_encoding, transformers.tokenization_utils_base.BatchEncoding) - or isinstance(batch_encoding, transformers.feature_extraction_utils.BatchFeature)): - image_tensor = batch_encoding['pixel_values'] - else: - image_tensor = batch_encoding # torch.Size([32, 4, 3, 224, 224]) - - # 如果图像张量的形状是5维，则无需添加额外维度 - if len(image_tensor.shape) == 4: - image_tensor = image_tensor.unsqueeze(0) # 添加批次维度 - - # 获取批次大小 - batch_size = image_tensor.size(0) - - with torch.no_grad(): - # 注册 hook 到模型的目标层（例如 vision_model 的倒数第二层） - layer = vision_model.vision_model.encoder - hook = layer.register_forward_hook(hook_fn) - - for i in range(batch_size): - # 取出当前批次中的单个图像 - single_image = image_tensor[i] # 添加批次维度 - # 调用 get_image_features 来获取图像特征 - _ = vision_model.get_image_features(single_image) - # 取消 hook - hook.remove() - - # 拼接所有特征向量成为一个张量 - all_embeddings = torch.stack(embeddings, dim=0).squeeze() - # torch.Size([32, 4, 50, 768]) or torch.Size([32, 2, 196, 768]) - return all_embeddings diff --git a/requirements.txt b/requirements.txt index ca7ba5c..20df3fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -datasets==2.16.1 +datasets==2.21.0 datasketch==1.6.4 Flask==3.0.3 Flask_Cors==4.0.0 @@ -9,7 +9,7 @@ matplotlib==3.5.1 ngrok==1.4.0 nltk==3.8 numpy==1.26.4 -openai==1.42.0 +openai==1.59.6 pandas==1.5.3 peft==0.7.1 psutil==5.9.8 @@ -19,12 +19,13 @@ scikit_learn==1.5.1 sentence_transformers==2.3.1 simhash==2.1.2 tiktoken==0.5.1 -torch==2.1.2 -torchvision==0.16.2 -transformers==4.44.0 +transformers==4.48.0 jinja2==3.1.2 jsonlines==4.0.0 -trl==0.8.6 +trl==0.13.0 ujson==5.1.0 wandb==0.18.3 +streamlit==1.30.0 +torch==2.2.2 +torchvision==0.17.2 gradio==5.0.0b5 diff --git a/1-pretrain_vlm.py b/train_pretrain_vlm.py similarity index 52% rename from 1-pretrain_vlm.py rename to train_pretrain_vlm.py index aca5408..0bc7766 100644 --- a/1-pretrain_vlm.py +++ b/train_pretrain_vlm.py @@ -12,59 +12,48 @@ import torch.nn.functional as F import torch.distributed as dist from contextlib import nullcontext -from torch import optim +from torch import optim, nn from torch.nn.parallel import DistributedDataParallel from torch.utils.data import DataLoader, DistributedSampler from transformers import AutoTokenizer, AutoModel -from model.model import Transformer -from model.LMConfig import LMConfig -from model.dataset import PretrainDataset -from model.vision_utils import get_vision_model, get_img_embedding +from model.model_vlm import MiniMindVLM +from model.VLMConfig import VLMConfig +from model.dataset import VLMDataset warnings.filterwarnings('ignore') -def count_parameters(model): - return sum(p.numel() for p in model.parameters() if p.requires_grad) - - def Logger(content): if not ddp or dist.get_rank() == 0: print(content) -def get_lr(it, all): - warmup_iters = args.warmup_iters - lr_decay_iters = all - min_lr = args.learning_rate / 10 - - if it < warmup_iters: - return args.learning_rate * it / warmup_iters - if it > lr_decay_iters: - return min_lr - decay_ratio = (it - warmup_iters) / (lr_decay_iters - warmup_iters) - assert 0 <= decay_ratio <= 1 - coeff = 0.5 * (1.0 + math.cos(math.pi * decay_ratio)) - return min_lr + coeff * (args.learning_rate - min_lr) +def get_lr(current_step, total_steps, lr): + return lr / 10 + 0.5 * lr * (1 + math.cos(math.pi * current_step / total_steps)) def train_epoch(epoch, wandb): + loss_fct = nn.CrossEntropyLoss(reduction='none') start_time = time.time() - for step, (X, Y, loss_mask, image_process) in enumerate(train_loader): + for step, (X, Y, loss_mask, pixel_tensors) in enumerate(train_loader): X = X.to(args.device) Y = Y.to(args.device) loss_mask = loss_mask.to(args.device) - image_process = image_process.to(args.device) - image_encoders = get_img_embedding(image_process, vision_model) - lr = get_lr(epoch * iter_per_epoch + step, args.epochs * iter_per_epoch) + pixel_tensors = pixel_tensors.to(args.device) + lr = get_lr(epoch * iter_per_epoch + step, args.epochs * iter_per_epoch, args.learning_rate) for param_group in optimizer.param_groups: param_group['lr'] = lr with ctx: - logits = model(X, Y, image_encoders=image_encoders).logits - loss = F.cross_entropy(logits.view(-1, logits.size(-1)), Y.view(-1), ignore_index=0, reduction='none') - loss_mask = loss_mask.view(-1) - loss = torch.sum(loss * loss_mask) / loss_mask.sum() + res = model(X, pixel_tensors=pixel_tensors) + loss = loss_fct( + res.logits.view(-1, res.logits.size(-1)), + Y.view(-1) + ).view(Y.size()) + + loss = (loss * loss_mask).sum() / loss_mask.sum() + loss += res.aux_loss + loss = loss / args.accumulation_steps scaler.scale(loss).backward() @@ -81,7 +70,7 @@ def train_epoch(epoch, wandb): spend_time = time.time() - start_time Logger( 'Epoch:[{}/{}]({}/{}) loss:{:.3f} lr:{:.7f} epoch_Time:{}min:'.format( - epoch, + epoch + 1, args.epochs, step, iter_per_epoch, @@ -96,41 +85,43 @@ def train_epoch(epoch, wandb): if (step + 1) % args.save_interval == 0 and (not ddp or dist.get_rank() == 0): model.eval() - moe_path = '_moe' if lm_config.use_moe else '' - ckp = f'{args.save_dir}/{lm_config.dim}{moe_path}_vlm_pretrain.pth' - + moe_path = '_moe' if model_config.use_moe else '' + ckp = f'{args.save_dir}/pretrain_vlm_{model_config.dim}{moe_path}.pth' if isinstance(model, torch.nn.parallel.DistributedDataParallel): state_dict = model.module.state_dict() else: state_dict = model.state_dict() - - torch.save(state_dict, ckp) + clean_state_dict = { + key: value for key, value in state_dict.items() if not key.startswith('vision_encoder.') + } + torch.save(clean_state_dict, ckp) model.train() -def init_model(lm_config): +def init_model(model_config: VLMConfig): tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer') - moe_path = '_moe' if lm_config.use_moe else '' - ckp = f'./out/{lm_config.dim}{moe_path}_llm.pth' - - model = Transformer(lm_config) + moe_path = '_moe' if model_config.use_moe else '' + # 加载纯语言模型权重 + ckp = f'./out/lm_{model_config.dim}{moe_path}.pth' + model = MiniMindVLM(model_config) state_dict = torch.load(ckp, map_location=args.device) - - # 处理不需要的前缀 - unwanted_prefix = '_orig_mod.' - for k, v in list(state_dict.items()): - if k.startswith(unwanted_prefix): - state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k) - - # 加载到模型中 model.load_state_dict(state_dict, strict=False) - model = model.to(args.device) - print(f'模型可学习参数: {count_parameters(model) / 1e6} 百万 = {count_parameters(model) / 1e9} B (Billion)') + # 冻结除 vision_proj 外的所有参数 + for name, param in model.named_parameters(): + if 'vision_proj' not in name: + param.requires_grad = False + # 可训练 + if hasattr(model, "layers"): + last_two_layers = model.layers[-1:] + for layer in last_two_layers: + for param in layer.parameters(): + param.requires_grad = True - (vision_model, preprocess) = get_vision_model(args.visual_encoder) - vision_model = vision_model.to(args.device) - return model, tokenizer, (vision_model, preprocess) + Logger(f'VLM可训练参数量：{sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e6:.3f} 百万') + + _, preprocess = MiniMindVLM.get_vision_model() + return model.to(args.device), tokenizer, preprocess def init_distributed_mode(): @@ -147,35 +138,32 @@ def init_distributed_mode(): if __name__ == "__main__": parser = argparse.ArgumentParser(description="MiniMind-V Pretrain") - parser.add_argument("--out_dir", type=str, default="out", help="Output directory") - parser.add_argument("--epochs", type=int, default=19, help="Number of epochs") - parser.add_argument("--batch_size", type=int, default=32, help="Batch size") - parser.add_argument("--learning_rate", type=float, default=4e-4, help="Learning rate") - parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu", - help="Device to use") - parser.add_argument("--dtype", type=str, default="bfloat16", help="Data type") - parser.add_argument("--use_wandb", default=False, action="store_true", help="Use Weights & Biases") - parser.add_argument("--wandb_project", type=str, default="MiniMind-V", help="Weights & Biases project name") - parser.add_argument("--num_workers", type=int, default=8, help="Number of workers for data loading") - parser.add_argument("--data_path", type=str, default="./dataset/LLaVA-Pretrain/chat-translated.json", - help="Path to training data") - parser.add_argument("--ddp", action="store_true", help="Use DistributedDataParallel") - parser.add_argument("--accumulation_steps", type=int, default=1, help="Gradient accumulation steps") - parser.add_argument("--grad_clip", type=float, default=1.0, help="Gradient clipping threshold") - parser.add_argument("--warmup_iters", type=int, default=0, help="Number of warmup iterations") - parser.add_argument("--log_interval", type=int, default=10, help="Logging interval") - parser.add_argument("--save_interval", type=int, default=100, help="Model saving interval") - parser.add_argument('--local_rank', type=int, default=-1, help='local rank for distributed training') - parser.add_argument('--visual_encoder', type=str, default="clip", help='type of visual endcoder') - + parser.add_argument("--out_dir", type=str, default="out") + parser.add_argument("--epochs", type=int, default=4) + parser.add_argument("--batch_size", type=int, default=16) + parser.add_argument("--learning_rate", type=float, default=4e-4) + parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu") + parser.add_argument("--dtype", type=str, default="bfloat16") + parser.add_argument("--use_wandb", default=False, action="store_true") + parser.add_argument("--wandb_project", type=str, default="MiniMind-V") + parser.add_argument("--num_workers", type=int, default=8) + parser.add_argument("--data_path", type=str, default="./dataset/pretrain_vlm_data.jsonl") + parser.add_argument("--images_path", type=str, default="./dataset/pretrain_images") + parser.add_argument("--ddp", action="store_true") + parser.add_argument("--accumulation_steps", type=int, default=1) + parser.add_argument("--grad_clip", type=float, default=1.0) + parser.add_argument("--warmup_iters", type=int, default=0) + parser.add_argument("--log_interval", type=int, default=100) + parser.add_argument("--save_interval", type=int, default=100) + parser.add_argument('--local_rank', type=int, default=-1) + parser.add_argument('--dim', default=512, type=int) + parser.add_argument('--n_layers', default=8, type=int) + parser.add_argument('--max_seq_len', default=640, type=int) + parser.add_argument('--use_moe', default=False, type=bool) args = parser.parse_args() - if args.visual_encoder == "clip": - lm_config = LMConfig() - else: - lm_config = LMConfig(image_special_token='<' * 98 + '>' * 98, image_ids=[30] * 98 + [32] * 98) - - max_seq_len = lm_config.max_seq_len + model_config = VLMConfig(dim=args.dim, n_layers=args.n_layers, max_seq_len=args.max_seq_len) + max_seq_len = model_config.max_seq_len args.save_dir = os.path.join(args.out_dir) os.makedirs(args.save_dir, exist_ok=True) os.makedirs(args.out_dir, exist_ok=True) @@ -199,12 +187,11 @@ if __name__ == "__main__": else: wandb = None - model, tokenizer, (vision_model, preprocess) = init_model(lm_config) + model, tokenizer, preprocess = init_model(model_config) - use_version = 0 - train_ds = PretrainDataset(args.data_path, tokenizer, vision_model=(vision_model, preprocess), - image_special_token=lm_config.image_special_token, - max_length=max_seq_len) + train_ds = VLMDataset(args.data_path, args.images_path, tokenizer, preprocess=preprocess, + image_special_token=model_config.image_special_token, + max_length=max_seq_len) train_sampler = DistributedSampler(train_ds) if ddp else None train_loader = DataLoader( train_ds, @@ -217,13 +204,7 @@ if __name__ == "__main__": ) scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype in ['float16', 'bfloat16'])) - optimizer = optim.Adam(model.parameters(), lr=args.learning_rate) - - if False and not lm_config.use_moe and platform.system() != 'Windows' and float( - torch.__version__.split('.')[0]) >= 2: - Logger("compiling the model... (takes a ~minute)") - unoptimized_model = model - model = torch.compile(model) + optimizer = optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=args.learning_rate) if ddp: model._ddp_params_and_buffers_to_ignore = {"pos_cis"} diff --git a/train_sft_vlm.py b/train_sft_vlm.py new file mode 100644 index 0000000..ea33c1a --- /dev/null +++ b/train_sft_vlm.py @@ -0,0 +1,205 @@ +import os +import platform +import argparse +import time +import math +import warnings +import json + +import pandas as pd +import torch +import torch.nn.functional as F +import torch.distributed as dist +from contextlib import nullcontext + +from torch import optim, nn +from torch.nn.parallel import DistributedDataParallel +from torch.utils.data import DataLoader, DistributedSampler +from transformers import AutoTokenizer, AutoModel +from model.model_vlm import MiniMindVLM +from model.VLMConfig import VLMConfig +from model.dataset import * + +warnings.filterwarnings('ignore') + + +def Logger(content): + if not ddp or dist.get_rank() == 0: + print(content) + + +def get_lr(current_step, total_steps, lr): + return lr / 10 + 0.5 * lr * (1 + math.cos(math.pi * current_step / total_steps)) + + +def train_epoch(epoch, wandb): + loss_fct = nn.CrossEntropyLoss(reduction='none') + start_time = time.time() + for step, (X, Y, loss_mask, pixel_tensors) in enumerate(train_loader): + X = X.to(args.device) + Y = Y.to(args.device) + loss_mask = loss_mask.to(args.device) + pixel_tensors = pixel_tensors.to(args.device) + lr = get_lr(epoch * iter_per_epoch + step, args.epochs * iter_per_epoch, args.learning_rate) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + with ctx: + res = model(X, pixel_tensors=pixel_tensors) + loss = loss_fct( + res.logits.view(-1, res.logits.size(-1)), + Y.view(-1) + ).view(Y.size()) + + loss = (loss * loss_mask).sum() / loss_mask.sum() + loss += res.aux_loss + loss = loss / args.accumulation_steps + + scaler.scale(loss).backward() + + if (step + 1) % args.accumulation_steps == 0: + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) + + scaler.step(optimizer) + scaler.update() + + optimizer.zero_grad(set_to_none=True) + + if step % args.log_interval == 0: + spend_time = time.time() - start_time + Logger( + 'Epoch:[{}/{}]({}/{}) loss:{:.3f} lr:{:.7f} epoch_Time:{}min:'.format( + epoch + 1, + args.epochs, + step, + iter_per_epoch, + loss.item(), + optimizer.param_groups[-1]['lr'], + spend_time / (step + 1) * iter_per_epoch // 60 - spend_time // 60)) + + if (wandb is not None) and (not ddp or dist.get_rank() == 0): + wandb.log({"loss": loss, + "lr": optimizer.param_groups[-1]['lr'], + "epoch_Time": spend_time / (step + 1) * iter_per_epoch // 60 - spend_time // 60}) + + if (step + 1) % args.save_interval == 0 and (not ddp or dist.get_rank() == 0): + model.eval() + moe_path = '_moe' if model_config.use_moe else '' + ckp = f'{args.save_dir}/sft_vlm_{model_config.dim}{moe_path}.pth' + if isinstance(model, torch.nn.parallel.DistributedDataParallel): + state_dict = model.module.state_dict() + else: + state_dict = model.state_dict() + clean_state_dict = { + key: value for key, value in state_dict.items() if not key.startswith('vision_encoder.') + } + torch.save(clean_state_dict, ckp) + model.train() + + +def init_model(model_config: VLMConfig): + tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer') + moe_path = '_moe' if model_config.use_moe else '' + ckp = f'./out/pretrain_vlm_{model_config.dim}{moe_path}.pth' + + model = MiniMindVLM(model_config) + state_dict = torch.load(ckp, map_location=args.device) + model.load_state_dict(state_dict, strict=False) + model = model.to(args.device) + + Logger(f'VLM可训练参数量：{sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e6:.3f} 百万') + + _, preprocess = MiniMindVLM.get_vision_model() + return model.to(args.device), tokenizer, preprocess + + +def init_distributed_mode(): + if not ddp: return + global ddp_local_rank, DEVICE + + dist.init_process_group(backend="nccl") + ddp_rank = int(os.environ["RANK"]) + ddp_local_rank = int(os.environ["LOCAL_RANK"]) + ddp_world_size = int(os.environ["WORLD_SIZE"]) + DEVICE = f"cuda:{ddp_local_rank}" + torch.cuda.set_device(DEVICE) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="MiniMind-V Pretrain") + parser.add_argument("--out_dir", type=str, default="out") + parser.add_argument("--epochs", type=int, default=6) + parser.add_argument("--batch_size", type=int, default=8) + parser.add_argument("--learning_rate", type=float, default=1e-6) + parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu") + parser.add_argument("--dtype", type=str, default="bfloat16") + parser.add_argument("--use_wandb", default=False, action="store_true") + parser.add_argument("--wandb_project", type=str, default="MiniMind-V") + parser.add_argument("--num_workers", type=int, default=8) + parser.add_argument("--data_path", type=str, default="./dataset/sft_vlm_data.jsonl") + parser.add_argument("--images_path", type=str, default="./dataset/sft_images") + parser.add_argument("--ddp", action="store_true") + parser.add_argument("--accumulation_steps", type=int, default=1) + parser.add_argument("--grad_clip", type=float, default=1.0) + parser.add_argument("--warmup_iters", type=int, default=0) + parser.add_argument("--log_interval", type=int, default=10) + parser.add_argument("--save_interval", type=int, default=10) + parser.add_argument('--local_rank', type=int, default=-1) + parser.add_argument('--dim', default=512, type=int) + parser.add_argument('--n_layers', default=8, type=int) + parser.add_argument('--max_seq_len', default=1536, type=int) + parser.add_argument('--use_moe', default=False, type=bool) + args = parser.parse_args() + + model_config = VLMConfig(dim=args.dim, n_layers=args.n_layers, max_seq_len=args.max_seq_len) + max_seq_len = model_config.max_seq_len + args.save_dir = os.path.join(args.out_dir) + os.makedirs(args.save_dir, exist_ok=True) + os.makedirs(args.out_dir, exist_ok=True) + tokens_per_iter = args.batch_size * max_seq_len + torch.manual_seed(1337) + device_type = "cuda" if "cuda" in args.device else "cpu" + + args.wandb_run_name = f"MiniMind-V SFT-Epoch-{args.epochs}-BatchSize-{args.batch_size}-LearningRate-{args.learning_rate}" + + ctx = nullcontext() if device_type == "cpu" else torch.cuda.amp.autocast() + ddp = int(os.environ.get("RANK", -1)) != -1 # is this a ddp run? + ddp_local_rank, DEVICE = 0, "cuda:0" + if ddp: + init_distributed_mode() + args.device = torch.device(DEVICE) + + if args.use_wandb and (not ddp or ddp_local_rank == 0): + import wandb + + wandb.init(project=args.wandb_project, name=args.wandb_run_name) + else: + wandb = None + + model, tokenizer, preprocess = init_model(model_config) + + train_ds = VLMDataset(args.data_path, args.images_path, tokenizer, preprocess=preprocess, + image_special_token=model_config.image_special_token, + max_length=max_seq_len) + train_sampler = DistributedSampler(train_ds) if ddp else None + train_loader = DataLoader( + train_ds, + batch_size=args.batch_size, + pin_memory=True, + drop_last=False, + shuffle=False, + num_workers=args.num_workers, + sampler=train_sampler + ) + + scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype in ['float16', 'bfloat16'])) + optimizer = optim.AdamW(model.parameters(), lr=args.learning_rate) + + if ddp: + model._ddp_params_and_buffers_to_ignore = {"pos_cis"} + model = DistributedDataParallel(model, device_ids=[ddp_local_rank]) + + iter_per_epoch = len(train_loader) + for epoch in range(args.epochs): + train_epoch(epoch, wandb) diff --git a/web_demo_vlm.py b/web_demo_vlm.py new file mode 100644 index 0000000..f343771 --- /dev/null +++ b/web_demo_vlm.py @@ -0,0 +1,176 @@ +import argparse +import torch +import warnings +import gradio as gr +from PIL import Image +from transformers import AutoTokenizer, AutoModelForCausalLM +from model.model_vlm import MiniMindVLM +from model.VLMConfig import VLMConfig +from transformers import logging as hf_logging + +hf_logging.set_verbosity_error() +warnings.filterwarnings('ignore') + + +def init_model(lm_config): + tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer') + + if args.load == 0: + moe_path = '_moe' if lm_config.use_moe else '' + ckp = f'./out/sft_vlm_{lm_config.dim}{moe_path}.pth' + model = MiniMindVLM(lm_config) + state_dict = torch.load(ckp, map_location=args.device) + model.load_state_dict({k: v for k, v in state_dict.items() if 'mask' not in k}, strict=False) + else: + transformers_model_path = './MiniMind2-V' + tokenizer = AutoTokenizer.from_pretrained(transformers_model_path) + model = AutoModelForCausalLM.from_pretrained(transformers_model_path, trust_remote_code=True) + + print(f'VLM参数量：{sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e6:.3f} 百万') + + vision_model, preprocess = MiniMindVLM.get_vision_model() + return model.eval().to(args.device), tokenizer, vision_model.to(args.device), preprocess + + +def chat(prompt, current_image_path): + global temperature, top_p + image = Image.open(current_image_path).convert('RGB') + pixel_tensors = MiniMindVLM.image2tensor(image, preprocess).to(model.device).unsqueeze(0) + + prompt = f'{lm_config.image_special_token}\n{prompt}' + messages = [{"role": "user", "content": prompt}] + + new_prompt = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True + )[-args.max_seq_len + 1:] + with torch.no_grad(): + x = torch.tensor(tokenizer(new_prompt)['input_ids'], device=model.device).unsqueeze(0) + outputs = model.generate( + x, + eos_token_id=tokenizer.eos_token_id, + max_new_tokens=args.max_seq_len, + temperature=temperature, + top_p=top_p, + stream=True, + pad_token_id=tokenizer.pad_token_id, + pixel_tensors=pixel_tensors + ) + try: + if not args.stream: + print(tokenizer.decode(outputs.squeeze()[x.shape[1]:].tolist(), skip_special_tokens=True), end='') + else: + history_idx = 0 + for y in outputs: + answer = tokenizer.decode(y[0].tolist(), skip_special_tokens=True) + if (answer and answer[-1] == '�') or not answer: + continue + yield answer[history_idx:] + history_idx = len(answer) + except StopIteration: + print("No answer") + print('\n') + + +def launch_gradio_server(server_name="0.0.0.0", server_port=7788): + global temperature, top_p + temperature = args.temperature + top_p = args.top_p + + with gr.Blocks() as demo: + gr.HTML(f""" +
+ + Hi, I'm MiniMind2-V +
+ """) + + with gr.Row(): + with gr.Column(scale=3): + def get_current_image_path(image): + global current_image_path + if image is None: + current_image_path = '' + return + current_image_path = image + return current_image_path + + with gr.Blocks() as iface: + with gr.Row(): + image_input = gr.Image(type="filepath", label="选择图片", height=650) + image_input.change(fn=get_current_image_path, inputs=image_input) + + def update_parameters(temperature_, top_p_): + global temperature, top_p + temperature = float(temperature_) + top_p = float(top_p_) + return temperature, top_p + + with gr.Blocks() as iface_param: + with gr.Row(): + temperature_slider = gr.Slider(label="Temperature", minimum=0.5, maximum=1.1, value=0.65) + top_p_slider = gr.Slider(label="Top-P", minimum=0.7, maximum=0.95, value=0.85) + + temperature_slider.change(fn=update_parameters, inputs=[temperature_slider, top_p_slider]) + top_p_slider.change(fn=update_parameters, inputs=[temperature_slider, top_p_slider]) + + with gr.Column(scale=6): + def chat_with_vlm(message, history): + if not message: + yield history + [("错误", "错误：提问不能为空。")] + return + if not current_image_path: + yield history + [("错误", "错误：图片不能为空。")] + return + + image_html = f'Image' + res_generator = chat(message, current_image_path) + response = '' + for res in res_generator: + response += res + yield history + [(f"{image_html} {message}", response)] + + chatbot = gr.Chatbot(label="MiniMind-Vision", height=680) + with gr.Row(): + with gr.Column(scale=8): + message_input = gr.Textbox( + placeholder="请输入你的问题...", + show_label=False, + container=False + ) + with gr.Column(scale=2, min_width=50): + submit_button = gr.Button("发送") + submit_button.click( + fn=chat_with_vlm, + inputs=[message_input, chatbot], + outputs=chatbot + ) + + # # 添加示例问题 + # gr.Examples( + # examples=["描述一下这个图片的内容。", "画面里面有什么？", "画面里的天气怎么样？"], + # inputs=message_input) + + demo.launch(server_name=server_name, server_port=server_port) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Chat with MiniMind") + parser.add_argument('--lora_name', default='None', type=str) + parser.add_argument('--out_dir', default='out', type=str) + parser.add_argument('--temperature', default=0.65, type=float) + parser.add_argument('--top_p', default=0.85, type=float) + parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu', type=str) + parser.add_argument('--dim', default=768, type=int) + parser.add_argument('--n_layers', default=16, type=int) + parser.add_argument('--max_seq_len', default=8192, type=int) + parser.add_argument('--use_moe', default=False, type=bool) + parser.add_argument('--stream', default=True, type=bool) + parser.add_argument('--load', default=0, type=int, help="0: 原生torch权重，1: transformers加载") + args = parser.parse_args() + + lm_config = VLMConfig(dim=args.dim, n_layers=args.n_layers, max_seq_len=args.max_seq_len, use_moe=args.use_moe) + model, tokenizer, vision_model, preprocess = init_model(lm_config) + launch_gradio_server(server_name="0.0.0.0", server_port=8888) diff --git a/web_server.py b/web_server.py deleted file mode 100644 index aad9213..0000000 --- a/web_server.py +++ /dev/null @@ -1,209 +0,0 @@ -import gradio as gr -import os -import random -import time - -import numpy as np -import torch -import warnings - -from PIL import Image -from transformers import AutoTokenizer, AutoModelForCausalLM -from model.model import Transformer -from model.LMConfig import LMConfig -from model.vision_utils import get_vision_model, get_img_process, get_img_embedding - -warnings.filterwarnings('ignore') - - -def count_parameters(model): - return sum(p.numel() for p in model.parameters() if p.requires_grad) - - -def init_model(lm_config): - tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer') - model_from = 2 # 1从权重，2用transformers - - if model_from == 1: - moe_path = '_moe' if lm_config.use_moe else '' - ckp = f'./out/{lm_config.dim}{moe_path}_vlm_sft.pth' - - model = Transformer(lm_config) - state_dict = torch.load(ckp, map_location=device) - - # 处理不需要的前缀 - unwanted_prefix = '_orig_mod.' - for k, v in list(state_dict.items()): - if k.startswith(unwanted_prefix): - state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k) - - for k, v in list(state_dict.items()): - if 'mask' in k: - del state_dict[k] - - # 加载到模型中 - model.load_state_dict(state_dict, strict=False) - else: - model = AutoModelForCausalLM.from_pretrained('minimind-v-v1', - trust_remote_code=True) - - model = model.to(device) - - print(f'模型参数: {count_parameters(model) / 1e6} 百万 = {count_parameters(model) / 1e9} B (Billion)') - - (vision_model, preprocess) = get_vision_model() - vision_model = vision_model.to(device) - return model, tokenizer, (vision_model, preprocess) - - -def chat(prompt, current_image_path): - # 打开图像并转换为RGB格式 - image = Image.open(current_image_path).convert('RGB') - image_process = get_img_process(image, preprocess).to(vision_model.device) - # 对图像进行编码 - image_encoder = get_img_embedding(image_process, vision_model).unsqueeze(0) - - prompt = f'{lm_config.image_special_token}\n{prompt}' - messages = [{"role": "user", "content": prompt}] - - # print(messages) - new_prompt = tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True - )[-(max_seq_len - 1):] - - x = tokenizer(new_prompt).data['input_ids'] - x = (torch.tensor(x, dtype=torch.long, device=device)[None, ...]) - - with torch.no_grad(): - res_y = model.generate(x, tokenizer.eos_token_id, max_new_tokens=max_seq_len, temperature=temperature, - top_k=top_k, stream=stream, image_encoders=image_encoder) - # print('[A]: ', end='') - try: - y = next(res_y) - except StopIteration: - print("No answer") - return "No answer" - - history_idx = 0 - while y != None: - answer = tokenizer.decode(y[0].tolist()) - if answer and answer[-1] == '�': - try: - y = next(res_y) - except: - break - continue - # print(answer) - if not len(answer): - try: - y = next(res_y) - except: - break - continue - - yield answer[history_idx:] - try: - y = next(res_y) - except: - break - history_idx = len(answer) - if not stream: - break - - -def launch_gradio_server(server_name="0.0.0.0", server_port=7788): - with gr.Blocks() as demo: - # # 顶部下拉菜单 - # model_choice = gr.Dropdown(label="选择模型", choices=["minimind-v1-v", "minimind-v1-small-v"], - # value="minimind-v1-v") - gr.HTML(f""" -
-

MiniMind-V ({(count_parameters(model) / 1e6):.2f} M) Test Demo 「单轮对话，无历史上下文」

-
- """) - # 左侧 - with gr.Row(): - with gr.Column(scale=1): - def get_current_image_path(image): - global current_image_path - if image is None: - current_image_path = '' - return - current_image_path = image - return current_image_path - - with gr.Blocks() as iface: - with gr.Row(): - image_input = gr.Image(type="filepath", label="选择图片", height=650) - image_input.change(fn=get_current_image_path, inputs=image_input) - - def update_parameters(temperature_, top_k_): - global temperature, top_k - temperature = temperature_ - top_k = top_k_ - - with gr.Blocks() as iface_param: - with gr.Row(): - temperature_slider = gr.Slider(label="Temperature", minimum=0, maximum=1.5, value=0.5) - top_k_slider = gr.Slider(label="Top K", minimum=1, maximum=20, value=12) - - temperature_slider.change(fn=update_parameters, inputs=[temperature_slider, top_k_slider]) - top_k_slider.change(fn=update_parameters, inputs=[temperature_slider, top_k_slider]) - - # 右侧聊天展示和输入框 - with gr.Column(scale=4): - def chat_with_vlm(message, history): - if not message: - yield history + [("错误", "错误：提问不能为空。")] - return - if not current_image_path: - yield history + [("错误", "错误：图片不能为空。")] - return - - image_html = f'Image' - res_generator = chat(message, current_image_path) - response = '' - for res in res_generator: - response += res - # 返回聊天历史，加上新的一轮对话 - yield history + [(f"{image_html} {message}", response)] - - # 创建ChatBot组件 - - chatbot = gr.Chatbot(label="MiniMind-V Chatbot", height=670) - - # 添加输入框和按钮 - with gr.Row(): - with gr.Column(scale=8): - message_input = gr.Textbox(placeholder="请输入你的问题...", show_label=False, container=False) - with gr.Column(scale=1, min_width=50): - submit_button = gr.Button("发送") - - # 设置按钮点击事件触发chat_with_vlm函数 - submit_button.click(fn=chat_with_vlm, - inputs=[message_input, chatbot], # 将消息输入框和聊天历史传入函数 - outputs=chatbot) # 更新Chatbot的输出内容 - - # 添加示例问题 - gr.Examples( - examples=["描述一下这个图片的内容。", "画面里面有什么？", "画面里的动物在干什么？", - "画面里的天气怎么样？"], - inputs=message_input) - - demo.launch(server_name=server_name, server_port=server_port) - - -if __name__ == '__main__': - device = 'cuda:0' if torch.cuda.is_available() else 'cpu' - # 获取模型配置 - lm_config = LMConfig() - max_seq_len = lm_config.max_seq_len - model, tokenizer, (vision_model, preprocess) = init_model(lm_config) - stream = True - current_image_path = '' - temperature = 0.5 - top_k = 12 - - launch_gradio_server(server_name="0.0.0.0", server_port=7788)