github
/
minimind-v
mirror of https://github.com/jingyaogong/minimind-v.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

multi-images training

pull/19/head
xinyanghuang7 2024-10-30 07:40:21 -04:00
parent 4babe0de30
commit 2adb3d15a0
12 changed files with 141 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.gitignore vendored
View File

@ -2,7 +2,6 @@ model/clip_model/clip-vit-base-patch32/
out/*.pth out/*.pth
full.json full.json
trans_json.py trans_json.py
dataset/sft_multi_images/ dataset/*
dataset/ !dataset/eval_images/
!dataset/eval_images !dataset/eval_multi_images/
!dataset/eval_multi_images

38
2-sft_vlm.py
View File

@ -18,7 +18,7 @@ from torch.utils.data import DataLoader, DistributedSampler
from transformers import AutoTokenizer, AutoModel from transformers import AutoTokenizer, AutoModel
from model.model import Transformer from model.model import Transformer
from model.LMConfig import LMConfig from model.LMConfig import LMConfig
from model.dataset import SFTDataset from model.dataset import SFTDataset, SFTDataset_multi
from model.vision_utils import get_vision_model, get_img_embedding from model.vision_utils import get_vision_model, get_img_embedding
warnings.filterwarnings('ignore') warnings.filterwarnings('ignore')
@ -97,7 +97,10 @@ def train_epoch(epoch, wandb):
if (step + 1) % args.save_interval == 0 and (not ddp or dist.get_rank() == 0): if (step + 1) % args.save_interval == 0 and (not ddp or dist.get_rank() == 0):
model.eval() model.eval()
moe_path = '_moe' if lm_config.use_moe else '' moe_path = '_moe' if lm_config.use_moe else ''
ckp = f'{args.save_dir}/{lm_config.dim}{moe_path}_vlm_sft.pth' if args.multi:
ckp = f'{args.save_dir}/{lm_config.dim}{moe_path}_vlm_sft_multi.pth'
else:
ckp = f'{args.save_dir}/{lm_config.dim}{moe_path}_vlm_sft.pth'
if isinstance(model, torch.nn.parallel.DistributedDataParallel): if isinstance(model, torch.nn.parallel.DistributedDataParallel):
state_dict = model.module.state_dict() state_dict = model.module.state_dict()
@ -106,12 +109,29 @@ def train_epoch(epoch, wandb):
torch.save(state_dict, ckp) torch.save(state_dict, ckp)
model.train() model.train()
if args.save_last and (not ddp or dist.get_rank() == 0) and args.multi:
model.eval()
moe_path = '_moe' if lm_config.use_moe else ''
ckp = f'{args.save_dir}/{lm_config.dim}{moe_path}_vlm_sft_multi.pth'
if isinstance(model, torch.nn.parallel.DistributedDataParallel):
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
torch.save(state_dict, ckp)
model.train()
def init_model(lm_config): def init_model(lm_config):
tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer') tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer')
moe_path = '_moe' if lm_config.use_moe else '' moe_path = '_moe' if lm_config.use_moe else ''
ckp = f'./out/{lm_config.dim}{moe_path}_vlm_pretrain.pth'
# 多图推理建议在单图推理之后
if args.multi:
ckp = f'./out/{lm_config.dim}{moe_path}_vlm_sft.pth'
else:
ckp = f'./out/{lm_config.dim}{moe_path}_vlm_pretrain.pth'
model = Transformer(lm_config) model = Transformer(lm_config)
state_dict = torch.load(ckp, map_location=args.device) state_dict = torch.load(ckp, map_location=args.device)
@ -159,6 +179,8 @@ if __name__ == "__main__":
parser.add_argument("--num_workers", type=int, default=8, help="Number of workers for data loading") parser.add_argument("--num_workers", type=int, default=8, help="Number of workers for data loading")
parser.add_argument("--data_path", type=str, default="./dataset/LLaVA-Instruct/llava_instruct_230k.json", parser.add_argument("--data_path", type=str, default="./dataset/LLaVA-Instruct/llava_instruct_230k.json",
help="Path to training data") help="Path to training data")
parser.add_argument("--data_path_multi", type=str, default="./dataset/sft_multi_images/output.json",
help="Path to multi images training data")
parser.add_argument("--ddp", action="store_true", help="Use DistributedDataParallel") parser.add_argument("--ddp", action="store_true", help="Use DistributedDataParallel")
parser.add_argument("--accumulation_steps", type=int, default=1, help="Gradient accumulation steps") parser.add_argument("--accumulation_steps", type=int, default=1, help="Gradient accumulation steps")
parser.add_argument("--grad_clip", type=float, default=1.0, help="Gradient clipping threshold") parser.add_argument("--grad_clip", type=float, default=1.0, help="Gradient clipping threshold")
@ -166,6 +188,8 @@ if __name__ == "__main__":
parser.add_argument("--log_interval", type=int, default=100, help="Logging interval") parser.add_argument("--log_interval", type=int, default=100, help="Logging interval")
parser.add_argument("--save_interval", type=int, default=100, help="Model saving interval") parser.add_argument("--save_interval", type=int, default=100, help="Model saving interval")
parser.add_argument('--local_rank', type=int, default=-1, help='local rank for distributed training') parser.add_argument('--local_rank', type=int, default=-1, help='local rank for distributed training')
parser.add_argument('--multi', type=bool, default=False, help='multi-images training')
parser.add_argument('--save_last', type=bool, default=True, help='save last step model')
args = parser.parse_args() args = parser.parse_args()
@ -197,9 +221,15 @@ if __name__ == "__main__":
model, tokenizer, (vision_model, preprocess) = init_model(lm_config) model, tokenizer, (vision_model, preprocess) = init_model(lm_config)
use_version = 0 use_version = 0
train_ds = SFTDataset(args.data_path, tokenizer, vision_model=(vision_model, preprocess), if args.multi:
print("进行多图训练，建议在指令微调后进行...")
train_ds = SFTDataset_multi(args.data_path_multi, tokenizer, vision_model=(vision_model, preprocess),
image_special_token=lm_config.image_special_token, image_special_token=lm_config.image_special_token,
max_length=max_seq_len) max_length=max_seq_len)
else:
train_ds = SFTDataset(args.data_path, tokenizer, vision_model=(vision_model, preprocess),
image_special_token=lm_config.image_special_token,
max_length=max_seq_len)
train_sampler = DistributedSampler(train_ds) if ddp else None train_sampler = DistributedSampler(train_ds) if ddp else None
train_loader = DataLoader( train_loader = DataLoader(
train_ds, train_ds,

159
3-eval_chat.py
View File

@ -1,11 +1,8 @@
import os import os
import random import random
import time
import numpy as np import numpy as np
import torch import torch
import warnings import warnings
from PIL import Image from PIL import Image
from transformers import AutoTokenizer, AutoModelForCausalLM from transformers import AutoTokenizer, AutoModelForCausalLM
from model.model import Transformer from model.model import Transformer
@ -19,14 +16,16 @@ def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad) return sum(p.numel() for p in model.parameters() if p.requires_grad)
def init_model(lm_config): def init_model(lm_config, device, multi):
tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer') tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer')
model_from = 1 # 1从权重，2用transformers model_from = 1 # 1从权重，2用transformers
if model_from == 1: if model_from == 1:
moe_path = '_moe' if lm_config.use_moe else '' moe_path = '_moe' if lm_config.use_moe else ''
ckp = f'./out/{lm_config.dim}{moe_path}_vlm_sft.pth' if multi:
ckp = f'./out/{lm_config.dim}{moe_path}_vlm_sft_multi.pth'
else:
ckp = f'./out/{lm_config.dim}{moe_path}_vlm_sft.pth'
model = Transformer(lm_config) model = Transformer(lm_config)
state_dict = torch.load(ckp, map_location=device) state_dict = torch.load(ckp, map_location=device)
@ -35,125 +34,103 @@ def init_model(lm_config):
for k, v in list(state_dict.items()): for k, v in list(state_dict.items()):
if k.startswith(unwanted_prefix): if k.startswith(unwanted_prefix):
state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k) state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k)
# 加载到模型中
model.load_state_dict(state_dict, strict=False) model.load_state_dict(state_dict, strict=False)
else: else:
model = AutoModelForCausalLM.from_pretrained('minimind-v-v1-small', model = AutoModelForCausalLM.from_pretrained('minimind-v-v1-small', trust_remote_code=True)
trust_remote_code=True)
model = model.to(device) model = model.to(device)
print(f'模型参数: {count_parameters(model) / 1e6} 百万 = {count_parameters(model) / 1e9} B (Billion)') print(f'模型参数: {count_parameters(model) / 1e6} 百万 = {count_parameters(model) / 1e9} B (Billion)')
(vision_model, preprocess) = get_vision_model() vision_model, preprocess = get_vision_model()
vision_model = vision_model.to(device) vision_model = vision_model.to(device)
return model, tokenizer, (vision_model, preprocess) return model, tokenizer, vision_model, preprocess
def setup_seed(seed): def setup_seed(seed):
random.seed(seed) # 设置 Python 的随机种子 random.seed(seed)
np.random.seed(seed) # 设置 NumPy 的随机种子 np.random.seed(seed)
torch.manual_seed(seed) # 设置 PyTorch 的随机种子 torch.manual_seed(seed)
torch.cuda.manual_seed(seed) # 为当前 GPU 设置随机种子（如果有） torch.cuda.manual_seed(seed)
torch.cuda.manual_seed_all(seed) # 为所有 GPU 设置随机种子（如果有） torch.cuda.manual_seed_all(seed)
torch.backends.cudnn.deterministic = True # 确保每次返回的卷积算法是确定的 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False # 关闭 cuDNN 的自动调优，避免不确定性 torch.backends.cudnn.benchmark = False
if __name__ == "__main__": if __name__ == "__main__":
# ----------------------------------------------------------------------------- # -------------------------- 基本参数设置 -----------------------------------
multi = False # 设置multi参数，控制单图或多图推理
out_dir = 'out' out_dir = 'out'
start = ""
temperature = 0.5 temperature = 0.5
top_k = 8 top_k = 8
setup_seed(1337) setup_seed(1337)
# device = 'cpu'
device = 'cuda:0' if torch.cuda.is_available() else 'cpu' device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
dtype = 'bfloat16' dtype = 'bfloat16'
max_seq_len = 1 * 1024 max_seq_len = 1024
lm_config = LMConfig() lm_config = LMConfig()
lm_config.max_seq_len = max_seq_len lm_config.max_seq_len = max_seq_len
# ----------------------------------------------------------------------------- model, tokenizer, vision_model, preprocess = init_model(lm_config, device, multi)
model.eval()
model, tokenizer, (vision_model, preprocess) = init_model(lm_config) # -------------------------- 问题和目录设置 -----------------------------------
if multi:
image_dir = './dataset/eval_multi_images/bird/'
prompt = "<image>\n<image>\nName all the differences between these two birds."
else:
image_dir = './dataset/eval_images/'
prompt = lm_config.image_special_token + '\n这个图片描述的是什么内容？'
model = model.eval()
# 推送到huggingface
# model.push_to_hub("minimind")
# tokenizer.push_to_hub("minimind")
stream = True
prompt = lm_config.image_special_token + '\n这个图片描述的是什么内容？'
i = 0
# 获取图像文件列表
image_dir = './dataset/eval_images/'
image_files = sorted(os.listdir(image_dir)) image_files = sorted(os.listdir(image_dir))
# 遍历图像文件 # -------------------------- 推理逻辑 -----------------------------------
for image_file in image_files: if multi:
image_path = os.path.join(image_dir, image_file) # 多图推理：所有图像编码一次性推理
image_encoders = []
# 打开图像并转换为RGB格式 for image_file in image_files:
image = Image.open(image_path).convert('RGB') image_path = os.path.join(image_dir, image_file)
image_process = get_img_process(image, preprocess).to(vision_model.device) image = Image.open(image_path).convert('RGB')
# 对图像进行编码 image_process = get_img_process(image, preprocess).to(vision_model.device)
image_encoder = get_img_embedding(image_process, vision_model).unsqueeze(0) image_encoder = get_img_embedding(image_process, vision_model).unsqueeze(0)
image_encoders.append(image_encoder)
print(f'[Image]: {image_file}') print(f'[Image]: {image_file}')
prompt_ = prompt.replace("\n", "") image_encoders = torch.cat(image_encoders, dim=0).unsqueeze(0)
print(f'[Q]: {prompt_}')
messages = [{"role": "user", "content": prompt}] messages = [{"role": "user", "content": prompt}]
new_prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)[-(max_seq_len - 1):]
# print(messages)
new_prompt = tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True
)[-(max_seq_len - 1):]
x = tokenizer(new_prompt).data['input_ids'] x = tokenizer(new_prompt).data['input_ids']
x = (torch.tensor(x, dtype=torch.long, device=device)[None, ...]) x = torch.tensor(x, dtype=torch.long, device=device)[None, ...]
answer = new_prompt
with torch.no_grad(): with torch.no_grad():
res_y = model.generate(x, tokenizer.eos_token_id, max_new_tokens=max_seq_len, temperature=temperature, res_y = model.generate(x, tokenizer.eos_token_id, max_new_tokens=max_seq_len, temperature=temperature,
top_k=top_k, stream=stream, image_encoders=image_encoder) top_k=top_k, stream=True, image_encoders=image_encoders)
print('[A]: ', end='') print('[A]: ', end='')
try:
y = next(res_y)
except StopIteration:
print("No answer")
continue
history_idx = 0 history_idx = 0
while y != None: for y in res_y:
answer = tokenizer.decode(y[0].tolist()) answer = tokenizer.decode(y[0].tolist())
if answer and answer[-1] == '<EFBFBD>':
try:
y = next(res_y)
except:
break
continue
# print(answer)
if not len(answer):
try:
y = next(res_y)
except:
break
continue
print(answer[history_idx:], end='', flush=True) print(answer[history_idx:], end='', flush=True)
try:
y = next(res_y)
except:
break
history_idx = len(answer) history_idx = len(answer)
if not stream:
break
print('\n') print('\n')
else:
# 单图推理：对每张图像单独推理
for image_file in image_files:
image_path = os.path.join(image_dir, image_file)
image = Image.open(image_path).convert('RGB')
image_process = get_img_process(image, preprocess).to(vision_model.device)
image_encoder = get_img_embedding(image_process, vision_model).unsqueeze(0)
messages = [{"role": "user", "content": prompt}]
new_prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)[-(max_seq_len - 1):]
x = tokenizer(new_prompt).data['input_ids']
x = torch.tensor(x, dtype=torch.long, device=device)[None, ...]
print(f'[Image]: {image_file}')
with torch.no_grad():
res_y = model.generate(x, tokenizer.eos_token_id, max_new_tokens=max_seq_len, temperature=temperature,
top_k=top_k, stream=True, image_encoders=image_encoder)
print('[A]: ', end='')
history_idx = 0
for y in res_y:
answer = tokenizer.decode(y[0].tolist())
print(answer[history_idx:], end='', flush=True)
history_idx = len(answer)
print('\n')

2
5-eval_chat_multi.py
View File

@ -85,8 +85,6 @@ if __name__ == "__main__":
stream = True stream = True
# prompt = lm_config.image_special_token + '\n这个图片描述的是什么内容？'
i = 0 i = 0
# 获取图像文件列表 # 获取图像文件列表
# image_dir = './dataset/eval_multi_images/' # 对于图像转换数据集 # image_dir = './dataset/eval_multi_images/' # 对于图像转换数据集

40
README.md
View File

@ -180,7 +180,7 @@ CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz
目录下，命名为`*_llm.pth` 目录下，命名为`*_llm.pth`
* 3.4 `python 1-pretrain_vlm.py` 执行预训练，得到 `*_vlm_pretrain.pth` 作为预训练的输出权重 * 3.4 `python 1-pretrain_vlm.py` 执行预训练，得到 `*_vlm_pretrain.pth` 作为预训练的输出权重
* 3.5 `python 2-sft_vlm.py` 执行指令微调，得到 `*_vlm_sft.pth` 作为指令微调的输出权重 * 3.5 `python 2-sft_vlm.py` 执行指令微调，得到 `*_vlm_sft.pth` 作为指令微调的输出权重
* 3.6 `python 4-sft_multi_vlm.py` 执行多图指令微调（效果仅参考），得到 `*_vlm_sft_multi.pth` 作为多图指令微调的输出权重，512+8 模型显存占用约为8198M * 3.6 `python 2-sft_vlm.py --multi True` 在指令微调的基础上执行多图指令微调（效果仅参考），得到 `*_vlm_sft_multi.pth` 作为多图指令微调的输出权重，512+8 模型显存占用约为8198M
* 4.测试自己训练的模型推理效果 * 4.测试自己训练的模型推理效果
* 确保需要使用的，训练完成的参数权重`*.pth`文件位于`./out/`目录下 * 确保需要使用的，训练完成的参数权重`*.pth`文件位于`./out/`目录下
@ -196,10 +196,7 @@ CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz
``` ```
* `python 3-eval_chat.py`测试模型的对话效果，其中测试图片在`./dataset/eval_images`下，可自行更换 * `python 3-eval_chat.py`测试模型的对话效果，其中测试图片在`./dataset/eval_images`下，可自行更换
![eval_chat](images/3-eval_chat.png) ![eval_chat](images/3-eval_chat.png)
<<<<<<< HEAD * `python 3-eval_chat.py`调整[multi](./3-eval_chat.py#L61)变量，可以测试模型的多图对话效果，其中测试图片在`./dataset/eval_multi_images`下，可自行更换（多图数据集规模相对较小且为英文对话，数据集仅包含两图对比的场景，因此微调效果有限）。
* `python 5-eval_chat_multi.py`测试模型的多图对话效果，其中测试图片在`./dataset/eval_multi_images`下，可自行更换（多图数据集规模相对较小且为英文对话，数据集仅包含两图对比的场景，因此微调效果有限）。
=======
>>>>>>> 606d58814fb72b49674ce4fdd47aed1e7f36d827
🍭 【Tip】预训练和全参指令微调pretrain和sft均支持多卡加速 🍭 【Tip】预训练和全参指令微调pretrain和sft均支持多卡加速
@ -316,7 +313,6 @@ minimind-v使用50个字符组成的 `<<<...>>>` 占位符代替图像，
因此minimind-v的prompt： 因此minimind-v的prompt：
```text ```text
<<<<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>>>>\n这个图片描述的是什么内容？
``` ```
计算完embedding和projection，并对图像部分token替换后 计算完embedding和projection，并对图像部分token替换后
@ -324,14 +320,12 @@ minimind-v使用50个字符组成的 `<<<...>>>` 占位符代替图像，
![input](./images/minimind-v-input.png) ![input](./images/minimind-v-input.png)
<<<<<<< HEAD
多图实现方法就是通过注入多个\<image\>图像占位符进行实现，不需要修改任何框架。 多图实现方法就是通过注入多个\<image\>图像占位符进行实现，不需要修改任何框架。
> ps: 唯一值得注意的点是，如果在训练过程中存在不同conversations插入图片数量不同的情况，需要利用空特征将较短的特征进行填充（对应[dataset的第267行](./model/dataset.py)），以保证能够在同样大小下被dataloader读取。 > ps: 唯一值得注意的点是，如果在训练过程中存在不同conversations插入图片数量不同的情况，需要利用空特征将较短的特征进行填充（对应[dataset的第267行](./model/dataset.py#267)），以保证能够在同样大小下被dataloader读取。
> pps: 在prompt中不需要如此做，仍旧是根据插入图像的数量来进行占位符的注入。因此，最终输入给LLM的input feature不会受填充特征的影响。 > pps: 在prompt中不需要如此做，仍旧是根据插入图像的数量来进行占位符的注入。因此，最终输入给LLM的input feature不会受填充特征的影响。
=======
>>>>>>> 606d58814fb72b49674ce4fdd47aed1e7f36d827
<u>至此，MiniMind-V的所有细节已经呈现完毕。</u> <u>至此，MiniMind-V的所有细节已经呈现完毕。</u>
<u>MiniMind-V的实现未参考 **任何** 第三方代码，完全基于MiniMind尽可能做最小改动产生，故代码实现和LlaVA等模型必然存在很大区别。 <u>MiniMind-V的实现未参考 **任何** 第三方代码，完全基于MiniMind尽可能做最小改动产生，故代码实现和LlaVA等模型必然存在很大区别。
@ -382,7 +376,6 @@ MiniMind-V与MiniMind的代码核心改动不超过100行，上手难度低。</
} }
``` ```
<<<<<<< HEAD
多图指令微调数据集格式： 多图指令微调数据集格式：
```json ```json
@ -422,32 +415,23 @@ MiniMind-V与MiniMind的代码核心改动不超过100行，上手难度低。</
注： 注：
+ 对于指令微调，仅保留了一轮对话，训练单轮对话模型，防止小模型性能被长文本拉低。 + 对于指令微调，仅保留了一轮对话，训练单轮对话模型，防止小模型性能被长文本拉低。
+ 多图数据集规模相对较小且为英文对话，数据集仅包含两图对比的场景，因此微调效果有限，这里只提供一种参考思路。 + 多图数据集规模相对较小且为英文对话，数据集仅包含两图对比的场景，因此微调效果有限，这里只提供一种参考思路。
=======
注：对于指令微调，仅保留了一轮对话，训练单轮对话模型，防止小模型性能被长文本拉低。
>>>>>>> 606d58814fb72b49674ce4fdd47aed1e7f36d827
最终的数据集下载地址：[百度网盘](https://pan.baidu.com/s/1Nz36OBBvVBGEx-PwIb7ofg?pwd=6666) | [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset) 最终的数据集下载地址：[百度网盘](https://pan.baidu.com/s/1Nz36OBBvVBGEx-PwIb7ofg?pwd=6666) | [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset)
多图对话数据集：[HuggingFace](https://hf-mirror.com/datasets/xinyanghuang/minimind-v_multi_image/tree/main)
## 训练 ## 训练
预训练从595K条数据集中学习图片的通用知识，比如鹿是鹿，狗是狗。 预训练从595K条数据集中学习图片的通用知识，比如鹿是鹿，狗是狗。
指令微调从230K条真实对话数据集中学习对图片提问的真实问答格式。 指令微调从230K条真实对话数据集中学习对图片提问的真实问答格式。
<<<<<<< HEAD 多图微调提供两个数据集，图像转换数据集和鸟类对比数据集，长度分别为3.5k和13.6k的真实问答格式。
多图微调提供两个数据集，长度分别为3.5k和13.6k的真实问答格式。
=======
>>>>>>> 606d58814fb72b49674ce4fdd47aed1e7f36d827
`1-pretrain_vlm.py` 执行预训练，得到 `*_vlm_pretrain.pth` 作为预训练的输出权重。 `1-pretrain_vlm.py` 执行预训练，得到 `*_vlm_pretrain.pth` 作为预训练的输出权重。
`2-sft_vlm.py` 执行指令微调，得到 `*_vlm_sft.pth` 作为指令微调的输出权重。 `2-sft_vlm.py` 执行指令微调，得到 `*_vlm_sft.pth` 作为指令微调的输出权重。
<<<<<<< HEAD
`4-sft_multi_vlm.py` 执行指令微调，得到 `*_vlm_sft_multi.pth` 作为指令微调的输出权重。
=======
>>>>>>> 606d58814fb72b49674ce4fdd47aed1e7f36d827
训练时均冻结visual encoder也就是clip模型，只微调Projection和LLM两部分。 训练时均冻结visual encoder也就是clip模型，只微调Projection和LLM两部分。
> Pretrain 512+8 模型 (训练时间和Loss参考图) > Pretrain 512+8 模型 (训练时间和Loss参考图)
@ -477,10 +461,7 @@ MiniMind-V与MiniMind的代码核心改动不超过100行，上手难度低。</
### 效果测试 ### 效果测试
<<<<<<< HEAD
#### 单图对话 #### 单图对话
=======
>>>>>>> 606d58814fb72b49674ce4fdd47aed1e7f36d827
<table> <table>
<thead> <thead>
<tr> <tr>
@ -565,7 +546,6 @@ MiniMind-V与MiniMind的代码核心改动不超过100行，上手难度低。</
</tbody> </tbody>
</table> </table>
<<<<<<< HEAD
#### 多图对话（效果十分有限） #### 多图对话（效果十分有限）
<table> <table>
@ -578,15 +558,13 @@ MiniMind-V与MiniMind的代码核心改动不超过100行，上手难度低。</
</thead> </thead>
<tbody> <tbody>
<tr> <tr>
<td><img src="./dataset/eval_multi_images/bird//0.jpg" alt="a-bird.png" style="width: 200px;"></td> <td><img src="./dataset/eval_multi_images/bird/0.jpg" alt="a-bird.png" style="width: 200px;"></td>
<td><img src="./dataset/eval_multi_images/bird//1.jpg" alt="a-bird.png" style="width: 200px;"></td> <td><img src="./dataset/eval_multi_images/bird/1.jpg" alt="a-bird.png" style="width: 200px;"></td>
<td>animal1 has a brown and black head with a black and white striped head . animal2 has a black head with a white stripe on its wings .</td> <td>animal1 has a brown and black head with a black and white striped head . animal2 has a black head with a white stripe on its wings .</td>
</tr> </tr>
</tbody> </tbody>
</table> </table>
=======
>>>>>>> 606d58814fb72b49674ce4fdd47aed1e7f36d827
### 启动推理 ### 启动推理
```bash ```bash

3
dataset/eval_multi_images/bird/0.jpg Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d34d90f79ad34a62ed46bf2ddcd2b348703c04817befceb6ece6629f508b8636
size 57989

3
dataset/eval_multi_images/bird/1.jpg Normal file
View File

@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:86f21fb466259b028c536a9814aee11165348df2f7443dff4e069b53b6c05a5f
size 96613

BIN
model/__pycache__/LMConfig.cpython-310.pyc
View File

Binary file not shown.

BIN
model/__pycache__/dataset.cpython-310.pyc
View File

Binary file not shown.

BIN
model/__pycache__/model.cpython-310.pyc
View File

Binary file not shown.

BIN
model/__pycache__/vision_utils.cpython-310.pyc
View File

Binary file not shown.

43
model/model.py
View File

@ -385,29 +385,26 @@ class Transformer(PreTrainedModel):
# 如果此时有图像编码 # 如果此时有图像编码
if image_encoders is not None: if image_encoders is not None:
vision_proj = self.vision_proj(image_encoders) # torch.Size([32, 4, 50, 512]) vision_proj = self.vision_proj(image_encoders)
if image_indices is not None: vision_proj = vision_proj.unsqueeze(0) if len(vision_proj.shape) == 3 else vision_proj
# 创建一个新的张量来存储拼接后的结果 if image_indices is not None:
new_h = [] # 创建一个新的张量来存储拼接后的结果
new_h = []
for i in range(h.size(0)): for i in range(h.size(0)):
current_h = h[i] # i即为current_batch_idx索引
img_idx = 0 img_idx = 0
for _, (batch_idx, start_idx, end_idx) in enumerate(image_indices): for batch_idx, start_idx, end_idx in image_indices:
if batch_idx == i: if batch_idx == i:
# 插入vision_proj特征 # 插入vision_proj特征
before = current_h[:start_idx, :] before = h[i][:start_idx, :]
after = current_h[end_idx + 1:, :] after = h[i][end_idx + 1:, :]
# 拼接 before, vision_proj, after # 拼接 before, vision_proj, after
if len(vision_proj.shape) == 4: h[i] = torch.cat((before, vision_proj[i][img_idx], after), dim=0)[:seqlen]
current_h = torch.cat((before, vision_proj[i][img_idx], after), dim=0)[:seqlen] img_idx += 1
else: new_h.append(h[i])
current_h = torch.cat((before, vision_proj[i], after), dim=0)[:seqlen] # 将所有拼接后的结果重新堆叠起来
img_idx += 1 new_h = torch.stack(new_h, dim=0) # torch.Size([32, 511, 512])
new_h.append(current_h) return new_h
# 将所有拼接后的结果堆叠起来
new_h = torch.stack(new_h, dim=0) # torch.Size([32, 511, 512])
return new_h
return h return h