add multi-image
xinyanghuang7
parent
dd3656de0f
commit
502fe28f33
|
|
@ -1 +0,0 @@
|
|||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
|
|
@ -1,4 +1,8 @@
|
|||
model/clip_model/clip-vit-base-patch32/
|
||||
out/*.pth
|
||||
full.json
|
||||
trans_json.py
|
||||
trans_json.py
|
||||
dataset/sft_multi_images/
|
||||
dataset/
|
||||
!dataset/eval_images
|
||||
!dataset/eval_multi_images
|
||||
|
|
@ -1,244 +0,0 @@
|
|||
import os
|
||||
import platform
|
||||
import argparse
|
||||
import time
|
||||
import math
|
||||
import warnings
|
||||
import json
|
||||
|
||||
import pandas as pd
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import torch.distributed as dist
|
||||
from contextlib import nullcontext
|
||||
|
||||
from torch import optim
|
||||
from torch.nn.parallel import DistributedDataParallel
|
||||
from torch.utils.data import DataLoader, DistributedSampler
|
||||
from transformers import AutoTokenizer, AutoModel
|
||||
from model.model import Transformer
|
||||
from model.LMConfig import LMConfig
|
||||
from model.dataset import SFTDataset_multi
|
||||
from model.vision_utils import get_vision_model, get_img_embedding
|
||||
|
||||
warnings.filterwarnings('ignore')
|
||||
|
||||
|
||||
def count_parameters(model):
|
||||
return sum(p.numel() for p in model.parameters() if p.requires_grad)
|
||||
|
||||
|
||||
def Logger(content):
|
||||
if not ddp or dist.get_rank() == 0:
|
||||
print(content)
|
||||
|
||||
|
||||
def get_lr(it, all):
|
||||
warmup_iters = args.warmup_iters
|
||||
lr_decay_iters = all
|
||||
min_lr = args.learning_rate / 10
|
||||
|
||||
if it < warmup_iters:
|
||||
return args.learning_rate * it / warmup_iters
|
||||
if it > lr_decay_iters:
|
||||
return min_lr
|
||||
decay_ratio = (it - warmup_iters) / (lr_decay_iters - warmup_iters)
|
||||
assert 0 <= decay_ratio <= 1
|
||||
coeff = 0.5 * (1.0 + math.cos(math.pi * decay_ratio))
|
||||
return min_lr + coeff * (args.learning_rate - min_lr)
|
||||
|
||||
|
||||
def train_epoch(epoch, wandb):
|
||||
start_time = time.time()
|
||||
for step, (X, Y, loss_mask, image_process) in enumerate(train_loader):
|
||||
X = X.to(args.device)
|
||||
Y = Y.to(args.device)
|
||||
loss_mask = loss_mask.to(args.device)
|
||||
image_process = image_process.to(args.device)
|
||||
image_encoders = get_img_embedding(image_process, vision_model) # torch.Size([32, 4, 50, 768])
|
||||
lr = get_lr(epoch * iter_per_epoch + step, args.epochs * iter_per_epoch)
|
||||
for param_group in optimizer.param_groups:
|
||||
param_group['lr'] = lr
|
||||
|
||||
with ctx:
|
||||
logits = model(X, Y, image_encoders=image_encoders).logits
|
||||
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), Y.view(-1), ignore_index=0, reduction='none')
|
||||
loss_mask = loss_mask.view(-1)
|
||||
loss = torch.sum(loss * loss_mask) / loss_mask.sum()
|
||||
|
||||
scaler.scale(loss).backward()
|
||||
|
||||
if (step + 1) % args.accumulation_steps == 0:
|
||||
scaler.unscale_(optimizer)
|
||||
torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip)
|
||||
|
||||
scaler.step(optimizer)
|
||||
scaler.update()
|
||||
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
|
||||
if step % args.log_interval == 0:
|
||||
spend_time = time.time() - start_time
|
||||
Logger(
|
||||
'Epoch:[{}/{}]({}/{}) loss:{:.3f} lr:{:.7f} epoch_Time:{}min:'.format(
|
||||
epoch,
|
||||
args.epochs,
|
||||
step,
|
||||
iter_per_epoch,
|
||||
loss.item(),
|
||||
optimizer.param_groups[-1]['lr'],
|
||||
spend_time / (step + 1) * iter_per_epoch // 60 - spend_time // 60))
|
||||
|
||||
if (wandb is not None) and (not ddp or dist.get_rank() == 0):
|
||||
wandb.log({"loss": loss,
|
||||
"lr": optimizer.param_groups[-1]['lr'],
|
||||
"epoch_Time": spend_time / (step + 1) * iter_per_epoch // 60 - spend_time // 60})
|
||||
|
||||
if (step + 1) % args.save_interval == 0 and (not ddp or dist.get_rank() == 0):
|
||||
model.eval()
|
||||
moe_path = '_moe' if lm_config.use_moe else ''
|
||||
ckp = f'{args.save_dir}/{lm_config.dim}{moe_path}_vlm_sft.pth'
|
||||
|
||||
if isinstance(model, torch.nn.parallel.DistributedDataParallel):
|
||||
state_dict = model.module.state_dict()
|
||||
else:
|
||||
state_dict = model.state_dict()
|
||||
|
||||
torch.save(state_dict, ckp)
|
||||
model.train()
|
||||
|
||||
if args.save_last and (not ddp or dist.get_rank() == 0):
|
||||
model.eval()
|
||||
moe_path = '_moe' if lm_config.use_moe else ''
|
||||
ckp = f'{args.save_dir}/{lm_config.dim}{moe_path}_vlm_sft_multi.pth'
|
||||
|
||||
if isinstance(model, torch.nn.parallel.DistributedDataParallel):
|
||||
state_dict = model.module.state_dict()
|
||||
else:
|
||||
state_dict = model.state_dict()
|
||||
|
||||
torch.save(state_dict, ckp)
|
||||
model.train()
|
||||
|
||||
|
||||
def init_model(lm_config):
|
||||
tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer')
|
||||
moe_path = '_moe' if lm_config.use_moe else ''
|
||||
# ckp = f'./out/{lm_config.dim}{moe_path}_vlm_pretrain.pth'
|
||||
ckp = f'./out/{lm_config.dim}{moe_path}_vlm_sft.pth'
|
||||
|
||||
model = Transformer(lm_config)
|
||||
state_dict = torch.load(ckp, map_location=args.device)
|
||||
|
||||
# 处理不需要的前缀
|
||||
unwanted_prefix = '_orig_mod.'
|
||||
for k, v in list(state_dict.items()):
|
||||
if k.startswith(unwanted_prefix):
|
||||
state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k)
|
||||
|
||||
# 加载到模型中
|
||||
model.load_state_dict(state_dict, strict=False)
|
||||
model = model.to(args.device)
|
||||
|
||||
print(f'模型可学习参数: {count_parameters(model) / 1e6} 百万 = {count_parameters(model) / 1e9} B (Billion)')
|
||||
|
||||
(vision_model, preprocess) = get_vision_model()
|
||||
vision_model = vision_model.to(args.device)
|
||||
return model, tokenizer, (vision_model, preprocess)
|
||||
|
||||
|
||||
def init_distributed_mode():
|
||||
if not ddp: return
|
||||
global ddp_local_rank, DEVICE
|
||||
|
||||
dist.init_process_group(backend="nccl")
|
||||
ddp_rank = int(os.environ["RANK"])
|
||||
ddp_local_rank = int(os.environ["LOCAL_RANK"])
|
||||
ddp_world_size = int(os.environ["WORLD_SIZE"])
|
||||
DEVICE = f"cuda:{ddp_local_rank}"
|
||||
torch.cuda.set_device(DEVICE)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="MiniMind-V-SFT")
|
||||
parser.add_argument("--out_dir", type=str, default="out", help="Output directory")
|
||||
parser.add_argument("--epochs", type=int, default=19, help="Number of epochs")
|
||||
parser.add_argument("--batch_size", type=int, default=32, help="Batch size")
|
||||
parser.add_argument("--learning_rate", type=float, default=1e-6, help="Learning rate")
|
||||
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu",
|
||||
help="Device to use")
|
||||
parser.add_argument("--dtype", type=str, default="bfloat16", help="Data type")
|
||||
parser.add_argument("--use_wandb", default=False, action="store_true", help="Use Weights & Biases")
|
||||
parser.add_argument("--wandb_project", type=str, default="MiniMind-V", help="Weights & Biases project name")
|
||||
parser.add_argument("--num_workers", type=int, default=8, help="Number of workers for data loading")
|
||||
parser.add_argument("--data_path", type=str, default="./dataset/sft_multi_images/output_cn.json",
|
||||
help="Path to training data")
|
||||
parser.add_argument("--ddp", action="store_true", help="Use DistributedDataParallel")
|
||||
parser.add_argument("--accumulation_steps", type=int, default=1, help="Gradient accumulation steps")
|
||||
parser.add_argument("--grad_clip", type=float, default=1.0, help="Gradient clipping threshold")
|
||||
parser.add_argument("--warmup_iters", type=int, default=0, help="Number of warmup iterations")
|
||||
parser.add_argument("--log_interval", type=int, default=100, help="Logging interval")
|
||||
parser.add_argument("--save_interval", type=int, default=100, help="Model saving interval")
|
||||
parser.add_argument('--local_rank', type=int, default=-1, help='local rank for distributed training')
|
||||
parser.add_argument('--save_last', type=bool, default=True, help='save last step model')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
lm_config = LMConfig()
|
||||
max_seq_len = lm_config.max_seq_len
|
||||
args.save_dir = os.path.join(args.out_dir)
|
||||
os.makedirs(args.save_dir, exist_ok=True)
|
||||
os.makedirs(args.out_dir, exist_ok=True)
|
||||
tokens_per_iter = args.batch_size * max_seq_len
|
||||
torch.manual_seed(1337)
|
||||
device_type = "cuda" if "cuda" in args.device else "cpu"
|
||||
|
||||
args.wandb_run_name = f"MiniMind-V-SFT-Epoch-{args.epochs}-BatchSize-{args.batch_size}-LearningRate-{args.learning_rate}"
|
||||
|
||||
ctx = nullcontext() if device_type == "cpu" else torch.cuda.amp.autocast()
|
||||
ddp = int(os.environ.get("RANK", -1)) != -1 # is this a ddp run?
|
||||
ddp_local_rank, DEVICE = 0, "cuda:0"
|
||||
if ddp:
|
||||
init_distributed_mode()
|
||||
args.device = torch.device(DEVICE)
|
||||
|
||||
if args.use_wandb and (not ddp or ddp_local_rank == 0):
|
||||
import wandb
|
||||
|
||||
wandb.init(project=args.wandb_project, name=args.wandb_run_name)
|
||||
else:
|
||||
wandb = None
|
||||
|
||||
model, tokenizer, (vision_model, preprocess) = init_model(lm_config)
|
||||
|
||||
use_version = 0
|
||||
train_ds = SFTDataset_multi(args.data_path, tokenizer, vision_model=(vision_model, preprocess),
|
||||
image_special_token=lm_config.image_special_token,
|
||||
max_length=max_seq_len)
|
||||
train_sampler = DistributedSampler(train_ds) if ddp else None
|
||||
train_loader = DataLoader(
|
||||
train_ds,
|
||||
batch_size=args.batch_size,
|
||||
pin_memory=True,
|
||||
drop_last=False,
|
||||
shuffle=False,
|
||||
num_workers=args.num_workers,
|
||||
sampler=train_sampler
|
||||
)
|
||||
|
||||
scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype in ['float16', 'bfloat16']))
|
||||
optimizer = optim.Adam(model.parameters(), lr=args.learning_rate)
|
||||
|
||||
if False and not lm_config.use_moe and platform.system() != 'Windows' and float(
|
||||
torch.__version__.split('.')[0]) >= 2:
|
||||
Logger("compiling the model... (takes a ~minute)")
|
||||
unoptimized_model = model
|
||||
model = torch.compile(model)
|
||||
|
||||
if ddp:
|
||||
model._ddp_params_and_buffers_to_ignore = {"pos_cis"}
|
||||
model = DistributedDataParallel(model, device_ids=[ddp_local_rank])
|
||||
|
||||
iter_per_epoch = len(train_loader)
|
||||
for epoch in range(args.epochs):
|
||||
train_epoch(epoch, wandb)
|
||||
|
|
@ -1,170 +0,0 @@
|
|||
import os
|
||||
import random
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import warnings
|
||||
|
||||
from PIL import Image
|
||||
from transformers import AutoTokenizer, AutoModelForCausalLM
|
||||
from model.model import Transformer
|
||||
from model.LMConfig import LMConfig
|
||||
from model.vision_utils import get_vision_model, get_img_process, get_img_embedding
|
||||
|
||||
warnings.filterwarnings('ignore')
|
||||
|
||||
|
||||
def count_parameters(model):
|
||||
return sum(p.numel() for p in model.parameters() if p.requires_grad)
|
||||
|
||||
|
||||
def init_model(lm_config):
|
||||
tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer')
|
||||
model_from = 1 # 1从权重，2用transformers
|
||||
|
||||
if model_from == 1:
|
||||
moe_path = '_moe' if lm_config.use_moe else ''
|
||||
ckp = f'./out/{lm_config.dim}{moe_path}_vlm_sft_multi.pth'
|
||||
|
||||
model = Transformer(lm_config)
|
||||
state_dict = torch.load(ckp, map_location=device)
|
||||
|
||||
# 处理不需要的前缀
|
||||
unwanted_prefix = '_orig_mod.'
|
||||
for k, v in list(state_dict.items()):
|
||||
if k.startswith(unwanted_prefix):
|
||||
state_dict[k[len(unwanted_prefix):]] = state_dict.pop(k)
|
||||
|
||||
# 加载到模型中
|
||||
model.load_state_dict(state_dict, strict=False)
|
||||
else:
|
||||
model = AutoModelForCausalLM.from_pretrained('minimind-v-v1-small',
|
||||
trust_remote_code=True)
|
||||
|
||||
model = model.to(device)
|
||||
|
||||
print(f'模型参数: {count_parameters(model) / 1e6} 百万 = {count_parameters(model) / 1e9} B (Billion)')
|
||||
|
||||
(vision_model, preprocess) = get_vision_model()
|
||||
vision_model = vision_model.to(device)
|
||||
return model, tokenizer, (vision_model, preprocess)
|
||||
|
||||
|
||||
def setup_seed(seed):
|
||||
random.seed(seed) # 设置 Python 的随机种子
|
||||
np.random.seed(seed) # 设置 NumPy 的随机种子
|
||||
torch.manual_seed(seed) # 设置 PyTorch 的随机种子
|
||||
torch.cuda.manual_seed(seed) # 为当前 GPU 设置随机种子（如果有）
|
||||
torch.cuda.manual_seed_all(seed) # 为所有 GPU 设置随机种子（如果有）
|
||||
torch.backends.cudnn.deterministic = True # 确保每次返回的卷积算法是确定的
|
||||
torch.backends.cudnn.benchmark = False # 关闭 cuDNN 的自动调优，避免不确定性
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# -----------------------------------------------------------------------------
|
||||
out_dir = 'out'
|
||||
start = ""
|
||||
temperature = 0.5
|
||||
top_k = 8
|
||||
setup_seed(1337)
|
||||
# device = 'cpu'
|
||||
device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
|
||||
dtype = 'bfloat16'
|
||||
max_seq_len = 1 * 1024
|
||||
lm_config = LMConfig()
|
||||
lm_config.max_seq_len = max_seq_len
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
model, tokenizer, (vision_model, preprocess) = init_model(lm_config)
|
||||
|
||||
model = model.eval()
|
||||
# 推送到huggingface
|
||||
# model.push_to_hub("minimind")
|
||||
# tokenizer.push_to_hub("minimind")
|
||||
|
||||
stream = True
|
||||
|
||||
# prompt = lm_config.image_special_token + '\n这个图片描述的是什么内容？'
|
||||
|
||||
i = 0
|
||||
# 获取图像文件列表
|
||||
image_dir = './dataset/eval_multi_images/'
|
||||
image_files = sorted(os.listdir(image_dir))
|
||||
|
||||
image_count = len(image_files)
|
||||
special_tokens = lm_config.image_special_token * image_count # 根据图片数量重复特殊标记
|
||||
|
||||
# 构建提示
|
||||
prompt = f"{special_tokens}\n这几张图片是否是同一个类别？"
|
||||
|
||||
# 遍历图像文件
|
||||
image_encoders = []
|
||||
for i, image_file in enumerate(image_files):
|
||||
image_path = os.path.join(image_dir, image_file)
|
||||
|
||||
# 打开图像并转换为RGB格式
|
||||
image = Image.open(image_path).convert('RGB')
|
||||
image_process = get_img_process(image, preprocess).to(vision_model.device)
|
||||
|
||||
# 对图像进行编码
|
||||
image_encoder = get_img_embedding(image_process, vision_model).unsqueeze(0)
|
||||
print(image_encoder.shape)
|
||||
image_encoders.append(image_encoder)
|
||||
print(f'[Image]: {image_file}')
|
||||
image_encoders = torch.cat(image_encoders, dim=0).unsqueeze(0)
|
||||
print(image_encoders.shape)
|
||||
|
||||
# 准备问题和消息
|
||||
prompt_ = prompt.replace("\n", "")
|
||||
print(f'[Q]: {prompt_}')
|
||||
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
|
||||
new_prompt = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True
|
||||
)[-(max_seq_len - 1):]
|
||||
|
||||
x = tokenizer(new_prompt).data['input_ids']
|
||||
x = (torch.tensor(x, dtype=torch.long, device=device)[None, ...])
|
||||
|
||||
with torch.no_grad():
|
||||
res_y = model.generate(x, tokenizer.eos_token_id, max_new_tokens=max_seq_len, temperature=temperature,
|
||||
top_k=top_k, stream=stream, image_encoders=image_encoders)
|
||||
|
||||
print('[A]: ', end='')
|
||||
try:
|
||||
y = next(res_y)
|
||||
except StopIteration:
|
||||
print("No answer")
|
||||
exit()
|
||||
|
||||
history_idx = 0
|
||||
while y is not None:
|
||||
answer = tokenizer.decode(y[0].tolist())
|
||||
if answer and answer[-1] == '<EFBFBD>':
|
||||
try:
|
||||
y = next(res_y)
|
||||
except StopIteration:
|
||||
break
|
||||
continue
|
||||
if not len(answer):
|
||||
try:
|
||||
y = next(res_y)
|
||||
except StopIteration:
|
||||
break
|
||||
continue
|
||||
|
||||
print(answer[history_idx:], end='', flush=True)
|
||||
try:
|
||||
y = next(res_y)
|
||||
except StopIteration:
|
||||
break
|
||||
history_idx = len(answer)
|
||||
|
||||
if not stream:
|
||||
break
|
||||
|
||||
print('\n')
|
||||
32
README.md
32
README.md
|
|
@ -180,6 +180,7 @@ CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz
|
|||
目录下，命名为`*_llm.pth`
|
||||
* 3.4 `python 1-pretrain_vlm.py` 执行预训练，得到 `*_vlm_pretrain.pth` 作为预训练的输出权重
|
||||
* 3.5 `python 2-sft_vlm.py` 执行指令微调，得到 `*_vlm_sft.pth` 作为指令微调的输出权重
|
||||
* 3.6 `python 4-sft_multi_vlm.py` 执行多图指令微调（效果仅参考），得到 `*_vlm_sft_multi.pth` 作为多图指令微调的输出权重，512+8 模型显存占用约为8198M
|
||||
|
||||
* 4.测试自己训练的模型推理效果
|
||||
* 确保需要使用的，训练完成的参数权重`*.pth`文件位于`./out/`目录下
|
||||
|
|
@ -195,6 +196,7 @@ CPU: Intel(R) Core(TM) i9-10980XE CPU @ 3.00GHz
|
|||
```
|
||||
* `python 3-eval_chat.py`测试模型的对话效果，其中测试图片在`./dataset/eval_images`下，可自行更换
|
||||
|
||||
* `python 5-eval_chat_multi.py`测试模型的多图对话效果，其中测试图片在`./dataset/eval_multi_images`下，可自行更换（多图数据集规模相对较小且为英文对话，数据集仅包含两图对比的场景，因此微调效果有限）。
|
||||
|
||||
🍭 【Tip】预训练和全参指令微调pretrain和sft均支持多卡加速
|
||||
|
||||
|
|
@ -319,6 +321,11 @@ minimind-v使用50个字符组成的 `<<<...>>>` 占位符代替图像，
|
|||
|
||||
|
||||
|
||||
多图实现方法就是通过注入多个\<image\>图像占位符进行实现。
|
||||
|
||||
> ps: 唯一值得注意的点是，如果在训练过程中存在不同conversations插入图片数量不同的情况，需要利用空特征将较短的特征进行填充（对应[dataset的第267行](./model/dataset.py)），以保证能够在同样大小下被dataloader读取。
|
||||
> pps: 在prompt中不需要如此做，仍旧是根据插入图像的数量来进行占位符的注入。因此，最终输入给LLM的input feature不会受填充特征的影响。
|
||||
|
||||
<u>至此，MiniMind-V的所有细节已经呈现完毕。</u>
|
||||
|
||||
<u>MiniMind-V的实现未参考 **任何** 第三方代码，完全基于MiniMind尽可能做最小改动产生，故代码实现和LlaVA等模型必然存在很大区别。
|
||||
|
|
@ -369,7 +376,28 @@ MiniMind-V与MiniMind的代码核心改动不超过100行，上手难度低。</
|
|||
}
|
||||
```
|
||||
|
||||
注：对于指令微调，仅保留了一轮对话，训练单轮对话模型，防止小模型性能被长文本拉低。
|
||||
多图指令微调数据集格式：
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "000000000000",
|
||||
"image": "0.jpg, 1.jpg",
|
||||
"conversations": [
|
||||
{
|
||||
"from": "human",
|
||||
"value": "context: Source Image: <image> Target Image: <image> Instruction: What is the correct image edit instruction that can transfrom the source image to target image?"
|
||||
},
|
||||
{
|
||||
"from": "gpt",
|
||||
"value": "take the people out of the back in the photo. Remove the two people behind the woman in the white dress and the man in the blue suit. remove people behind the couple in the centre"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
注：
|
||||
+ 对于指令微调，仅保留了一轮对话，训练单轮对话模型，防止小模型性能被长文本拉低。
|
||||
+ 多图数据集规模相对较小且为英文对话，数据集仅包含两图对比的场景，因此微调效果有限，这里只提供一种参考思路。
|
||||
|
||||
最终的数据集下载地址：[百度网盘](https://pan.baidu.com/s/1Nz36OBBvVBGEx-PwIb7ofg?pwd=6666) | [HuggingFace](https://huggingface.co/datasets/jingyaogong/minimind-v_dataset)
|
||||
|
||||
|
|
@ -383,6 +411,8 @@ MiniMind-V与MiniMind的代码核心改动不超过100行，上手难度低。</
|
|||
|
||||
`2-sft_vlm.py` 执行指令微调，得到 `*_vlm_sft.pth` 作为指令微调的输出权重。
|
||||
|
||||
`4-sft_multi_vlm.py` 执行指令微调，得到 `*_vlm_sft_multi.pth` 作为指令微调的输出权重。
|
||||
|
||||
训练时均冻结visual encoder也就是clip模型，只微调Projection和LLM两部分。
|
||||
|
||||
> Pretrain 512+8 模型 (训练时间和Loss参考图)
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ class LMConfig(PretrainedConfig):
|
|||
|
||||
def __init__(
|
||||
self,
|
||||
dim: int = 512,
|
||||
n_layers: int = 8,
|
||||
dim: int = 512, # 768
|
||||
n_layers: int = 8, # 16
|
||||
n_heads: int = 16,
|
||||
n_kv_heads: int = 8,
|
||||
vocab_size: int = 6400,
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -197,7 +197,7 @@ class SFTDataset_multi(Dataset):
|
|||
self.vision_model, self.preprocess = vision_model
|
||||
self.padding = 0
|
||||
self.bos_id = self.tokenizer('<s>assistant').data['input_ids']
|
||||
self.dataset_path = './dataset/sft2_images/'
|
||||
self.dataset_path = './dataset/sft_multi_images/'
|
||||
self.image_special_token = image_special_token
|
||||
|
||||
def __len__(self):
|
||||
|
|
@ -264,7 +264,7 @@ class SFTDataset_multi(Dataset):
|
|||
image_encoders.append(get_img_process(image, self.preprocess)['pixel_values'])
|
||||
|
||||
# 确定目标形状
|
||||
max_images = 4 # 根据你的需求设置 一次性最大输入几张图片
|
||||
max_images = 2 # 根据你的需求设置 一次性最大输入几张图片
|
||||
target_shape = (max_images, 3, 224, 224)
|
||||
|
||||
# 创建填充张量
|
||||
|
|
|
|||
|
|
@ -399,7 +399,10 @@ class Transformer(PreTrainedModel):
|
|||
before = current_h[:start_idx, :]
|
||||
after = current_h[end_idx + 1:, :]
|
||||
# 拼接 before, vision_proj, after
|
||||
current_h = torch.cat((before, vision_proj[i][img_idx], after), dim=0)[:seqlen]
|
||||
if len(vision_proj.shape) == 4:
|
||||
current_h = torch.cat((before, vision_proj[i][img_idx], after), dim=0)[:seqlen]
|
||||
else:
|
||||
current_h = torch.cat((before, vision_proj[i], after), dim=0)[:seqlen]
|
||||
img_idx += 1
|
||||
new_h.append(current_h)
|
||||
# 将所有拼接后的结果堆叠起来
|
||||
|
|
@ -440,7 +443,8 @@ class Transformer(PreTrainedModel):
|
|||
self.last_loss = None
|
||||
|
||||
self.OUT.__setitem__('logits', logits)
|
||||
self.OUT.__setitem__('last_loss', self.last_loss)
|
||||
# self.OUT.__setitem__('last_loss', self.last_loss)
|
||||
self.OUT.__setitem__('loss', self.last_loss)
|
||||
return self.OUT
|
||||
|
||||
@torch.inference_mode()
|
||||
|
|
|
|||
|
|
@ -7,37 +7,21 @@ with open('full.json', 'r') as f:
|
|||
# 创建新的格式
|
||||
new_data = []
|
||||
|
||||
# 定义选择列表的翻译字典
|
||||
translation_dict = {
|
||||
"True": "是的",
|
||||
"False": "不是",
|
||||
"painted": "涂鸦",
|
||||
"ripe": "成熟"
|
||||
}
|
||||
|
||||
for item in data['data']:
|
||||
sample_id = str(item['sample_id']).zfill(12) # 格式化为12位数字
|
||||
images = [f"<image>" for _ in item['task_instance']['images_path']]
|
||||
|
||||
# 直接从task_instance中获取choice_list
|
||||
choice_list = item['task_instance']['choice_list']
|
||||
|
||||
# 判断是否需要翻译
|
||||
if all(choice in translation_dict for choice in choice_list):
|
||||
choice_list = [translation_dict[choice] for choice in choice_list]
|
||||
response = translation_dict[item['response']]
|
||||
else:
|
||||
response = item['response']
|
||||
|
||||
source_image = f"<image>" # 源图像占位符
|
||||
target_image = f"<image>" # 目标图像占位符
|
||||
instruction = data['metadata']['task_instruction'][item['task_instruction_id']]
|
||||
|
||||
# 构造新的对话格式
|
||||
conversation = [
|
||||
{
|
||||
"from": "human",
|
||||
"value": f"{''.join(images)}\n{data['metadata']['task_instruction'][item['task_instruction_id']]} 选择列表: {', '.join(choice_list)}."
|
||||
"value": f"context: Source Image: {source_image} Target Image: {target_image} Instruction: {instruction}."
|
||||
},
|
||||
{
|
||||
"from": "gpt",
|
||||
"value": response
|
||||
"value": item['response']
|
||||
}
|
||||
]
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue