From 4d0fc87022b2ff5dbf1793fece4a262a0a506126 Mon Sep 17 00:00:00 2001 From: xinyanghuang7 Date: Tue, 29 Oct 2024 11:51:03 -0400 Subject: [PATCH] add multi-images --- 4-sft_multi_vlm.py | 2 +- 5-eval_chat_multi.py | 9 +++-- README.md | 41 +++++++++++++++++++++- model/__pycache__/dataset.cpython-310.pyc | Bin 6234 -> 6234 bytes trans_json.py | 2 +- 5 files changed, 49 insertions(+), 5 deletions(-) diff --git a/4-sft_multi_vlm.py b/4-sft_multi_vlm.py index 3c820e2..7ed3d23 100644 --- a/4-sft_multi_vlm.py +++ b/4-sft_multi_vlm.py @@ -237,7 +237,7 @@ if __name__ == "__main__": if ddp: model._ddp_params_and_buffers_to_ignore = {"pos_cis"} - model = DistributedDataParallel(model, device_ids=[ddp_local_rank], find_unused_parameters=True) + model = DistributedDataParallel(model, device_ids=[ddp_local_rank]) iter_per_epoch = len(train_loader) for epoch in range(args.epochs): diff --git a/5-eval_chat_multi.py b/5-eval_chat_multi.py index fdf4d86..9134e10 100644 --- a/5-eval_chat_multi.py +++ b/5-eval_chat_multi.py @@ -89,10 +89,15 @@ if __name__ == "__main__": i = 0 # 获取图像文件列表 - image_dir = './dataset/eval_multi_images/' + # image_dir = './dataset/eval_multi_images/' # 对于图像转换数据集 + image_dir = './dataset/eval_multi_images/bird/' # 对于鸟类数据集 image_files = sorted(os.listdir(image_dir)) - prompt = "context: Source Image: Target Image: Instruction: What is the correct image edit instruction that can transfrom the source image to target image?" + # 对于图像转换数据集 + # prompt = "context: Source Image: Target Image: Instruction: What is the correct image edit instruction that can transfrom the source image to target image?" + + # 对于鸟类数据集 + prompt = "\n\nName all the differences between these two birds." # 遍历图像文件 image_encoders = [] diff --git a/README.md b/README.md index e9aeda6..92cc82c 100644 --- a/README.md +++ b/README.md @@ -321,7 +321,7 @@ minimind-v使用50个字符组成的 `<<<...>>>` 占位符代替图像， ![input](./images/minimind-v-input.png) -多图实现方法就是通过注入多个\图像占位符进行实现。 +多图实现方法就是通过注入多个\图像占位符进行实现，不需要修改任何框架。 > ps: 唯一值得注意的点是，如果在训练过程中存在不同conversations插入图片数量不同的情况，需要利用空特征将较短的特征进行填充（对应[dataset的第267行](./model/dataset.py)），以保证能够在同样大小下被dataloader读取。 > pps: 在prompt中不需要如此做，仍旧是根据插入图像的数量来进行占位符的注入。因此，最终输入给LLM的input feature不会受填充特征的影响。 @@ -395,6 +395,23 @@ MiniMind-V与MiniMind的代码核心改动不超过100行，上手难度低。\n\nName all the differences between these two birds." + }, + { + "from": "gpt", + "value": "animal1 is brown with white tuft while animal2 is white with dark brown wings" + } + ] + } +``` + 注： + 对于指令微调，仅保留了一轮对话，训练单轮对话模型，防止小模型性能被长文本拉低。 + 多图数据集规模相对较小且为英文对话，数据集仅包含两图对比的场景，因此微调效果有限，这里只提供一种参考思路。 @@ -407,6 +424,8 @@ MiniMind-V与MiniMind的代码核心改动不超过100行，上手难度低。 @@ -526,6 +546,25 @@ MiniMind-V与MiniMind的代码核心改动不超过100行，上手难度低。 +#### 多图对话（效果十分有限） + + + + + + + + + + + + + + + + +
图片1图片2512_sft_multi
a-bird.pnga-bird.pnganimal1 has a brown and black head with a black and white striped head . animal2 has a black head with a white stripe on its wings .
+ ### 启动推理 ```bash diff --git a/model/__pycache__/dataset.cpython-310.pyc b/model/__pycache__/dataset.cpython-310.pyc index dfec1d5c9f887f5e2d4e4cf3577f2e3f842b311a..ea280d71674552fe8db25f982de9003c39998347 100644 GIT binary patch delta 19 Zcmca*aLa%zpO=@50SFo%Y~%`%001~}1vLNw delta 19 Zcmca*aLa%zpO=@50SJulZ{!M)001_Z1m*w$ diff --git a/trans_json.py b/trans_json.py index 3f22352..a62f394 100644 --- a/trans_json.py +++ b/trans_json.py @@ -17,7 +17,7 @@ for item in data['data']: conversation = [ { "from": "human", - "value": f"context: Source Image: {source_image} Target Image: {target_image} Instruction: {instruction}." + "value": f"\n\n{instruction}" }, { "from": "gpt",