diff --git a/3-eval_chat.py b/3-eval_chat.py index 84fa53f..f505979 100644 --- a/3-eval_chat.py +++ b/3-eval_chat.py @@ -41,7 +41,7 @@ def init_model(lm_config, device, multi): model = model.to(device) print(f'模型参数: {count_parameters(model) / 1e6} 百万 = {count_parameters(model) / 1e9} B (Billion)') - vision_model, preprocess = get_vision_model() + vision_model, preprocess = get_vision_model(encoder_type="clip") vision_model = vision_model.to(device) return model, tokenizer, vision_model, preprocess @@ -66,7 +66,8 @@ if __name__ == "__main__": device = 'cuda:0' if torch.cuda.is_available() else 'cpu' dtype = 'bfloat16' max_seq_len = 1024 - lm_config = LMConfig() + # lm_config = LMConfig() + lm_config = LMConfig(image_special_token='<'*2+'>'*2, image_ids=[30]*2+[32]*2) lm_config.max_seq_len = max_seq_len model, tokenizer, vision_model, preprocess = init_model(lm_config, device, multi) model.eval() diff --git a/model/__pycache__/LMConfig.cpython-310.pyc b/model/__pycache__/LMConfig.cpython-310.pyc index fdc47c4..098838d 100644 Binary files a/model/__pycache__/LMConfig.cpython-310.pyc and b/model/__pycache__/LMConfig.cpython-310.pyc differ diff --git a/model/__pycache__/model.cpython-310.pyc b/model/__pycache__/model.cpython-310.pyc index 7d7909d..7a19d9a 100644 Binary files a/model/__pycache__/model.cpython-310.pyc and b/model/__pycache__/model.cpython-310.pyc differ diff --git a/model/model.py b/model/model.py index 6f8fdbb..bfa4c65 100644 --- a/model/model.py +++ b/model/model.py @@ -374,6 +374,11 @@ class Transformer(PreTrainedModel): image_ids_tensor = torch.tensor(image_ids).to(tokens.device) len_image_ids = len(image_ids) + # .generate时,在初始化后直接跳过 + if len_image_ids > tokens.size(1): + # print(f"len_image_ids ({len_image_ids}) is greater than sequence length ({tokens.size(1)}), skipping.") + return None + # 使用view来创建一个视图,便于处理滑动窗口 tokens_view = tokens.unfold(1, len_image_ids, 1) # 在第二维度创建滑动窗口 # 检查每个滑动窗口是否与image_ids_tensor相等