github
/
minimind-v
mirror of https://github.com/jingyaogong/minimind-v.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge b696a7f084 into 26de815253

pull/45/merge
ckq 2025-04-11 13:07:21 +08:00 committed by GitHub
parent 26de815253 b696a7f084
commit 6ae1b3e6ae
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
model/model_vlm.py
View File

@ -3,7 +3,7 @@ from .model import *
from typing import Optional, Tuple, List from typing import Optional, Tuple, List
from torch import nn from torch import nn
import warnings import warnings
from transformers import CLIPProcessor, CLIPModel from transformers import CLIPProcessor, CLIPVisionModel
import torch import torch
warnings.filterwarnings('ignore') warnings.filterwarnings('ignore')
@ -36,7 +36,7 @@ class MiniMindVLM(MiniMindLM):
@staticmethod @staticmethod
def get_vision_model(model_path="./model/vision_model/clip-vit-base-patch16"): def get_vision_model(model_path="./model/vision_model/clip-vit-base-patch16"):
model = CLIPModel.from_pretrained(model_path) model = CLIPVisionModel.from_pretrained(model_path)
processor = CLIPProcessor.from_pretrained(model_path) processor = CLIPProcessor.from_pretrained(model_path)
# 冻结 vision_encoder 的所有参数 # 冻结 vision_encoder 的所有参数
for param in model.parameters(): for param in model.parameters():
@ -52,7 +52,7 @@ class MiniMindVLM(MiniMindLM):
@staticmethod @staticmethod
def get_image_embeddings(image_tensors, vision_model): def get_image_embeddings(image_tensors, vision_model):
with torch.no_grad(): with torch.no_grad():
outputs = vision_model.vision_model(pixel_values=image_tensors) outputs = vision_model(pixel_values=image_tensors)
img_embedding = outputs.last_hidden_state[:, 1:, :].squeeze() img_embedding = outputs.last_hidden_state[:, 1:, :].squeeze()
return img_embedding return img_embedding