minimind-v/rag/retriever.py

"""
CLIP-based Image Retriever for RAG System
Uses frozen CLIP ViT-B/16 from MiniMindVLM for visual similarity
"""
import os
import sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

import torch
import numpy as np
from PIL import Image
from typing import List, Union, Dict, Tuple
from model.model_vlm import MiniMindVLM
from rag.vector_store import VectorStore


class CLIPRetriever:
    """
    CLIP-based retriever for visual similarity search
    
    Uses the same CLIP ViT-B/16 model as MiniMindVLM for consistency
    """
    
    def __init__(
        self,
        vision_model_path: str = "./model/vision_model/clip-vit-base-patch16",
        device: str = "cuda" if torch.cuda.is_available() else "cpu"
    ):
        """
        Initialize CLIP retriever
        
        Args:
            vision_model_path: Path to CLIP model
            device: Device to run on
        """
        self.device = device
        
        # Load CLIP model (reuse MiniMindVLM's CLIP loader)
        self.vision_model, self.processor = MiniMindVLM.get_vision_model(vision_model_path)
        
        if self.vision_model is None:
            raise ValueError(f"CLIP model not found at {vision_model_path}")
        
        self.vision_model = self.vision_model.to(device)
        self.vision_model.eval()
        
        print(f"✅ Loaded CLIP ViT-B/16 on {device}")
    
    def encode_image(self, image: Union[str, Image.Image]) -> np.ndarray:
        """
        Encode a single image to CLIP embedding
        
        Args:
            image: PIL Image or path to image
        
        Returns:
            Embedding as numpy array of shape (768,)
        """
        # Load image if path provided
        if isinstance(image, str):
            image = Image.open(image)
        
        # Convert image to tensor
        image_tensor = MiniMindVLM.image2tensor(image, self.processor).to(self.device)
        
        # Get CLIP embedding
        with torch.no_grad():
            embedding = MiniMindVLM.get_image_embeddings(image_tensor, self.vision_model)
            # Pool patch embeddings to single vector
            embedding = embedding.mean(dim=0).cpu().numpy()  # Shape: (768,)
        
        return embedding
    
    def encode_images_batch(self, images: List[Union[str, Image.Image]], batch_size: int = 32) -> np.ndarray:
        """
        Encode multiple images in batches
        
        Args:
            images: List of PIL Images or paths
            batch_size: Batch size for encoding
        
        Returns:
            Embeddings as numpy array of shape (N, 768)
        """
        embeddings = []
        
        for i in range(0, len(images), batch_size):
            batch = images[i:i + batch_size]
            
            # Load images
            batch_images = []
            for img in batch:
                if isinstance(img, str):
                    img = Image.open(img)
                batch_images.append(img)
            
            # Convert to tensors
            # image2tensor returns (1, C, H, W), so we concat on batch dim
            batch_tensors = torch.cat([
                MiniMindVLM.image2tensor(img, self.processor)
                for img in batch_images
            ], dim=0).to(self.device)
            
            # Encode
            with torch.no_grad():
                batch_embeddings = MiniMindVLM.get_image_embeddings(batch_tensors, self.vision_model)
                # Pool: (batch, patches, 768) -> (batch, 768)
                batch_embeddings = batch_embeddings.mean(dim=1).cpu().numpy()
            
            embeddings.append(batch_embeddings)
        
        return np.vstack(embeddings)
    
    def build_index(
        self,
        images: List[Union[str, Image.Image]],
        metadata: List[Dict],
        save_path: str,
        batch_size: int = 32
    ) -> VectorStore:
        """
        Build FAISS index from images
        
        Args:
            images: List of images or paths
            metadata: List of metadata dicts (one per image)
            save_path: Where to save index
            batch_size: Encoding batch size
        
        Returns:
            VectorStore with built index
        """
        print(f"🔨 Building index from {len(images)} images...")
        
        # Encode all images
        embeddings = self.encode_images_batch(images, batch_size=batch_size)
        print(f"✅ Encoded {len(embeddings)} images to {embeddings.shape[1]}-dim vectors")
        
        # Build vector store
        store = VectorStore(dimension=embeddings.shape[1])
        store.add(embeddings, metadata)
        
        # Save
        store.save(save_path)
        
        return store
    
    def retrieve(
        self,
        query_image: Union[str, Image.Image],
        vector_store: VectorStore,
        k: int = 5
    ) -> Tuple[np.ndarray, List[Dict]]:
        """
        Retrieve k most similar images from index
        
        Args:
            query_image: Query image or path
            vector_store: VectorStore to search in
            k: Number of results to return
        
        Returns:
            distances: Array of shape (k,) with distances
            metadata: List of k metadata dicts
        """
        # Encode query
        query_embedding = self.encode_image(query_image)
        query_embedding = query_embedding.reshape(1, -1)  # (1, 768)
        
        # Search
        distances, indices, metadata = vector_store.search(query_embedding, k=k)
        
        # Return flattened results (since we only have 1 query)
        return distances[0], metadata[0]


# Example usage
if __name__ == "__main__":
    # Create retriever
    retriever = CLIPRetriever()
    
    # Example: encode single image
    # img_path = "dataset/eval_images/城市车水马龙-city-traffic.jpg"
    # embedding = retriever.encode_image(img_path)
    # print(f"Image embedding shape: {embedding.shape}")