334 lines
14 KiB
Python
334 lines
14 KiB
Python
"""
|
|
DriveMind-V DPO Evaluation Script
|
|
Compares SFT baseline vs DPO-aligned model on driving preference dataset.
|
|
|
|
Metrics:
|
|
- Preference accuracy (% of pairs where chosen log-prob > rejected)
|
|
- Per-category accuracy breakdown
|
|
- Reward margin (chosen - rejected log-prob)
|
|
- Qualitative generation samples
|
|
|
|
Usage:
|
|
cd /workspace/minimind-v/minimind-v
|
|
uv run python eval_dpo_driving.py
|
|
"""
|
|
import os
|
|
import sys
|
|
import json
|
|
import copy
|
|
import warnings
|
|
import argparse
|
|
from collections import defaultdict
|
|
|
|
import torch
|
|
import torch.nn.functional as F
|
|
from transformers import AutoTokenizer
|
|
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
from model.model_vlm import MiniMindVLM, VLMConfig
|
|
from trainer.lora_utils import apply_lora, get_lora_config
|
|
from trainer.trainer_utils import setup_seed
|
|
|
|
warnings.filterwarnings('ignore')
|
|
|
|
|
|
# ── Helpers ───────────────────────────────────────────────────────────────────
|
|
|
|
def load_sft_model(weight_path: str, hidden_size: int, num_layers: int, device: str):
|
|
config = VLMConfig(hidden_size=hidden_size, num_hidden_layers=num_layers)
|
|
model = MiniMindVLM(config, vision_model_path='./model/vision_model/clip-vit-base-patch16')
|
|
weights = torch.load(weight_path, map_location=device)
|
|
model.load_state_dict(weights, strict=False)
|
|
return model.eval().to(device)
|
|
|
|
|
|
def load_dpo_model(sft_model, adapter_dir: str, hidden_size: int,
|
|
num_layers: int, device: str, lora_r: int, lora_alpha: int):
|
|
from peft import PeftModel
|
|
# Start from a fresh copy of the SFT weights, then attach adapters
|
|
config = VLMConfig(hidden_size=hidden_size, num_hidden_layers=num_layers)
|
|
base = MiniMindVLM(config, vision_model_path='./model/vision_model/clip-vit-base-patch16')
|
|
# Copy weights from already-loaded SFT model
|
|
base.load_state_dict(
|
|
{k: v for k, v in sft_model.state_dict().items()}, strict=False
|
|
)
|
|
base = base.to(device)
|
|
# Attach PEFT adapters
|
|
lora_config = get_lora_config(r=lora_r, lora_alpha=lora_alpha)
|
|
base = apply_lora(base, lora_config, verbose=False)
|
|
from peft import set_peft_model_state_dict
|
|
import safetensors.torch as sf
|
|
adapter_file = os.path.join(adapter_dir, 'adapter_model.safetensors')
|
|
adapter_weights = sf.load_file(adapter_file)
|
|
set_peft_model_state_dict(base, adapter_weights)
|
|
return base.eval().to(device)
|
|
|
|
|
|
def compute_log_probs(model, input_ids, attention_mask, prompt_len, device):
|
|
"""
|
|
Returns (sum_logp, mean_logp) over response tokens only.
|
|
mean_logp is length-normalized and used for preference accuracy
|
|
to avoid the bias where shorter rejected responses win by default.
|
|
"""
|
|
with torch.no_grad():
|
|
attn_float = attention_mask.float()
|
|
outputs = model(input_ids=input_ids, attention_mask=attn_float)
|
|
logits = outputs.logits # (B, L, V)
|
|
|
|
shift_logits = logits[:, :-1, :].contiguous()
|
|
shift_labels = input_ids[:, 1:].contiguous()
|
|
shift_mask = attention_mask[:, 1:].contiguous()
|
|
|
|
log_probs_all = F.log_softmax(shift_logits, dim=-1)
|
|
token_log_probs = log_probs_all.gather(
|
|
2, shift_labels.unsqueeze(-1)
|
|
).squeeze(-1)
|
|
|
|
response_mask = shift_mask.clone()
|
|
response_start = max(0, prompt_len - 1)
|
|
response_mask[0, :response_start] = 0
|
|
|
|
token_log_probs = token_log_probs * response_mask.float()
|
|
n_response_tokens = response_mask.float().sum().item()
|
|
n_response_tokens = max(n_response_tokens, 1)
|
|
|
|
sum_logp = token_log_probs.sum().item()
|
|
mean_logp = sum_logp / n_response_tokens
|
|
return sum_logp, mean_logp
|
|
|
|
|
|
def tokenize(text, tokenizer, max_length, device):
|
|
enc = tokenizer(
|
|
text, truncation=True, max_length=max_length,
|
|
return_tensors='pt', add_special_tokens=False,
|
|
)
|
|
return enc['input_ids'].to(device), enc['attention_mask'].to(device)
|
|
|
|
|
|
def generate_response(model, tokenizer, prompt: str, device: str,
|
|
max_new_tokens: int = 128) -> str:
|
|
inputs = tokenizer(prompt, return_tensors='pt', add_special_tokens=False)
|
|
input_ids = inputs['input_ids'].to(device)
|
|
attention_mask = inputs['attention_mask'].to(device)
|
|
with torch.no_grad():
|
|
output_ids = model.generate(
|
|
inputs=input_ids,
|
|
attention_mask=attention_mask,
|
|
max_new_tokens=max_new_tokens,
|
|
do_sample=False,
|
|
pad_token_id=tokenizer.pad_token_id,
|
|
eos_token_id=tokenizer.eos_token_id,
|
|
)
|
|
new_tokens = output_ids[0][input_ids.shape[1]:]
|
|
return tokenizer.decode(new_tokens, skip_special_tokens=True).strip()
|
|
|
|
|
|
# ── Evaluation ────────────────────────────────────────────────────────────────
|
|
|
|
def evaluate_preferences(model, tokenizer, samples, max_length, device, label):
|
|
per_category = defaultdict(lambda: {'correct': 0, 'total': 0, 'margins': []})
|
|
overall = {'correct': 0, 'total': 0, 'margins': []}
|
|
|
|
for sample in samples:
|
|
prompt = sample['prompt']
|
|
chosen = sample['chosen']
|
|
rejected = sample['rejected']
|
|
category = sample.get('category', 'unknown')
|
|
|
|
prompt_ids, _ = tokenize(prompt, tokenizer, max_length, device)
|
|
prompt_len = prompt_ids.shape[1]
|
|
|
|
chosen_ids, chosen_mask = tokenize(prompt + chosen, tokenizer, max_length, device)
|
|
rejected_ids, rejected_mask = tokenize(prompt + rejected, tokenizer, max_length, device)
|
|
|
|
chosen_sum, chosen_mean = compute_log_probs(model, chosen_ids, chosen_mask, prompt_len, device)
|
|
rejected_sum, rejected_mean = compute_log_probs(model, rejected_ids, rejected_mask, prompt_len, device)
|
|
|
|
# Use length-normalized log-prob to avoid bias toward shorter rejected responses
|
|
correct = chosen_mean > rejected_mean
|
|
margin = chosen_mean - rejected_mean # per-token margin
|
|
|
|
overall['total'] += 1
|
|
overall['correct'] += int(correct)
|
|
overall['margins'].append(margin)
|
|
|
|
per_category[category]['total'] += 1
|
|
per_category[category]['correct'] += int(correct)
|
|
per_category[category]['margins'].append(margin)
|
|
|
|
return overall, dict(per_category)
|
|
|
|
|
|
# ── Report printing ────────────────────────────────────────────────────────────
|
|
|
|
def print_report(sft_overall, sft_cats, dpo_overall, dpo_cats, samples, tokenizer,
|
|
sft_model, dpo_model, device, args):
|
|
sep = '-' * 68
|
|
|
|
print('\n' + '=' * 68)
|
|
print(' DriveMind-V DPO Evaluation Report')
|
|
print('=' * 68)
|
|
|
|
# Overall
|
|
sft_acc = sft_overall['correct'] / sft_overall['total']
|
|
dpo_acc = dpo_overall['correct'] / dpo_overall['total']
|
|
sft_avg_margin = sum(sft_overall['margins']) / len(sft_overall['margins'])
|
|
dpo_avg_margin = sum(dpo_overall['margins']) / len(dpo_overall['margins'])
|
|
|
|
print(f'\n{"Model":<20} {"Pref Acc":>10} {"Avg Margin/tok":>16} {"Pairs":>8}')
|
|
print(sep)
|
|
print(f'{"SFT (baseline)":<20} {sft_acc:>9.1%} {sft_avg_margin:>16.4f} {sft_overall["total"]:>8}')
|
|
print(f'{"DPO (aligned)":<20} {dpo_acc:>9.1%} {dpo_avg_margin:>16.4f} {dpo_overall["total"]:>8}')
|
|
delta_acc = dpo_acc - sft_acc
|
|
delta_margin = dpo_avg_margin - sft_avg_margin
|
|
print(f'{"Delta (DPO - SFT)":<20} {delta_acc:>+9.1%} {delta_margin:>+16.4f}')
|
|
print(f'\n Note: Margin = mean(chosen_logp) - mean(rejected_logp) per response token.')
|
|
print(f' Length-normalized to remove bias toward shorter rejected responses.')
|
|
|
|
# Per-category
|
|
print(f'\n{"Category":<24} {"SFT Acc":>9} {"DPO Acc":>9} {"Delta":>9} {"SFT Margin":>12} {"DPO Margin":>12}')
|
|
print(sep)
|
|
all_cats = sorted(set(list(sft_cats.keys()) + list(dpo_cats.keys())))
|
|
for cat in all_cats:
|
|
s = sft_cats.get(cat, {'correct': 0, 'total': 1, 'margins': [0]})
|
|
d = dpo_cats.get(cat, {'correct': 0, 'total': 1, 'margins': [0]})
|
|
s_acc = s['correct'] / s['total']
|
|
d_acc = d['correct'] / d['total']
|
|
s_margin = sum(s['margins']) / len(s['margins'])
|
|
d_margin = sum(d['margins']) / len(d['margins'])
|
|
delta = d_acc - s_acc
|
|
print(f'{cat:<24} {s_acc:>8.1%} {d_acc:>9.1%} {delta:>+9.1%} {s_margin:>12.2f} {d_margin:>12.2f}')
|
|
|
|
# Margin distribution
|
|
print(f'\nMargin Distribution (mean chosen_logp/tok - mean rejected_logp/tok):')
|
|
print(sep)
|
|
for label, overall in [('SFT', sft_overall), ('DPO', dpo_overall)]:
|
|
margins = overall['margins']
|
|
pos = sum(1 for m in margins if m > 0)
|
|
neg = sum(1 for m in margins if m <= 0)
|
|
mn = min(margins)
|
|
mx = max(margins)
|
|
med = sorted(margins)[len(margins) // 2]
|
|
print(f' {label}: min={mn:.1f} median={med:.1f} max={mx:.1f} pos={pos} neg={neg}')
|
|
|
|
# Qualitative samples
|
|
print(f'\nQualitative Generation Samples (DPO model):')
|
|
print(sep)
|
|
# Pick one example per category (first occurrence)
|
|
seen_cats = set()
|
|
qual_samples = []
|
|
for s in samples:
|
|
cat = s.get('category', 'unknown')
|
|
if cat not in seen_cats:
|
|
seen_cats.add(cat)
|
|
qual_samples.append(s)
|
|
if len(qual_samples) >= 6:
|
|
break
|
|
|
|
for s in qual_samples:
|
|
prompt = s['prompt']
|
|
chosen = s['chosen']
|
|
rejected = s['rejected']
|
|
cat = s.get('category', 'unknown')
|
|
# Extract scenario text from prompt
|
|
scenario_start = prompt.find('Scenario:')
|
|
scenario_end = prompt.find('[/INST]')
|
|
scenario = prompt[scenario_start:scenario_end].strip() if scenario_start != -1 else prompt[-120:]
|
|
|
|
print(f'\n [{cat}]')
|
|
print(f' Scenario: {scenario[10:].strip()}')
|
|
sft_resp = generate_response(sft_model, tokenizer, prompt, device, max_new_tokens=args.max_new_tokens)
|
|
dpo_resp = generate_response(dpo_model, tokenizer, prompt, device, max_new_tokens=args.max_new_tokens)
|
|
print(f' SFT : {sft_resp}')
|
|
print(f' DPO : {dpo_resp}')
|
|
print(f' Chosen : {chosen}')
|
|
print(f' Rejected: {rejected}')
|
|
|
|
# Summary
|
|
print(f'\n{"=" * 68}')
|
|
print(' Summary')
|
|
print(f'{"=" * 68}')
|
|
print(f' Dataset : {sft_overall["total"]} preference pairs across {len(all_cats)} categories')
|
|
print(f' SFT baseline preference accuracy : {sft_acc:.1%}')
|
|
print(f' DPO aligned preference accuracy : {dpo_acc:.1%}')
|
|
print(f' Accuracy improvement : {delta_acc:+.1%}')
|
|
print(f' Avg margin improvement : {delta_margin:+.2f} log-prob units')
|
|
print()
|
|
relative_gain = (dpo_acc - sft_acc) / max(1 - sft_acc, 1e-6)
|
|
if dpo_acc >= 0.90:
|
|
grade = 'STRONG — DPO above 90% preference accuracy'
|
|
elif dpo_acc >= 0.75:
|
|
grade = 'MODERATE — DPO above 75% preference accuracy'
|
|
elif relative_gain >= 0.50:
|
|
grade = f'MEANINGFUL — {relative_gain:.0%} relative gain over SFT; more epochs or data would push higher'
|
|
else:
|
|
grade = 'WEAK — consider more training epochs or a larger/better preference dataset'
|
|
print(f' Assessment: {grade}')
|
|
print('=' * 68 + '\n')
|
|
|
|
|
|
# ── Main ──────────────────────────────────────────────────────────────────────
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='DriveMind-V DPO Evaluation')
|
|
parser.add_argument('--data_path', default='dataset/driving_preferences.jsonl')
|
|
parser.add_argument('--sft_weight', default='out/sft_vlm_512.pth')
|
|
parser.add_argument('--dpo_adapter_dir', default='out/dpo_driving_lora_512_adapters')
|
|
parser.add_argument('--hidden_size', type=int, default=512)
|
|
parser.add_argument('--num_hidden_layers', type=int, default=8)
|
|
parser.add_argument('--lora_r', type=int, default=16)
|
|
parser.add_argument('--lora_alpha', type=int, default=32)
|
|
parser.add_argument('--max_length', type=int, default=256)
|
|
parser.add_argument('--max_new_tokens', type=int, default=80)
|
|
parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu')
|
|
parser.add_argument('--seed', type=int, default=42)
|
|
args = parser.parse_args()
|
|
|
|
setup_seed(args.seed)
|
|
device = args.device
|
|
|
|
print(f'Device: {device}')
|
|
print(f'Loading tokenizer...')
|
|
tokenizer = AutoTokenizer.from_pretrained('./model')
|
|
|
|
# Load samples
|
|
samples = []
|
|
with open(args.data_path) as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
samples.append(json.loads(line))
|
|
print(f'Loaded {len(samples)} preference pairs')
|
|
|
|
# Load SFT model
|
|
print(f'Loading SFT model from {args.sft_weight}...')
|
|
sft_model = load_sft_model(args.sft_weight, args.hidden_size, args.num_hidden_layers, device)
|
|
print(' SFT model loaded.')
|
|
|
|
# Load DPO model
|
|
print(f'Loading DPO model from {args.dpo_adapter_dir}...')
|
|
dpo_model = load_dpo_model(
|
|
sft_model, args.dpo_adapter_dir, args.hidden_size,
|
|
args.num_hidden_layers, device, args.lora_r, args.lora_alpha
|
|
)
|
|
print(' DPO model loaded.')
|
|
|
|
# Evaluate both models
|
|
print('\nEvaluating SFT model...')
|
|
sft_overall, sft_cats = evaluate_preferences(
|
|
sft_model, tokenizer, samples, args.max_length, device, 'SFT'
|
|
)
|
|
|
|
print('Evaluating DPO model...')
|
|
dpo_overall, dpo_cats = evaluate_preferences(
|
|
dpo_model, tokenizer, samples, args.max_length, device, 'DPO'
|
|
)
|
|
|
|
print('\nGenerating qualitative samples...')
|
|
print_report(sft_overall, sft_cats, dpo_overall, dpo_cats, samples,
|
|
tokenizer, sft_model, dpo_model, device, args)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|