github
/
minimind-v
mirror of https://github.com/jingyaogong/minimind-v.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[feat] test parquet

pull/107/head
jingyaogong 2025-12-27 10:56:07 +08:00
parent fef284776f
commit d2bdff3269
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dataset/lm_dataset.py
View File

@ -76,6 +76,7 @@ class VLMDataset(Dataset):
return X, Y, loss_mask, image_tensor return X, Y, loss_mask, image_tensor
# 测试parquet数据读取和可视化
if __name__ == '__main__': if __name__ == '__main__':
import matplotlib.pyplot as plt; plt.rcParams['font.sans-serif'] = ['Arial Unicode MS', 'SimHei'] import matplotlib.pyplot as plt; plt.rcParams['font.sans-serif'] = ['Arial Unicode MS', 'SimHei']
for path in ['pretrain_data.parquet', 'sft_data.parquet']: for path in ['pretrain_data.parquet', 'sft_data.parquet']: