[feat] test parquet

2025-12-27 10:56:07 +08:00 · 2025-12-27 10:56:07 +08:00 · d2bdff3269
parent fef284776f
commit d2bdff3269
1 changed files with 1 additions and 0 deletions
--- a/dataset/lm_dataset.py
+++ b/dataset/lm_dataset.py
@ -76,6 +76,7 @@ class VLMDataset(Dataset):

        return X, Y, loss_mask, image_tensor

+# 测试parquet数据读取和可视化
 if __name__ == '__main__':
    import matplotlib.pyplot as plt; plt.rcParams['font.sans-serif'] = ['Arial Unicode MS', 'SimHei']
    for path in ['pretrain_data.parquet', 'sft_data.parquet']: