From ad741354b013884942e3421f8fe7bb0000de85b1 Mon Sep 17 00:00:00 2001
From: cn-farmer <sjinqun@sina.com>
Date: Wed, 2 Apr 2025 10:37:58 +0800
Subject: [PATCH] =?UTF-8?q?fix=20(=20model/dataset.py=20)=20=E5=9C=A8?=
 =?UTF-8?q?=E9=A2=84=E8=AE=AD=E7=BB=83=E9=98=B6=E6=AE=B5=EF=BC=8Cpretrain?=
 =?UTF-8?q?=5Fhq.jsonl=E6=96=87=E4=BB=B6=E4=B8=AD=E5=B7=B2=E7=BB=8F?=
 =?UTF-8?q?=E7=94=A8=E5=92=8C=E5=B0=86=E5=8F=A5=E5=AD=90=E5=88=86=E5=89=B2?=
 =?UTF-8?q?=EF=BC=8C=E5=9C=A8dataset.py=E6=96=87=E4=BB=B6=E6=9E=84?=
 =?UTF-8?q?=E5=BB=BA=E8=BE=93=E5=85=A5=E6=96=87=E6=9C=AC=E6=97=B6=EF=BC=8C?=
 =?UTF-8?q?=E5=86=8D=E6=AC=A1=E5=9C=A8=E5=A4=B4=E9=83=A8=E5=92=8C=E5=B0=BE?=
 =?UTF-8?q?=E9=83=A8=E6=B7=BB=E5=8A=A0=E4=BA=86=E5=92=8C=EF=BC=8C=E6=9C=89?=
 =?UTF-8?q?=E9=87=8D=E5=A4=8D=E3=80=82?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

fix #314
---
 model/dataset.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/model/dataset.py b/model/dataset.py
index 4336c7e..a0a3b73 100644
--- a/model/dataset.py
+++ b/model/dataset.py
@@ -35,7 +35,7 @@ class PretrainDataset(Dataset):
         sample = self.samples[index]
 
         # 构建输入文本
-        text = f"{self.tokenizer.bos_token}{str(sample['text'])}{self.tokenizer.eos_token}"
+        text = sample['text']
         encoding = self.tokenizer(
             text,
             max_length=self.max_length,