From 2d7bb67ce284c13c85abe547f4742838b145d514 Mon Sep 17 00:00:00 2001
From: cn-farmer <sjinqun@sina.com>
Date: Wed, 2 Apr 2025 10:18:04 +0800
Subject: [PATCH] =?UTF-8?q?fix=EF=BC=88model/dataset.py)=20=E5=8E=9F?=
 =?UTF-8?q?=E5=A7=8B=E6=96=87=E4=BB=B6=E4=B8=AD=E5=B7=B2=E7=BB=8F=E5=8C=85?=
 =?UTF-8?q?=E5=90=AB=E4=BA=86<s>=E5=92=8C</s>=E5=8F=A5=E5=AD=90=E5=88=86?=
 =?UTF-8?q?=E5=89=B2=E7=AC=A6=EF=BC=8C=E6=97=A0=E9=9C=80=E5=9C=A8=E4=BB=A3?=
 =?UTF-8?q?=E7=A0=81=E4=B8=AD=E9=87=8D=E5=A4=8D=E6=B7=BB=E5=8A=A0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes #314
---
 model/dataset.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/model/dataset.py b/model/dataset.py
index 7750789..052a519 100644
--- a/model/dataset.py
+++ b/model/dataset.py
@@ -35,7 +35,7 @@ class PretrainDataset(Dataset):
         sample = self.samples[index]
 
         # 构建输入文本
-        text = f"{self.tokenizer.bos_token}{str(sample['text'])}{self.tokenizer.eos_token}"
+        text = sample['text']
         encoding = self.tokenizer(
             text,
             max_length=self.max_length,