fix(model/dataset.py)

原始文件中已经包含了<s>和</s>句子分割符,无需在代码中重复添加

Fixes #314
pull/331/head
cn-farmer 2025-04-02 10:18:04 +08:00
parent 04b56ea86c
commit 2d7bb67ce2
1 changed files with 1 additions and 1 deletions

View File

@ -35,7 +35,7 @@ class PretrainDataset(Dataset):
sample = self.samples[index]
# 构建输入文本
text = f"{self.tokenizer.bos_token}{str(sample['text'])}{self.tokenizer.eos_token}"
text = sample['text']
encoding = self.tokenizer(
text,
max_length=self.max_length,