parent
04b56ea86c
commit
2d7bb67ce2
|
|
@ -35,7 +35,7 @@ class PretrainDataset(Dataset):
|
|||
sample = self.samples[index]
|
||||
|
||||
# 构建输入文本
|
||||
text = f"{self.tokenizer.bos_token}{str(sample['text'])}{self.tokenizer.eos_token}"
|
||||
text = sample['text']
|
||||
encoding = self.tokenizer(
|
||||
text,
|
||||
max_length=self.max_length,
|
||||
|
|
|
|||
Loading…
Reference in New Issue