parent
04b56ea86c
commit
2d7bb67ce2
|
|
@ -35,7 +35,7 @@ class PretrainDataset(Dataset):
|
||||||
sample = self.samples[index]
|
sample = self.samples[index]
|
||||||
|
|
||||||
# 构建输入文本
|
# 构建输入文本
|
||||||
text = f"{self.tokenizer.bos_token}{str(sample['text'])}{self.tokenizer.eos_token}"
|
text = sample['text']
|
||||||
encoding = self.tokenizer(
|
encoding = self.tokenizer(
|
||||||
text,
|
text,
|
||||||
max_length=self.max_length,
|
max_length=self.max_length,
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue