github
/
minimind
mirror of https://github.com/jingyaogong/minimind.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update model/dataset.py

pull/825/head
gongjy 2024-09-14 16:09:42 +08:00
parent 2a78f002e5
commit 49da2cec5c
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
model/dataset.py
View File

@ -80,18 +80,18 @@ class SFTDataset(Dataset):
# #
sample = self.df.iloc[index] sample = self.df.iloc[index]
history = self.safe_eval(sample['history']) history = self.safe_eval(sample['history'])
q = sample['q'] q = str(sample['q'])
a = sample['a'] a = str(sample['a'])
messages = [] messages = []
for history_message in history: for history_message in history:
if len(history_message) <= 1: if len(history_message) <= 1:
continue continue
messages.append( messages.append(
{"role": 'user', "content": history_message[0][:self.max_length // 2]} {"role": 'user', "content": str(history_message[0])[:self.max_length // 2]}
) )
messages.append( messages.append(
{"role": 'assistant', "content": history_message[1][:self.max_length // 2]} {"role": 'assistant', "content": str(history_message[1])[:self.max_length // 2]}
) )
messages += [ messages += [