321 lines
12 KiB
Plaintext
321 lines
12 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 16,
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"import pandas as pd\n",
|
||
"import os"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"<div>\n",
|
||
"<style scoped>\n",
|
||
" .dataframe tbody tr th:only-of-type {\n",
|
||
" vertical-align: middle;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe tbody tr th {\n",
|
||
" vertical-align: top;\n",
|
||
" }\n",
|
||
"\n",
|
||
" .dataframe thead th {\n",
|
||
" text-align: right;\n",
|
||
" }\n",
|
||
"</style>\n",
|
||
"<table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: right;\">\n",
|
||
" <th></th>\n",
|
||
" <th>text</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <th>0</th>\n",
|
||
" <td><|im_start|>鉴别一组中文文章的风格和特点，例如官方、口语、文言等。需要提供样例文...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>1</th>\n",
|
||
" <td><|im_start|>根据输入的内容，编写一个类别标签。\\n这是一篇介绍如何阅读心电图的文...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>2</th>\n",
|
||
" <td><|im_start|>客户要求看一份报告，但他没有说明他需要哪份报告。他是指哪份报告？这个...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>3</th>\n",
|
||
" <td><|im_start|>请从这首诗选出三句话，重新组成另一首新的诗歌。秋天美景尽收眼底，枫叶...</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <th>4</th>\n",
|
||
" <td><|im_start|>引述一段启发人心的一句话，用于鼓励团队士气。“团队的力量不在于每个人...</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table>\n",
|
||
"</div>"
|
||
],
|
||
"text/plain": [
|
||
" text\n",
|
||
"0 <|im_start|>鉴别一组中文文章的风格和特点，例如官方、口语、文言等。需要提供样例文...\n",
|
||
"1 <|im_start|>根据输入的内容，编写一个类别标签。\\n这是一篇介绍如何阅读心电图的文...\n",
|
||
"2 <|im_start|>客户要求看一份报告，但他没有说明他需要哪份报告。他是指哪份报告？这个...\n",
|
||
"3 <|im_start|>请从这首诗选出三句话，重新组成另一首新的诗歌。秋天美景尽收眼底，枫叶...\n",
|
||
"4 <|im_start|>引述一段启发人心的一句话，用于鼓励团队士气。“团队的力量不在于每个人..."
|
||
]
|
||
},
|
||
"execution_count": 2,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"df=pd.read_json(\"dataset/pretrain_hq.jsonl\",lines=True)\n",
|
||
"df.head()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 17,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"'<|im_start|>鉴别一组中文文章的风格和特点，例如官方、口语、文言等。需要提供样例文章才能准确鉴别不同的风格和特点。<|im_end|> <|im_start|>好的，现在帮我查一下今天的天气怎么样?今天的天气依据地区而异。请问你需要我帮你查询哪个地区的天气呢？<|im_end|> <|im_start|>打开闹钟功能，定一个明天早上七点的闹钟。好的，我已经帮您打开闹钟功能，闹钟将在明天早上七点准时响起。<|im_end|> <|im_start|>为以下场景写一句话描述：一个孤独的老人坐在公园长椅上看着远处。一位孤独的老人坐在公园长椅上凝视远方。<|im_end|> <|im_start|>非常感谢你的回答。请告诉我，这些数据是关于什么主题的？这些数据是关于不同年龄段的男女人口比例分布的。<|im_end|> <|im_start|>帮我想一个有趣的标题。这个挺有趣的：\"如何成为一名成功的魔术师\" 调皮的标题往往会吸引读者的注意力。<|im_end|> <|im_start|>回答一个问题，地球的半径是多少？地球的平均半径约为6371公里，这是地球自赤道到两极的距离的平均值。<|im_end|> <|im_start|>识别文本中的语气，并将其分类为喜悦、悲伤、惊异等。\\n文本：“今天是我的生日！”这个文本的语气是喜悦。<|im_end|>'"
|
||
]
|
||
},
|
||
"execution_count": 17,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"df.iloc[0]['text']"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 28,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"'已处理并保存 252 条数据'"
|
||
]
|
||
},
|
||
"execution_count": 28,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"def process_and_save_dataset(dataset_list, max_length=512, output_file=\"dataset/pretrain_hq.jsonl\"):\n",
|
||
" import json\n",
|
||
" \n",
|
||
" combined_text = \"\"\n",
|
||
" result_list = []\n",
|
||
" \n",
|
||
" for data in dataset_list:\n",
|
||
" # 添加特殊Token\n",
|
||
" processed_text = \"<|im_start|>\" + data.strip() + \"<|im_end|>\"\n",
|
||
" \n",
|
||
" # 如果当前文本加上新样本会超过最大长度\n",
|
||
" if len(combined_text + \" \" + processed_text) > max_length and combined_text:\n",
|
||
" # 保存当前拼接的文本\n",
|
||
" result_list.append({\"text\": combined_text+\"<|endoftext|>\"*10})\n",
|
||
" # 重置拼接文本\n",
|
||
" combined_text = processed_text\n",
|
||
" else:\n",
|
||
" # 添加空格分隔符（如果combined_text不为空）\n",
|
||
" if combined_text:\n",
|
||
" combined_text += \" \" + processed_text\n",
|
||
" else:\n",
|
||
" combined_text = processed_text\n",
|
||
" \n",
|
||
" # 处理最后剩余的文本\n",
|
||
" if combined_text:\n",
|
||
" result_list.append({\"text\": combined_text + \"<|endoftext|>\"*10})\n",
|
||
" \n",
|
||
" # 以最小IO方式保存到jsonl文件\n",
|
||
" with open(output_file, 'a', encoding='utf-8') as f:\n",
|
||
" for item in result_list:\n",
|
||
" f.write(json.dumps(item, ensure_ascii=False) + '\\n')\n",
|
||
" \n",
|
||
" return f\"已处理并保存 {len(result_list)} 条数据\"\n",
|
||
"\n",
|
||
"# 使用示例\n",
|
||
"raw=\"\"\n",
|
||
"for file in os.listdir():\n",
|
||
" if file.endswith(\".txt\"):\n",
|
||
" with open(file, \"r\", encoding=\"utf-8\") as f:\n",
|
||
" raw += f.read()\n",
|
||
"\n",
|
||
"cleaned_raw = raw.replace(\"\"\",\"\").replace(\"\"\",\"\").replace(\"'\",\"\").replace(\"'\",\"\").replace(\"，\",\"\").replace(\"\\n\",\"\")\n",
|
||
"dataset = cleaned_raw.split(\"。\")\n",
|
||
"\n",
|
||
"# 处理并保存数据\n",
|
||
"process_and_save_dataset(dataset[1:], max_length=512)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"## remove"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 27,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"'已从 dataset/pretrain_hq.jsonl 中删除最后 362 条数据，剩余 1413259 条'"
|
||
]
|
||
},
|
||
"execution_count": 27,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"def remove_last_n_entries(jsonl_file, n=362):\n",
|
||
" import json\n",
|
||
" \n",
|
||
" # 读取所有行\n",
|
||
" with open(jsonl_file, 'r', encoding='utf-8') as f:\n",
|
||
" lines = f.readlines()\n",
|
||
" \n",
|
||
" # 计算要保留的行数\n",
|
||
" keep_count = max(0, len(lines) - n)\n",
|
||
" remaining_lines = lines[:keep_count]\n",
|
||
" \n",
|
||
" # 重写文件\n",
|
||
" with open(jsonl_file, 'w', encoding='utf-8') as f:\n",
|
||
" f.writelines(remaining_lines)\n",
|
||
" \n",
|
||
" return f\"已从 {jsonl_file} 中删除最后 {n} 条数据，剩余 {keep_count} 条\"\n",
|
||
"\n",
|
||
"# 使用示例\n",
|
||
"remove_last_n_entries(\"dataset/pretrain_hq.jsonl\", 362)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"metadata": {},
|
||
"source": [
|
||
"## 添加结束符"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 29,
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"已处理 1413511 条数据，修改了 1413511 条数据添加结束标记\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"def add_endoftext_to_jsonl(jsonl_file, batch_size=1000):\n",
|
||
" import json\n",
|
||
" import os\n",
|
||
" import tempfile\n",
|
||
" \n",
|
||
" # 创建临时文件\n",
|
||
" temp_file = tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', delete=False)\n",
|
||
" temp_filename = temp_file.name\n",
|
||
" \n",
|
||
" endoftext_str = \"<|endoftext|>\" * 10 # 重复150次\n",
|
||
" \n",
|
||
" # 记录处理的行数和修改的行数\n",
|
||
" processed = 0\n",
|
||
" modified = 0\n",
|
||
" \n",
|
||
" try:\n",
|
||
" # 逐批次处理文件\n",
|
||
" with open(jsonl_file, 'r', encoding='utf-8') as in_file:\n",
|
||
" batch = []\n",
|
||
" for line in in_file:\n",
|
||
" processed += 1\n",
|
||
" try:\n",
|
||
" data = json.loads(line.strip())\n",
|
||
" # 检查是否以<|endoftext|>结尾\n",
|
||
" if \"text\" in data and not data[\"text\"].endswith(\"<|endoftext|>\"):\n",
|
||
" data[\"text\"] += endoftext_str\n",
|
||
" modified += 1\n",
|
||
" batch.append(json.dumps(data, ensure_ascii=False) + '\\n')\n",
|
||
" \n",
|
||
" # 批处理写入\n",
|
||
" if len(batch) >= batch_size:\n",
|
||
" temp_file.writelines(batch)\n",
|
||
" batch = []\n",
|
||
" except json.JSONDecodeError:\n",
|
||
" # 处理无效JSON的情况\n",
|
||
" temp_file.write(line)\n",
|
||
" \n",
|
||
" # 写入最后一批\n",
|
||
" if batch:\n",
|
||
" temp_file.writelines(batch)\n",
|
||
" \n",
|
||
" # 关闭临时文件\n",
|
||
" temp_file.close()\n",
|
||
" \n",
|
||
" # 替换原始文件\n",
|
||
" os.replace(temp_filename, jsonl_file)\n",
|
||
" \n",
|
||
" return f\"已处理 {processed} 条数据，修改了 {modified} 条数据添加结束标记\"\n",
|
||
" \n",
|
||
" except Exception as e:\n",
|
||
" # 发生错误时删除临时文件\n",
|
||
" if os.path.exists(temp_filename):\n",
|
||
" os.unlink(temp_filename)\n",
|
||
" return f\"处理出错: {str(e)}\"\n",
|
||
"\n",
|
||
"# 使用示例\n",
|
||
"file_path = \"dataset/pretrain_hq.jsonl\"\n",
|
||
"result = add_endoftext_to_jsonl(file_path)\n",
|
||
"print(result)"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "minimind",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.11.0"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 2
|
||
}
|