{ "cells": [ { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import os" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
text
0<|im_start|>鉴别一组中文文章的风格和特点，例如官方、口语、文言等。需要提供样例文...
1<|im_start|>根据输入的内容，编写一个类别标签。\\n这是一篇介绍如何阅读心电图的文...
2<|im_start|>客户要求看一份报告，但他没有说明他需要哪份报告。他是指哪份报告？这个...
3<|im_start|>请从这首诗选出三句话，重新组成另一首新的诗歌。秋天美景尽收眼底，枫叶...
4<|im_start|>引述一段启发人心的一句话，用于鼓励团队士气。“团队的力量不在于每个人...
\n", "
" ], "text/plain": [ " text\n", "0 <|im_start|>鉴别一组中文文章的风格和特点，例如官方、口语、文言等。需要提供样例文...\n", "1 <|im_start|>根据输入的内容，编写一个类别标签。\\n这是一篇介绍如何阅读心电图的文...\n", "2 <|im_start|>客户要求看一份报告，但他没有说明他需要哪份报告。他是指哪份报告？这个...\n", "3 <|im_start|>请从这首诗选出三句话，重新组成另一首新的诗歌。秋天美景尽收眼底，枫叶...\n", "4 <|im_start|>引述一段启发人心的一句话，用于鼓励团队士气。“团队的力量不在于每个人..." ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df=pd.read_json(\"dataset/pretrain_hq.jsonl\",lines=True)\n", "df.head()" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'<|im_start|>鉴别一组中文文章的风格和特点，例如官方、口语、文言等。需要提供样例文章才能准确鉴别不同的风格和特点。<|im_end|> <|im_start|>好的，现在帮我查一下今天的天气怎么样?今天的天气依据地区而异。请问你需要我帮你查询哪个地区的天气呢？<|im_end|> <|im_start|>打开闹钟功能，定一个明天早上七点的闹钟。好的，我已经帮您打开闹钟功能，闹钟将在明天早上七点准时响起。<|im_end|> <|im_start|>为以下场景写一句话描述：一个孤独的老人坐在公园长椅上看着远处。一位孤独的老人坐在公园长椅上凝视远方。<|im_end|> <|im_start|>非常感谢你的回答。请告诉我，这些数据是关于什么主题的？这些数据是关于不同年龄段的男女人口比例分布的。<|im_end|> <|im_start|>帮我想一个有趣的标题。这个挺有趣的：\"如何成为一名成功的魔术师\" 调皮的标题往往会吸引读者的注意力。<|im_end|> <|im_start|>回答一个问题，地球的半径是多少？地球的平均半径约为6371公里，这是地球自赤道到两极的距离的平均值。<|im_end|> <|im_start|>识别文本中的语气，并将其分类为喜悦、悲伤、惊异等。\\n文本：“今天是我的生日！”这个文本的语气是喜悦。<|im_end|>'" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.iloc[0]['text']" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'已处理并保存 252 条数据'" ] }, "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def process_and_save_dataset(dataset_list, max_length=512, output_file=\"dataset/pretrain_hq.jsonl\"):\n", " import json\n", " \n", " combined_text = \"\"\n", " result_list = []\n", " \n", " for data in dataset_list:\n", " # 添加特殊Token\n", " processed_text = \"<|im_start|>\" + data.strip() + \"<|im_end|>\"\n", " \n", " # 如果当前文本加上新样本会超过最大长度\n", " if len(combined_text + \" \" + processed_text) > max_length and combined_text:\n", " # 保存当前拼接的文本\n", " result_list.append({\"text\": combined_text+\"<|endoftext|>\"*10})\n", " # 重置拼接文本\n", " combined_text = processed_text\n", " else:\n", " # 添加空格分隔符（如果combined_text不为空）\n", " if combined_text:\n", " combined_text += \" \" + processed_text\n", " else:\n", " combined_text = processed_text\n", " \n", " # 处理最后剩余的文本\n", " if combined_text:\n", " result_list.append({\"text\": combined_text + \"<|endoftext|>\"*10})\n", " \n", " # 以最小IO方式保存到jsonl文件\n", " with open(output_file, 'a', encoding='utf-8') as f:\n", " for item in result_list:\n", " f.write(json.dumps(item, ensure_ascii=False) + '\\n')\n", " \n", " return f\"已处理并保存 {len(result_list)} 条数据\"\n", "\n", "# 使用示例\n", "raw=\"\"\n", "for file in os.listdir():\n", " if file.endswith(\".txt\"):\n", " with open(file, \"r\", encoding=\"utf-8\") as f:\n", " raw += f.read()\n", "\n", "cleaned_raw = raw.replace(\"\"\",\"\").replace(\"\"\",\"\").replace(\"'\",\"\").replace(\"'\",\"\").replace(\"，\",\"\").replace(\"\\n\",\"\")\n", "dataset = cleaned_raw.split(\"。\")\n", "\n", "# 处理并保存数据\n", "process_and_save_dataset(dataset[1:], max_length=512)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## remove" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'已从 dataset/pretrain_hq.jsonl 中删除最后 362 条数据，剩余 1413259 条'" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def remove_last_n_entries(jsonl_file, n=362):\n", " import json\n", " \n", " # 读取所有行\n", " with open(jsonl_file, 'r', encoding='utf-8') as f:\n", " lines = f.readlines()\n", " \n", " # 计算要保留的行数\n", " keep_count = max(0, len(lines) - n)\n", " remaining_lines = lines[:keep_count]\n", " \n", " # 重写文件\n", " with open(jsonl_file, 'w', encoding='utf-8') as f:\n", " f.writelines(remaining_lines)\n", " \n", " return f\"已从 {jsonl_file} 中删除最后 {n} 条数据，剩余 {keep_count} 条\"\n", "\n", "# 使用示例\n", "remove_last_n_entries(\"dataset/pretrain_hq.jsonl\", 362)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 添加结束符" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "已处理 1413511 条数据，修改了 1413511 条数据添加结束标记\n" ] } ], "source": [ "def add_endoftext_to_jsonl(jsonl_file, batch_size=1000):\n", " import json\n", " import os\n", " import tempfile\n", " \n", " # 创建临时文件\n", " temp_file = tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', delete=False)\n", " temp_filename = temp_file.name\n", " \n", " endoftext_str = \"<|endoftext|>\" * 10 # 重复150次\n", " \n", " # 记录处理的行数和修改的行数\n", " processed = 0\n", " modified = 0\n", " \n", " try:\n", " # 逐批次处理文件\n", " with open(jsonl_file, 'r', encoding='utf-8') as in_file:\n", " batch = []\n", " for line in in_file:\n", " processed += 1\n", " try:\n", " data = json.loads(line.strip())\n", " # 检查是否以<|endoftext|>结尾\n", " if \"text\" in data and not data[\"text\"].endswith(\"<|endoftext|>\"):\n", " data[\"text\"] += endoftext_str\n", " modified += 1\n", " batch.append(json.dumps(data, ensure_ascii=False) + '\\n')\n", " \n", " # 批处理写入\n", " if len(batch) >= batch_size:\n", " temp_file.writelines(batch)\n", " batch = []\n", " except json.JSONDecodeError:\n", " # 处理无效JSON的情况\n", " temp_file.write(line)\n", " \n", " # 写入最后一批\n", " if batch:\n", " temp_file.writelines(batch)\n", " \n", " # 关闭临时文件\n", " temp_file.close()\n", " \n", " # 替换原始文件\n", " os.replace(temp_filename, jsonl_file)\n", " \n", " return f\"已处理 {processed} 条数据，修改了 {modified} 条数据添加结束标记\"\n", " \n", " except Exception as e:\n", " # 发生错误时删除临时文件\n", " if os.path.exists(temp_filename):\n", " os.unlink(temp_filename)\n", " return f\"处理出错: {str(e)}\"\n", "\n", "# 使用示例\n", "file_path = \"dataset/pretrain_hq.jsonl\"\n", "result = add_endoftext_to_jsonl(file_path)\n", "print(result)" ] } ], "metadata": { "kernelspec": { "display_name": "minimind", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.0" } }, "nbformat": 4, "nbformat_minor": 2 }