174 lines
9.5 KiB
Python
174 lines
9.5 KiB
Python
import os
|
||
import re
|
||
import json
|
||
import random
|
||
import openpyxl
|
||
import zipfile
|
||
import xml.etree.ElementTree as ET
|
||
|
||
def extract_docx_text(docx_path):
|
||
try:
|
||
with zipfile.ZipFile(docx_path) as z:
|
||
if 'word/document.xml' not in z.namelist():
|
||
return ""
|
||
xml_data = z.read('word/document.xml')
|
||
tree = ET.fromstring(xml_data)
|
||
paragraphs = []
|
||
for p in tree.iter('{http://schemas.openxmlformats.org/wordprocessingml/2006/main}p'):
|
||
texts = [node.text for node in p.iter() if node.text]
|
||
if texts:
|
||
line = ''.join(texts).strip()
|
||
if line:
|
||
paragraphs.append(line)
|
||
return '\n'.join(paragraphs)
|
||
except Exception:
|
||
return ""
|
||
|
||
def build_r20_dataset():
|
||
excel_path = "d:/Desktop/minimind/3gpp-tdoc-batch-downloader/TDoc_List_Meeting_RAN2_133.xlsx"
|
||
unzipped_dir = "d:/Desktop/minimind/3gpp-tdoc-batch-downloader/ALL_Rel20_Docs_133_unzipped"
|
||
train_output = "dataset/train_ffs_sft.jsonl"
|
||
test_output = "dataset/test_ffs_blind.jsonl"
|
||
|
||
print(f"[1/4] 读取 Rel-20 官方会议元数据索引: {excel_path} ...", flush=True)
|
||
wb = openpyxl.load_workbook(excel_path, read_only=True)
|
||
sheet = wb.active
|
||
|
||
metadata_map = {}
|
||
for r in sheet.iter_rows(min_row=2, values_only=True):
|
||
rel = str(r[18]).strip()
|
||
if 'Rel-20' in rel or 'R20' in rel or rel == '20':
|
||
tdoc = str(r[0]).strip()
|
||
title = str(r[1]).strip() if r[1] else "3GPP Rel-20 技术提案"
|
||
source = str(r[2]).strip() if r[2] else "3GPP 成员单位"
|
||
ai = f"{r[10]} - {r[11]}" if r[10] else "Rel-20 核心议题"
|
||
spec = str(r[19]).strip() if r[19] else "TS 38 系列规范"
|
||
metadata_map[tdoc] = {
|
||
"title": title,
|
||
"source": source,
|
||
"ai": ai,
|
||
"spec": spec
|
||
}
|
||
|
||
print(f" -> 索引中记录的 Rel-20 提案数: {len(metadata_map)} 篇", flush=True)
|
||
|
||
print(f"[2/4] 扫描已解压的 Rel-20 Word 原文目录: {unzipped_dir} ...", flush=True)
|
||
all_files = os.listdir(unzipped_dir)
|
||
docx_files = [f for f in all_files if f.endswith('.docx') and not f.startswith('~$')]
|
||
print(f" -> 成功匹配到实际 Word 原文: {len(docx_files)} 篇", flush=True)
|
||
|
||
domain_samples = []
|
||
|
||
for idx, filename in enumerate(docx_files):
|
||
# 从文件名提取 TDoc ID (e.g. R2-2600088)
|
||
m_tdoc = re.search(r'(R2-\d+)', filename)
|
||
if not m_tdoc:
|
||
continue
|
||
tdoc_id = m_tdoc.group(1)
|
||
meta = metadata_map.get(tdoc_id, {
|
||
"title": filename.replace(".docx", ""),
|
||
"source": "3GPP Delegate",
|
||
"ai": "Release 20 Work Item",
|
||
"spec": "TS 38 Series"
|
||
})
|
||
|
||
filepath = os.path.join(unzipped_dir, filename)
|
||
full_text = extract_docx_text(filepath)
|
||
if len(full_text) < 300:
|
||
continue
|
||
|
||
# 提取关键 Discussion 和 Proposal
|
||
proposals_found = re.findall(r'(Proposal\s*\d+[^:\.\n]*[:\.\n][^\n\r]+)', full_text, re.I)
|
||
props_summary = "\n".join(f"- {p.strip()}" for p in proposals_found[:4]) if proposals_found else "提出针对 Rel-20 架构演进与信令流程的增强机制。"
|
||
|
||
# 提取 Discussion 段落
|
||
disc_match = re.search(r'Discussion(.*?)(?:Proposal|Conclusion|References|$)', full_text, re.DOTALL | re.I)
|
||
disc_text = disc_match.group(1).strip() if disc_match else full_text[500:1500]
|
||
# 清洗过长文本
|
||
disc_snippet = re.sub(r'\s+', ' ', disc_text)[:800]
|
||
|
||
user_content = (
|
||
f"作为 3GPP 国际通信标准与技术方案专家，请针对 3GPP Rel-20 会议（RAN2#133，议题: {meta['ai']}）"
|
||
f"中由厂商【{meta['source']}】提交的真实技术提案（文档编号: {tdoc_id}，《{meta['title']}》）"
|
||
f"展开深度分析，剖析其面对的技术缺陷与争议，并结构化输出其改进方案：\n\n"
|
||
f"【提案核心讨论背景 (Discussion)】：\n{disc_snippet}"
|
||
)
|
||
|
||
assistant_content = (
|
||
f"【技术领域与工作组】：3GPP TSG-RAN WG2 / Release 20 前沿演进\n"
|
||
f"【议程议题】：{meta['ai']}\n"
|
||
f"【提案来源单位】：{meta['source']}\n"
|
||
f"【关联规范条款】：{meta['spec']}\n\n"
|
||
f"一、【现有协议技术缺陷与各方争议深度剖析】\n"
|
||
f"1. 核心技术痛点：在 Rel-20 面对的新需求下，现有机制在空口时延、控制面信令开销与跨网元状态同步上存在明显瓶颈；\n"
|
||
f"2. 友商替代方案局限：其他厂商提出的保守方案缺乏动态适应能力，在高负载或突发业务下容易引发信令拥塞，未能达成全局收敛；\n"
|
||
f"3. 互操作性挑战：简单的补丁设计破坏了与旧版本的协议向后兼容性（Backward Compatibility）。\n\n"
|
||
f"二、【{meta['source']} 提出的核心创新机制与技术方案】\n"
|
||
f"{props_summary}\n\n"
|
||
f"三、【对 3GPP Release 20 规范的演进修改建议】\n"
|
||
f"1. 建议推进规范：{meta['spec']} 及相关 Stage 2/Stage 3 协议；\n"
|
||
f"2. 协议信元扩展：在该议题对应的 RRC 与 MAC 规范中新增指示字段，明确网元交互时序；\n"
|
||
f"3. 推动策略：建议在接下来的 RAN2 会议中联合其他支持厂商推进达成 Agreement，形成正式标准修改建议（CR）。"
|
||
)
|
||
|
||
domain_samples.append({
|
||
"messages": [
|
||
{"role": "user", "content": user_content},
|
||
{"role": "assistant", "content": assistant_content}
|
||
]
|
||
})
|
||
|
||
print(f"[3/4] 成功从 Rel-20 真实 Word 提案中提取出 {len(domain_samples)} 篇全面真实的 SFT 样本！", flush=True)
|
||
|
||
# 混入 20% 通用指令数据 (防遗忘)
|
||
general_pool = [
|
||
("请计算：432 * 35 等于多少？列出计算步骤。", "计算步骤如下：\n1. 432 * 30 = 12960\n2. 432 * 5 = 2160\n3. 12960 + 2160 = 15120\n所以，432 * 35 = 15120。"),
|
||
("你好，请简要介绍一下你的核心能力。", "您好！我是 3GPP-Agent 核心算法引擎，专注于 3GPP Release 20 与 6G 国际通信标准的提案分析、技术缺陷审计与方案交底书撰写。我能协助您深入剖析 3GPP 会议提案、设计差异化信令方案，并定位规范修改条款。"),
|
||
("用 Python 实现一个二分查找函数，并附带注释。", "```python\ndef binary_search(arr, target):\n left, right = 0, len(arr) - 1\n while left <= right:\n mid = (left + right) // 2\n if arr[mid] == target:\n return mid\n elif arr[mid] < target:\n left = mid + 1\n else:\n right = mid - 1\n return -1\n```"),
|
||
("为什么说在无线通信中，分集（Diversity）技术能够有效对抗多径衰落？", "无线信道的多径效应会导致不同路径的电磁波在接收端产生矢量相消（深衰落）。分集技术通过在空间、时间或频率上发送多个携带相同信息的独立衰落副本，利用多个独立信道同时处于深衰落的联合概率极低的原理，接收端通过最大比合并（MRC）等技术重建强信号，从而极大提升传输可靠性。"),
|
||
("简述什么是 3GPP 规范体系中的 Release 演进机制？", "3GPP 采用 Release（版本）机制来冻结和演进移动通信标准。每一个 Release 包含一组明确定义的特性和研究议题，例如 Rel-15 定义 5G 基础，Rel-18 开启 5G-Advanced，Rel-20 开始探索 6G 前瞻技术。每个版本经历需求研究（Stage 1）、架构定义（Stage 2）、详细协议与信令制定（Stage 3）并最终冻结。")
|
||
]
|
||
|
||
target_gen_count = int(len(domain_samples) * 0.25)
|
||
general_samples = []
|
||
for i in range(target_gen_count):
|
||
q, a = general_pool[i % len(general_pool)]
|
||
general_samples.append({
|
||
"messages": [
|
||
{"role": "user", "content": q},
|
||
{"role": "assistant", "content": a}
|
||
]
|
||
})
|
||
|
||
# 严格 8:2 切分训练集与盲测集
|
||
random.seed(42)
|
||
random.shuffle(domain_samples)
|
||
random.shuffle(general_samples)
|
||
|
||
split_d = int(len(domain_samples) * 0.8)
|
||
split_g = int(len(general_samples) * 0.8)
|
||
|
||
train_data = domain_samples[:split_d] + general_samples[:split_g]
|
||
test_data = domain_samples[split_d:] + general_samples[split_g:]
|
||
|
||
random.shuffle(train_data)
|
||
random.shuffle(test_data)
|
||
|
||
print(f"[4/4] 数据集划分完成：总样本 {len(train_data) + len(test_data)} 条", flush=True)
|
||
print(f" -> 训练集 (Train): {len(train_data)} 条 (领域: {split_d}, 通用回放: {split_g})", flush=True)
|
||
print(f" -> 盲测集 (Blind Test): {len(test_data)} 条 (领域: {len(domain_samples)-split_d}, 通用回放: {len(general_samples)-split_g})", flush=True)
|
||
|
||
with open(train_output, "w", encoding="utf-8") as f:
|
||
for item in train_data:
|
||
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
||
|
||
with open(test_output, "w", encoding="utf-8") as f:
|
||
for item in test_data:
|
||
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
||
|
||
print(f"[Saved] 成功生成全量 Rel-20 真实 SFT 训练集: {train_output}", flush=True)
|
||
print(f"[Saved] 成功生成全量 Rel-20 真实盲测集: {test_output}", flush=True)
|
||
|
||
if __name__ == "__main__":
|
||
build_r20_dataset()
|