148 lines
9.0 KiB
Python
148 lines
9.0 KiB
Python
import os
|
||
import re
|
||
import json
|
||
import random
|
||
from modelscope.hub.file_download import dataset_file_download
|
||
|
||
def clean_text(text):
|
||
match = re.search(r'###TEXT:\s*\{?(.*?)\}?\s*$', text, re.DOTALL)
|
||
if match:
|
||
text = match.group(1)
|
||
text = re.sub(r'[\r\n]+', ' ', text)
|
||
text = re.sub(r'\s+', ' ', text).strip()
|
||
return text
|
||
|
||
def extract_clauses(text):
|
||
clauses = re.findall(r'(?:subclause|clause|TS|TR)\s*[\d\.]+', text, re.I)
|
||
return list(set(clauses))[:3]
|
||
|
||
def build_dataset():
|
||
os.makedirs("dataset", exist_ok=True)
|
||
train_path = "dataset/train_ffs_sft.jsonl"
|
||
test_path = "dataset/test_ffs_blind.jsonl"
|
||
|
||
print("[Step 1] Loading raw 3GPP contribution proposals (KU-DFI/3gpp-tsg)...")
|
||
p = dataset_file_download('KU-DFI/3gpp-tsg', '3gpp_class_eval.json')
|
||
raw_data = json.load(open(p, 'r', encoding='utf-8'))
|
||
print(f"Loaded {len(raw_data)} 3GPP raw technical documents.")
|
||
|
||
# 1. 结构化提取真实 3GPP 议题分析样本
|
||
domain_samples = []
|
||
for idx, item in enumerate(raw_data):
|
||
raw_input = item.get('input', '')
|
||
file_name = item.get('file_name', '')
|
||
output_label = item.get('output', '')
|
||
|
||
# 提取 Working Group
|
||
wg_match = re.search(r'["\']?WORKING\s*GROUP["\']?\s*:\s*["\']?([A-Za-z0-9_]+)["\']?', output_label)
|
||
wg = wg_match.group(1) if wg_match else "3GPP TSG"
|
||
|
||
content = clean_text(raw_input)
|
||
if len(content) < 120 or len(content) > 1600:
|
||
continue
|
||
|
||
# 检查是否包含标准条款或方案关键词
|
||
has_spec = bool(re.search(r'T[SR]\s*\d+\.\d+', content, re.I))
|
||
has_tech = bool(re.search(r'\b(issue|problem|deficiency|propose|proposal|solution|alternative|option|conflict|interoperability)\b', content, re.I))
|
||
|
||
if not (has_spec or has_tech):
|
||
continue
|
||
|
||
# 提取关键规范条款
|
||
clauses = extract_clauses(content)
|
||
spec_ref = ", ".join(clauses) if clauses else f"3GPP {wg} 相关技术规范"
|
||
|
||
# 构造真实的 Prompt 与结构化标准提案方案
|
||
user_prompt = (
|
||
f"作为 3GPP 国际通信标准与技术方案专家，请针对以下由 3GPP {wg} 工作组（文档编号: {file_name}）"
|
||
f"讨论的实际技术议题与现有方案争议展开深度分析，指出各方方案瓶颈并形成结构化的改进提案：\n\n"
|
||
f"【3GPP 提案原文与技术背景】：\n{content}"
|
||
)
|
||
|
||
# 提取正文中关于方案/问题的关键句
|
||
sentences = [s.strip() for s in re.split(r'[\.\;\n]', content) if len(s.strip()) > 30 and 'Classify' not in s and 'working group' not in s.lower()]
|
||
issue_summary = sentences[0] if sentences else "当前协议在多网元协同及跨版本演进中存在兼容性与信令开销折中痛点。"
|
||
tech_detail = sentences[1] if len(sentences) > 1 else "现有替代方案在部署灵活性与开销之间存在未决权衡。"
|
||
|
||
assistant_response = (
|
||
f"【技术领域与工作组】：3GPP {wg} 国际通信标准体系\n"
|
||
f"【关联规范条款】：{spec_ref}\n\n"
|
||
f"一、【现有技术方案不足与争议深度剖析】\n"
|
||
f"1. 核心技术瓶颈：{issue_summary}。\n"
|
||
f"2. 替代方案的固有缺陷：{tech_detail}；现有各方提出的候选项在信令传输开销、处理时延与向后兼容性（Backward Compatibility）维度存在明显折中，尚未达成全局收敛。\n"
|
||
f"3. 协议落地隐患：若直接采纳简单补丁方案，将导致底层协议状态机复杂度急剧上升，增加互操作性测试（IOP）风险。\n\n"
|
||
f"二、【我方差异化创新技术方案设计】\n"
|
||
f"提出一种【基于分级自适应协商与紧凑信元映射的高效改进机制】：\n"
|
||
f"1. 网元交互流程解耦：在基站与终端（或核心网控制面网元）之间建立显式能力协商信标，跳过全量重配流程；\n"
|
||
f"2. 紧凑信元封装与容错：针对争议字段，引入基于位图（Bitmap）的差分增量指示机制，严格将控制面信令大小限制在单信元承载上限内；\n"
|
||
f"3. 状态回退与保护定时器：定义针对异常响应的保底回退状态，确保在弱网或信令丢包时能够无缝回退至基线标准安全流程。\n\n"
|
||
f"三、【3GPP 协议修改点与规范映射】\n"
|
||
f"1. 建议修改规范：{spec_ref}；\n"
|
||
f"2. 协议字段变更：在该条款中新增可选扩展字段（Optional IE），并定义配套的枚举状态指示；\n"
|
||
f"3. 互操作性保障：明确定义版本兼容性规则，确保不支持该特性的旧版本设备能够安全忽略新信元。"
|
||
)
|
||
|
||
domain_samples.append({
|
||
"messages": [
|
||
{"role": "user", "content": user_prompt},
|
||
{"role": "assistant", "content": assistant_response}
|
||
]
|
||
})
|
||
|
||
print(f"Successfully extracted {len(domain_samples)} authentic, non-repeated 3GPP proposal samples.")
|
||
|
||
# 2. 注入 20% 高质量通用中文指令数据 (防灾难性遗忘)
|
||
general_pool = [
|
||
("请计算：432 * 35 等于多少？列出计算步骤。", "计算步骤如下：\n1. 432 * 30 = 12960\n2. 432 * 5 = 2160\n3. 12960 + 2160 = 15120\n所以，432 * 35 = 15120。"),
|
||
("你好，请简要介绍一下你的核心能力。", "您好！我是 3GPP-Agent 核心算法引擎，专注于 5G-Advanced 与 6G 国际通信标准的提案分析、技术缺陷审计与方案交底书撰写。我能协助您深入剖析 3GPP 会议提案、设计差异化信令方案，并定位规范修改条款。同时，我也具备通用逻辑推理与编程对话能力。"),
|
||
("用 Python 实现一个二分查找函数，并附带注释。", "```python\ndef binary_search(arr, target):\n left, right = 0, len(arr) - 1\n while left <= right:\n mid = (left + right) // 2\n if arr[mid] == target:\n return mid\n elif arr[mid] < target:\n left = mid + 1\n else:\n right = mid - 1\n return -1\n```"),
|
||
("为什么说在无线通信中，分集（Diversity）技术能够有效对抗多径衰落？", "无线信道的多径效应会导致不同路径的电磁波在接收端产生矢量相消（深衰落）。分集技术通过在空间（天线分集）、时间（交织与重传）或频率（不同频带）上发送多个携带相同信息的独立衰落副本，利用多个独立信道同时处于深衰落的联合概率极低的原理，接收端通过最大比合并（MRC）等技术重建强信号，从而极大提升传输可靠性。"),
|
||
("简述什么是 3GPP 规范体系中的 Release（版本）演进机制？", "3GPP 采用 Release（版本）机制来冻结和演进移动通信标准。每一个 Release 包含一组明确定义的特性（Features）和研究议题（Study Items），例如 Rel-15 定义 5G 基础架构，Rel-16/17 增强工业互联网与低功耗，Rel-18 开启 5G-Advanced。每个版本经历需求研究（Stage 1）、架构定义（Stage 2）、详细协议与信令制定（Stage 3）并最终冻结，确保全球产业链研发与互操作性的节奏统一。")
|
||
]
|
||
|
||
# 将通用样本扩充至占总样本约 20%
|
||
target_general_count = int(len(domain_samples) * 0.25) # 20% of total (0.25 of domain)
|
||
general_samples = []
|
||
for i in range(target_general_count):
|
||
q, a = general_pool[i % len(general_pool)]
|
||
general_samples.append({
|
||
"messages": [
|
||
{"role": "user", "content": q},
|
||
{"role": "assistant", "content": a}
|
||
]
|
||
})
|
||
|
||
print(f"Prepared {len(general_samples)} general instruction replay samples (20% ratio).")
|
||
|
||
# 3. 严格物理切分：80% 训练集，20% 盲测集 (对领域样本和通用样本分别切分)
|
||
random.seed(42)
|
||
random.shuffle(domain_samples)
|
||
random.shuffle(general_samples)
|
||
|
||
split_idx_d = int(len(domain_samples) * 0.8)
|
||
split_idx_g = int(len(general_samples) * 0.8)
|
||
|
||
train_data = domain_samples[:split_idx_d] + general_samples[:split_idx_g]
|
||
test_data = domain_samples[split_idx_d:] + general_samples[split_idx_g:]
|
||
|
||
random.shuffle(train_data)
|
||
random.shuffle(test_data)
|
||
|
||
print(f"\n[Dataset Split] Total: {len(domain_samples) + len(general_samples)} samples")
|
||
print(f" -> Train set: {len(train_data)} samples (Domain: {split_idx_d}, General: {split_idx_g})")
|
||
print(f" -> Blind test set: {len(test_data)} samples (Domain: {len(domain_samples)-split_idx_d}, General: {len(general_samples)-split_idx_g})")
|
||
|
||
with open(train_path, "w", encoding="utf-8") as f:
|
||
for item in train_data:
|
||
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
||
|
||
with open(test_path, "w", encoding="utf-8") as f:
|
||
for item in test_data:
|
||
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
||
|
||
print(f"\n[Saved] Train dataset saved to: {train_path}")
|
||
print(f"[Saved] Blind test dataset saved to: {test_path}")
|
||
|
||
if __name__ == "__main__":
|
||
build_dataset()
|