91 lines
2.6 KiB
Python
91 lines
2.6 KiB
Python
import re
|
|
from typing import Dict, Iterable, List, Set
|
|
|
|
import ftfy
|
|
from langdetect import detect, DetectorFactory
|
|
from datasketch import MinHash, MinHashLSH
|
|
|
|
|
|
DetectorFactory.seed = 0
|
|
|
|
|
|
_CONTROL_CHARS = re.compile(r"[\u0000-\u001F\u007F]")
|
|
_MULTI_SPACE = re.compile(r"\s{2,}")
|
|
_URL = re.compile(r"https?://\S+")
|
|
|
|
|
|
def normalize_text(text: str) -> str:
|
|
"""Basic normalization: fix encoding, strip control chars, canonical whitespace."""
|
|
text = ftfy.fix_text(text)
|
|
text = _CONTROL_CHARS.sub(" ", text)
|
|
text = text.replace("\t", " ").replace("\r", " ")
|
|
text = _MULTI_SPACE.sub(" ", text)
|
|
return text.strip()
|
|
|
|
|
|
def is_language(text: str, lang: str = "zh", min_chars: int = 64) -> bool:
|
|
"""Heuristic language detection using langdetect.
|
|
Require minimal length to avoid misclassification.
|
|
"""
|
|
if len(text) < min_chars:
|
|
return False
|
|
try:
|
|
return detect(text) == lang
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def filter_text(text: str) -> bool:
|
|
"""Simple quality filters: drop extremely short, URL-only, code-only content."""
|
|
if len(text) < 64:
|
|
return False
|
|
if len(text) < 256 and _URL.fullmatch(text):
|
|
return False
|
|
# Drop text with excessive symbols/noise
|
|
noise_ratio = sum(ch in "{}[]<>|~^`" for ch in text) / max(1, len(text))
|
|
if noise_ratio > 0.2:
|
|
return False
|
|
return True
|
|
|
|
|
|
def build_minhash(text: str, num_perm: int = 128) -> MinHash:
|
|
mh = MinHash(num_perm=num_perm)
|
|
# Use 5-gram shingles
|
|
for i in range(max(0, len(text) - 4)):
|
|
shingle = text[i : i + 5]
|
|
mh.update(shingle.encode("utf-8", errors="ignore"))
|
|
return mh
|
|
|
|
|
|
class Deduper:
|
|
def __init__(self, threshold: float = 0.85, num_perm: int = 128):
|
|
self.lsh = MinHashLSH(threshold=threshold, num_perm=num_perm)
|
|
self.num_perm = num_perm
|
|
self._keys: Set[str] = set()
|
|
|
|
def is_duplicate(self, key: str, text: str) -> bool:
|
|
mh = build_minhash(text, num_perm=self.num_perm)
|
|
if key in self._keys:
|
|
return True
|
|
if self.lsh.query(mh):
|
|
return True
|
|
self.lsh.insert(key, mh)
|
|
self._keys.add(key)
|
|
return False
|
|
|
|
|
|
def clean_record(rec: Dict, target_lang: str = "zh") -> Dict:
|
|
"""Clean a single record: normalize, filter, language check."""
|
|
text = normalize_text(rec.get("text", ""))
|
|
if not filter_text(text):
|
|
return {}
|
|
# Prefer records that are in target language, but allow mixed content in Wikipedia
|
|
if not is_language(text, target_lang):
|
|
return {}
|
|
cleaned = {
|
|
"source": rec.get("source"),
|
|
"title": rec.get("title"),
|
|
"text": text,
|
|
}
|
|
return cleaned
|