diff --git a/.gitignore b/.gitignore
index de6bd7a..6e9815f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,9 @@
-/model/__pycache__
+.idea/
+
+__pycache__
+
/dataset
-/out
\ No newline at end of file
+/out
+/logs
+
+/MiniMind2
diff --git a/eval_model.py b/eval_model.py
index fc71257..bf2c302 100644
--- a/eval_model.py
+++ b/eval_model.py
@@ -13,17 +13,24 @@ warnings.filterwarnings('ignore')
def init_model(args):
- tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer')
if args.load == 0:
+ tokenizer = AutoTokenizer.from_pretrained(f'./model/{args.vocab}_tokenizer')
moe_path = '_moe' if args.use_moe else ''
modes = {0: 'pretrain', 1: 'full_sft', 2: 'rlhf', 3: 'reason'}
- ckp = f'./{args.out_dir}/{modes[args.model_mode]}_{args.dim}{moe_path}.pth'
+ suffix = f'_{args.model_suffix}' if args.model_suffix else ''
+ ckp = f'./{args.out_dir}/{modes[args.model_mode]}_{args.dim}{moe_path}{suffix}.pth'
model = MiniMindLM(LMConfig(
dim=args.dim,
n_layers=args.n_layers,
+ n_heads=args.n_heads,
+ n_kv_heads=args.n_kv_heads,
max_seq_len=args.max_seq_len,
- use_moe=args.use_moe
+ use_moe=args.use_moe,
+ vocab_size=tokenizer.vocab_size,
+ use_mla=args.use_mla,
+ use_cache=True,
+ torch_dtype=args.dtype,
))
state_dict = torch.load(ckp, map_location=args.device)
@@ -105,8 +112,11 @@ def main():
parser = argparse.ArgumentParser(description="Chat with MiniMind")
parser.add_argument('--lora_name', default='None', type=str)
parser.add_argument('--out_dir', default='out', type=str)
+ parser.add_argument('--model_suffix', default='', type=str)
parser.add_argument('--temperature', default=0.85, type=float)
parser.add_argument('--top_p', default=0.85, type=float)
+ parser.add_argument('--rp', default=1., type=float)
+ parser.add_argument('--dtype', default='bfloat16', type=str)
parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu', type=str)
# 此处max_seq_len（最大允许输入长度）并不意味模型具有对应的长文本的性能，仅防止QA出现被截断的问题
# MiniMind2-moe (145M)：(dim=640, n_layers=8, use_moe=True)
@@ -114,38 +124,50 @@ def main():
# MiniMind2 (104M)：(dim=768, n_layers=16)
parser.add_argument('--dim', default=512, type=int)
parser.add_argument('--n_layers', default=8, type=int)
+ parser.add_argument('--n_heads', default=8, type=int)
+ parser.add_argument('--n_kv_heads', default=2, type=int)
parser.add_argument('--max_seq_len', default=8192, type=int)
parser.add_argument('--use_moe', default=False, type=bool)
+ parser.add_argument('--vocab', default='minimind', type=str)
+ parser.add_argument('--use_mla', default=False, action='store_true')
# 携带历史对话上下文条数
# history_cnt需要设为偶数，即【用户问题, 模型回答】为1组；设置为0时，即当前query不携带历史上文
# 模型未经过外推微调时，在更长的上下文的chat_template时难免出现性能的明显退化，因此需要注意此处设置
parser.add_argument('--history_cnt', default=0, type=int)
parser.add_argument('--stream', default=True, type=bool)
parser.add_argument('--load', default=0, type=int, help="0: 原生torch权重，1: transformers加载")
- parser.add_argument('--model_mode', default=1, type=int,
+ parser.add_argument('--model_mode', default=0, type=int,
help="0: 预训练模型，1: SFT-Chat模型，2: RLHF-Chat模型，3: Reason模型")
+ parser.add_argument('--test_mode', default=0, type=int, choices=[0, 1], help="测试模式；0：自动测试，1：手动测试")
args = parser.parse_args()
+ if args.device == 'cpu':
+ # intel i5 12490f 发现 bfloat16 速度明显慢于 float32
+ args.dtype = 'float32'
model, tokenizer = init_model(args)
prompts = get_prompt_datas(args)
- test_mode = int(input('[0] 自动测试\n[1] 手动输入\n'))
+ test_mode = args.test_mode
messages = []
for idx, prompt in enumerate(prompts if test_mode == 0 else iter(lambda: input('👶: '), '')):
setup_seed(random.randint(0, 2048))
# setup_seed(2025) # 如需固定每次输出则换成【固定】的随机种子
if test_mode == 0: print(f'👶: {prompt}')
- messages = messages[-args.history_cnt:] if args.history_cnt else []
- messages.append({"role": "user", "content": prompt})
+ if args.model_mode == 0:
+ new_prompt = tokenizer.bos_token + prompt
+ else:
+ messages = messages[-args.history_cnt:] if args.history_cnt else []
+ messages.append({"role": "user", "content": prompt})
- new_prompt = tokenizer.apply_chat_template(
- messages,
- tokenize=False,
- add_generation_prompt=True
- )[-args.max_seq_len + 1:] if args.model_mode != 0 else (tokenizer.bos_token + prompt)
+ new_prompt = tokenizer.apply_chat_template(
+ messages,
+ tokenize=False,
+ add_generation_prompt=True
+ )[-args.max_seq_len + 1:]
answer = new_prompt
+
with torch.no_grad():
x = torch.tensor(tokenizer(new_prompt)['input_ids'], device=args.device).unsqueeze(0)
outputs = model.generate(
@@ -154,22 +176,25 @@ def main():
max_new_tokens=args.max_seq_len,
temperature=args.temperature,
top_p=args.top_p,
- stream=True,
+ rp=args.rp,
+ stream=args.stream,
pad_token_id=tokenizer.pad_token_id
)
print('🤖️: ', end='')
try:
- if not args.stream:
- print(tokenizer.decode(outputs.squeeze()[x.shape[1]:].tolist(), skip_special_tokens=True), end='')
- else:
+ if args.stream:
history_idx = 0
for y in outputs:
- answer = tokenizer.decode(y[0].tolist(), skip_special_tokens=True)
+ lis = y[0].tolist()
+ answer = tokenizer.decode(lis, skip_special_tokens=True)
if (answer and answer[-1] == '�') or not answer:
continue
- print(answer[history_idx:], end='', flush=True)
+ # 此处兼容自定义的按字分词表的多余空格
+ print(answer[history_idx + 1 if args.vocab == 'char' and history_idx > 0 else history_idx:], end='', flush=True)
history_idx = len(answer)
+ else:
+ print(tokenizer.decode(outputs.squeeze()[x.shape[1]:].tolist(), skip_special_tokens=True), end='')
except StopIteration:
print("No answer")
print('\n')
diff --git a/mem_utils.py b/mem_utils.py
new file mode 100644
index 0000000..ce64eb2
--- /dev/null
+++ b/mem_utils.py
@@ -0,0 +1,96 @@
+"""
+一些显存调试的工具函数
+"""
+import torch
+
+
+def get_model_memory_usage(model):
+ total_param_memory = 0
+ param_info = {}
+
+ for name, param in model.named_parameters():
+ # 计算单个参数的显存（单位：MB）
+ param_size = param.data.element_size() * param.data.nelement() / 1024 ** 2
+ total_param_memory += param_size
+ param_info[name] = {
+ "size_mb": f"{param_size:.2f} MB",
+ "shape": param.shape,
+ "dtype": param.dtype,
+ }
+
+ param_info["total_param_memory"] = f"{total_param_memory:.2f} MB"
+ return param_info
+
+
+def get_optimizer_memory_usage(optimizer):
+ total_state_memory = 0
+ state_info = {}
+
+ for group in optimizer.param_groups:
+ for p in group["params"]:
+ if p in optimizer.state:
+ state = optimizer.state[p]
+ for key, value in state.items():
+ if isinstance(value, torch.Tensor):
+ # 计算优化器状态张量的显存
+ state_size = value.element_size() * value.nelement() / 1024 ** 2
+ total_state_memory += state_size
+ state_info[f"Param_{id(p)}_{key}"] = {
+ "size_mb": f"{state_size:.2f} MB",
+ "shape": value.shape,
+ "dtype": value.dtype,
+ }
+
+ state_info["total_optimizer_state_memory"] = f"{total_state_memory:.2f} MB"
+ return state_info
+
+
+def log_memory_snapshot(model, optimizer, step):
+ print(f"--- Memory Snapshot (Step {step}) ---")
+
+ # 模型参数显存
+ model_mem = get_model_memory_usage(model)
+ print("Model Parameters:")
+ for name, info in model_mem.items():
+ print(f" {name}: {info}")
+
+ # 优化器状态显存
+ optimizer_mem = get_optimizer_memory_usage(optimizer)
+ print("\nOptimizer State:")
+ for name, info in optimizer_mem.items():
+ print(f" {name}: {info}")
+
+ # 总显存占用（模型 + 优化器）
+ total_memory = float(model_mem["total_param_memory"].split()[0]) + \
+ float(optimizer_mem["total_optimizer_state_memory"].split()[0])
+ print(f"\nTotal Memory: {total_memory:.2f} MB")
+
+
+class ActivationMemoryTracker:
+ def __init__(self):
+ self.activation_info = {}
+ self.hooks = []
+
+ def track_activations(self, model):
+ for name, module in model.named_modules():
+ hook = module.register_forward_hook(self._hook_fn(name))
+ self.hooks.append(hook)
+
+ def _hook_fn(self, name):
+ def hook(_, __, output):
+ # 计算激活值的显存（单位：MB）
+ if isinstance(output, torch.Tensor):
+ mem = output.element_size() * output.nelement() / 1024 ** 2
+ self.activation_info[name] = {
+ "size_mb": f"{mem:.2f} MB",
+ "size": mem,
+ "shape": output.shape,
+ "dtype": output.dtype,
+ }
+ return hook
+
+ def clear(self):
+ for hook in self.hooks:
+ hook.remove()
+ self.activation_info = {}
+ self.hooks = []
diff --git a/model/LMConfig.py b/model/LMConfig.py
index fb0a76d..f150acf 100644
--- a/model/LMConfig.py
+++ b/model/LMConfig.py
@@ -19,6 +19,14 @@ class LMConfig(PretrainedConfig):
rope_theta: int = 1e6,
dropout: float = 0.0,
flash_attn: bool = True,
+ # MLA相关配置
+ use_mla: bool = False,
+ q_lora_rank: int = 720, # query压缩的维度
+ kv_lora_rank: int = 14,
+ qk_nope_head_dim: int = 38,
+ qk_rope_head_dim: int = 32,
+ v_head_dim: int = 64,
+ original_seq_len = 512,
####################################################
# Here are the specific configurations of MOE
# When use_moe is false, the following is invalid
@@ -32,6 +40,7 @@ class LMConfig(PretrainedConfig):
aux_loss_alpha: float = 0.1,
seq_aux: bool = True,
norm_topk_prob: bool = True,
+ use_cache: bool = False,
**kwargs,
):
self.dim = dim
@@ -46,6 +55,17 @@ class LMConfig(PretrainedConfig):
self.rope_theta = rope_theta
self.dropout = dropout
self.flash_attn = flash_attn
+
+ self.use_mla = use_mla
+ self.q_lora_rank = q_lora_rank
+ self.kv_lora_rank = kv_lora_rank
+ self.qk_nope_head_dim = qk_nope_head_dim
+ self.qk_rope_head_dim = qk_rope_head_dim
+ self.v_head_dim = v_head_dim
+ self.original_seq_len = original_seq_len
+ self.mscale: float = 1.
+ self.rope_factor = 20
+ self.max_batch_size = 1
####################################################
# Here are the specific configurations of MOE
# When use_moe is false, the following is invalid
@@ -58,4 +78,5 @@ class LMConfig(PretrainedConfig):
self.aux_loss_alpha = aux_loss_alpha # 辅助损失的alpha参数
self.seq_aux = seq_aux # 是否在序列级别上计算辅助损失
self.norm_topk_prob = norm_topk_prob # 是否标准化top-k概率
+ self.use_cache = use_cache
super().__init__(**kwargs)
diff --git a/model/attention.py b/model/attention.py
new file mode 100644
index 0000000..b1130a9
--- /dev/null
+++ b/model/attention.py
@@ -0,0 +1,191 @@
+import math
+from typing import Optional, Literal, Tuple
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from .LMConfig import LMConfig
+
+
+block_size = 128
+attn_impl: Literal["naive", "absorb"] = "absorb"
+
+
+class Linear(nn.Module):
+ """
+ 这是deepseek的线性层实现，暂时没有使用
+ Custom linear layer with support for quantized weights and optional bias.
+
+ Args:
+ in_features (int): Number of input features.
+ out_features (int): Number of output features.
+ bias (bool): Whether to include a bias term. Defaults to False.
+ dtype (optional): Data type for the layer. Defaults to `torch.bfloat16`.
+ """
+ dtype = torch.bfloat16
+
+ def __init__(self, in_features: int, out_features: int, bias: bool = False, dtype = None):
+ super().__init__()
+ self.in_features = in_features
+ self.out_features = out_features
+ self.weight = nn.Parameter(torch.empty(out_features, in_features, dtype=dtype or Linear.dtype))
+ nn.init.kaiming_normal_(self.weight, mode='fan_out', nonlinearity='linear')
+ self.register_parameter("scale", None)
+ if bias:
+ self.bias = nn.Parameter(torch.empty(out_features))
+ else:
+ self.register_parameter("bias", None)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ """
+ Forward pass for the custom linear layer.
+
+ Args:
+ x (torch.Tensor): Input tensor.
+
+ Returns:
+ torch.Tensor: Transformed tensor after linear computation.
+ """
+ return F.linear(x, self.weight, self.bias)
+
+
+def apply_rotary_emb(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor:
+ """
+ Applies rotary positional embeddings to the input tensor.
+
+ Args:
+ x (torch.Tensor): Input tensor with positional embeddings to be applied.
+ freqs_cis (torch.Tensor): Precomputed complex exponential values for positional embeddings.
+
+ Returns:
+ torch.Tensor: Tensor with rotary embeddings applied.
+ """
+ dtype = x.dtype
+ x = torch.view_as_complex(x.float().view(*x.shape[:-1], -1, 2))
+ freqs_cis = freqs_cis.view(1, x.size(1), 1, x.size(-1))
+ y = torch.view_as_real(x * freqs_cis).flatten(3)
+ return y.to(dtype)
+
+
+class MLA(nn.Module):
+ """
+ Multi-Headed Attention Layer (MLA).
+
+ Attributes:
+ dim (int): Dimensionality of the input features.
+ n_heads (int): Number of attention heads.
+ n_local_heads (int): Number of local attention heads for distributed systems.
+ q_lora_rank (int): Rank for low-rank query projection.
+ kv_lora_rank (int): Rank for low-rank key/value projection.
+ qk_nope_head_dim (int): Dimensionality of non-positional query/key projections.
+ qk_rope_head_dim (int): Dimensionality of rotary-positional query/key projections.
+ qk_head_dim (int): Total dimensionality of query/key projections.
+ v_head_dim (int): Dimensionality of value projections.
+ softmax_scale (float): Scaling factor for softmax in attention computation.
+ """
+ def __init__(self, args: LMConfig):
+ super().__init__()
+ self.dim = args.dim
+ self.n_heads = args.n_heads
+ self.n_local_heads = args.n_heads
+ self.q_lora_rank = args.q_lora_rank
+ self.kv_lora_rank = args.kv_lora_rank
+ self.qk_nope_head_dim = args.qk_nope_head_dim
+ self.qk_rope_head_dim = args.qk_rope_head_dim
+ self.qk_head_dim = args.qk_nope_head_dim + args.qk_rope_head_dim
+ self.v_head_dim = args.v_head_dim
+
+
+ dtype = args.torch_dtype
+ if self.q_lora_rank == 0:
+ self.wq = nn.Linear(self.dim, self.n_heads * self.qk_head_dim, bias=False, dtype=dtype)
+ else:
+ self.wq_a = nn.Linear(self.dim, self.q_lora_rank, bias=False, dtype=dtype)
+ self.q_norm = nn.RMSNorm(self.q_lora_rank, dtype=dtype)
+ self.wq_b = nn.Linear(self.q_lora_rank, self.n_heads * self.qk_head_dim, bias=False, dtype=dtype)
+ self.wkv_a = nn.Linear(self.dim, self.kv_lora_rank + self.qk_rope_head_dim, bias=False, dtype=dtype)
+ self.kv_norm = nn.RMSNorm(self.kv_lora_rank, dtype=dtype)
+ self.wkv_b = nn.Linear(self.kv_lora_rank, self.n_heads * (self.qk_nope_head_dim + self.v_head_dim), bias=False, dtype=dtype)
+ self.wo = nn.Linear(self.n_heads * self.v_head_dim, self.dim, bias=False, dtype=dtype)
+ self.softmax_scale = self.qk_head_dim ** -0.5
+ if args.max_seq_len > args.original_seq_len:
+ mscale = 0.1 * args.mscale * math.log(args.rope_factor) + 1.0
+ self.softmax_scale = self.softmax_scale * mscale * mscale
+
+ self.use_cache = args.use_cache
+ if self.use_cache:
+ if attn_impl == "naive":
+ self.register_buffer("k_cache", torch.zeros(args.max_batch_size, args.max_seq_len, self.n_local_heads, self.qk_head_dim, dtype=dtype), persistent=False)
+ self.register_buffer("v_cache", torch.zeros(args.max_batch_size, args.max_seq_len, self.n_local_heads, self.v_head_dim, dtype=dtype), persistent=False)
+ else:
+ self.register_buffer("kv_cache", torch.zeros(args.max_batch_size, args.max_seq_len, self.kv_lora_rank, dtype=dtype), persistent=False)
+ self.register_buffer("pe_cache", torch.zeros(args.max_batch_size, args.max_seq_len, self.qk_rope_head_dim, dtype=dtype), persistent=False)
+
+ def forward(self,
+ x: torch.Tensor,
+ start_pos: int,
+ freqs_cis: torch.Tensor,
+ past_key_value: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+ mask: Optional[torch.Tensor] = None):
+ """
+ Forward pass for the Multi-Headed Attention Layer (MLA).
+
+ Args:
+ x (torch.Tensor): Input tensor of shape (batch_size, seq_len, dim).
+ start_pos (int): Starting position in the sequence for caching.
+ freqs_cis (torch.Tensor): Precomputed complex exponential values for rotary embeddings.
+ mask (Optional[torch.Tensor]): Mask tensor to exclude certain positions from attention.
+
+ Returns:
+ torch.Tensor: Output tensor with the same shape as the input.
+ """
+ bsz, seqlen, _ = x.size()
+ end_pos = start_pos + seqlen
+ if self.q_lora_rank == 0:
+ q = self.wq(x)
+ else:
+ q = self.wq_b(self.q_norm(self.wq_a(x)))
+ q = q.view(bsz, seqlen, self.n_local_heads, self.qk_head_dim)
+ q_nope, q_pe = torch.split(q, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1)
+ q_pe = apply_rotary_emb(q_pe, freqs_cis)
+ kv = self.wkv_a(x)
+ kv, k_pe = torch.split(kv, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
+ k_pe = apply_rotary_emb(k_pe.unsqueeze(2), freqs_cis)
+ v = None
+ if attn_impl == "naive":
+ q = torch.cat([q_nope, q_pe], dim=-1)
+ kv = self.wkv_b(self.kv_norm(kv))
+ kv = kv.view(bsz, seqlen, self.n_local_heads, self.qk_nope_head_dim + self.v_head_dim)
+ k_nope, v = torch.split(kv, [self.qk_nope_head_dim, self.v_head_dim], dim=-1)
+ k = torch.cat([k_nope, k_pe.expand(-1, -1, self.n_local_heads, -1)], dim=-1)
+ if self.use_cache:
+ self.k_cache[:bsz, start_pos:end_pos] = k
+ self.v_cache[:bsz, start_pos:end_pos] = v
+ k = self.k_cache[:bsz, :end_pos]
+ v = self.k_cache[:bsz, :end_pos]
+ scores = torch.einsum("bshd,bthd->bsht", q, k) * self.softmax_scale
+ else:
+ # wkv_b = self.wkv_b.weight if self.wkv_b.scale is None else weight_dequant(self.wkv_b.weight, self.wkv_b.scale, block_size)
+ wkv_b = self.wkv_b.weight
+ wkv_b = wkv_b.view(self.n_local_heads, -1, self.kv_lora_rank)
+ q_nope = torch.einsum("bshd,hdc->bshc", q_nope, wkv_b[:, :self.qk_nope_head_dim])
+ kv = self.kv_norm(kv)
+ k_pe = k_pe.squeeze(2)
+ if self.use_cache:
+ self.kv_cache[:bsz, start_pos:end_pos] = kv
+ self.pe_cache[:bsz, start_pos:end_pos] = k_pe
+ kv = self.kv_cache[:bsz, :end_pos]
+ k_pe = self.pe_cache[:bsz, :end_pos]
+ scores = (torch.einsum("bshc,btc->bsht", q_nope, kv) +
+ torch.einsum("bshr,btr->bsht", q_pe, k_pe)) * self.softmax_scale
+ if mask is not None:
+ scores += mask.unsqueeze(1)
+ scores = scores.softmax(dim=-1, dtype=torch.float32).type_as(x)
+ if attn_impl == "naive":
+ x = torch.einsum("bsht,bthd->bshd", scores, v)
+ else:
+ x = torch.einsum("bsht,btc->bshc", scores, kv)
+ x = torch.einsum("bshc,hdc->bshd", x, wkv_b[:, -self.v_head_dim:])
+ x = self.wo(x.flatten(2))
+ return x, None
diff --git a/model/char_tokenizer/special_tokens_map.json b/model/char_tokenizer/special_tokens_map.json
new file mode 100644
index 0000000..9bfed75
--- /dev/null
+++ b/model/char_tokenizer/special_tokens_map.json
@@ -0,0 +1,6 @@
+{
+ "bos_token": "",
+ "eos_token": "",
+ "pad_token": "",
+ "unk_token": ""
+}
diff --git a/model/char_tokenizer/tokenizer.json b/model/char_tokenizer/tokenizer.json
new file mode 100644
index 0000000..9a2f79e
--- /dev/null
+++ b/model/char_tokenizer/tokenizer.json
@@ -0,0 +1,6451 @@
+{
+ "version": "1.0",
+ "truncation": null,
+ "padding": null,
+ "added_tokens": [
+ {
+ "id": 0,
+ "content": "",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 1,
+ "content": "",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 2,
+ "content": "",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ }
+ ],
+ "normalizer": null,
+ "pre_tokenizer": {
+ "type": "Split",
+ "pattern": {
+ "String": ""
+ },
+ "behavior": "Isolated",
+ "invert": false
+ },
+ "post_processor": null,
+ "decoder": null,
+ "model": {
+ "type": "WordLevel",
+ "vocab": {
+ "": 0,
+ "": 1,
+ "": 2,
+ "，": 3,
+ "的": 4,
+ "。": 5,
+ "一": 6,
+ "\n": 7,
+ "是": 8,
+ "了": 9,
+ "s": 10,
+ ">": 11,
+ "<": 12,
+ "我": 13,
+ "和": 14,
+ "在": 15,
+ "不": 16,
+ "个": 17,
+ "人": 18,
+ "这": 19,
+ "有": 20,
+ " ": 21,
+ "以": 22,
+ "：": 23,
+ "、": 24,
+ "来": 25,
+ "可": 26,
+ "/": 27,
+ "中": 28,
+ "…": 29,
+ "要": 30,
+ "能": 31,
+ "你": 32,
+ "们": 33,
+ "上": 34,
+ "下": 35,
+ "他": 36,
+ "到": 37,
+ "为": 38,
+ "着": 39,
+ "小": 40,
+ "出": 41,
+ "子": 42,
+ "用": 43,
+ "文": 44,
+ "大": 45,
+ "地": 46,
+ "生": 47,
+ "时": 48,
+ "好": 49,
+ "自": 50,
+ "她": 51,
+ "？": 52,
+ "「": 53,
+ "对": 54,
+ "」": 55,
+ "也": 56,
+ "会": 57,
+ "如": 58,
+ "成": 59,
+ "道": 60,
+ "过": 61,
+ "么": 62,
+ "然": 63,
+ "动": 64,
+ "得": 65,
+ "于": 66,
+ "那": 67,
+ "说": 68,
+ "就": 69,
+ "天": 70,
+ "行": 71,
+ "多": 72,
+ "面": 73,
+ "发": 74,
+ "学": 75,
+ "些": 76,
+ "提": 77,
+ "方": 78,
+ "更": 79,
+ "里": 80,
+ "分": 81,
+ ".": 82,
+ "本": 83,
+ "后": 84,
+ "看": 85,
+ "感": 86,
+ "！": 87,
+ "身": 88,
+ "心": 89,
+ "还": 90,
+ "等": 91,
+ "现": 92,
+ "体": 93,
+ "进": 94,
+ "起": 95,
+ "题": 96,
+ "情": 97,
+ "作": 98,
+ "力": 99,
+ "”": 100,
+ "“": 101,
+ "将": 102,
+ "去": 103,
+ "手": 104,
+ "之": 105,
+ "而": 106,
+ "给": 107,
+ "都": 108,
+ "想": 109,
+ "种": 110,
+ "并": 111,
+ "最": 112,
+ "关": 113,
+ "物": 114,
+ "家": 115,
+ "定": 116,
+ "高": 117,
+ "开": 118,
+ "其": 119,
+ "问": 120,
+ "明": 121,
+ "果": 122,
+ "工": 123,
+ "法": 124,
+ "没": 125,
+ "经": 126,
+ "点": 127,
+ "同": 128,
+ "己": 129,
+ "意": 130,
+ "美": 131,
+ "性": 132,
+ "间": 133,
+ "很": 134,
+ "应": 135,
+ "加": 136,
+ "因": 137,
+ "理": 138,
+ "主": 139,
+ "气": 140,
+ "让": 141,
+ "重": 142,
+ "但": 143,
+ "它": 144,
+ "使": 145,
+ "常": 146,
+ "被": 147,
+ "新": 148,
+ "1": 149,
+ "事": 150,
+ "数": 151,
+ "此": 152,
+ "女": 153,
+ "只": 154,
+ "机": 155,
+ "据": 156,
+ "需": 157,
+ "色": 158,
+ "类": 159,
+ "化": 160,
+ "样": 161,
+ "从": 162,
+ "什": 163,
+ "保": 164,
+ "语": 165,
+ "或": 166,
+ "影": 167,
+ "头": 168,
+ "句": 169,
+ "所": 170,
+ "公": 171,
+ "2": 172,
+ "实": 173,
+ "电": 174,
+ "前": 175,
+ "通": 176,
+ "两": 177,
+ "技": 178,
+ "无": 179,
+ "解": 180,
+ "年": 181,
+ "当": 182,
+ "表": 183,
+ "度": 184,
+ "话": 185,
+ "水": 186,
+ "正": 187,
+ "0": 188,
+ "知": 189,
+ "品": 190,
+ "智": 191,
+ "全": 192,
+ "入": 193,
+ "章": 194,
+ "已": 195,
+ "内": 196,
+ "回": 197,
+ "国": 198,
+ "请": 199,
+ "您": 200,
+ "量": 201,
+ "长": 202,
+ "外": 203,
+ "合": 204,
+ "相": 205,
+ "名": 206,
+ "该": 207,
+ "啊": 208,
+ "比": 209,
+ "与": 210,
+ "信": 211,
+ "者": 212,
+ "根": 213,
+ "少": 214,
+ "术": 215,
+ "活": 216,
+ "把": 217,
+ "变": 218,
+ "取": 219,
+ "声": 220,
+ "习": 221,
+ "计": 222,
+ "受": 223,
+ "词": 224,
+ "业": 225,
+ "包": 226,
+ "口": 227,
+ "快": 228,
+ "非": 229,
+ "部": 230,
+ "处": 231,
+ "特": 232,
+ "又": 233,
+ "别": 234,
+ "息": 235,
+ "何": 236,
+ "供": 237,
+ "欢": 238,
+ "轻": 239,
+ "觉": 240,
+ "真": 241,
+ "建": 242,
+ "次": 243,
+ "球": 244,
+ "老": 245,
+ "段": 246,
+ "放": 247,
+ "儿": 248,
+ "式": 249,
+ "目": 250,
+ "系": 251,
+ "展": 252,
+ "环": 253,
+ "三": 254,
+ "食": 255,
+ "制": 256,
+ "算": 257,
+ "眼": 258,
+ "3": 259,
+ "帮": 260,
+ "确": 261,
+ "结": 262,
+ "喜": 263,
+ "白": 264,
+ "几": 265,
+ "具": 266,
+ "程": 267,
+ "做": 268,
+ "每": 269,
+ "持": 270,
+ "服": 271,
+ "向": 272,
+ "述": 273,
+ "再": 274,
+ "日": 275,
+ "红": 276,
+ "位": 277,
+ "风": 278,
+ "写": 279,
+ "住": 280,
+ "容": 281,
+ "及": 282,
+ "助": 283,
+ "强": 284,
+ "市": 285,
+ "爱": 286,
+ "场": 287,
+ "见": 288,
+ "太": 289,
+ "交": 290,
+ "笑": 291,
+ "肉": 292,
+ "深": 293,
+ "产": 294,
+ "光": 295,
+ "平": 296,
+ "带": 297,
+ "边": 298,
+ "指": 299,
+ "才": 300,
+ "任": 301,
+ "吗": 302,
+ "满": 303,
+ "务": 304,
+ "答": 305,
+ "花": 306,
+ "男": 307,
+ "篇": 308,
+ "由": 309,
+ "流": 310,
+ "5": 311,
+ "张": 312,
+ "完": 313,
+ "总": 314,
+ "直": 315,
+ "接": 316,
+ "适": 317,
+ "神": 318,
+ "求": 319,
+ "安": 320,
+ "器": 321,
+ "利": 322,
+ "打": 323,
+ "海": 324,
+ "领": 325,
+ "响": 326,
+ "代": 327,
+ "候": 328,
+ "健": 329,
+ "选": 330,
+ "件": 331,
+ "听": 332,
+ "医": 333,
+ "科": 334,
+ "言": 335,
+ "\"": 336,
+ "断": 337,
+ "极": 338,
+ "推": 339,
+ "便": 340,
+ "紧": 341,
+ "括": 342,
+ "音": 343,
+ "清": 344,
+ "原": 345,
+ "质": 346,
+ "标": 347,
+ "乐": 348,
+ "思": 349,
+ "像": 350,
+ "型": 351,
+ "单": 352,
+ "-": 353,
+ "未": 354,
+ "步": 355,
+ "字": 356,
+ "司": 357,
+ "游": 358,
+ "议": 359,
+ "车": 360,
+ "决": 361,
+ "境": 362,
+ "够": 363,
+ "论": 364,
+ "门": 365,
+ "找": 366,
+ "识": 367,
+ "案": 368,
+ "妈": 369,
+ "走": 370,
+ "简": 371,
+ "空": 372,
+ "越": 373,
+ "始": 374,
+ "观": 375,
+ "吧": 376,
+ "先": 377,
+ "怎": 378,
+ "改": 379,
+ "例": 380,
+ "注": 381,
+ "双": 382,
+ "达": 383,
+ "形": 384,
+ "社": 385,
+ "传": 386,
+ "友": 387,
+ "期": 388,
+ "效": 389,
+ "精": 390,
+ "世": 391,
+ "设": 392,
+ "难": 393,
+ "落": 394,
+ "今": 395,
+ "星": 396,
+ "且": 397,
+ "统": 398,
+ "姐": 399,
+ "域": 400,
+ "描": 401,
+ "节": 402,
+ "认": 403,
+ "创": 404,
+ "却": 405,
+ "景": 406,
+ "n": 407,
+ "热": 408,
+ "脸": 409,
+ "近": 410,
+ "月": 411,
+ "望": 412,
+ "康": 413,
+ "温": 414,
+ "许": 415,
+ "4": 416,
+ "模": 417,
+ "呢": 418,
+ "视": 419,
+ "味": 420,
+ "立": 421,
+ "反": 422,
+ "城": 423,
+ "吃": 424,
+ "运": 425,
+ "二": 426,
+ "第": 427,
+ "试": 428,
+ "十": 429,
+ "管": 430,
+ "减": 431,
+ "阳": 432,
+ "李": 433,
+ "随": 434,
+ "细": 435,
+ "户": 436,
+ "员": 437,
+ "准": 438,
+ "各": 439,
+ "告": 440,
+ "金": 441,
+ "西": 442,
+ "功": 443,
+ "调": 444,
+ "哪": 445,
+ "整": 446,
+ "导": 447,
+ "格": 448,
+ "教": 449,
+ "护": 450,
+ "料": 451,
+ "足": 452,
+ "房": 453,
+ "错": 454,
+ "疗": 455,
+ "丽": 456,
+ "干": 457,
+ "续": 458,
+ "界": 459,
+ "义": 460,
+ "示": 461,
+ "微": 462,
+ "战": 463,
+ "山": 464,
+ "书": 465,
+ "；": 466,
+ "客": 467,
+ ",": 468,
+ "叫": 469,
+ "历": 470,
+ "基": 471,
+ "态": 472,
+ "研": 473,
+ "A": 474,
+ "备": 475,
+ "吸": 476,
+ "刚": 477,
+ "巴": 478,
+ "师": 479,
+ "源": 480,
+ "短": 481,
+ "网": 482,
+ "造": 483,
+ "条": 484,
+ "首": 485,
+ "尽": 486,
+ "玉": 487,
+ "至": 488,
+ "香": 489,
+ "亲": 490,
+ "衣": 491,
+ "》": 492,
+ "《": 493,
+ "键": 494,
+ "组": 495,
+ "区": 496,
+ "择": 497,
+ "病": 498,
+ "似": 499,
+ "究": 500,
+ "象": 501,
+ "引": 502,
+ "转": 503,
+ "考": 504,
+ "黑": 505,
+ "则": 506,
+ "）": 507,
+ "连": 508,
+ "素": 509,
+ "价": 510,
+ "含": 511,
+ "王": 512,
+ "死": 513,
+ "故": 514,
+ "马": 515,
+ "路": 516,
+ "验": 517,
+ "旅": 518,
+ "片": 519,
+ "林": 520,
+ ":": 521,
+ "东": 522,
+ "兴": 523,
+ "优": 524,
+ "商": 525,
+ "命": 526,
+ "跟": 527,
+ "餐": 528,
+ "项": 529,
+ "速": 530,
+ "周": 531,
+ "支": 532,
+ "角": 533,
+ "火": 534,
+ "广": 535,
+ "致": 536,
+ "嘴": 537,
+ "预": 538,
+ "增": 539,
+ "约": 540,
+ "收": 541,
+ "拉": 542,
+ "复": 543,
+ "元": 544,
+ "（": 545,
+ "孩": 546,
+ "谢": 547,
+ "离": 548,
+ "乳": 549,
+ "飞": 550,
+ "资": 551,
+ "况": 552,
+ "失": 553,
+ "低": 554,
+ "朋": 555,
+ "早": 556,
+ "克": 557,
+ "阴": 558,
+ "狗": 559,
+ "存": 560,
+ "线": 561,
+ "玩": 562,
+ "四": 563,
+ "慢": 564,
+ "即": 565,
+ "斯": 566,
+ "龙": 567,
+ "易": 568,
+ "晚": 569,
+ "虽": 570,
+ "探": 571,
+ "腿": 572,
+ "评": 573,
+ "激": 574,
+ "报": 575,
+ "列": 576,
+ "测": 577,
+ "记": 578,
+ "棒": 579,
+ "鸡": 580,
+ "读": 581,
+ "压": 582,
+ "往": 583,
+ "编": 584,
+ "消": 585,
+ "菜": 586,
+ "排": 587,
+ "否": 588,
+ "般": 589,
+ "雪": 590,
+ "糖": 591,
+ "共": 592,
+ "份": 593,
+ "刻": 594,
+ "—": 595,
+ "终": 596,
+ "股": 597,
+ "娇": 598,
+ "充": 599,
+ "积": 600,
+ "摘": 601,
+ "状": 602,
+ "穿": 603,
+ "哥": 604,
+ "远": 605,
+ "率": 606,
+ "划": 607,
+ "显": 608,
+ "势": 609,
+ "软": 610,
+ "险": 611,
+ "析": 612,
+ "史": 613,
+ "母": 614,
+ "趣": 615,
+ "松": 616,
+ "宝": 617,
+ "\\": 618,
+ "享": 619,
+ "采": 620,
+ "淫": 621,
+ "插": 622,
+ "柔": 623,
+ "演": 624,
+ "修": 625,
+ "华": 626,
+ "布": 627,
+ "丝": 628,
+ "买": 629,
+ "舒": 630,
+ "规": 631,
+ "8": 632,
+ "配": 633,
+ "联": 634,
+ "较": 635,
+ "抽": 636,
+ "参": 637,
+ "练": 638,
+ "装": 639,
+ "·": 640,
+ "图": 641,
+ "输": 642,
+ "突": 643,
+ "施": 644,
+ "饮": 645,
+ "诉": 646,
+ "静": 647,
+ "五": 648,
+ "害": 649,
+ "古": 650,
+ "必": 651,
+ "尔": 652,
+ "诗": 653,
+ "查": 654,
+ "索": 655,
+ "闻": 656,
+ "队": 657,
+ "值": 658,
+ "独": 659,
+ "介": 660,
+ "士": 661,
+ "乎": 662,
+ "英": 663,
+ "o": 664,
+ "虑": 665,
+ "除": 666,
+ "半": 667,
+ "呼": 668,
+ "治": 669,
+ "获": 670,
+ "阿": 671,
+ "歌": 672,
+ "另": 673,
+ "坐": 674,
+ "e": 675,
+ "令": 676,
+ "云": 677,
+ "黄": 678,
+ "雨": 679,
+ "站": 680,
+ "维": 681,
+ "穴": 682,
+ "6": 683,
+ "青": 684,
+ "挑": 685,
+ "惊": 686,
+ "a": 687,
+ "著": 688,
+ "酒": 689,
+ "露": 690,
+ "照": 691,
+ "政": 692,
+ "构": 693,
+ "围": 694,
+ "称": 695,
+ "灵": 696,
+ "南": 697,
+ "哈": 698,
+ "脑": 699,
+ "万": 700,
+ "尝": 701,
+ "睡": 702,
+ "室": 703,
+ "I": 704,
+ "释": 705,
+ "猫": 706,
+ "阵": 707,
+ "蛋": 708,
+ "床": 709,
+ "9": 710,
+ "材": 711,
+ "希": 712,
+ "奇": 713,
+ "脚": 714,
+ "园": 715,
+ "停": 716,
+ "夏": 717,
+ "免": 718,
+ "抱": 719,
+ "育": 720,
+ "密": 721,
+ "弄": 722,
+ "巧": 723,
+ "叶": 724,
+ "集": 725,
+ "夫": 726,
+ "背": 727,
+ "待": 728,
+ "控": 729,
+ "奶": 730,
+ "互": 731,
+ "射": 732,
+ "款": 733,
+ "号": 734,
+ "液": 735,
+ "戏": 736,
+ "春": 737,
+ "嗯": 738,
+ "丰": 739,
+ "概": 740,
+ "洁": 741,
+ "序": 742,
+ "费": 743,
+ "讨": 744,
+ "切": 745,
+ "缓": 746,
+ "仅": 747,
+ "富": 748,
+ "绝": 749,
+ "倒": 750,
+ "升": 751,
+ "策": 752,
+ "证": 753,
+ "底": 754,
+ "击": 755,
+ "办": 756,
+ "营": 757,
+ "民": 758,
+ "德": 759,
+ "北": 760,
+ "夜": 761,
+ "讲": 762,
+ "雄": 763,
+ "专": 764,
+ "苹": 765,
+ "众": 766,
+ "购": 767,
+ "牛": 768,
+ "欲": 769,
+ "误": 770,
+ "竟": 771,
+ "痛": 772,
+ "寻": 773,
+ "绍": 774,
+ "冷": 775,
+ "烈": 776,
+ "亮": 777,
+ "居": 778,
+ "台": 779,
+ "团": 780,
+ "置": 781,
+ "养": 782,
+ "浪": 783,
+ "换": 784,
+ "念": 785,
+ "负": 786,
+ "～": 787,
+ "按": 788,
+ "超": 789,
+ "厅": 790,
+ "米": 791,
+ "留": 792,
+ "百": 793,
+ "赛": 794,
+ "拿": 795,
+ "冰": 796,
+ "某": 797,
+ "荐": 798,
+ "渐": 799,
+ "i": 800,
+ "销": 801,
+ "际": 802,
+ "石": 803,
+ "翻": 804,
+ "防": 805,
+ "急": 806,
+ "武": 807,
+ "亚": 808,
+ "妹": 809,
+ "季": 810,
+ "染": 811,
+ "陈": 812,
+ "7": 813,
+ "拥": 814,
+ "怕": 815,
+ "送": 816,
+ "散": 817,
+ "血": 818,
+ "泛": 819,
+ "军": 820,
+ "皮": 821,
+ "裤": 822,
+ "善": 823,
+ "魔": 824,
+ "梦": 825,
+ "艺": 826,
+ "嫩": 827,
+ "依": 828,
+ "伤": 829,
+ "滑": 830,
+ "画": 831,
+ "刺": 832,
+ "久": 833,
+ "破": 834,
+ "胸": 835,
+ "哦": 836,
+ "巨": 837,
+ "药": 838,
+ "暖": 839,
+ "冲": 840,
+ "忙": 841,
+ "律": 842,
+ "沙": 843,
+ "异": 844,
+ "草": 845,
+ "宫": 846,
+ "兰": 847,
+ "挺": 848,
+ "暗": 849,
+ "唇": 850,
+ "避": 851,
+ "跑": 852,
+ "粉": 853,
+ "职": 854,
+ "措": 855,
+ "迷": 856,
+ "继": 857,
+ "坚": 858,
+ "树": 859,
+ "降": 860,
+ "摸": 861,
+ "举": 862,
+ "店": 863,
+ "谁": 864,
+ "限": 865,
+ "属": 866,
+ "伸": 867,
+ "钱": 868,
+ "训": 869,
+ "京": 870,
+ "剧": 871,
+ "融": 872,
+ "潮": 873,
+ "投": 874,
+ "娘": 875,
+ "秋": 876,
+ "块": 877,
+ "层": 878,
+ "怀": 879,
+ "顶": 880,
+ "院": 881,
+ "弟": 882,
+ "征": 883,
+ "喝": 884,
+ "止": 885,
+ "严": 886,
+ "差": 887,
+ "顺": 888,
+ "屁": 889,
+ "毛": 890,
+ "忍": 891,
+ "腰": 892,
+ "呀": 893,
+ "羞": 894,
+ "追": 895,
+ "级": 896,
+ "熟": 897,
+ "鱼": 898,
+ "乱": 899,
+ "争": 900,
+ "钟": 901,
+ "沉": 902,
+ "若": 903,
+ "假": 904,
+ "济": 905,
+ "福": 906,
+ "油": 907,
+ "隐": 908,
+ "脱": 909,
+ "舌": 910,
+ "掌": 911,
+ "波": 912,
+ "遇": 913,
+ "蓝": 914,
+ "湿": 915,
+ "辑": 916,
+ "族": 917,
+ "雅": 918,
+ "父": 919,
+ "仍": 920,
+ "庭": 921,
+ "鲜": 922,
+ "曲": 923,
+ "污": 924,
+ "绿": 925,
+ "木": 926,
+ "颜": 927,
+ "宇": 928,
+ "杯": 929,
+ "罗": 930,
+ "良": 931,
+ "咖": 932,
+ "迎": 933,
+ "企": 934,
+ "r": 935,
+ "坏": 936,
+ "睛": 937,
+ "络": 938,
+ "妇": 939,
+ "敢": 940,
+ "透": 941,
+ "卡": 942,
+ "威": 943,
+ "啡": 944,
+ "牌": 945,
+ ";": 946,
+ "府": 947,
+ "P": 948,
+ "愿": 949,
+ "跳": 950,
+ "秘": 951,
+ "江": 952,
+ "疑": 953,
+ "范": 954,
+ "努": 955,
+ "B": 956,
+ "秀": 957,
+ "校": 958,
+ "官": 959,
+ "齐": 960,
+ "硬": 961,
+ "苦": 962,
+ "洞": 963,
+ "洗": 964,
+ "杂": 965,
+ "疾": 966,
+ "野": 967,
+ "佳": 968,
+ "千": 969,
+ "怪": 970,
+ "峰": 971,
+ "普": 972,
+ "顿": 973,
+ "饭": 974,
+ "顾": 975,
+ "板": 976,
+ "警": 977,
+ "右": 978,
+ "左": 979,
+ "植": 980,
+ "织": 981,
+ "均": 982,
+ "握": 983,
+ "奋": 984,
+ "圆": 985,
+ "套": 986,
+ "略": 987,
+ "临": 988,
+ "爸": 989,
+ "摇": 990,
+ "颗": 991,
+ "阅": 992,
+ "休": 993,
+ "楚": 994,
+ "t": 995,
+ "礼": 996,
+ "检": 997,
+ "馆": 998,
+ "卫": 999,
+ "醒": 1000,
+ "洋": 1001,
+ "群": 1002,
+ "初": 1003,
+ "笔": 1004,
+ "爽": 1005,
+ "益": 1006,
+ "狂": 1007,
+ "符": 1008,
+ "归": 1009,
+ "吻": 1010,
+ "铁": 1011,
+ "幸": 1012,
+ "权": 1013,
+ "判": 1014,
+ "彩": 1015,
+ "端": 1016,
+ "杨": 1017,
+ "杀": 1018,
+ "耳": 1019,
+ "朝": 1020,
+ "岁": 1021,
+ "鸟": 1022,
+ "促": 1023,
+ "掉": 1024,
+ "座": 1025,
+ "靠": 1026,
+ "楼": 1027,
+ "润": 1028,
+ "抓": 1029,
+ "责": 1030,
+ "甚": 1031,
+ "须": 1032,
+ "蜜": 1033,
+ "担": 1034,
+ "替": 1035,
+ "贵": 1036,
+ "胜": 1037,
+ "缺": 1038,
+ "暴": 1039,
+ "l": 1040,
+ "舞": 1041,
+ "昨": 1042,
+ "志": 1043,
+ "操": 1044,
+ "媚": 1045,
+ "典": 1046,
+ "恶": 1047,
+ "臀": 1048,
+ "困": 1049,
+ "诊": 1050,
+ "~": 1051,
+ "欣": 1052,
+ "逐": 1053,
+ "粗": 1054,
+ "T": 1055,
+ "猛": 1056,
+ "尼": 1057,
+ "佛": 1058,
+ "禁": 1059,
+ "烤": 1060,
+ "狠": 1061,
+ "艳": 1062,
+ "汽": 1063,
+ "吟": 1064,
+ "售": 1065,
+ "赵": 1066,
+ "详": 1067,
+ "聊": 1068,
+ "莉": 1069,
+ "盖": 1070,
+ "监": 1071,
+ "尚": 1072,
+ "患": 1073,
+ "余": 1074,
+ "移": 1075,
+ "剩": 1076,
+ "救": 1077,
+ "炼": 1078,
+ "纪": 1079,
+ "虚": 1080,
+ "麻": 1081,
+ "印": 1082,
+ "忘": 1083,
+ "C": 1084,
+ "私": 1085,
+ "码": 1086,
+ "舔": 1087,
+ "旁": 1088,
+ "摄": 1089,
+ "苏": 1090,
+ "娜": 1091,
+ "拍": 1092,
+ "土": 1093,
+ "毒": 1094,
+ "察": 1095,
+ "荡": 1096,
+ "浓": 1097,
+ "曾": 1098,
+ "局": 1099,
+ "颤": 1100,
+ "八": 1101,
+ "甜": 1102,
+ "挥": 1103,
+ "混": 1104,
+ "伦": 1105,
+ "贝": 1106,
+ "午": 1107,
+ "漫": 1108,
+ "河": 1109,
+ "瑶": 1110,
+ "博": 1111,
+ "敏": 1112,
+ "触": 1113,
+ "银": 1114,
+ "抚": 1115,
+ "尊": 1116,
+ "呵": 1117,
+ "肥": 1118,
+ "淡": 1119,
+ "抗": 1120,
+ "媒": 1121,
+ "肯": 1122,
+ "摆": 1123,
+ "妙": 1124,
+ "班": 1125,
+ "肤": 1126,
+ "寒": 1127,
+ "剑": 1128,
+ "鼓": 1129,
+ "恐": 1130,
+ "皇": 1131,
+ "搭": 1132,
+ "闪": 1133,
+ "付": 1134,
+ "袋": 1135,
+ "绪": 1136,
+ "圣": 1137,
+ "招": 1138,
+ "闭": 1139,
+ "攻": 1140,
+ "惯": 1141,
+ "衡": 1142,
+ "鞋": 1143,
+ "承": 1144,
+ "缩": 1145,
+ "h": 1146,
+ "胡": 1147,
+ "频": 1148,
+ "疫": 1149,
+ "扭": 1150,
+ "仿": 1151,
+ "肌": 1152,
+ "谈": 1153,
+ "尖": 1154,
+ "录": 1155,
+ "茶": 1156,
+ "骚": 1157,
+ "症": 1158,
+ "拟": 1159,
+ "洲": 1160,
+ "c": 1161,
+ "宁": 1162,
+ "州": 1163,
+ "m": 1164,
+ "刘": 1165,
+ "询": 1166,
+ "滩": 1167,
+ "唔": 1168,
+ "眠": 1169,
+ "弹": 1170,
+ "幻": 1171,
+ "姑": 1172,
+ "汉": 1173,
+ "既": 1174,
+ "珠": 1175,
+ "协": 1176,
+ "六": 1177,
+ "守": 1178,
+ "危": 1179,
+ "龟": 1180,
+ "抖": 1181,
+ "垃": 1182,
+ "婆": 1183,
+ "圾": 1184,
+ "灯": 1185,
+ "S": 1186,
+ "针": 1187,
+ "筑": 1188,
+ "贴": 1189,
+ "呜": 1190,
+ "藏": 1191,
+ "摩": 1192,
+ "译": 1193,
+ "财": 1194,
+ "稳": 1195,
+ "偏": 1196,
+ "纳": 1197,
+ "羽": 1198,
+ "驾": 1199,
+ "赏": 1200,
+ "驶": 1201,
+ "震": 1202,
+ "折": 1203,
+ "牙": 1204,
+ "紫": 1205,
+ "派": 1206,
+ "退": 1207,
+ "欧": 1208,
+ "柳": 1209,
+ "邮": 1210,
+ "仙": 1211,
+ "恩": 1212,
+ "奥": 1213,
+ "=": 1214,
+ "抬": 1215,
+ "蔬": 1216,
+ "伴": 1217,
+ "桌": 1218,
+ "趋": 1219,
+ "毫": 1220,
+ "耐": 1221,
+ "励": 1222,
+ "池": 1223,
+ "忽": 1224,
+ "梅": 1225,
+ "犯": 1226,
+ "货": 1227,
+ "焦": 1228,
+ "核": 1229,
+ "嘿": 1230,
+ "哼": 1231,
+ "兵": 1232,
+ "勇": 1233,
+ "潜": 1234,
+ "课": 1235,
+ "妻": 1236,
+ "聚": 1237,
+ "爷": 1238,
+ "赶": 1239,
+ "塞": 1240,
+ "锻": 1241,
+ "执": 1242,
+ "拼": 1243,
+ "岛": 1244,
+ "农": 1245,
+ "酸": 1246,
+ "喘": 1247,
+ "沟": 1248,
+ "添": 1249,
+ "裙": 1250,
+ "烟": 1251,
+ "萧": 1252,
+ "氧": 1253,
+ "宣": 1254,
+ "夹": 1255,
+ "揉": 1256,
+ "七": 1257,
+ "芳": 1258,
+ "默": 1259,
+ "额": 1260,
+ "豆": 1261,
+ "骨": 1262,
+ "票": 1263,
+ "偷": 1264,
+ "毕": 1265,
+ "森": 1266,
+ "擦": 1267,
+ "吴": 1268,
+ "九": 1269,
+ "汇": 1270,
+ "磨": 1271,
+ "冒": 1272,
+ "斗": 1273,
+ "姿": 1274,
+ "秦": 1275,
+ "束": 1276,
+ "厚": 1277,
+ "屄": 1278,
+ "啦": 1279,
+ "玛": 1280,
+ "唐": 1281,
+ "咬": 1282,
+ "涉": 1283,
+ "伙": 1284,
+ "尤": 1285,
+ "叹": 1286,
+ "雷": 1287,
+ "漂": 1288,
+ "珍": 1289,
+ "疼": 1290,
+ "姨": 1291,
+ "献": 1292,
+ "副": 1293,
+ "凤": 1294,
+ "距": 1295,
+ "盘": 1296,
+ "呻": 1297,
+ "湖": 1298,
+ "永": 1299,
+ "愉": 1300,
+ "估": 1301,
+ "畅": 1302,
+ "宜": 1303,
+ "X": 1304,
+ "链": 1305,
+ "D": 1306,
+ "汁": 1307,
+ "悦": 1308,
+ "凉": 1309,
+ "杰": 1310,
+ "菲": 1311,
+ "罪": 1312,
+ "孤": 1313,
+ "谷": 1314,
+ "M": 1315,
+ "村": 1316,
+ "怒": 1317,
+ "熊": 1318,
+ "屋": 1319,
+ "莫": 1320,
+ "慧": 1321,
+ "喔": 1322,
+ "婚": 1323,
+ "烧": 1324,
+ "净": 1325,
+ "纤": 1326,
+ "刀": 1327,
+ "架": 1328,
+ "遍": 1329,
+ "省": 1330,
+ "骑": 1331,
+ "纠": 1332,
+ "忆": 1333,
+ "．": 1334,
+ "壮": 1335,
+ "版": 1336,
+ "封": 1337,
+ "肏": 1338,
+ "灾": 1339,
+ "虎": 1340,
+ "础": 1341,
+ "旨": 1342,
+ "末": 1343,
+ "登": 1344,
+ "橙": 1345,
+ "喷": 1346,
+ "萨": 1347,
+ "饰": 1348,
+ "浑": 1349,
+ "闲": 1350,
+ "繁": 1351,
+ "幽": 1352,
+ "宠": 1353,
+ "撞": 1354,
+ "塑": 1355,
+ "街": 1356,
+ "眉": 1357,
+ "堂": 1358,
+ "竞": 1359,
+ "G": 1360,
+ "搜": 1361,
+ "番": 1362,
+ "y": 1363,
+ "纯": 1364,
+ "君": 1365,
+ "敬": 1366,
+ "朵": 1367,
+ "凌": 1368,
+ "塔": 1369,
+ "逃": 1370,
+ "N": 1371,
+ "b": 1372,
+ "遗": 1373,
+ "宗": 1374,
+ "肩": 1375,
+ "琳": 1376,
+ "冬": 1377,
+ "田": 1378,
+ "帝": 1379,
+ "晰": 1380,
+ "弱": 1381,
+ "劲": 1382,
+ "薄": 1383,
+ "朗": 1384,
+ "纸": 1385,
+ "嘛": 1386,
+ "途": 1387,
+ "轮": 1388,
+ "姆": 1389,
+ "躺": 1390,
+ "弃": 1391,
+ "捏": 1392,
+ "奖": 1393,
+ "航": 1394,
+ "鬼": 1395,
+ "旧": 1396,
+ "侧": 1397,
+ "骤": 1398,
+ "托": 1399,
+ "汤": 1400,
+ "奴": 1401,
+ "桃": 1402,
+ "泳": 1403,
+ "泄": 1404,
+ "懂": 1405,
+ "陆": 1406,
+ "隔": 1407,
+ "赞": 1408,
+ "诱": 1409,
+ "腹": 1410,
+ "泪": 1411,
+ "兄": 1412,
+ "爆": 1413,
+ "玲": 1414,
+ "劳": 1415,
+ "燕": 1416,
+ "宽": 1417,
+ "祝": 1418,
+ "飘": 1419,
+ "鼻": 1420,
+ "艾": 1421,
+ "窗": 1422,
+ "韩": 1423,
+ "播": 1424,
+ "碳": 1425,
+ "卖": 1426,
+ "躯": 1427,
+ "哭": 1428,
+ "叔": 1429,
+ "借": 1430,
+ "败": 1431,
+ "爬": 1432,
+ "吐": 1433,
+ "晴": 1434,
+ "燃": 1435,
+ "附": 1436,
+ "损": 1437,
+ ")": 1438,
+ "童": 1439,
+ "'": 1440,
+ "扬": 1441,
+ "糕": 1442,
+ "稍": 1443,
+ "丁": 1444,
+ "箱": 1445,
+ "魂": 1446,
+ "挂": 1447,
+ "辆": 1448,
+ "盛": 1449,
+ "映": 1450,
+ "迹": 1451,
+ "径": 1452,
+ "_": 1453,
+ "启": 1454,
+ "圈": 1455,
+ "惠": 1456,
+ "仔": 1457,
+ "尾": 1458,
+ "d": 1459,
+ "厉": 1460,
+ "酱": 1461,
+ "晓": 1462,
+ "L": 1463,
+ "痒": 1464,
+ "拔": 1465,
+ "蒂": 1466,
+ "绕": 1467,
+ "镜": 1468,
+ "孙": 1469,
+ "碰": 1470,
+ "滴": 1471,
+ "培": 1472,
+ "脂": 1473,
+ "滚": 1474,
+ "p": 1475,
+ "占": 1476,
+ "妮": 1477,
+ "迅": 1478,
+ "黎": 1479,
+ "狼": 1480,
+ "累": 1481,
+ "卷": 1482,
+ "瑜": 1483,
+ "u": 1484,
+ "腻": 1485,
+ "翼": 1486,
+ "镇": 1487,
+ "洛": 1488,
+ "壁": 1489,
+ "扩": 1490,
+ "淋": 1491,
+ "瞬": 1492,
+ "凡": 1493,
+ "陪": 1494,
+ "裸": 1495,
+ "涌": 1496,
+ "歉": 1497,
+ "泡": 1498,
+ "船": 1499,
+ "凝": 1500,
+ "废": 1501,
+ "滋": 1502,
+ "汗": 1503,
+ "醉": 1504,
+ "辣": 1505,
+ "补": 1506,
+ "抵": 1507,
+ "乘": 1508,
+ "旋": 1509,
+ "革": 1510,
+ "仰": 1511,
+ "伽": 1512,
+ "豪": 1513,
+ "臂": 1514,
+ "逗": 1515,
+ "诚": 1516,
+ "琴": 1517,
+ "犹": 1518,
+ "宙": 1519,
+ "酥": 1520,
+ "脏": 1521,
+ "墙": 1522,
+ "伊": 1523,
+ "固": 1524,
+ "撑": 1525,
+ "晶": 1526,
+ "赤": 1527,
+ "扎": 1528,
+ "丹": 1529,
+ "挤": 1530,
+ "迫": 1531,
+ "疯": 1532,
+ "宾": 1533,
+ "伯": 1534,
+ "纽": 1535,
+ "呈": 1536,
+ "幅": 1537,
+ "烦": 1538,
+ "吹": 1539,
+ "恨": 1540,
+ "倾": 1541,
+ "惑": 1542,
+ "浮": 1543,
+ "浴": 1544,
+ "诺": 1545,
+ "郁": 1546,
+ "惜": 1547,
+ "枪": 1548,
+ "伟": 1549,
+ "障": 1550,
+ "呆": 1551,
+ "J": 1552,
+ "敌": 1553,
+ "庆": 1554,
+ "伏": 1555,
+ "泰": 1556,
+ "缠": 1557,
+ "币": 1558,
+ "昊": 1559,
+ "幕": 1560,
+ "搂": 1561,
+ "兽": 1562,
+ "坦": 1563,
+ "奈": 1564,
+ "吮": 1565,
+ "唱": 1566,
+ "甲": 1567,
+ "亡": 1568,
+ "谓": 1569,
+ "锅": 1570,
+ "披": 1571,
+ "(": 1572,
+ "氛": 1573,
+ "貌": 1574,
+ "篮": 1575,
+ "奏": 1576,
+ "盈": 1577,
+ "冠": 1578,
+ "悠": 1579,
+ "恋": 1580,
+ "瓶": 1581,
+ "陷": 1582,
+ "朱": 1583,
+ "糊": 1584,
+ "籍": 1585,
+ "炎": 1586,
+ "喊": 1587,
+ "骄": 1588,
+ "蕾": 1589,
+ "龄": 1590,
+ "岳": 1591,
+ "扑": 1592,
+ "麦": 1593,
+ "歧": 1594,
+ "尿": 1595,
+ "g": 1596,
+ "纷": 1597,
+ "戴": 1598,
+ "晃": 1599,
+ "煮": 1600,
+ "茎": 1601,
+ "遥": 1602,
+ "递": 1603,
+ "贡": 1604,
+ "庄": 1605,
+ "乡": 1606,
+ "胁": 1607,
+ "缝": 1608,
+ "罩": 1609,
+ "悲": 1610,
+ "莎": 1611,
+ "吞": 1612,
+ "啪": 1613,
+ "库": 1614,
+ "唯": 1615,
+ "乖": 1616,
+ "侯": 1617,
+ "零": 1618,
+ "彼": 1619,
+ "阻": 1620,
+ "缘": 1621,
+ "霜": 1622,
+ "延": 1623,
+ "盐": 1624,
+ "疲": 1625,
+ "饱": 1626,
+ "签": 1627,
+ "彻": 1628,
+ "洪": 1629,
+ "聪": 1630,
+ "蒙": 1631,
+ "甘": 1632,
+ "晨": 1633,
+ "奸": 1634,
+ "怡": 1635,
+ "储": 1636,
+ "冥": 1637,
+ "莲": 1638,
+ "侵": 1639,
+ "吓": 1640,
+ "卧": 1641,
+ "综": 1642,
+ "腾": 1643,
+ "莹": 1644,
+ "残": 1645,
+ "蓉": 1646,
+ "搞": 1647,
+ "徒": 1648,
+ "勾": 1649,
+ "碎": 1650,
+ "绩": 1651,
+ "绘": 1652,
+ "O": 1653,
+ "苗": 1654,
+ "踪": 1655,
+ "肚": 1656,
+ "脉": 1657,
+ "仪": 1658,
+ "荣": 1659,
+ "E": 1660,
+ "涨": 1661,
+ "订": 1662,
+ "曼": 1663,
+ "樱": 1664,
+ "挣": 1665,
+ "晕": 1666,
+ "铅": 1667,
+ "灭": 1668,
+ "恢": 1669,
+ "翘": 1670,
+ "访": 1671,
+ "愈": 1672,
+ "饼": 1673,
+ "吉": 1674,
+ "娱": 1675,
+ "炒": 1676,
+ "邪": 1677,
+ "扶": 1678,
+ "犬": 1679,
+ "拒": 1680,
+ "❤": 1681,
+ "川": 1682,
+ "涛": 1683,
+ "尺": 1684,
+ "乔": 1685,
+ "阶": 1686,
+ "颈": 1687,
+ "裹": 1688,
+ "垂": 1689,
+ "瓜": 1690,
+ "申": 1691,
+ "姓": 1692,
+ "扣": 1693,
+ "亦": 1694,
+ "%": 1695,
+ "循": 1696,
+ "R": 1697,
+ "咱": 1698,
+ "怜": 1699,
+ "俩": 1700,
+ "寿": 1701,
+ "x": 1702,
+ "县": 1703,
+ "渴": 1704,
+ "袜": 1705,
+ "椒": 1706,
+ "宋": 1707,
+ "鲁": 1708,
+ "辉": 1709,
+ "遭": 1710,
+ "慰": 1711,
+ "噢": 1712,
+ "+": 1713,
+ "胆": 1714,
+ "寸": 1715,
+ "殊": 1716,
+ "舍": 1717,
+ "椅": 1718,
+ "薇": 1719,
+ "迪": 1720,
+ "谱": 1721,
+ "嘻": 1722,
+ "薛": 1723,
+ "迟": 1724,
+ "丈": 1725,
+ "践": 1726,
+ "琼": 1727,
+ "昏": 1728,
+ "钢": 1729,
+ "绵": 1730,
+ "奔": 1731,
+ "妖": 1732,
+ "虫": 1733,
+ "孔": 1734,
+ "郎": 1735,
+ "胀": 1736,
+ "沿": 1737,
+ "跪": 1738,
+ "萝": 1739,
+ "韵": 1740,
+ "拜": 1741,
+ "载": 1742,
+ "悄": 1743,
+ "炸": 1744,
+ "灰": 1745,
+ "逻": 1746,
+ "委": 1747,
+ "骂": 1748,
+ "剂": 1749,
+ "裂": 1750,
+ "０": 1751,
+ "浅": 1752,
+ "振": 1753,
+ "翰": 1754,
+ "闹": 1755,
+ "瓣": 1756,
+ "遵": 1757,
+ "誉": 1758,
+ "扮": 1759,
+ "哎": 1760,
+ "*": 1761,
+ "乏": 1762,
+ "菊": 1763,
+ "耗": 1764,
+ "厨": 1765,
+ "锁": 1766,
+ "批": 1767,
+ "搅": 1768,
+ "匀": 1769,
+ "逼": 1770,
+ "亿": 1771,
+ "弯": 1772,
+ "填": 1773,
+ "跨": 1774,
+ "氏": 1775,
+ "衫": 1776,
+ "悉": 1777,
+ "忠": 1778,
+ "‘": 1779,
+ "乌": 1780,
+ "扰": 1781,
+ "倩": 1782,
+ "胯": 1783,
+ "忧": 1784,
+ "丢": 1785,
+ "涵": 1786,
+ "粒": 1787,
+ "躲": 1788,
+ "哺": 1789,
+ "鸣": 1790,
+ "涂": 1791,
+ "郑": 1792,
+ "徐": 1793,
+ "横": 1794,
+ "鼠": 1795,
+ "抹": 1796,
+ "赢": 1797,
+ "桥": 1798,
+ "赖": 1799,
+ "庞": 1800,
+ "肖": 1801,
+ "墨": 1802,
+ "欺": 1803,
+ "肠": 1804,
+ "违": 1805,
+ "趴": 1806,
+ "脖": 1807,
+ "仁": 1808,
+ "堡": 1809,
+ "魅": 1810,
+ "蕉": 1811,
+ "抑": 1812,
+ "扫": 1813,
+ "猜": 1814,
+ "审": 1815,
+ "脆": 1816,
+ "沈": 1817,
+ "膀": 1818,
+ "俏": 1819,
+ "’": 1820,
+ "羊": 1821,
+ "猪": 1822,
+ "予": 1823,
+ "竹": 1824,
+ "烫": 1825,
+ "皱": 1826,
+ "撒": 1827,
+ "跃": 1828,
+ "辛": 1829,
+ "傲": 1830,
+ "港": 1831,
+ "k": 1832,
+ "盯": 1833,
+ "督": 1834,
+ "袭": 1835,
+ "袖": 1836,
+ "暂": 1837,
+ "绳": 1838,
+ "芬": 1839,
+ "档": 1840,
+ "慌": 1841,
+ "览": 1842,
+ "覆": 1843,
+ "馨": 1844,
+ "勃": 1845,
+ "逸": 1846,
+ "葱": 1847,
+ "俊": 1848,
+ "耸": 1849,
+ "凯": 1850,
+ "扯": 1851,
+ "莱": 1852,
+ "兔": 1853,
+ "辅": 1854,
+ "猎": 1855,
+ "蝶": 1856,
+ "悬": 1857,
+ "齿": 1858,
+ "勒": 1859,
+ "恒": 1860,
+ "梁": 1861,
+ "莓": 1862,
+ "慕": 1863,
+ "辈": 1864,
+ "纹": 1865,
+ "掩": 1866,
+ "婷": 1867,
+ "姬": 1868,
+ "仇": 1869,
+ "１": 1870,
+ "碧": 1871,
+ "拌": 1872,
+ "哲": 1873,
+ "凑": 1874,
+ "搓": 1875,
+ "捷": 1876,
+ "癌": 1877,
+ "诸": 1878,
+ "钻": 1879,
+ "瑞": 1880,
+ "闷": 1881,
+ "浸": 1882,
+ "睁": 1883,
+ "捕": 1884,
+ "哀": 1885,
+ "胞": 1886,
+ "倍": 1887,
+ "腔": 1888,
+ "耻": 1889,
+ "罚": 1890,
+ "偶": 1891,
+ "盒": 1892,
+ "婉": 1893,
+ "F": 1894,
+ "堪": 1895,
+ "泽": 1896,
+ "肢": 1897,
+ "茄": 1898,
+ "罢": 1899,
+ "拖": 1900,
+ "枝": 1901,
+ "岸": 1902,
+ "佩": 1903,
+ "肪": 1904,
+ "骗": 1905,
+ "孟": 1906,
+ "丫": 1907,
+ "翠": 1908,
+ "涩": 1909,
+ "屈": 1910,
+ "芒": 1911,
+ "遮": 1912,
+ "喉": 1913,
+ "蜂": 1914,
+ "臭": 1915,
+ "夺": 1916,
+ "杜": 1917,
+ "『": 1918,
+ "泉": 1919,
+ "返": 1920,
+ "锐": 1921,
+ "辱": 1922,
+ "屏": 1923,
+ "眸": 1924,
+ "拳": 1925,
+ "侍": 1926,
+ "厌": 1927,
+ "衬": 1928,
+ "蛇": 1929,
+ "拨": 1930,
+ "咨": 1931,
+ "挡": 1932,
+ "怨": 1933,
+ "耀": 1934,
+ "俗": 1935,
+ "吁": 1936,
+ "尘": 1937,
+ "碍": 1938,
+ "惨": 1939,
+ "敦": 1940,
+ "喂": 1941,
+ "惧": 1942,
+ "盟": 1943,
+ "』": 1944,
+ "肺": 1945,
+ "荷": 1946,
+ "咕": 1947,
+ "凭": 1948,
+ "祖": 1949,
+ "颊": 1950,
+ "授": 1951,
+ "席": 1952,
+ "抢": 1953,
+ "胖": 1954,
+ "烂": 1955,
+ "雯": 1956,
+ "埃": 1957,
+ "烹": 1958,
+ "芝": 1959,
+ "柱": 1960,
+ "瞧": 1961,
+ "瞪": 1962,
+ "玄": 1963,
+ "谋": 1964,
+ "棉": 1965,
+ "耍": 1966,
+ "贱": 1967,
+ "央": 1968,
+ "咪": 1969,
+ "凶": 1970,
+ "隆": 1971,
+ "宿": 1972,
+ "傻": 1973,
+ "盾": 1974,
+ "御": 1975,
+ "葡": 1976,
+ "寂": 1977,
+ "【": 1978,
+ "颖": 1979,
+ "邀": 1980,
+ "】": 1981,
+ "刑": 1982,
+ "萄": 1983,
+ "堆": 1984,
+ "碗": 1985,
+ "硕": 1986,
+ "溜": 1987,
+ "盗": 1988,
+ "铃": 1989,
+ "绑": 1990,
+ "刷": 1991,
+ "巾": 1992,
+ "愤": 1993,
+ "牢": 1994,
+ "H": 1995,
+ "埋": 1996,
+ "讶": 1997,
+ "馈": 1998,
+ "雾": 1999,
+ "岩": 2000,
+ "丧": 2001,
+ "纵": 2002,
+ "嫌": 2003,
+ "殿": 2004,
+ "嫣": 2005,
+ "慈": 2006,
+ "愣": 2007,
+ "黛": 2008,
+ "税": 2009,
+ "幼": 2010,
+ "堵": 2011,
+ "沾": 2012,
+ "匹": 2013,
+ "串": 2014,
+ "铺": 2015,
+ "霍": 2016,
+ "拓": 2017,
+ "账": 2018,
+ "玫": 2019,
+ "洒": 2020,
+ "穆": 2021,
+ "浩": 2022,
+ "豫": 2023,
+ "矛": 2024,
+ "蒸": 2025,
+ "痕": 2026,
+ "秒": 2027,
+ "贯": 2028,
+ "澡": 2029,
+ "肿": 2030,
+ "琪": 2031,
+ "阔": 2032,
+ "卢": 2033,
+ "租": 2034,
+ "轩": 2035,
+ "粘": 2036,
+ "矿": 2037,
+ "璃": 2038,
+ "挖": 2039,
+ "痴": 2040,
+ "剪": 2041,
+ "厂": 2042,
+ "V": 2043,
+ "喻": 2044,
+ "辞": 2045,
+ "噗": 2046,
+ "匆": 2047,
+ "腐": 2048,
+ "唤": 2049,
+ "]": 2050,
+ "柜": 2051,
+ "[": 2052,
+ "穷": 2053,
+ "陶": 2054,
+ "箭": 2055,
+ "臣": 2056,
+ "泥": 2057,
+ "恼": 2058,
+ "舰": 2059,
+ "截": 2060,
+ "妃": 2061,
+ "珊": 2062,
+ "荒": 2063,
+ "?": 2064,
+ "催": 2065,
+ "狮": 2066,
+ "斜": 2067,
+ "懒": 2068,
+ "肴": 2069,
+ "讯": 2070,
+ "逊": 2071,
+ "狐": 2072,
+ "贪": 2073,
+ "霞": 2074,
+ "帅": 2075,
+ "瑟": 2076,
+ "娃": 2077,
+ "饶": 2078,
+ "页": 2079,
+ "昂": 2080,
+ "踏": 2081,
+ "溢": 2082,
+ "瘦": 2083,
+ "饪": 2084,
+ "２": 2085,
+ "瑰": 2086,
+ "勤": 2087,
+ "肆": 2088,
+ "哇": 2089,
+ "湾": 2090,
+ "牵": 2091,
+ "嘉": 2092,
+ "孕": 2093,
+ "贼": 2094,
+ "狱": 2095,
+ "陌": 2096,
+ "裁": 2097,
+ "毁": 2098,
+ "咳": 2099,
+ "v": 2100,
+ "驱": 2101,
+ "漏": 2102,
+ "侠": 2103,
+ "揭": 2104,
+ "晒": 2105,
+ "援": 2106,
+ "董": 2107,
+ "淑": 2108,
+ "爹": 2109,
+ "允": 2110,
+ "厘": 2111,
+ "贾": 2112,
+ "汪": 2113,
+ "霸": 2114,
+ "贫": 2115,
+ "党": 2116,
+ "藤": 2117,
+ "咯": 2118,
+ "溪": 2119,
+ "咽": 2120,
+ "聘": 2121,
+ "傅": 2122,
+ "掘": 2123,
+ "怖": 2124,
+ "叠": 2125,
+ "炮": 2126,
+ "苍": 2127,
+ "函": 2128,
+ "殖": 2129,
+ "嫂": 2130,
+ "趁": 2131,
+ "窄": 2132,
+ "撰": 2133,
+ "卓": 2134,
+ "糟": 2135,
+ "帽": 2136,
+ "凸": 2137,
+ "茨": 2138,
+ "梨": 2139,
+ "灌": 2140,
+ "恰": 2141,
+ "谐": 2142,
+ "梯": 2143,
+ "漠": 2144,
+ "捉": 2145,
+ "膜": 2146,
+ "茫": 2147,
+ "卜": 2148,
+ "尸": 2149,
+ "妆": 2150,
+ "谊": 2151,
+ "芙": 2152,
+ "鸭": 2153,
+ "旦": 2154,
+ "菌": 2155,
+ "拂": 2156,
+ "芸": 2157,
+ "攀": 2158,
+ "玻": 2159,
+ "乃": 2160,
+ "虹": 2161,
+ "戒": 2162,
+ "稀": 2163,
+ "拾": 2164,
+ "吊": 2165,
+ "携": 2166,
+ "┅": 2167,
+ "惹": 2168,
+ "址": 2169,
+ "K": 2170,
+ "肛": 2171,
+ "轨": 2172,
+ "割": 2173,
+ "谨": 2174,
+ "贺": 2175,
+ "俯": 2176,
+ "浆": 2177,
+ "f": 2178,
+ "喃": 2179,
+ "÷": 2180,
+ "敲": 2181,
+ "燥": 2182,
+ "奉": 2183,
+ "贸": 2184,
+ "锋": 2185,
+ "妳": 2186,
+ "盆": 2187,
+ "柠": 2188,
+ "蒜": 2189,
+ "忌": 2190,
+ "踢": 2191,
+ "叉": 2192,
+ "郭": 2193,
+ "悟": 2194,
+ "帐": 2195,
+ "旗": 2196,
+ "俱": 2197,
+ "萍": 2198,
+ "撩": 2199,
+ "夸": 2200,
+ "磁": 2201,
+ "雕": 2202,
+ "檬": 2203,
+ "悔": 2204,
+ "迈": 2205,
+ "娟": 2206,
+ "绒": 2207,
+ "蕊": 2208,
+ "擎": 2209,
+ "抛": 2210,
+ "胶": 2211,
+ "W": 2212,
+ " ": 2213,
+ "慎": 2214,
+ "扔": 2215,
+ "膝": 2216,
+ "弥": 2217,
+ "鞭": 2218,
+ "蹲": 2219,
+ "泊": 2220,
+ "擅": 2221,
+ "撕": 2222,
+ "赋": 2223,
+ "津": 2224,
+ "艰": 2225,
+ "弗": 2226,
+ "愁": 2227,
+ "鲸": 2228,
+ "夕": 2229,
+ "姊": 2230,
+ "蹈": 2231,
+ "疏": 2232,
+ "昆": 2233,
+ "桶": 2234,
+ "丛": 2235,
+ "蕴": 2236,
+ "鹰": 2237,
+ "纱": 2238,
+ "蒋": 2239,
+ "删": 2240,
+ "岂": 2241,
+ "舅": 2242,
+ "曹": 2243,
+ "吕": 2244,
+ "笼": 2245,
+ "垫": 2246,
+ "嗔": 2247,
+ "恭": 2248,
+ "吼": 2249,
+ "翅": 2250,
+ "哟": 2251,
+ "婴": 2252,
+ "械": 2253,
+ "鹅": 2254,
+ "斑": 2255,
+ "邦": 2256,
+ "侣": 2257,
+ "茹": 2258,
+ "袍": 2259,
+ "娅": 2260,
+ "赫": 2261,
+ "丑": 2262,
+ "伍": 2263,
+ "跌": 2264,
+ "逆": 2265,
+ "逍": 2266,
+ "胃": 2267,
+ "壤": 2268,
+ "煎": 2269,
+ "淇": 2270,
+ "愧": 2271,
+ "唉": 2272,
+ "妍": 2273,
+ "黏": 2274,
+ "皆": 2275,
+ "寺": 2276,
+ "惩": 2277,
+ "棋": 2278,
+ "虐": 2279,
+ "阐": 2280,
+ "酷": 2281,
+ "醋": 2282,
+ "丘": 2283,
+ "井": 2284,
+ "伐": 2285,
+ "宴": 2286,
+ "踩": 2287,
+ "扇": 2288,
+ "咙": 2289,
+ "饿": 2290,
+ "姜": 2291,
+ "翔": 2292,
+ "粮": 2293,
+ "啥": 2294,
+ "逛": 2295,
+ "栖": 2296,
+ "珑": 2297,
+ "泣": 2298,
+ "魏": 2299,
+ "鹿": 2300,
+ "胴": 2301,
+ "毅": 2302,
+ "爵": 2303,
+ "贤": 2304,
+ "棍": 2305,
+ "绷": 2306,
+ "弦": 2307,
+ "颇": 2308,
+ "滤": 2309,
+ "夷": 2310,
+ "３": 2311,
+ "U": 2312,
+ "召": 2313,
+ "鉴": 2314,
+ "菇": 2315,
+ "帆": 2316,
+ "酪": 2317,
+ "锦": 2318,
+ "瓦": 2319,
+ "蛮": 2320,
+ "麟": 2321,
+ "─": 2322,
+ "猴": 2323,
+ "胎": 2324,
+ "趾": 2325,
+ "宛": 2326,
+ "邻": 2327,
+ "壶": 2328,
+ "焰": 2329,
+ "劣": 2330,
+ "捧": 2331,
+ "尬": 2332,
+ "冯": 2333,
+ "尴": 2334,
+ "崩": 2335,
+ "炉": 2336,
+ "窝": 2337,
+ "枚": 2338,
+ "枕": 2339,
+ "宏": 2340,
+ "贞": 2341,
+ "涯": 2342,
+ "弓": 2343,
+ "庙": 2344,
+ "妩": 2345,
+ "伞": 2346,
+ "爪": 2347,
+ "筋": 2348,
+ "蹭": 2349,
+ "稿": 2350,
+ "杏": 2351,
+ "豹": 2352,
+ "框": 2353,
+ "谜": 2354,
+ "匙": 2355,
+ "斥": 2356,
+ "w": 2357,
+ "嫁": 2358,
+ "️": 2359,
+ "赎": 2360,
+ "掏": 2361,
+ "甩": 2362,
+ "橘": 2363,
+ "眯": 2364,
+ "魄": 2365,
+ "壳": 2366,
+ "赌": 2367,
+ "沐": 2368,
+ "×": 2369,
+ "澈": 2370,
+ "廷": 2371,
+ "叙": 2372,
+ "寓": 2373,
+ "婶": 2374,
+ "５": 2375,
+ "劫": 2376,
+ "帘": 2377,
+ "啸": 2378,
+ "♡": 2379,
+ "蝴": 2380,
+ "卿": 2381,
+ "饥": 2382,
+ "兼": 2383,
+ "缚": 2384,
+ "勉": 2385,
+ "媳": 2386,
+ "韧": 2387,
+ "靡": 2388,
+ "摔": 2389,
+ "沸": 2390,
+ "颠": 2391,
+ "坛": 2392,
+ "鹏": 2393,
+ "狸": 2394,
+ "搬": 2395,
+ "阁": 2396,
+ "瘫": 2397,
+ "矩": 2398,
+ "矮": 2399,
+ "吵": 2400,
+ "晋": 2401,
+ "丸": 2402,
+ "乙": 2403,
+ "俄": 2404,
+ "迁": 2405,
+ "铜": 2406,
+ "绽": 2407,
+ "腕": 2408,
+ "诈": 2409,
+ "偿": 2410,
+ "坡": 2411,
+ "沃": 2412,
+ "岗": 2413,
+ "桑": 2414,
+ "旱": 2415,
+ "杉": 2416,
+ "窃": 2417,
+ "崇": 2418,
+ "劝": 2419,
+ "肝": 2420,
+ "寄": 2421,
+ "歇": 2422,
+ "帕": 2423,
+ "轰": 2424,
+ "亏": 2425,
+ "绮": 2426,
+ "杆": 2427,
+ "脊": 2428,
+ "刮": 2429,
+ "畜": 2430,
+ "宅": 2431,
+ "渗": 2432,
+ "媛": 2433,
+ "惫": 2434,
+ "彦": 2435,
+ "膨": 2436,
+ "沫": 2437,
+ "隙": 2438,
+ "眨": 2439,
+ "逝": 2440,
+ "孝": 2441,
+ "溃": 2442,
+ "舐": 2443,
+ "砍": 2444,
+ "皙": 2445,
+ "稠": 2446,
+ "肃": 2447,
+ "渠": 2448,
+ "漆": 2449,
+ "虾": 2450,
+ "钥": 2451,
+ "膛": 2452,
+ "碑": 2453,
+ "滥": 2454,
+ "刹": 2455,
+ "册": 2456,
+ "囊": 2457,
+ "筛": 2458,
+ "赚": 2459,
+ "桂": 2460,
+ "澳": 2461,
+ "契": 2462,
+ "呃": 2463,
+ "烘": 2464,
+ "赐": 2465,
+ "辨": 2466,
+ "♥": 2467,
+ "枫": 2468,
+ "蘑": 2469,
+ "衰": 2470,
+ "僵": 2471,
+ "捂": 2472,
+ "辰": 2473,
+ "淌": 2474,
+ "霈": 2475,
+ "渡": 2476,
+ "朴": 2477,
+ "Y": 2478,
+ "泌": 2479,
+ "凄": 2480,
+ "榄": 2481,
+ "陵": 2482,
+ "橄": 2483,
+ "捆": 2484,
+ "祸": 2485,
+ "祈": 2486,
+ "雇": 2487,
+ "雀": 2488,
+ "胳": 2489,
+ "屌": 2490,
+ "浇": 2491,
+ "氢": 2492,
+ "瓷": 2493,
+ "筒": 2494,
+ "腊": 2495,
+ "蠕": 2496,
+ "芯": 2497,
+ "褪": 2498,
+ "烁": 2499,
+ "薯": 2500,
+ "驰": 2501,
+ "柿": 2502,
+ "％": 2503,
+ "拦": 2504,
+ "柴": 2505,
+ "畏": 2506,
+ "剥": 2507,
+ "灼": 2508,
+ "挫": 2509,
+ "掀": 2510,
+ "巫": 2511,
+ "枯": 2512,
+ "辩": 2513,
+ "漉": 2514,
+ "拆": 2515,
+ "搐": 2516,
+ "叛": 2517,
+ "泼": 2518,
+ "℃": 2519,
+ "聂": 2520,
+ "狭": 2521,
+ "祭": 2522,
+ "姻": 2523,
+ "亭": 2524,
+ "!": 2525,
+ "挪": 2526,
+ "腴": 2527,
+ "腥": 2528,
+ "厦": 2529,
+ "耶": 2530,
+ "矶": 2531,
+ "瘾": 2532,
+ "湘": 2533,
+ "慨": 2534,
+ "膊": 2535,
+ "盼": 2536,
+ "贷": 2537,
+ "逢": 2538,
+ "茂": 2539,
+ "牧": 2540,
+ "署": 2541,
+ "寞": 2542,
+ "嘟": 2543,
+ "噪": 2544,
+ "捅": 2545,
+ "欠": 2546,
+ "碌": 2547,
+ "ｅ": 2548,
+ "溶": 2549,
+ "萌": 2550,
+ "哑": 2551,
+ "Q": 2552,
+ "诞": 2553,
+ "隶": 2554,
+ "４": 2555,
+ "妥": 2556,
+ "嘶": 2557,
+ "挨": 2558,
+ "嗅": 2559,
+ "ａ": 2560,
+ "棕": 2561,
+ "拢": 2562,
+ "彤": 2563,
+ "恤": 2564,
+ "沮": 2565,
+ "溯": 2566,
+ "咐": 2567,
+ "翩": 2568,
+ "毯": 2569,
+ "伪": 2570,
+ "押": 2571,
+ "雌": 2572,
+ "旺": 2573,
+ "蓬": 2574,
+ "葛": 2575,
+ "仆": 2576,
+ "券": 2577,
+ "拯": 2578,
+ "痉": 2579,
+ "誓": 2580,
+ "斤": 2581,
+ "敞": 2582,
+ "暑": 2583,
+ "吾": 2584,
+ "滨": 2585,
+ "漾": 2586,
+ "届": 2587,
+ "脯": 2588,
+ "祥": 2589,
+ "坠": 2590,
+ "挛": 2591,
+ "倦": 2592,
+ "挽": 2593,
+ "掠": 2594,
+ "薪": 2595,
+ "缕": 2596,
+ "钓": 2597,
+ "恍": 2598,
+ "妾": 2599,
+ "闯": 2600,
+ "削": 2601,
+ "廉": 2602,
+ "熬": 2603,
+ "潺": 2604,
+ "绣": 2605,
+ "赔": 2606,
+ "曰": 2607,
+ "雁": 2608,
+ "啼": 2609,
+ "赠": 2610,
+ "兮": 2611,
+ "冻": 2612,
+ "债": 2613,
+ "柏": 2614,
+ "寡": 2615,
+ "舟": 2616,
+ "笨": 2617,
+ "啤": 2618,
+ "蠢": 2619,
+ "蚁": 2620,
+ "梳": 2621,
+ "咒": 2622,
+ "厕": 2623,
+ "塌": 2624,
+ "醇": 2625,
+ "廊": 2626,
+ "溅": 2627,
+ "诡": 2628,
+ "芽": 2629,
+ "橡": 2630,
+ "詹": 2631,
+ "役": 2632,
+ "#": 2633,
+ "靖": 2634,
+ "６": 2635,
+ "仓": 2636,
+ "撼": 2637,
+ "谅": 2638,
+ "熙": 2639,
+ "婪": 2640,
+ "腺": 2641,
+ "怯": 2642,
+ "浊": 2643,
+ "烛": 2644,
+ "粱": 2645,
+ "辐": 2646,
+ "殷": 2647,
+ "匪": 2648,
+ "卑": 2649,
+ "唾": 2650,
+ "芷": 2651,
+ "倡": 2652,
+ "灿": 2653,
+ "啧": 2654,
+ "摧": 2655,
+ "竭": 2656,
+ "歪": 2657,
+ "躁": 2658,
+ "韦": 2659,
+ "挠": 2660,
+ "茵": 2661,
+ "芊": 2662,
+ "卵": 2663,
+ "咿": 2664,
+ "栋": 2665,
+ "懈": 2666,
+ "柄": 2667,
+ "※": 2668,
+ "摊": 2669,
+ "淘": 2670,
+ "兜": 2671,
+ "僧": 2672,
+ "靴": 2673,
+ "湛": 2674,
+ "袁": 2675,
+ "趟": 2676,
+ "哄": 2677,
+ "鹤": 2678,
+ "８": 2679,
+ "撤": 2680,
+ "砸": 2681,
+ "衷": 2682,
+ "啾": 2683,
+ "狄": 2684,
+ "嗽": 2685,
+ "筹": 2686,
+ "巡": 2687,
+ "浏": 2688,
+ "卉": 2689,
+ "憋": 2690,
+ "兹": 2691,
+ "栏": 2692,
+ "侄": 2693,
+ "缀": 2694,
+ "墓": 2695,
+ "怔": 2696,
+ "潘": 2697,
+ "漓": 2698,
+ "瞒": 2699,
+ "Ａ": 2700,
+ "吩": 2701,
+ "咏": 2702,
+ "淀": 2703,
+ "昔": 2704,
+ "罐": 2705,
+ "亵": 2706,
+ "柯": 2707,
+ "佣": 2708,
+ "尹": 2709,
+ "拱": 2710,
+ "扒": 2711,
+ "羡": 2712,
+ "９": 2713,
+ "伺": 2714,
+ "竖": 2715,
+ "脾": 2716,
+ "棵": 2717,
+ "捣": 2718,
+ "婊": 2719,
+ "阮": 2720,
+ "墅": 2721,
+ "蔓": 2722,
+ "斩": 2723,
+ "蚀": 2724,
+ "蛛": 2725,
+ "噜": 2726,
+ "迭": 2727,
+ "牲": 2728,
+ "曦": 2729,
+ "厢": 2730,
+ "娥": 2731,
+ "嘲": 2732,
+ "蓄": 2733,
+ "膳": 2734,
+ "咸": 2735,
+ "巢": 2736,
+ "芹": 2737,
+ "畔": 2738,
+ "菠": 2739,
+ "珂": 2740,
+ "哩": 2741,
+ "崖": 2742,
+ "妨": 2743,
+ "嗦": 2744,
+ "侦": 2745,
+ "朔": 2746,
+ "嗓": 2747,
+ "嗤": 2748,
+ "谎": 2749,
+ "哗": 2750,
+ "妓": 2751,
+ "昭": 2752,
+ "冤": 2753,
+ "凹": 2754,
+ "拘": 2755,
+ "裆": 2756,
+ "拭": 2757,
+ "炽": 2758,
+ "哧": 2759,
+ "巷": 2760,
+ "戳": 2761,
+ "噬": 2762,
+ "掐": 2763,
+ "癸": 2764,
+ "宵": 2765,
+ "暮": 2766,
+ "吱": 2767,
+ "幺": 2768,
+ "钮": 2769,
+ "惕": 2770,
+ "憾": 2771,
+ "蔡": 2772,
+ "嬉": 2773,
+ "鲨": 2774,
+ "屑": 2775,
+ "∶": 2776,
+ "峡": 2777,
+ "琐": 2778,
+ "潇": 2779,
+ "仗": 2780,
+ "闺": 2781,
+ "坑": 2782,
+ "砰": 2783,
+ "郊": 2784,
+ "稻": 2785,
+ "茸": 2786,
+ "鼎": 2787,
+ "盲": 2788,
+ "旷": 2789,
+ "呐": 2790,
+ "郡": 2791,
+ "糙": 2792,
+ "宰": 2793,
+ "镖": 2794,
+ "ｏ": 2795,
+ "ｉ": 2796,
+ "唧": 2797,
+ "姚": 2798,
+ "瀑": 2799,
+ "渊": 2800,
+ "缸": 2801,
+ "娶": 2802,
+ "罕": 2803,
+ "z": 2804,
+ "椎": 2805,
+ "窥": 2806,
+ "轴": 2807,
+ "渔": 2808,
+ "履": 2809,
+ "煞": 2810,
+ "亢": 2811,
+ "栽": 2812,
+ "螺": 2813,
+ "瑚": 2814,
+ "叮": 2815,
+ "蚂": 2816,
+ "汹": 2817,
+ "捞": 2818,
+ "ｎ": 2819,
+ "佐": 2820,
+ "鲍": 2821,
+ "拽": 2822,
+ "铭": 2823,
+ "榨": 2824,
+ "刃": 2825,
+ "窜": 2826,
+ "裔": 2827,
+ "劈": 2828,
+ "翁": 2829,
+ "奢": 2830,
+ "嗨": 2831,
+ "７": 2832,
+ "昧": 2833,
+ "艘": 2834,
+ "篷": 2835,
+ "拐": 2836,
+ "瘤": 2837,
+ "驻": 2838,
+ "彷": 2839,
+ "妞": 2840,
+ "捡": 2841,
+ "彰": 2842,
+ "栗": 2843,
+ "谦": 2844,
+ "撅": 2845,
+ "捐": 2846,
+ "禅": 2847,
+ "遣": 2848,
+ "嚣": 2849,
+ "倚": 2850,
+ "绸": 2851,
+ "勺": 2852,
+ "婢": 2853,
+ "弧": 2854,
+ "撸": 2855,
+ "绯": 2856,
+ "榜": 2857,
+ "茜": 2858,
+ "搏": 2859,
+ "咋": 2860,
+ "砂": 2861,
+ "甫": 2862,
+ "扁": 2863,
+ "岭": 2864,
+ "叽": 2865,
+ "艇": 2866,
+ "蜡": 2867,
+ "滞": 2868,
+ "酵": 2869,
+ "辕": 2870,
+ "挚": 2871,
+ "猩": 2872,
+ "粥": 2873,
+ "咧": 2874,
+ "仲": 2875,
+ "饲": 2876,
+ "儒": 2877,
+ "巍": 2878,
+ "戈": 2879,
+ "撇": 2880,
+ "囚": 2881,
+ "讽": 2882,
+ "膏": 2883,
+ "裳": 2884,
+ "狞": 2885,
+ "蝉": 2886,
+ "屿": 2887,
+ "帖": 2888,
+ "娴": 2889,
+ "蚊": 2890,
+ "蹦": 2891,
+ "妒": 2892,
+ "驳": 2893,
+ "煌": 2894,
+ "Ｍ": 2895,
+ "朦": 2896,
+ "寨": 2897,
+ "咦": 2898,
+ "秽": 2899,
+ "喧": 2900,
+ "焚": 2901,
+ "胧": 2902,
+ "钩": 2903,
+ "悍": 2904,
+ "Ｓ": 2905,
+ "杖": 2906,
+ "霄": 2907,
+ "揽": 2908,
+ "褶": 2909,
+ "筱": 2910,
+ "赴": 2911,
+ "昌": 2912,
+ "呕": 2913,
+ "阜": 2914,
+ "稚": 2915,
+ "骇": 2916,
+ "凰": 2917,
+ "暇": 2918,
+ "筠": 2919,
+ "榻": 2920,
+ "巅": 2921,
+ "噫": 2922,
+ "梵": 2923,
+ "桐": 2924,
+ "纲": 2925,
+ "堕": 2926,
+ "峻": 2927,
+ "抠": 2928,
+ "蹂": 2929,
+ "杭": 2930,
+ "芦": 2931,
+ "淹": 2932,
+ "狩": 2933,
+ "钰": 2934,
+ "泻": 2935,
+ "崎": 2936,
+ "牺": 2937,
+ "凛": 2938,
+ "﹐": 2939,
+ "浦": 2940,
+ "傍": 2941,
+ "哆": 2942,
+ "瑕": 2943,
+ "Ｘ": 2944,
+ "鸿": 2945,
+ "鄙": 2946,
+ "岚": 2947,
+ "蛙": 2948,
+ "躏": 2949,
+ "愚": 2950,
+ "瞳": 2951,
+ "琦": 2952,
+ "嵌": 2953,
+ "碱": 2954,
+ "廓": 2955,
+ "沁": 2956,
+ "甸": 2957,
+ "檀": 2958,
+ "禽": 2959,
+ "莺": 2960,
+ "蟒": 2961,
+ "咚": 2962,
+ "Ｃ": 2963,
+ "齁": 2964,
+ "昀": 2965,
+ "诀": 2966,
+ "弊": 2967,
+ "妄": 2968,
+ "Ｄ": 2969,
+ "砖": 2970,
+ "瞥": 2971,
+ "塘": 2972,
+ "剔": 2973,
+ "甬": 2974,
+ "薰": 2975,
+ "Ｂ": 2976,
+ "`": 2977,
+ "暧": 2978,
+ "瞄": 2979,
+ "氓": 2980,
+ "梭": 2981,
+ "甄": 2982,
+ "曳": 2983,
+ "蜘": 2984,
+ "邃": 2985,
+ "嘤": 2986,
+ "嗡": 2987,
+ "眩": 2988,
+ "彪": 2989,
+ "Ｏ": 2990,
+ "钉": 2991,
+ "悚": 2992,
+ "焙": 2993,
+ "唷": 2994,
+ "衍": 2995,
+ "豚": 2996,
+ "锤": 2997,
+ "乞": 2998,
+ "蔽": 2999,
+ "Ｔ": 3000,
+ "顽": 3001,
+ "遂": 3002,
+ "沦": 3003,
+ "拧": 3004,
+ "笛": 3005,
+ "陛": 3006,
+ "Z": 3007,
+ "芭": 3008,
+ "抄": 3009,
+ "鳞": 3010,
+ "戚": 3011,
+ "锈": 3012,
+ "$": 3013,
+ "°": 3014,
+ "踝": 3015,
+ "舱": 3016,
+ "襟": 3017,
+ "朕": 3018,
+ "瞎": 3019,
+ "酬": 3020,
+ "ｒ": 3021,
+ "扼": 3022,
+ "渍": 3023,
+ "鸦": 3024,
+ "璐": 3025,
+ "狡": 3026,
+ "坪": 3027,
+ "歹": 3028,
+ "轿": 3029,
+ "彭": 3030,
+ "丐": 3031,
+ "吨": 3032,
+ "辜": 3033,
+ "窒": 3034,
+ "蹬": 3035,
+ "嘘": 3036,
+ "嚷": 3037,
+ "谣": 3038,
+ "拇": 3039,
+ "粹": 3040,
+ "偎": 3041,
+ "凳": 3042,
+ " ": 3043,
+ "谭": 3044,
+ "蓓": 3045,
+ "嚼": 3046,
+ "圳": 3047,
+ "濒": 3048,
+ "掰": 3049,
+ "礁": 3050,
+ "陡": 3051,
+ "嘱": 3052,
+ "煤": 3053,
+ "蹄": 3054,
+ "陋": 3055,
+ "涎": 3056,
+ "Ｐ": 3057,
+ "坊": 3058,
+ "骼": 3059,
+ "靥": 3060,
+ "菱": 3061,
+ "卸": 3062,
+ "翎": 3063,
+ "樊": 3064,
+ "饺": 3065,
+ "璇": 3066,
+ "萱": 3067,
+ "荫": 3068,
+ "筷": 3069,
+ "厮": 3070,
+ "驯": 3071,
+ "坤": 3072,
+ "愕": 3073,
+ "鸽": 3074,
+ "疚": 3075,
+ "狰": 3076,
+ "椰": 3077,
+ "颁": 3078,
+ "惶": 3079,
+ "绚": 3080,
+ "腮": 3081,
+ "寝": 3082,
+ "裕": 3083,
+ "崔": 3084,
+ "俺": 3085,
+ "诧": 3086,
+ "飓": 3087,
+ "擒": 3088,
+ "磕": 3089,
+ "邓": 3090,
+ "苞": 3091,
+ "揪": 3092,
+ "澜": 3093,
+ "婿": 3094,
+ "绫": 3095,
+ "矜": 3096,
+ "嫉": 3097,
+ "逮": 3098,
+ "睹": 3099,
+ "庸": 3100,
+ "箍": 3101,
+ "炭": 3102,
+ "铐": 3103,
+ "炫": 3104,
+ "嗷": 3105,
+ "焉": 3106,
+ "槽": 3107,
+ "佑": 3108,
+ "蹙": 3109,
+ "屠": 3110,
+ "耕": 3111,
+ "贩": 3112,
+ "躬": 3113,
+ "姗": 3114,
+ "抿": 3115,
+ "ｔ": 3116,
+ "猥": 3117,
+ "曝": 3118,
+ "巩": 3119,
+ "恬": 3120,
+ "魁": 3121,
+ "茅": 3122,
+ "栈": 3123,
+ "梗": 3124,
+ "斐": 3125,
+ "宪": 3126,
+ "邵": 3127,
+ "柚": 3128,
+ "瞅": 3129,
+ "扳": 3130,
+ "敛": 3131,
+ "ｙ": 3132,
+ "匠": 3133,
+ "ｌ": 3134,
+ "倏": 3135,
+ "汀": 3136,
+ "桩": 3137,
+ "髓": 3138,
+ "秩": 3139,
+ "荧": 3140,
+ "鳄": 3141,
+ "黯": 3142,
+ "疤": 3143,
+ "绞": 3144,
+ "芍": 3145,
+ "裴": 3146,
+ "啜": 3147,
+ "盏": 3148,
+ "霉": 3149,
+ "蜷": 3150,
+ "骆": 3151,
+ "勿": 3152,
+ "－": 3153,
+ "炖": 3154,
+ "蓦": 3155,
+ "惟": 3156,
+ "惬": 3157,
+ "晌": 3158,
+ "褐": 3159,
+ "搔": 3160,
+ "讼": 3161,
+ "肾": 3162,
+ "氮": 3163,
+ "潭": 3164,
+ "炙": 3165,
+ "葬": 3166,
+ "糯": 3167,
+ "衔": 3168,
+ "熏": 3169,
+ "株": 3170,
+ "漱": 3171,
+ "馅": 3172,
+ "腋": 3173,
+ "阱": 3174,
+ "枉": 3175,
+ "腌": 3176,
+ "藉": 3177,
+ "矣": 3178,
+ "汝": 3179,
+ "沧": 3180,
+ "|": 3181,
+ "馀": 3182,
+ "睫": 3183,
+ "奠": 3184,
+ "嚎": 3185,
+ "疆": 3186,
+ "刊": 3187,
+ "耽": 3188,
+ "ｓ": 3189,
+ "抒": 3190,
+ "拚": 3191,
+ "渺": 3192,
+ "坎": 3193,
+ "蟹": 3194,
+ "沥": 3195,
+ "涡": 3196,
+ "沛": 3197,
+ "晖": 3198,
+ "•": 3199,
+ "浙": 3200,
+ "睾": 3201,
+ "恳": 3202,
+ "兆": 3203,
+ "嘀": 3204,
+ "搁": 3205,
+ "圭": 3206,
+ "琢": 3207,
+ "涟": 3208,
+ "钦": 3209,
+ "眶": 3210,
+ "徽": 3211,
+ "蔚": 3212,
+ "皂": 3213,
+ "崽": 3214,
+ "钙": 3215,
+ "驴": 3216,
+ "螓": 3217,
+ "惚": 3218,
+ "纺": 3219,
+ "渣": 3220,
+ "碟": 3221,
+ "筝": 3222,
+ "葵": 3223,
+ "蒲": 3224,
+ "谧": 3225,
+ "猿": 3226,
+ "焕": 3227,
+ "仑": 3228,
+ "臊": 3229,
+ "聆": 3230,
+ "挟": 3231,
+ "缴": 3232,
+ "蝠": 3233,
+ "匿": 3234,
+ "惮": 3235,
+ "熔": 3236,
+ "澄": 3237,
+ "萎": 3238,
+ "慷": 3239,
+ "秃": 3240,
+ "乒": 3241,
+ "慵": 3242,
+ "酿": 3243,
+ "朽": 3244,
+ "辽": 3245,
+ "闰": 3246,
+ "锡": 3247,
+ "脐": 3248,
+ "玥": 3249,
+ "乓": 3250,
+ "啃": 3251,
+ "菁": 3252,
+ "峦": 3253,
+ "盎": 3254,
+ "椭": 3255,
+ "瑾": 3256,
+ "沂": 3257,
+ "霖": 3258,
+ "荆": 3259,
+ "畴": 3260,
+ "咛": 3261,
+ "侮": 3262,
+ "j": 3263,
+ "磊": 3264,
+ "卒": 3265,
+ "邱": 3266,
+ "钠": 3267,
+ "耿": 3268,
+ "梢": 3269,
+ "钗": 3270,
+ "峨": 3271,
+ "溺": 3272,
+ "嘎": 3273,
+ "哮": 3274,
+ "狈": 3275,
+ "Ｅ": 3276,
+ "茉": 3277,
+ "Ｋ": 3278,
+ "侃": 3279,
+ "僻": 3280,
+ "哨": 3281,
+ "悸": 3282,
+ "瞩": 3283,
+ "剖": 3284,
+ "崛": 3285,
+ "钘": 3286,
+ "藕": 3287,
+ "盔": 3288,
+ "篡": 3289,
+ "霏": 3290,
+ "徊": 3291,
+ "q": 3292,
+ "Ｊ": 3293,
+ "觅": 3294,
+ "硅": 3295,
+ "俘": 3296,
+ "喵": 3297,
+ "梆": 3298,
+ "蝙": 3299,
+ "冗": 3300,
+ "楠": 3301,
+ "婕": 3302,
+ "壑": 3303,
+ "乍": 3304,
+ "淮": 3305,
+ "堤": 3306,
+ "遏": 3307,
+ "徘": 3308,
+ "ｈ": 3309,
+ "烷": 3310,
+ "硫": 3311,
+ "琉": 3312,
+ "棘": 3313,
+ "涕": 3314,
+ "鬓": 3315,
+ "募": 3316,
+ "扛": 3317,
+ "肘": 3318,
+ "馒": 3319,
+ "牝": 3320,
+ "邢": 3321,
+ "犀": 3322,
+ "Ｎ": 3323,
+ "箫": 3324,
+ "兀": 3325,
+ "涤": 3326,
+ "苔": 3327,
+ "菈": 3328,
+ "憨": 3329,
+ "伶": 3330,
+ "啰": 3331,
+ "逞": 3332,
+ "窟": 3333,
+ "ｍ": 3334,
+ "俞": 3335,
+ "鸥": 3336,
+ "瑛": 3337,
+ "磅": 3338,
+ "垢": 3339,
+ "豁": 3340,
+ "窍": 3341,
+ "舜": 3342,
+ "黝": 3343,
+ "酶": 3344,
+ "呸": 3345,
+ "挲": 3346,
+ "霓": 3347,
+ "Ｖ": 3348,
+ "Ｉ": 3349,
+ "氨": 3350,
+ "ｃ": 3351,
+ "漪": 3352,
+ "颂": 3353,
+ "穗": 3354,
+ "叨": 3355,
+ "雏": 3356,
+ "铝": 3357,
+ "瞟": 3358,
+ "惭": 3359,
+ "灶": 3360,
+ "霎": 3361,
+ "缎": 3362,
+ "鸾": 3363,
+ "娆": 3364,
+ "哒": 3365,
+ "怦": 3366,
+ "璨": 3367,
+ "坞": 3368,
+ "夭": 3369,
+ "喱": 3370,
+ "凋": 3371,
+ "熄": 3372,
+ "弘": 3373,
+ "穹": 3374,
+ "汩": 3375,
+ "亨": 3376,
+ "枣": 3377,
+ "尉": 3378,
+ "寇": 3379,
+ "楞": 3380,
+ "拙": 3381,
+ "礿": 3382,
+ "颐": 3383,
+ "煸": 3384,
+ "滔": 3385,
+ "藻": 3386,
+ "诶": 3387,
+ "掷": 3388,
+ "鸢": 3389,
+ "捻": 3390,
+ "峭": 3391,
+ "彗": 3392,
+ "璀": 3393,
+ "祷": 3394,
+ "苟": 3395,
+ "ｕ": 3396,
+ "冈": 3397,
+ "钞": 3398,
+ "弛": 3399,
+ "濡": 3400,
+ "驼": 3401,
+ "绅": 3402,
+ "絮": 3403,
+ "拎": 3404,
+ "冉": 3405,
+ "岔": 3406,
+ "呦": 3407,
+ "伫": 3408,
+ "祁": 3409,
+ "逾": 3410,
+ "妲": 3411,
+ "陀": 3412,
+ "肮": 3413,
+ "绎": 3414,
+ "冶": 3415,
+ "Ｌ": 3416,
+ "鹦": 3417,
+ "棱": 3418,
+ "熠": 3419,
+ "簇": 3420,
+ "牡": 3421,
+ "臻": 3422,
+ "缤": 3423,
+ "骸": 3424,
+ "虏": 3425,
+ "叱": 3426,
+ "踹": 3427,
+ "敷": 3428,
+ "斧": 3429,
+ "隧": 3430,
+ "昼": 3431,
+ "呗": 3432,
+ "姥": 3433,
+ "孽": 3434,
+ "垒": 3435,
+ "鹉": 3436,
+ "恣": 3437,
+ "奚": 3438,
+ "屉": 3439,
+ "龚": 3440,
+ "辖": 3441,
+ "叭": 3442,
+ "澎": 3443,
+ "Ｇ": 3444,
+ "禀": 3445,
+ "梧": 3446,
+ "嗜": 3447,
+ "枢": 3448,
+ "颅": 3449,
+ "咀": 3450,
+ "呲": 3451,
+ "捶": 3452,
+ "吏": 3453,
+ "鬟": 3454,
+ "彬": 3455,
+ "铸": 3456,
+ "Ｈ": 3457,
+ "媾": 3458,
+ "禹": 3459,
+ "毙": 3460,
+ "攒": 3461,
+ "陕": 3462,
+ "庐": 3463,
+ "蕙": 3464,
+ "屎": 3465,
+ "诏": 3466,
+ "呛": 3467,
+ "葫": 3468,
+ "匕": 3469,
+ "珀": 3470,
+ "绰": 3471,
+ "屡": 3472,
+ "戎": 3473,
+ "倘": 3474,
+ "贬": 3475,
+ "辟": 3476,
+ "喀": 3477,
+ "镶": 3478,
+ "喽": 3479,
+ "噩": 3480,
+ "喳": 3481,
+ "棚": 3482,
+ "眷": 3483,
+ "棠": 3484,
+ "剃": 3485,
+ "矫": 3486,
+ "茗": 3487,
+ "阎": 3488,
+ "缭": 3489,
+ "汲": 3490,
+ "Ｒ": 3491,
+ "诵": 3492,
+ "厄": 3493,
+ "虞": 3494,
+ "栓": 3495,
+ "皓": 3496,
+ "蝎": 3497,
+ "仕": 3498,
+ "稽": 3499,
+ "窘": 3500,
+ "氯": 3501,
+ "浣": 3502,
+ "厥": 3503,
+ "嗒": 3504,
+ "讪": 3505,
+ "缪": 3506,
+ "肋": 3507,
+ "猾": 3508,
+ "禄": 3509,
+ "丙": 3510,
+ "垮": 3511,
+ "兑": 3512,
+ "蔷": 3513,
+ "雍": 3514,
+ "喇": 3515,
+ "蜀": 3516,
+ "舆": 3517,
+ "顷": 3518,
+ "乾": 3519,
+ "扉": 3520,
+ "斌": 3521,
+ "锥": 3522,
+ "卦": 3523,
+ "攥": 3524,
+ "哉": 3525,
+ "霾": 3526,
+ "湃": 3527,
+ "糜": 3528,
+ "磷": 3529,
+ "寥": 3530,
+ "杵": 3531,
+ "寐": 3532,
+ "遐": 3533,
+ "蜗": 3534,
+ "珏": 3535,
+ "迦": 3536,
+ "刁": 3537,
+ "嗲": 3538,
+ "ｄ": 3539,
+ "矢": 3540,
+ "铲": 3541,
+ "袅": 3542,
+ "嘈": 3543,
+ "秉": 3544,
+ "橱": 3545,
+ "叩": 3546,
+ "瘙": 3547,
+ "笋": 3548,
+ "骏": 3549,
+ "赁": 3550,
+ "斋": 3551,
+ "恕": 3552,
+ "霭": 3553,
+ "懊": 3554,
+ "膣": 3555,
+ "怅": 3556,
+ "妤": 3557,
+ "缅": 3558,
+ "萤": 3559,
+ "Ｙ": 3560,
+ "汐": 3561,
+ "晤": 3562,
+ "姣": 3563,
+ "苑": 3564,
+ "绛": 3565,
+ "莽": 3566,
+ "杠": 3567,
+ "惰": 3568,
+ "诅": 3569,
+ "蔑": 3570,
+ "旬": 3571,
+ "驭": 3572,
+ "勋": 3573,
+ "汰": 3574,
+ "晾": 3575,
+ "咻": 3576,
+ "檐": 3577,
+ "斓": 3578,
+ "墟": 3579,
+ "忖": 3580,
+ "亩": 3581,
+ "嬴": 3582,
+ "烙": 3583,
+ "徙": 3584,
+ "蚕": 3585,
+ "赘": 3586,
+ "羁": 3587,
+ "榴": 3588,
+ "迸": 3589,
+ "襄": 3590,
+ "郝": 3591,
+ "Ｑ": 3592,
+ "柑": 3593,
+ "窦": 3594,
+ "侈": 3595,
+ "蝇": 3596,
+ "谚": 3597,
+ "＝": 3598,
+ "鞠": 3599,
+ "衅": 3600,
+ "泞": 3601,
+ "鸳": 3602,
+ "赧": 3603,
+ "吭": 3604,
+ "谛": 3605,
+ "蛊": 3606,
+ "怠": 3607,
+ "炕": 3608,
+ "昵": 3609,
+ "淳": 3610,
+ "坟": 3611,
+ "咆": 3612,
+ "惺": 3613,
+ "粼": 3614,
+ "庵": 3615,
+ "揍": 3616,
+ "靓": 3617,
+ "^": 3618,
+ "祀": 3619,
+ "搀": 3620,
+ "倪": 3621,
+ "叼": 3622,
+ "萦": 3623,
+ "孜": 3624,
+ "竿": 3625,
+ "颔": 3626,
+ "飙": 3627,
+ "谴": 3628,
+ "捺": 3629,
+ "梓": 3630,
+ "酣": 3631,
+ "胭": 3632,
+ "倔": 3633,
+ "勘": 3634,
+ "颓": 3635,
+ "揣": 3636,
+ "庇": 3637,
+ "婵": 3638,
+ "谍": 3639,
+ "钳": 3640,
+ "钾": 3641,
+ "咔": 3642,
+ "癫": 3643,
+ "兢": 3644,
+ "懵": 3645,
+ "贲": 3646,
+ "衩": 3647,
+ "琅": 3648,
+ "谬": 3649,
+ "宕": 3650,
+ "婀": 3651,
+ "嫖": 3652,
+ "睐": 3653,
+ "弩": 3654,
+ "覃": 3655,
+ "粪": 3656,
+ "胺": 3657,
+ "翱": 3658,
+ "瘩": 3659,
+ "衙": 3660,
+ "疙": 3661,
+ "碾": 3662,
+ "皎": 3663,
+ "沌": 3664,
+ "芥": 3665,
+ "晦": 3666,
+ "Ｆ": 3667,
+ "胰": 3668,
+ "漩": 3669,
+ "旭": 3670,
+ "崭": 3671,
+ "忑": 3672,
+ "→": 3673,
+ "棺": 3674,
+ "讳": 3675,
+ "瞰": 3676,
+ "洽": 3677,
+ "佬": 3678,
+ "疵": 3679,
+ "蹑": 3680,
+ "ｋ": 3681,
+ "咂": 3682,
+ "偌": 3683,
+ "唬": 3684,
+ "忐": 3685,
+ "踉": 3686,
+ "慑": 3687,
+ "桔": 3688,
+ "蛤": 3689,
+ "紊": 3690,
+ "跄": 3691,
+ "哽": 3692,
+ "瀚": 3693,
+ "纬": 3694,
+ "眺": 3695,
+ "琥": 3696,
+ "氦": 3697,
+ "麒": 3698,
+ "尻": 3699,
+ "鸯": 3700,
+ "鹃": 3701,
+ "凿": 3702,
+ "滕": 3703,
+ "呤": 3704,
+ "栩": 3705,
+ "觑": 3706,
+ "菩": 3707,
+ "啐": 3708,
+ "攘": 3709,
+ "²": 3710,
+ "烯": 3711,
+ "钧": 3712,
+ "俐": 3713,
+ "坍": 3714,
+ "ｇ": 3715,
+ "馋": 3716,
+ "蜥": 3717,
+ "癖": 3718,
+ "奎": 3719,
+ "霆": 3720,
+ "痹": 3721,
+ "讥": 3722,
+ "翡": 3723,
+ "羔": 3724,
+ "祇": 3725,
+ "诛": 3726,
+ "婧": 3727,
+ "丞": 3728,
+ "髦": 3729,
+ "瞻": 3730,
+ "憧": 3731,
+ "睿": 3732,
+ "萋": 3733,
+ "奕": 3734,
+ "溉": 3735,
+ "葳": 3736,
+ "跋": 3737,
+ "忿": 3738,
+ "沼": 3739,
+ "炯": 3740,
+ "坝": 3741,
+ "祉": 3742,
+ "飒": 3743,
+ "憎": 3744,
+ "煲": 3745,
+ "聋": 3746,
+ "骋": 3747,
+ "腓": 3748,
+ "悴": 3749,
+ "ｐ": 3750,
+ "苛": 3751,
+ "啵": 3752,
+ "阑": 3753,
+ "憬": 3754,
+ "璧": 3755,
+ "孵": 3756,
+ "诠": 3757,
+ "粤": 3758,
+ "陨": 3759,
+ "褥": 3760,
+ "籽": 3761,
+ "鑫": 3762,
+ "剌": 3763,
+ "蚌": 3764,
+ "茧": 3765,
+ "滢": 3766,
+ "寰": 3767,
+ "幢": 3768,
+ "嚓": 3769,
+ "廖": 3770,
+ "斟": 3771,
+ "淆": 3772,
+ "嗣": 3773,
+ "憔": 3774,
+ "跺": 3775,
+ "绊": 3776,
+ "簧": 3777,
+ "捋": 3778,
+ "萼": 3779,
+ "酮": 3780,
+ "甥": 3781,
+ "腑": 3782,
+ "咫": 3783,
+ "枷": 3784,
+ "囔": 3785,
+ "鞘": 3786,
+ "涓": 3787,
+ "酌": 3788,
+ "隋": 3789,
+ "蜿": 3790,
+ "遛": 3791,
+ "戟": 3792,
+ "贿": 3793,
+ "祟": 3794,
+ "鹭": 3795,
+ "💗": 3796,
+ "鲤": 3797,
+ "锯": 3798,
+ "痰": 3799,
+ "擀": 3800,
+ "涝": 3801,
+ "羚": 3802,
+ "嫦": 3803,
+ "芜": 3804,
+ "煽": 3805,
+ "垦": 3806,
+ "篝": 3807,
+ "钝": 3808,
+ "": 3809,
+ "鸵": 3810,
+ "槛": 3811,
+ "＂": 3812,
+ "壹": 3813,
+ "蜒": 3814,
+ "涅": 3815,
+ "铛": 3816,
+ "拷": 3817,
+ "煜": 3818,
+ "撬": 3819,
+ "悻": 3820,
+ "痘": 3821,
+ "苯": 3822,
+ "酚": 3823,
+ "痞": 3824,
+ "啮": 3825,
+ "俑": 3826,
+ "屯": 3827,
+ "啄": 3828,
+ "擂": 3829,
+ "缆": 3830,
+ "髅": 3831,
+ "掺": 3832,
+ "篱": 3833,
+ "萃": 3834,
+ "垄": 3835,
+ "惦": 3836,
+ "镂": 3837,
+ "窈": 3838,
+ "舵": 3839,
+ "貂": 3840,
+ "呓": 3841,
+ "蕃": 3842,
+ "渲": 3843,
+ "骷": 3844,
+ "悯": 3845,
+ "匣": 3846,
+ "跤": 3847,
+ "谑": 3848,
+ "硝": 3849,
+ "阪": 3850,
+ "渎": 3851,
+ "祺": 3852,
+ "痪": 3853,
+ "麿": 3854,
+ "撮": 3855,
+ "酯": 3856,
+ "佼": 3857,
+ "窕": 3858,
+ "裘": 3859,
+ "馥": 3860,
+ "泵": 3861,
+ "涧": 3862,
+ "娑": 3863,
+ "辗": 3864,
+ "诫": 3865,
+ "炳": 3866,
+ "娲": 3867,
+ "簸": 3868,
+ "戮": 3869,
+ "迄": 3870,
+ "蹋": 3871,
+ "鸠": 3872,
+ "叟": 3873,
+ "幂": 3874,
+ "惇": 3875,
+ "蒨": 3876,
+ "荔": 3877,
+ "睦": 3878,
+ "鲑": 3879,
+ "蜴": 3880,
+ "蛟": 3881,
+ "驸": 3882,
+ "殆": 3883,
+ "揩": 3884,
+ "髻": 3885,
+ "皑": 3886,
+ "尧": 3887,
+ "恪": 3888,
+ "遁": 3889,
+ "暄": 3890,
+ "虔": 3891,
+ "闸": 3892,
+ "懦": 3893,
+ "呱": 3894,
+ "吠": 3895,
+ "}": 3896,
+ "赟": 3897,
+ "□": 3898,
+ "霹": 3899,
+ "嗖": 3900,
+ "洼": 3901,
+ "冕": 3902,
+ "袤": 3903,
+ "抉": 3904,
+ "铠": 3905,
+ "倭": 3906,
+ "翕": 3907,
+ "赦": 3908,
+ "帷": 3909,
+ "焯": 3910,
+ "籁": 3911,
+ "胚": 3912,
+ "殴": 3913,
+ "涣": 3914,
+ "渚": 3915,
+ "阙": 3916,
+ "镣": 3917,
+ "栅": 3918,
+ "蘸": 3919,
+ "辫": 3920,
+ "π": 3921,
+ "簌": 3922,
+ "猝": 3923,
+ "蔻": 3924,
+ "噼": 3925,
+ "弈": 3926,
+ "垠": 3927,
+ "靶": 3928,
+ "桦": 3929,
+ "摁": 3930,
+ "唠": 3931,
+ "禾": 3932,
+ "帜": 3933,
+ "羹": 3934,
+ "鲈": 3935,
+ "鞍": 3936,
+ "嬷": 3937,
+ "沅": 3938,
+ "杞": 3939,
+ "缇": 3940,
+ "淅": 3941,
+ "疹": 3942,
+ "锌": 3943,
+ "邸": 3944,
+ "Ｕ": 3945,
+ "阀": 3946,
+ "绢": 3947,
+ "辙": 3948,
+ "饵": 3949,
+ "茱": 3950,
+ "袒": 3951,
+ "酡": 3952,
+ "嘬": 3953,
+ "玷": 3954,
+ "雳": 3955,
+ "坂": 3956,
+ "邑": 3957,
+ "冽": 3958,
+ "蛾": 3959,
+ "蔼": 3960,
+ "瘪": 3961,
+ "韶": 3962,
+ "桀": 3963,
+ "嵩": 3964,
+ "镯": 3965,
+ "锺": 3966,
+ "黠": 3967,
+ "宸": 3968,
+ "牠": 3969,
+ "唰": 3970,
+ "祠": 3971,
+ "蠡": 3972,
+ "颚": 3973,
+ "惆": 3974,
+ "奄": 3975,
+ "茁": 3976,
+ "寅": 3977,
+ "踱": 3978,
+ "蘅": 3979,
+ "蜓": 3980,
+ "坷": 3981,
+ "醛": 3982,
+ "褒": 3983,
+ "镁": 3984,
+ "{": 3985,
+ "侬": 3986,
+ "汶": 3987,
+ "蜻": 3988,
+ "熵": 3989,
+ "翟": 3990,
+ "颌": 3991,
+ "纶": 3992,
+ "睽": 3993,
+ "樽": 3994,
+ "窑": 3995,
+ "缉": 3996,
+ "罡": 3997,
+ "拗": 3998,
+ "邬": 3999,
+ "冀": 4000,
+ "踮": 4001,
+ "娼": 4002,
+ "狙": 4003,
+ "螃": 4004,
+ "焖": 4005,
+ "&": 4006,
+ "拣": 4007,
+ "椿": 4008,
+ "锂": 4009,
+ "驿": 4010,
+ "俨": 4011,
+ "掳": 4012,
+ "阖": 4013,
+ "瓢": 4014,
+ "挞": 4015,
+ "旖": 4016,
+ "褂": 4017,
+ "矗": 4018,
+ "Ｚ": 4019,
+ "剁": 4020,
+ "嗟": 4021,
+ "腆": 4022,
+ "侏": 4023,
+ "迢": 4024,
+ "瑄": 4025,
+ "宦": 4026,
+ "沓": 4027,
+ "嚏": 4028,
+ "咄": 4029,
+ "睬": 4030,
+ "笃": 4031,
+ "簪": 4032,
+ "芋": 4033,
+ "剐": 4034,
+ "曙": 4035,
+ "@": 4036,
+ "淤": 4037,
+ "龌": 4038,
+ "侥": 4039,
+ "榆": 4040,
+ "铎": 4041,
+ "澹": 4042,
+ "悖": 4043,
+ "闵": 4044,
+ "拴": 4045,
+ "ｗ": 4046,
+ "拈": 4047,
+ "剿": 4048,
+ "燎": 4049,
+ "醺": 4050,
+ "恻": 4051,
+ "ｂ": 4052,
+ "簿": 4053,
+ "荤": 4054,
+ "莞": 4055,
+ "旎": 4056,
+ "舶": 4057,
+ "僚": 4058,
+ "圃": 4059,
+ "敝": 4060,
+ "卤": 4061,
+ "壬": 4062,
+ "苇": 4063,
+ "峙": 4064,
+ "忒": 4065,
+ "筐": 4066,
+ "砌": 4067,
+ "氟": 4068,
+ "噎": 4069,
+ "吝": 4070,
+ "桓": 4071,
+ "鹊": 4072,
+ "槟": 4073,
+ "旌": 4074,
+ "溟": 4075,
+ "瞑": 4076,
+ "匈": 4077,
+ "姝": 4078,
+ "疮": 4079,
+ "诣": 4080,
+ "龊": 4081,
+ "惘": 4082,
+ "茬": 4083,
+ "釉": 4084,
+ "炊": 4085,
+ "箐": 4086,
+ "榭": 4087,
+ "冢": 4088,
+ "迥": 4089,
+ "俭": 4090,
+ "跷": 4091,
+ "笙": 4092,
+ "庶": 4093,
+ "卯": 4094,
+ "徨": 4095,
+ "忡": 4096,
+ "匡": 4097,
+ "嘌": 4098,
+ "吒": 4099,
+ "亟": 4100,
+ "悼": 4101,
+ "袂": 4102,
+ "晞": 4103,
+ "ｘ": 4104,
+ "涸": 4105,
+ "浒": 4106,
+ "粟": 4107,
+ "柬": 4108,
+ "肇": 4109,
+ "豌": 4110,
+ "屹": 4111,
+ "祐": 4112,
+ "锢": 4113,
+ "耘": 4114,
+ "恃": 4115,
+ "岖": 4116,
+ "秤": 4117,
+ "槐": 4118,
+ "汨": 4119,
+ "挎": 4120,
+ "眈": 4121,
+ "灐": 4122,
+ "缰": 4123,
+ "谄": 4124,
+ "烽": 4125,
+ "噙": 4126,
+ "罹": 4127,
+ "\t": 4128,
+ "瀛": 4129,
+ "ｆ": 4130,
+ "枭": 4131,
+ "瞿": 4132,
+ "沪": 4133,
+ "炬": 4134,
+ "恙": 4135,
+ "毋": 4136,
+ "稼": 4137,
+ "袄": 4138,
+ "惋": 4139,
+ "鸨": 4140,
+ "巳": 4141,
+ "睨": 4142,
+ "佯": 4143,
+ "庾": 4144,
+ "噘": 4145,
+ "孰": 4146,
+ "苷": 4147,
+ "庚": 4148,
+ "酋": 4149,
+ "幔": 4150,
+ "阂": 4151,
+ "锣": 4152,
+ "邹": 4153,
+ "桨": 4154,
+ "荟": 4155,
+ "磋": 4156,
+ "﹗": 4157,
+ "帛": 4158,
+ "腼": 4159,
+ "嗫": 4160,
+ "缔": 4161,
+ "―": 4162,
+ "吆": 4163,
+ "肽": 4164,
+ "灸": 4165,
+ "躇": 4166,
+ "猕": 4167,
+ "骰": 4168,
+ "藩": 4169,
+ "畸": 4170,
+ "馏": 4171,
+ "钛": 4172,
+ "蒿": 4173,
+ "伎": 4174,
+ "娓": 4175,
+ "欸": 4176,
+ "熨": 4177,
+ "逅": 4178,
+ "粽": 4179,
+ "鳕": 4180,
+ "镀": 4181,
+ "飕": 4182,
+ "邂": 4183,
+ "踌": 4184,
+ "焗": 4185,
+ "耷": 4186,
+ "怂": 4187,
+ "嚅": 4188,
+ "酝": 4189,
+ "晏": 4190,
+ "§": 4191,
+ "譬": 4192,
+ "鹂": 4193,
+ "墩": 4194,
+ "颦": 4195,
+ "–": 4196,
+ "徉": 4197,
+ "陇": 4198,
+ "ｖ": 4199,
+ "鳗": 4200,
+ "咎": 4201,
+ "鳍": 4202,
+ "札": 4203,
+ "扈": 4204,
+ "羑": 4205,
+ "瑙": 4206,
+ "哝": 4207,
+ "抡": 4208,
+ "匮": 4209,
+ "徜": 4210,
+ "烬": 4211,
+ "缨": 4212,
+ "遽": 4213,
+ "啬": 4214,
+ "焊": 4215,
+ "揖": 4216,
+ "痣": 4217,
+ "煦": 4218,
+ "琶": 4219,
+ "Ｗ": 4220,
+ "憩": 4221,
+ "喙": 4222,
+ "缥": 4223,
+ "嗳": 4224,
+ "佻": 4225,
+ "忏": 4226,
+ "谏": 4227,
+ "泓": 4228,
+ "贮": 4229,
+ "驹": 4230,
+ "锵": 4231,
+ "胱": 4232,
+ "☆": 4233,
+ "衲": 4234,
+ "掣": 4235,
+ "盹": 4236,
+ "唏": 4237,
+ "啶": 4238,
+ "昕": 4239,
+ "涔": 4240,
+ "噔": 4241,
+ "耦": 4242,
+ "瘟": 4243,
+ "惴": 4244,
+ "袱": 4245,
+ "麝": 4246,
+ "玟": 4247,
+ "堇": 4248,
+ "岐": 4249,
+ "嘭": 4250,
+ "孪": 4251,
+ "掂": 4252,
+ "轼": 4253,
+ "蚓": 4254,
+ "／": 4255,
+ "涮": 4256,
+ "阉": 4257,
+ "缈": 4258,
+ "琵": 4259,
+ "荀": 4260,
+ "娄": 4261,
+ "赂": 4262,
+ "鹫": 4263,
+ "］": 4264,
+ "［": 4265,
+ "邯": 4266,
+ "嬛": 4267,
+ "螂": 4268,
+ "赃": 4269,
+ "廿": 4270,
+ "蹊": 4271,
+ "诲": 4272,
+ "痊": 4273,
+ "垣": 4274,
+ "吋": 4275,
+ "笈": 4276,
+ "笠": 4277,
+ "荑": 4278,
+ "毡": 4279,
+ "蚯": 4280,
+ "礴": 4281,
+ "辄": 4282,
+ "樵": 4283,
+ "髯": 4284,
+ "茏": 4285,
+ "＊": 4286,
+ "蟆": 4287,
+ "筵": 4288,
+ "镑": 4289,
+ "纨": 4290,
+ "渝": 4291,
+ "嘧": 4292,
+ "蹿": 4293,
+ "渤": 4294,
+ "杳": 4295,
+ "闽": 4296,
+ "媲": 4297,
+ "戛": 4298,
+ "亥": 4299,
+ "嗬": 4300,
+ "砚": 4301,
+ "钏": 4302,
+ "瘀": 4303,
+ "揄": 4304,
+ "芮": 4305,
+ "鬃": 4306,
+ "敖": 4307,
+ "嫡": 4308,
+ "枸": 4309,
+ "诬": 4310,
+ "蟠": 4311,
+ "嗄": 4312,
+ "潢": 4313,
+ "禧": 4314,
+ "姹": 4315,
+ "峥": 4316,
+ "＃": 4317,
+ "瓮": 4318,
+ "蜕": 4319,
+ "笺": 4320,
+ "麸": 4321,
+ "囡": 4322,
+ "狎": 4323,
+ "唆": 4324,
+ "臃": 4325,
+ "愫": 4326,
+ "榈": 4327,
+ "掬": 4328,
+ "弼": 4329,
+ "锄": 4330,
+ "苒": 4331,
+ "迳": 4332,
+ "湍": 4333,
+ "殃": 4334,
+ "楷": 4335,
+ "瞠": 4336,
+ "嗝": 4337,
+ "摒": 4338,
+ "奂": 4339,
+ "猖": 4340,
+ "胥": 4341,
+ "荼": 4342,
+ "崂": 4343,
+ "犊": 4344,
+ "捍": 4345,
+ "烨": 4346,
+ "酉": 4347,
+ "歼": 4348,
+ "酗": 4349,
+ "偕": 4350,
+ "骁": 4351,
+ "佟": 4352,
+ "铮": 4353,
+ "刨": 4354,
+ "砧": 4355,
+ "嫔": 4356,
+ "傀": 4357,
+ "祛": 4358,
+ "俾": 4359,
+ "趐": 4360,
+ "辘": 4361,
+ "嵋": 4362,
+ "隘": 4363,
+ "麓": 4364,
+ "觞": 4365,
+ "烩": 4366,
+ "睑": 4367,
+ "膺": 4368,
+ "揶": 4369,
+ "隅": 4370,
+ "搡": 4371,
+ "瘸": 4372,
+ "阈": 4373,
+ "哔": 4374,
+ "埔": 4375,
+ "楣": 4376,
+ "痫": 4377,
+ "胤": 4378,
+ "荏": 4379,
+ "谕": 4380,
+ "荃": 4381,
+ "雹": 4382,
+ "ㄚ": 4383,
+ "聿": 4384,
+ "戾": 4385,
+ "勐": 4386,
+ "坨": 4387,
+ "渭": 4388,
+ "暹": 4389,
+ "榛": 4390,
+ "鳃": 4391,
+ "喋": 4392,
+ "潦": 4393,
+ "缮": 4394,
+ "蕨": 4395,
+ "虬": 4396,
+ "匍": 4397,
+ "侨": 4398,
+ "毗": 4399,
+ "脓": 4400,
+ "谙": 4401,
+ "镰": 4402,
+ "踵": 4403,
+ "幌": 4404,
+ "◇": 4405,
+ "羲": 4406,
+ "酰": 4407,
+ "瞌": 4408,
+ "讷": 4409,
+ "鄂": 4410,
+ "羌": 4411,
+ "砺": 4412,
+ "犁": 4413,
+ "珞": 4414,
+ "鼾": 4415,
+ "洱": 4416,
+ "帚": 4417,
+ "锚": 4418,
+ "儡": 4419,
+ "祢": 4420,
+ "岑": 4421,
+ "蹶": 4422,
+ "叁": 4423,
+ "鳖": 4424,
+ "①": 4425,
+ "幡": 4426,
+ "噤": 4427,
+ "轧": 4428,
+ "匐": 4429,
+ "迂": 4430,
+ "砾": 4431,
+ "娣": 4432,
+ "擞": 4433,
+ "馁": 4434,
+ "疡": 4435,
+ "苓": 4436,
+ "葩": 4437,
+ "绻": 4438,
+ "俚": 4439,
+ "嶂": 4440,
+ "沽": 4441,
+ "麾": 4442,
+ "珈": 4443,
+ "峒": 4444,
+ "蜈": 4445,
+ "窿": 4446,
+ "妊": 4447,
+ "菖": 4448,
+ "甭": 4449,
+ "犷": 4450,
+ "玺": 4451,
+ "铿": 4452,
+ "衮": 4453,
+ "疟": 4454,
+ "鳌": 4455,
+ "羿": 4456,
+ "︰": 4457,
+ "踞": 4458,
+ "ｑ": 4459,
+ "氲": 4460,
+ "喏": 4461,
+ "隍": 4462,
+ "亘": 4463,
+ "骊": 4464,
+ "潞": 4465,
+ "窖": 4466,
+ "蝗": 4467,
+ "娠": 4468,
+ "缱": 4469,
+ "岫": 4470,
+ "戍": 4471,
+ "蟾": 4472,
+ "氤": 4473,
+ "鹳": 4474,
+ "谩": 4475,
+ "盅": 4476,
+ "蔫": 4477,
+ "韭": 4478,
+ "呷": 4479,
+ "蔗": 4480,
+ "璋": 4481,
+ "祗": 4482,
+ "暝": 4483,
+ "釜": 4484,
+ "蚝": 4485,
+ "舀": 4486,
+ "獠": 4487,
+ "琰": 4488,
+ "骡": 4489,
+ "②": 4490,
+ "仄": 4491,
+ "糗": 4492,
+ "橇": 4493,
+ "泠": 4494,
+ "恿": 4495,
+ "豺": 4496,
+ "跚": 4497,
+ "霁": 4498,
+ "臆": 4499,
+ "崆": 4500,
+ "弋": 4501,
+ "糠": 4502,
+ "鱿": 4503,
+ "偃": 4504,
+ "忪": 4505,
+ "槌": 4506,
+ "咲": 4507,
+ "攫": 4508,
+ "痿": 4509,
+ "贻": 4510,
+ "〞": 4511,
+ "亅": 4512,
+ "琏": 4513,
+ "窣": 4514,
+ "〝": 4515,
+ "ｊ": 4516,
+ "劾": 4517,
+ "觊": 4518,
+ "滟": 4519,
+ "臾": 4520,
+ "篆": 4521,
+ "翌": 4522,
+ "é": 4523,
+ "稷": 4524,
+ "婠": 4525,
+ "懿": 4526,
+ "槿": 4527,
+ "戊": 4528,
+ "汕": 4529,
+ "皋": 4530,
+ "蹴": 4531,
+ "掖": 4532,
+ "诽": 4533,
+ "町": 4534,
+ "觎": 4535,
+ "谔": 4536,
+ "蹒": 4537,
+ "搪": 4538,
+ "帧": 4539,
+ "撷": 4540,
+ "猬": 4541,
+ "箔": 4542,
+ "腚": 4543,
+ "蟑": 4544,
+ "闾": 4545,
+ "孺": 4546,
+ "诩": 4547,
+ "谒": 4548,
+ "窸": 4549,
+ "诙": 4550,
+ "暨": 4551,
+ "鳅": 4552,
+ "稣": 4553,
+ "棹": 4554,
+ "魇": 4555,
+ "皿": 4556,
+ "绥": 4557,
+ "钵": 4558,
+ "嫀": 4559,
+ "拄": 4560,
+ "绡": 4561,
+ "汞": 4562,
+ "俪": 4563,
+ "臼": 4564,
+ "芪": 4565,
+ "纫": 4566,
+ "裟": 4567,
+ "牟": 4568,
+ "舫": 4569,
+ "竺": 4570,
+ "镍": 4571,
+ "菡": 4572,
+ "蛰": 4573,
+ "熹": 4574,
+ "濑": 4575,
+ "孢": 4576,
+ "谤": 4577,
+ "嘹": 4578,
+ "烃": 4579,
+ "殉": 4580,
+ "刽": 4581,
+ "汴": 4582,
+ "攸": 4583,
+ "闫": 4584,
+ "敕": 4585,
+ "跹": 4586,
+ "韬": 4587,
+ "锭": 4588,
+ "👍": 4589,
+ "仞": 4590,
+ "赳": 4591,
+ "馍": 4592,
+ "瘴": 4593,
+ "龈": 4594,
+ "瞇": 4595,
+ "藜": 4596,
+ "夙": 4597,
+ "浃": 4598,
+ "恁": 4599,
+ "湮": 4600,
+ "藓": 4601,
+ "胄": 4602,
+ "蚤": 4603,
+ "娩": 4604,
+ "渥": 4605,
+ "衹": 4606,
+ "龇": 4607,
+ "瘠": 4608,
+ "嫒": 4609,
+ "饯": 4610,
+ "撂": 4611,
+ "屐": 4612,
+ "溥": 4613,
+ "﹖": 4614,
+ "淼": 4615,
+ "・": 4616,
+ "怩": 4617,
+ "泫": 4618,
+ "β": 4619,
+ "讹": 4620,
+ "翊": 4621,
+ "脔": 4622,
+ "曜": 4623,
+ "磺": 4624,
+ "≈": 4625,
+ "黔": 4626,
+ "殒": 4627,
+ "颀": 4628,
+ "戌": 4629,
+ "嫙": 4630,
+ "滇": 4631,
+ "玖": 4632,
+ "濯": 4633,
+ "岌": 4634,
+ "琛": 4635,
+ "倌": 4636,
+ "铳": 4637,
+ "捎": 4638,
+ "筏": 4639,
+ "🌹": 4640,
+ "箕": 4641,
+ "哂": 4642,
+ "绔": 4643,
+ "遑": 4644,
+ "淙": 4645,
+ "纾": 4646,
+ "泯": 4647,
+ "罔": 4648,
+ "辍": 4649,
+ "蓁": 4650,
+ "靳": 4651,
+ "樟": 4652,
+ "雉": 4653,
+ "吖": 4654,
+ "怼": 4655,
+ "愠": 4656,
+ "泾": 4657,
+ "○": 4658,
+ "浚": 4659,
+ "匝": 4660,
+ "彝": 4661,
+ "宓": 4662,
+ "蛹": 4663,
+ "剽": 4664,
+ "啷": 4665,
+ "噌": 4666,
+ "抨": 4667,
+ "毓": 4668,
+ "傥": 4669,
+ "缜": 4670,
+ "恺": 4671,
+ "忸": 4672,
+ "陂": 4673,
+ "啕": 4674,
+ "夯": 4675,
+ "遨": 4676,
+ "宥": 4677,
+ "怆": 4678,
+ "蚩": 4679,
+ "跶": 4680,
+ "淬": 4681,
+ "嗥": 4682,
+ "獒": 4683,
+ "笆": 4684,
+ "沱": 4685,
+ "僮": 4686,
+ "娉": 4687,
+ "垩": 4688,
+ "秧": 4689,
+ "黍": 4690,
+ "琇": 4691,
+ "栾": 4692,
+ "磐": 4693,
+ "咣": 4694,
+ "麋": 4695,
+ "骥": 4696,
+ "锏": 4697,
+ "匾": 4698,
+ "晗": 4699,
+ "蛀": 4700,
+ "驮": 4701,
+ "鲫": 4702,
+ "镭": 4703,
+ "犒": 4704,
+ "昙": 4705,
+ "篓": 4706,
+ "埠": 4707,
+ "沏": 4708,
+ "痔": 4709,
+ "蜊": 4710,
+ "ｚ": 4711,
+ "砥": 4712,
+ "芡": 4713,
+ "朷": 4714,
+ "腩": 4715,
+ "骅": 4716,
+ "嘞": 4717,
+ "胛": 4718,
+ "衾": 4719,
+ "逋": 4720,
+ "蹉": 4721,
+ "怏": 4722,
+ "莴": 4723,
+ "呶": 4724,
+ "囧": 4725,
+ "粲": 4726,
+ "榕": 4727,
+ "坯": 4728,
+ "俸": 4729,
+ "ㄧ": 4730,
+ "蜚": 4731,
+ "槃": 4732,
+ "栉": 4733,
+ "饨": 4734,
+ "哐": 4735,
+ "〔": 4736,
+ "羟": 4737,
+ "俟": 4738,
+ "濮": 4739,
+ "裾": 4740,
+ "〕": 4741,
+ "晟": 4742,
+ "": 4743,
+ "啁": 4744,
+ "纣": 4745,
+ "扪": 4746,
+ "膻": 4747,
+ "隼": 4748,
+ "撵": 4749,
+ "蟋": 4750,
+ "峑": 4751,
+ "蚣": 4752,
+ "辇": 4753,
+ "嵘": 4754,
+ "帼": 4755,
+ "罄": 4756,
+ "楔": 4757,
+ "蟀": 4758,
+ "忱": 4759,
+ "轶": 4760,
+ "坳": 4761,
+ "珺": 4762,
+ "玮": 4763,
+ "莆": 4764,
+ "俦": 4765,
+ "恸": 4766,
+ "碘": 4767,
+ "谟": 4768,
+ "趄": 4769,
+ "③": 4770,
+ "嗑": 4771,
+ "刍": 4772,
+ "舷": 4773,
+ "馄": 4774,
+ "斛": 4775,
+ "癞": 4776,
+ "骝": 4777,
+ "💕": 4778,
+ "〉": 4779,
+ "滂": 4780,
+ "缄": 4781,
+ "肱": 4782,
+ "螨": 4783,
+ "蜃": 4784,
+ "脍": 4785,
+ "卟": 4786,
+ "趔": 4787,
+ "飚": 4788,
+ "〈": 4789,
+ "褚": 4790,
+ "熽": 4791,
+ "橦": 4792,
+ "摹": 4793,
+ "楂": 4794,
+ "鹄": 4795,
+ "晁": 4796,
+ "蓑": 4797,
+ "螳": 4798,
+ "赅": 4799,
+ "仨": 4800,
+ "蓂": 4801,
+ "汾": 4802,
+ "锰": 4803,
+ "砦": 4804,
+ "竣": 4805,
+ "熘": 4806,
+ "璞": 4807,
+ "祯": 4808,
+ "迩": 4809,
+ "靛": 4810,
+ "＋": 4811,
+ "唑": 4812,
+ "榔": 4813,
+ "诘": 4814,
+ "毂": 4815,
+ "饷": 4816,
+ "�": 4817,
+ "燧": 4818,
+ "跆": 4819,
+ "壕": 4820,
+ "翳": 4821,
+ "鞑": 4822,
+ "髋": 4823,
+ "俎": 4824,
+ "铀": 4825,
+ "α": 4826,
+ "囱": 4827,
+ "诃": 4828,
+ "饕": 4829,
+ "硼": 4830,
+ "谆": 4831,
+ "魈": 4832,
+ "衢": 4833,
+ "湄": 4834,
+ "铂": 4835,
+ "鬣": 4836,
+ "怵": 4837,
+ "谪": 4838,
+ "囤": 4839,
+ "诋": 4840,
+ "√": 4841,
+ "嶙": 4842,
+ "跻": 4843,
+ "棂": 4844,
+ "堰": 4845,
+ "戬": 4846,
+ "荻": 4847,
+ "镳": 4848,
+ "殡": 4849,
+ "蝌": 4850,
+ "钊": 4851,
+ "忤": 4852,
+ "秆": 4853,
+ "峋": 4854,
+ "艹": 4855,
+ "讵": 4856,
+ "鹜": 4857,
+ "搽": 4858,
+ "苣": 4859,
+ "笉": 4860,
+ "陲": 4861,
+ "刖": 4862,
+ "桉": 4863,
+ "泗": 4864,
+ "煳": 4865,
+ "炷": 4866,
+ "掴": 4867,
+ "岱": 4868,
+ "呯": 4869,
+ "谗": 4870,
+ "凫": 4871,
+ "咤": 4872,
+ "瘁": 4873,
+ "殇": 4874,
+ "摞": 4875,
+ "罂": 4876,
+ "裨": 4877,
+ "逡": 4878,
+ "殄": 4879,
+ "槎": 4880,
+ "跎": 4881,
+ "晔": 4882,
+ "漳": 4883,
+ "褛": 4884,
+ "嬅": 4885,
+ "蚪": 4886,
+ "郸": 4887,
+ "钜": 4888,
+ "莅": 4889,
+ "钴": 4890,
+ "帏": 4891,
+ "绉": 4892,
+ "铬": 4893,
+ "蓟": 4894,
+ "孱": 4895,
+ "逵": 4896,
+ "嚯": 4897,
+ "苜": 4898,
+ "掇": 4899,
+ "踊": 4900,
+ "赣": 4901,
+ "迤": 4902,
+ "●": 4903,
+ "嘣": 4904,
+ "籘": 4905,
+ "绺": 4906,
+ "抻": 4907,
+ "闩": 4908,
+ "★": 4909,
+ "盂": 4910,
+ "昱": 4911,
+ "辔": 4912,
+ "艮": 4913,
+ "遢": 4914,
+ "夔": 4915,
+ "妪": 4916,
+ "磬": 4917,
+ "赊": 4918,
+ "汛": 4919,
+ "羸": 4920,
+ "烊": 4921,
+ "谲": 4922,
+ "溷": 4923,
+ "蝼": 4924,
+ "谥": 4925,
+ "驷": 4926,
+ "Ⅱ": 4927,
+ "砣": 4928,
+ "飏": 4929,
+ "呣": 4930,
+ "咝": 4931,
+ "潋": 4932,
+ "纂": 4933,
+ "萸": 4934,
+ "墉": 4935,
+ "孚": 4936,
+ "悱": 4937,
+ "偻": 4938,
+ "钿": 4939,
+ "镌": 4940,
+ "盥": 4941,
+ "稔": 4942,
+ "裎": 4943,
+ "骠": 4944,
+ "弭": 4945,
+ "璟": 4946,
+ "佘": 4947,
+ "姈": 4948,
+ "倜": 4949,
+ "偈": 4950,
+ "绶": 4951,
+ "泸": 4952,
+ "漕": 4953,
+ "獭": 4954,
+ "潸": 4955,
+ "潼": 4956,
+ "銮": 4957,
+ "颍": 4958,
+ "擘": 4959,
+ "濠": 4960,
+ "祂": 4961,
+ "荥": 4962,
+ "厩": 4963,
+ "羯": 4964,
+ "闱": 4965,
+ "嫚": 4966,
+ "霰": 4967,
+ "咩": 4968,
+ "翦": 4969,
+ "藐": 4970,
+ "嫈": 4971,
+ "炜": 4972,
+ "喟": 4973,
+ "栀": 4974,
+ "贰": 4975,
+ "邋": 4976,
+ "芫": 4977,
+ "獗": 4978,
+ "堑": 4979,
+ "畿": 4980,
+ "桅": 4981,
+ "鎏": 4982,
+ "龋": 4983,
+ "檄": 4984,
+ "镫": 4985,
+ "蛆": 4986,
+ "轲": 4987,
+ "铆": 4988,
+ "锹": 4989,
+ "耆": 4990,
+ "珚": 4991,
+ "邈": 4992,
+ "瞭": 4993,
+ "鹑": 4994,
+ "寮": 4995,
+ "玑": 4996,
+ "堀": 4997,
+ "讴": 4998,
+ "啻": 4999,
+ "腱": 5000,
+ "铵": 5001,
+ "眦": 5002,
+ "谀": 5003,
+ "镐": 5004,
+ "🙏": 5005,
+ "珐": 5006,
+ "粑": 5007,
+ "掸": 5008,
+ "蹇": 5009,
+ "邺": 5010,
+ "峪": 5011,
+ "拮": 5012,
+ "祚": 5013,
+ "觐": 5014,
+ "榫": 5015,
+ "笫": 5016,
+ "鼹": 5017,
+ "卞": 5018,
+ "苡": 5019,
+ "龛": 5020,
+ "疱": 5021,
+ "荞": 5022,
+ "幄": 5023,
+ "骞": 5024,
+ "丕": 5025,
+ "兖": 5026,
+ "啭": 5027,
+ "蜇": 5028,
+ "咭": 5029,
+ "茌": 5030,
+ "郢": 5031,
+ "骢": 5032,
+ "硒": 5033,
+ "骛": 5034,
+ "槁": 5035,
+ "淄": 5036,
+ "秸": 5037,
+ "饴": 5038,
+ "﹌": 5039,
+ "噶": 5040,
+ "垛": 5041,
+ "擢": 5042,
+ "棣": 5043,
+ "沢": 5044,
+ "箸": 5045,
+ "胫": 5046,
+ "桫": 5047,
+ "褴": 5048,
+ "佝": 5049,
+ "摀": 5050,
+ "蹩": 5051,
+ "隗": 5052,
+ "笳": 5053,
+ "舂": 5054,
+ "鸪": 5055,
+ "氅": 5056,
+ "琨": 5057,
+ "嗞": 5058,
+ "葭": 5059,
+ "😭": 5060,
+ "佰": 5061,
+ "糅": 5062,
+ "岷": 5063,
+ "楫": 5064,
+ "薏": 5065,
+ "剜": 5066,
+ "孀": 5067,
+ "恹": 5068,
+ "旻": 5069,
+ "颧": 5070,
+ "琍": 5071,
+ "瓯": 5072,
+ "孑": 5073,
+ "臧": 5074,
+ "👏": 5075,
+ "湜": 5076,
+ "癜": 5077,
+ "桎": 5078,
+ "碴": 5079,
+ "荚": 5080,
+ "薜": 5081,
+ "姌": 5082,
+ "绾": 5083,
+ "鹧": 5084,
+ "馊": 5085,
+ "黜": 5086,
+ "¥": 5087,
+ "伢": 5088,
+ "疽": 5089,
+ "蠹": 5090,
+ "诮": 5091,
+ "梏": 5092,
+ "琖": 5093,
+ "ā": 5094,
+ "呎": 5095,
+ "嵬": 5096,
+ "佞": 5097,
+ "鄱": 5098,
+ "钨": 5099,
+ "è": 5100,
+ "糍": 5101,
+ "蚺": 5102,
+ "铢": 5103,
+ "④": 5104,
+ "佤": 5105,
+ "骜": 5106,
+ "吣": 5107,
+ "殛": 5108,
+ "乜": 5109,
+ "睇": 5110,
+ "铉": 5111,
+ "榉": 5112,
+ "赡": 5113,
+ "飨": 5114,
+ "泱": 5115,
+ "纡": 5116,
+ "聒": 5117,
+ "侗": 5118,
+ "睥": 5119,
+ "诰": 5120,
+ "纭": 5121,
+ "跛": 5122,
+ "弑": 5123,
+ "斡": 5124,
+ "氰": 5125,
+ "虱": 5126,
+ "袈": 5127,
+ "桧": 5128,
+ "蛎": 5129,
+ "挝": 5130,
+ "蚱": 5131,
+ "楮": 5132,
+ "阡": 5133,
+ "徇": 5134,
+ "锲": 5135,
+ "ì": 5136,
+ "玳": 5137,
+ "钺": 5138,
+ "牒": 5139,
+ "＞": 5140,
+ "忻": 5141,
+ "耙": 5142,
+ "螯": 5143,
+ "隽": 5144,
+ "😊": 5145,
+ "μ": 5146,
+ "哞": 5147,
+ "畦": 5148,
+ "蕈": 5149,
+ "璜": 5150,
+ "簟": 5151,
+ "缃": 5152,
+ "醪": 5153,
+ "鹘": 5154,
+ "橹": 5155,
+ "卮": 5156,
+ "懑": 5157,
+ "笞": 5158,
+ "褓": 5159,
+ "钋": 5160,
+ "虢": 5161,
+ "餮": 5162,
+ "浔": 5163,
+ "鏖": 5164,
+ "骧": 5165,
+ "￥": 5166,
+ "亳": 5167,
+ "狒": 5168,
+ "葆": 5169,
+ "锃": 5170,
+ "塽": 5171,
+ "潆": 5172,
+ "蜍": 5173,
+ "醚": 5174,
+ "钼": 5175,
+ "＜": 5176,
+ "〇": 5177,
+ "裱": 5178,
+ "骈": 5179,
+ "杈": 5180,
+ "馔": 5181,
+ "鳝": 5182,
+ "鸮": 5183,
+ "奘": 5184,
+ "皈": 5185,
+ "唳": 5186,
+ "峤": 5187,
+ "鲟": 5188,
+ "歆": 5189,
+ "姘": 5190,
+ "＆": 5191,
+ "牖": 5192,
+ "丨": 5193,
+ "叵": 5194,
+ "滓": 5195,
+ "贽": 5196,
+ "铄": 5197,
+ "冼": 5198,
+ "嘁": 5199,
+ "擡": 5200,
+ "荇": 5201,
+ "崴": 5202,
+ "椽": 5203,
+ "茴": 5204,
+ "楹": 5205,
+ "砝": 5206,
+ "鲛": 5207,
+ "猞": 5208,
+ "砷": 5209,
+ "庖": 5210,
+ "珣": 5211,
+ "诌": 5212,
+ "邛": 5213,
+ "镕": 5214,
+ "羧": 5215,
+ "赈": 5216,
+ "牍": 5217,
+ "崧": 5218,
+ "箧": 5219,
+ "篑": 5220,
+ "媔": 5221,
+ "鳟": 5222,
+ "陟": 5223,
+ "猁": 5224,
+ "骐": 5225,
+ "疸": 5226,
+ "箴": 5227,
+ "鹞": 5228,
+ "篁": 5229,
+ "馐": 5230,
+ "妘": 5231,
+ "猷": 5232,
+ "纰": 5233,
+ "á": 5234,
+ "佥": 5235,
+ "↓": 5236,
+ "驽": 5237,
+ "皖": 5238,
+ "襁": 5239,
+ "噱": 5240,
+ "溴": 5241,
+ "煨": 5242,
+ "雩": 5243,
+ "à": 5244,
+ "醴": 5245,
+ "侑": 5246,
+ "挹": 5247,
+ "锉": 5248,
+ "镉": 5249,
+ "缁": 5250,
+ "琮": 5251,
+ "铤": 5252,
+ "鹌": 5253,
+ "琊": 5254,
+ "氩": 5255,
+ "蕤": 5256,
+ "鲲": 5257,
+ "🎉": 5258,
+ "蘼": 5259,
+ "岘": 5260,
+ "桡": 5261,
+ "痂": 5262,
+ "蓼": 5263,
+ "吡": 5264,
+ "酊": 5265,
+ "圜": 5266,
+ "佗": 5267,
+ "膈": 5268,
+ "蛩": 5269,
+ "袢": 5270,
+ "㭴": 5271,
+ "襦": 5272,
+ "觥": 5273,
+ "鸩": 5274,
+ "倨": 5275,
+ "樯": 5276,
+ "豉": 5277,
+ "鄢": 5278,
+ "＿": 5279,
+ "膘": 5280,
+ "苕": 5281,
+ "琚": 5282,
+ "θ": 5283,
+ "≤": 5284,
+ "啖": 5285,
+ "嫱": 5286,
+ "爰": 5287,
+ "莘": 5288,
+ "蓿": 5289,
+ "忝": 5290,
+ "斫": 5291,
+ "痢": 5292,
+ "眇": 5293,
+ "磴": 5294,
+ "鼬": 5295,
+ "箩": 5296,
+ "骖": 5297,
+ "屦": 5298,
+ "犄": 5299,
+ "鄄": 5300,
+ "舛": 5301,
+ "í": 5302,
+ "姫": 5303,
+ "扃": 5304,
+ "僖": 5305,
+ "僭": 5306,
+ "扦": 5307,
+ "湫": 5308,
+ "😘": 5309,
+ "歙": 5310,
+ "豢": 5311,
+ "玠": 5312,
+ "旆": 5313,
+ "欹": 5314,
+ "碜": 5315,
+ "铨": 5316,
+ "杓": 5317,
+ "洵": 5318,
+ "矍": 5319,
+ "踟": 5320,
+ "镊": 5321,
+ "鹗": 5322,
+ "崃": 5323,
+ "橐": 5324,
+ "瑭": 5325,
+ "癣": 5326,
+ "😂": 5327,
+ "仃": 5328,
+ "莼": 5329,
+ "伥": 5330,
+ "悝": 5331,
+ "碉": 5332,
+ "≥": 5333,
+ "耒": 5334,
+ "菅": 5335,
+ "仝": 5336,
+ "盱": 5337,
+ "糸": 5338,
+ "莒": 5339,
+ "貔": 5340,
+ "魃": 5341,
+ "鸱": 5342,
+ "噻": 5343,
+ "翮": 5344,
+ "衿": 5345,
+ "颞": 5346,
+ "麂": 5347,
+ "刈": 5348,
+ "懋": 5349,
+ "疣": 5350,
+ "痺": 5351,
+ "舸": 5352,
+ "蒹": 5353,
+ "轭": 5354,
+ "郗": 5355,
+ "の": 5356,
+ "儆": 5357,
+ "睪": 5358,
+ "硌": 5359,
+ "郓": 5360,
+ "铰": 5361,
+ "阆": 5362,
+ "赭": 5363,
+ "蹰": 5364,
+ "齑": 5365,
+ "囗": 5366,
+ "囫": 5367,
+ "囵": 5368,
+ "秾": 5369,
+ "诟": 5370,
+ "酩": 5371,
+ "悭": 5372,
+ "栊": 5373,
+ "椁": 5374,
+ "筇": 5375,
+ "蛔": 5376,
+ "³": 5377,
+ "犟": 5378,
+ "貉": 5379,
+ "杲": 5380,
+ "楸": 5381,
+ "醍": 5382,
+ "嗵": 5383,
+ "禺": 5384,
+ "篙": 5385,
+ "粕": 5386,
+ "荠": 5387,
+ "钎": 5388,
+ "炀": 5389,
+ "猊": 5390,
+ "萩": 5391,
+ "藿": 5392,
+ "蚋": 5393,
+ "逶": 5394,
+ "🌺": 5395,
+ "伈": 5396,
+ "侪": 5397,
+ "匂": 5398,
+ "厝": 5399,
+ "墀": 5400,
+ "孬": 5401,
+ "佚": 5402,
+ "梃": 5403,
+ "獐": 5404,
+ "挈": 5405,
+ "芎": 5406,
+ "苎": 5407,
+ "﹑": 5408,
+ "抟": 5409,
+ "桠": 5410,
+ "榷": 5411,
+ "濂": 5412,
+ "笏": 5413,
+ "傣": 5414,
+ "泮": 5415,
+ "箓": 5416,
+ "菰": 5417,
+ "邕": 5418,
+ "鞅": 5419,
+ "黢": 5420,
+ "弁": 5421,
+ "蒯": 5422,
+ "醐": 5423,
+ "卅": 5424,
+ "枱": 5425,
+ "泷": 5426,
+ "螭": 5427,
+ "餍": 5428,
+ "垆": 5429,
+ "晢": 5430,
+ "🍎": 5431,
+ "‥": 5432,
+ "燮": 5433,
+ "衽": 5434,
+ "谶": 5435,
+ "ǎ": 5436,
+ "珪": 5437,
+ "芩": 5438,
+ "蛐": 5439,
+ "蹼": 5440,
+ "塾": 5441,
+ "豕": 5442,
+ "🍃": 5443,
+ "刎": 5444,
+ "杪": 5445,
+ "淞": 5446,
+ "璘": 5447,
+ "煅": 5448,
+ "氖": 5449,
+ "绀": 5450,
+ "逦": 5451,
+ "髭": 5452,
+ "帙": 5453,
+ "杷": 5454,
+ "鸷": 5455,
+ "ī": 5456,
+ "呒": 5457,
+ "恫": 5458,
+ "潍": 5459,
+ "翛": 5460,
+ "菽": 5461,
+ "蚜": 5462,
+ "诳": 5463,
+ "リ": 5464,
+ "兕": 5465,
+ "诨": 5466,
+ "粳": 5467,
+ "绦": 5468,
+ "蒻": 5469,
+ "躅": 5470,
+ "｝": 5471,
+ "嗪": 5472,
+ "睢": 5473,
+ "硏": 5474,
+ "秣": 5475,
+ "茯": 5476,
+ "貅": 5477,
+ "阊": 5478,
+ "﹔": 5479,
+ "🌞": 5480,
+ "怱": 5481,
+ "谘": 5482,
+ "赓": 5483,
+ "徂": 5484,
+ "缙": 5485,
+ "踫": 5486,
+ "揆": 5487,
+ "淖": 5488,
+ "罅": 5489,
+ "😁": 5490,
+ "奁": 5491,
+ "婺": 5492,
+ "殚": 5493,
+ "碣": 5494,
+ "绌": 5495,
+ "荨": 5496,
+ "酹": 5497,
+ "饽": 5498,
+ "｛": 5499,
+ "晷": 5500,
+ "瞋": 5501,
+ "磔": 5502,
+ "Ⅰ": 5503,
+ "娈": 5504,
+ "楝": 5505,
+ "浵": 5506,
+ "谯": 5507,
+ "忄": 5508,
+ "缟": 5509,
+ "葺": 5510,
+ "趺": 5511,
+ "酞": 5512,
+ "壅": 5513,
+ "桁": 5514,
+ "讧": 5515,
+ "谌": 5516,
+ "跖": 5517,
+ "孛": 5518,
+ "痍": 5519,
+ "巽": 5520,
+ "辎": 5521,
+ "♀": 5522,
+ "伉": 5523,
+ "殁": 5524,
+ "蒟": 5525,
+ "⑤": 5526,
+ "怿": 5527,
+ "晩": 5528,
+ "杼": 5529,
+ "樾": 5530,
+ "殓": 5531,
+ "隈": 5532,
+ "麈": 5533,
+ "🐶": 5534,
+ "搾": 5535,
+ "栎": 5536,
+ "雠": 5537,
+ "嫪": 5538,
+ "拊": 5539,
+ "掾": 5540,
+ "祎": 5541,
+ "莳": 5542,
+ "轸": 5543,
+ "阕": 5544,
+ "栘": 5545,
+ "曷": 5546,
+ "湲": 5547,
+ "滁": 5548,
+ "灞": 5549,
+ "寤": 5550,
+ "嵇": 5551,
+ "珥": 5552,
+ "窠": 5553,
+ "ㄛ": 5554,
+ "撚": 5555,
+ "槊": 5556,
+ "涪": 5557,
+ "甾": 5558,
+ "柩": 5559,
+ "爻": 5560,
+ "璁": 5561,
+ "瓠": 5562,
+ "痈": 5563,
+ "蜢": 5564,
+ "钒": 5565,
+ "侩": 5566,
+ "囿": 5567,
+ "尙": 5568,
+ "旄": 5569,
+ "洄": 5570,
+ "鹈": 5571,
+ "忾": 5572,
+ "琬": 5573,
+ "碛": 5574,
+ "邝": 5575,
+ "颉": 5576,
+ "鲶": 5577,
+ "亓": 5578,
+ "喁": 5579,
+ "ǐ": 5580,
+ "牦": 5581,
+ "睚": 5582,
+ "隳": 5583,
+ "唁": 5584,
+ "枇": 5585,
+ "柰": 5586,
+ "欤": 5587,
+ "砲": 5588,
+ "镗": 5589,
+ "👩": 5590,
+ "怛": 5591,
+ "愆": 5592,
+ "雎": 5593,
+ "い": 5594,
+ "劭": 5595,
+ "桢": 5596,
+ "辊": 5597,
+ "颢": 5598,
+ "鳏": 5599,
+ "柒": 5600,
+ "縻": 5601,
+ "肓": 5602,
+ "腭": 5603,
+ "莠": 5604,
+ "讫": 5605,
+ "僯": 5606,
+ "喑": 5607,
+ "帔": 5608,
+ "矽": 5609,
+ "葚": 5610,
+ "啉": 5611,
+ "佃": 5612,
+ "逑": 5613,
+ "⼩": 5614,
+ "ー": 5615,
+ "柘": 5616,
+ "稹": 5617,
+ "锆": 5618,
+ "饬": 5619,
+ "剡": 5620,
+ "徼": 5621,
+ "撺": 5622,
+ "溽": 5623,
+ "猗": 5624,
+ "闼": 5625,
+ "鹕": 5626,
+ "Ⅲ": 5627,
+ "サ": 5628,
+ "涠": 5629,
+ "玕": 5630,
+ "痤": 5631,
+ "诿": 5632,
+ "＄": 5633,
+ "勖": 5634,
+ "厓": 5635,
+ "湎": 5636,
+ "瑷": 5637,
+ "遒": 5638,
+ "邙": 5639,
+ "閟": 5640,
+ "💔": 5641,
+ "マ": 5642,
+ "郦": 5643,
+ "🌸": 5644,
+ "俳": 5645,
+ "愔": 5646,
+ "搥": 5647,
+ "曛": 5648,
+ "煖": 5649,
+ "瓒": 5650,
+ "腈": 5651,
+ "诓": 5652,
+ "阗": 5653,
+ "鬻": 5654,
+ "鹓": 5655,
+ "圄": 5656,
+ "堞": 5657,
+ "嵯": 5658,
+ "荽": 5659,
+ "赝": 5660,
+ "驺": 5661,
+ "囹": 5662,
+ "埙": 5663,
+ "昶": 5664,
+ "沤": 5665,
+ "辋": 5666,
+ "偾": 5667,
+ "嘚": 5668,
+ "孳": 5669,
+ "庠": 5670,
+ "祜": 5671,
+ "舄": 5672,
+ "茕": 5673,
+ "嘅": 5674,
+ "慉": 5675,
+ "眬": 5676,
+ "Δ": 5677,
+ "γ": 5678,
+ "幪": 5679,
+ "枳": 5680,
+ "洇": 5681,
+ "脘": 5682,
+ "鞣": 5683,
+ "渌": 5684,
+ "狻": 5685,
+ "獾": 5686,
+ "瑗": 5687,
+ "瓤": 5688,
+ "箪": 5689,
+ "蹁": 5690,
+ "酐": 5691,
+ "🌳": 5692,
+ "焱": 5693,
+ "筮": 5694,
+ "薨": 5695,
+ "赍": 5696,
+ "ù": 5697,
+ "ㄟ": 5698,
+ "峇": 5699,
+ "箝": 5700,
+ "芾": 5701,
+ "±": 5702,
+ "⋯": 5703,
+ "☀": 5704,
+ "み": 5705,
+ "仟": 5706,
+ "戢": 5707,
+ "涿": 5708,
+ "癯": 5709,
+ "芰": 5710,
+ "暍": 5711,
+ "穰": 5712,
+ "纥": 5713,
+ "芈": 5714,
+ "蕖": 5715,
+ "褰": 5716,
+ "钯": 5717,
+ "𪩘": 5718,
+ "叻": 5719,
+ "喹": 5720,
+ "搧": 5721,
+ "旸": 5722,
+ "氵": 5723,
+ "沆": 5724,
+ "脲": 5725,
+ "′": 5726,
+ "徕": 5727,
+ "怄": 5728,
+ "荩": 5729,
+ "豸": 5730,
+ "酲": 5731,
+ "驲": 5732,
+ "‧": 5733,
+ "俅": 5734,
+ "缢": 5735,
+ "苻": 5736,
+ "菏": 5737,
+ "鲷": 5738,
+ "や": 5739,
+ "亶": 5740,
+ "岬": 5741,
+ "曩": 5742,
+ "灏": 5743,
+ "疴": 5744,
+ "竽": 5745,
+ "趿": 5746,
+ "遴": 5747,
+ "鏊": 5748,
+ "屣": 5749,
+ "瑁": 5750,
+ "繇": 5751,
+ "薮": 5752,
+ "踽": 5753,
+ "郤": 5754,
+ "旮": 5755,
+ "犸": 5756,
+ "盍": 5757,
+ "觚": 5758,
+ "埂": 5759,
+ "摭": 5760,
+ "珉": 5761,
+ "粝": 5762,
+ "缛": 5763,
+ "胪": 5764,
+ "蛭": 5765,
+ "铡": 5766,
+ "唿": 5767,
+ "珙": 5768,
+ "疝": 5769,
+ "翥": 5770,
+ "℉": 5771,
+ "搴": 5772,
+ "燠": 5773,
+ "砭": 5774,
+ "苋": 5775,
+ "邳": 5776,
+ "旒": 5777,
+ "湟": 5778,
+ "痨": 5779,
+ "籐": 5780,
+ "蛳": 5781,
+ "呖": 5782,
+ "醮": 5783,
+ "鼙": 5784,
+ "叒": 5785,
+ "聩": 5786,
+ "蝓": 5787,
+ "钤": 5788,
+ "铣": 5789,
+ "丌": 5790,
+ "彊": 5791,
+ "悌": 5792,
+ "旯": 5793,
+ "澪": 5794,
+ "瘳": 5795,
+ "骎": 5796,
+ "💪": 5797,
+ "♪": 5798,
+ "⼀": 5799,
+ "瀣": 5800,
+ "蔺": 5801,
+ "洙": 5802,
+ "澧": 5803,
+ "矾": 5804,
+ "趸": 5805,
+ "鳜": 5806,
+ "ω": 5807,
+ "■": 5808,
+ "攲": 5809,
+ "溏": 5810,
+ "昴": 5811,
+ "楉": 5812,
+ "祼": 5813,
+ "荜": 5814,
+ "饧": 5815,
+ "ü": 5816,
+ "哢": 5817,
+ "猱": 5818,
+ "魑": 5819,
+ "￣": 5820,
+ "⿊": 5821,
+ "廪": 5822,
+ "秫": 5823,
+ "蚵": 5824,
+ "轱": 5825,
+ "锑": 5826,
+ "鹬": 5827,
+ "丼": 5828,
+ "夤": 5829,
+ "恽": 5830,
+ "椋": 5831,
+ "沬": 5832,
+ "竦": 5833,
+ "蜮": 5834,
+ "觌": 5835,
+ "鲢": 5836,
+ "鼍": 5837,
+ "👨": 5838,
+ "岿": 5839,
+ "恂": 5840,
+ "掼": 5841,
+ "旃": 5842,
+ "昉": 5843,
+ "簋": 5844,
+ "罍": 5845,
+ "蛞": 5846,
+ "醅": 5847,
+ "锷": 5848,
+ "＇": 5849,
+ "😍": 5850,
+ "栞": 5851,
+ "纮": 5852,
+ "缗": 5853,
+ "鄞": 5854,
+ "｀": 5855,
+ "姪": 5856,
+ "峣": 5857,
+ "薅": 5858,
+ "薤": 5859,
+ "蝈": 5860,
+ "鲔": 5861,
+ "λ": 5862,
+ "⊸": 5863,
+ "诹": 5864,
+ "⼈": 5865,
+ "妗": 5866,
+ "暱": 5867,
+ "瘢": 5868,
+ "蜉": 5869,
+ "蟊": 5870,
+ "趼": 5871,
+ "阃": 5872,
+ "ε": 5873,
+ "ろ": 5874,
+ "幵": 5875,
+ "彀": 5876,
+ "檠": 5877,
+ "爨": 5878,
+ "吚": 5879,
+ "斝": 5880,
+ "氐": 5881,
+ "汜": 5882,
+ "珰": 5883,
+ "瑧": 5884,
+ "耄": 5885,
+ "锗": 5886,
+ "鹢": 5887,
+ "洌": 5888,
+ "甑": 5889,
+ "踅": 5890,
+ "馗": 5891,
+ "璎": 5892,
+ "绂": 5893,
+ "罴": 5894,
+ "胍": 5895,
+ "☕": 5896,
+ "帻": 5897,
+ "徭": 5898,
+ "懔": 5899,
+ "橛": 5900,
+ "蘧": 5901,
+ "蝮": 5902,
+ "铗": 5903,
+ "莜": 5904,
+ "蕲": 5905,
+ "郫": 5906,
+ "吽": 5907,
+ "桷": 5908,
+ "浥": 5909,
+ "蘖": 5910,
+ "蛉": 5911,
+ "赀": 5912,
+ "钡": 5913,
+ "铙": 5914,
+ "龅": 5915,
+ "＠": 5916,
+ "傩": 5917,
+ "儋": 5918,
+ "嬗": 5919,
+ "抺": 5920,
+ "沄": 5921,
+ "螈": 5922,
+ "螫": 5923,
+ "踯": 5924,
+ "镔": 5925,
+ "∞": 5926,
+ "呋": 5927,
+ "廛": 5928,
+ "戗": 5929,
+ "滦": 5930,
+ "邨": 5931,
+ "鲠": 5932,
+ "ó": 5933,
+ "愍": 5934,
+ "枰": 5935,
+ "泚": 5936,
+ "铩": 5937,
+ "🖤": 5938,
+ "嵴": 5939,
+ "氪": 5940,
+ "浼": 5941,
+ "穑": 5942,
+ "蠲": 5943,
+ "颡": 5944,
+ "鼋": 5945,
+ "₂": 5946,
+ "堃": 5947,
+ "彧": 5948,
+ "瘆": 5949,
+ "膦": 5950,
+ "茆": 5951,
+ "菟": 5952,
+ "辂": 5953,
+ "洮": 5954,
+ "猢": 5955,
+ "笄": 5956,
+ "荦": 5957,
+ "蒽": 5958,
+ "轫": 5959,
+ "😄": 5960,
+ "垓": 5961,
+ "摈": 5962,
+ "枥": 5963,
+ "毽": 5964,
+ "淲": 5965,
+ "獣": 5966,
+ "瘘": 5967,
+ "篪": 5968,
+ "纛": 5969,
+ "缫": 5970,
+ "蜱": 5971,
+ "鳐": 5972,
+ "♂": 5973,
+ "坼": 5974,
+ "扥": 5975,
+ "燄": 5976,
+ "砀": 5977,
+ "缣": 5978,
+ "迓": 5979,
+ "鲭": 5980,
+ "嘏": 5981,
+ "晳": 5982,
+ "沭": 5983,
+ "疋": 5984,
+ "皤": 5985,
+ "颙": 5986,
+ "ö": 5987,
+ "◆": 5988,
+ "唻": 5989,
+ "毐": 5990,
+ "焘": 5991,
+ "珮": 5992,
+ "绗": 5993,
+ "喆": 5994,
+ "樗": 5995,
+ "狲": 5996,
+ "畹": 5997,
+ "皴": 5998,
+ "莪": 5999,
+ "啯": 6000,
+ "渑": 6001,
+ "犍": 6002,
+ "钹": 6003,
+ "｜": 6004,
+ "☉": 6005,
+ "狯": 6006,
+ "菝": 6007,
+ "蝰": 6008,
+ "轺": 6009,
+ "铌": 6010,
+ "陉": 6011,
+ "魉": 6012,
+ "佶": 6013,
+ "甍": 6014,
+ "砒": 6015,
+ "荪": 6016,
+ "螟": 6017,
+ "阏": 6018,
+ "阒": 6019,
+ "鬈": 6020,
+ "ē": 6021,
+ "吔": 6022,
+ "炔": 6023,
+ "痧": 6024,
+ "眄": 6025,
+ "谵": 6026,
+ "锶": 6027,
+ "ō": 6028,
+ "唵": 6029,
+ "嗉": 6030,
+ "媄": 6031,
+ "巉": 6032,
+ "沔": 6033,
+ "萘": 6034,
+ "蝾": 6035,
+ "鎗": 6036,
+ "镬": 6037,
+ "ǔ": 6038,
+ "勰": 6039,
+ "秪": 6040,
+ "臜": 6041,
+ "飧": 6042,
+ "髂": 6043,
+ "━": 6044,
+ "す": 6045,
+ "亹": 6046,
+ "栐": 6047,
+ "炅": 6048,
+ "筌": 6049,
+ "蒺": 6050,
+ "郞": 6051,
+ "郴": 6052,
+ "鸰": 6053,
+ "爿": 6054,
+ "笥": 6055,
+ "縠": 6056,
+ "纻": 6057,
+ "轾": 6058,
+ "郅": 6059,
+ "阍": 6060,
+ "隰": 6061,
+ "鲵": 6062,
+ "燊": 6063,
+ "😃": 6064,
+ "😔": 6065,
+ "呔": 6066,
+ "姮": 6067,
+ "柢": 6068,
+ "疥": 6069,
+ "糁": 6070,
+ "茔": 6071,
+ "趵": 6072,
+ "跸": 6073,
+ "骶": 6074,
+ "な": 6075,
+ "妁": 6076,
+ "嫰": 6077,
+ "孥": 6078,
+ "旰": 6079,
+ "柝": 6080,
+ "溧": 6081,
+ "菀": 6082,
+ "蓍": 6083,
+ "裣": 6084,
+ "谠": 6085,
+ "靼": 6086,
+ "韫": 6087,
+ "し": 6088,
+ "愎": 6089,
+ "戕": 6090,
+ "柞": 6091,
+ "絷": 6092,
+ "臌": 6093,
+ "荳": 6094,
+ "蛱": 6095,
+ "圩": 6096,
+ "屙": 6097,
+ "庑": 6098,
+ "揸": 6099,
+ "茭": 6100,
+ "葇": 6101,
+ "蓊": 6102,
+ "迨": 6103,
+ "酢": 6104,
+ "陬": 6105,
+ "鲇": 6106,
+ "鸬": 6107,
+ "ǒ": 6108,
+ "丶": 6109,
+ "啫": 6110,
+ "囷": 6111,
+ "岧": 6112,
+ "擤": 6113,
+ "桖": 6114,
+ "欷": 6115,
+ "獬": 6116,
+ "珯": 6117,
+ "缦": 6118,
+ "缯": 6119,
+ "邠": 6120,
+ "👦": 6121,
+ "ú": 6122,
+ "ρ": 6123,
+ "∩": 6124,
+ "掹": 6125,
+ "舣": 6126,
+ "苫": 6127,
+ "赉": 6128,
+ "魍": 6129,
+ "ū": 6130,
+ "−": 6131,
+ "嶷": 6132,
+ "漯": 6133,
+ "狺": 6134,
+ "贶": 6135,
+ "霪": 6136,
+ "黒": 6137,
+ "😉": 6138,
+ "〗": 6139,
+ "た": 6140,
+ "ン": 6141,
+ "匏": 6142,
+ "彘": 6143,
+ "毵": 6144,
+ "涴": 6145,
+ "篾": 6146,
+ "て": 6147,
+ "冇": 6148,
+ "圻": 6149,
+ "媪": 6150,
+ "廋": 6151,
+ "悛": 6152,
+ "搦": 6153,
+ "朐": 6154,
+ "椀": 6155,
+ "椹": 6156,
+ "楦": 6157,
+ "睃": 6158,
+ "禊": 6159,
+ "绋": 6160,
+ "诜": 6161,
+ "轳": 6162,
+ "颏": 6163,
+ "檗": 6164,
+ "瘗": 6165,
+ "羭": 6166,
+ "胝": 6167,
+ "誊": 6168,
+ "骘": 6169,
+ "＾": 6170,
+ "侔": 6171,
+ "呑": 6172,
+ "嬲": 6173,
+ "崦": 6174,
+ "恚": 6175,
+ "捩": 6176,
+ "燔": 6177,
+ "犽": 6178,
+ "瓴": 6179,
+ "碓": 6180,
+ "稞": 6181,
+ "窅": 6182,
+ "箬": 6183,
+ "臞": 6184,
+ "觜": 6185,
+ "辚": 6186,
+ "髫": 6187,
+ "佢": 6188,
+ "俛": 6189,
+ "呴": 6190,
+ "抃": 6191,
+ "焓": 6192,
+ "牯": 6193,
+ "狳": 6194,
+ "蟥": 6195,
+ "踼": 6196,
+ "镞": 6197,
+ "骀": 6198,
+ "倮": 6199,
+ "呧": 6200,
+ "椟": 6201,
+ "淦": 6202,
+ "羰": 6203,
+ "苴": 6204,
+ "钚": 6205,
+ "鹚": 6206,
+ "黼": 6207,
+ "値": 6208,
+ "囓": 6209,
+ "嶋": 6210,
+ "暾": 6211,
+ "朅": 6212,
+ "潟": 6213,
+ "疠": 6214,
+ "籼": 6215,
+ "绨": 6216,
+ "闳": 6217,
+ "颛": 6218,
+ "│": 6219,
+ "と": 6220,
+ "哋": 6221,
+ "弇": 6222,
+ "撄": 6223,
+ "箨": 6224,
+ "髀": 6225,
+ "鳙": 6226,
+ "🌊": 6227,
+ "🌷": 6228,
+ "𣨼": 6229,
+ "⑥": 6230,
+ "〖": 6231,
+ "圉": 6232,
+ "婌": 6233,
+ "搆": 6234,
+ "曈": 6235,
+ "汊": 6236,
+ "澍": 6237,
+ "熳": 6238,
+ "犰": 6239,
+ "锒": 6240,
+ "鲙": 6241,
+ "鲱": 6242,
+ "⼥": 6243,
+ "姒": 6244,
+ "嬿": 6245,
+ "嵚": 6246,
+ "怙": 6247,
+ "缬": 6248,
+ "聃": 6249,
+ "舖": 6250,
+ "薙": 6251,
+ "蝣": 6252,
+ "镒": 6253,
+ "黥": 6254,
+ "🌍": 6255,
+ "💝": 6256,
+ "φ": 6257,
+ "乇": 6258,
+ "刕": 6259,
+ "慇": 6260,
+ "挻": 6261,
+ "暌": 6262,
+ "浜": 6263,
+ "簷": 6264,
+ "腠": 6265,
+ "萁": 6266,
+ "藁": 6267,
+ "跏": 6268,
+ "跬": 6269,
+ "躜": 6270,
+ "铋": 6271,
+ "铯": 6272,
+ "阄": 6273,
+ "韪": 6274,
+ "𦶟": 6275,
+ "卺": 6276,
+ "埽": 6277,
+ "珲": 6278,
+ "痱": 6279,
+ "缶": 6280,
+ "荅": 6281,
+ "讣": 6282,
+ "讬": 6283,
+ "骓": 6284,
+ "髙": 6285,
+ "龃": 6286,
+ "厶": 6287,
+ "哙": 6288,
+ "柁": 6289,
+ "泅": 6290,
+ "浠": 6291,
+ "溲": 6292,
+ "烝": 6293,
+ "玦": 6294,
+ "瑀": 6295,
+ "痼": 6296,
+ "眙": 6297,
+ "矧": 6298,
+ "耋": 6299,
+ "髌": 6300,
+ "👧": 6301,
+ "𫖯": 6302,
+ "⊙": 6303,
+ "亸": 6304,
+ "狍": 6305,
+ "猡": 6306,
+ "睺": 6307,
+ "粿": 6308,
+ "绠": 6309,
+ "苌": 6310,
+ "衄": 6311,
+ "裒": 6312,
+ "鄯": 6313,
+ "镝": 6314,
+ "雒": 6315,
+ "💋": 6316,
+ "唢": 6317,
+ "啍": 6318,
+ "潴": 6319,
+ "瘿": 6320,
+ "遘": 6321,
+ "钕": 6322,
+ "顼": 6323,
+ "飖": 6324,
+ "鸸": 6325,
+ "🌻": 6326,
+ "😌": 6327,
+ "り": 6328,
+ "屧": 6329,
+ "榼": 6330,
+ "籓": 6331,
+ "镛": 6332,
+ "霅": 6333,
+ "🎓": 6334,
+ "σ": 6335,
+ "‰": 6336,
+ "乂": 6337,
+ "咗": 6338,
+ "塍": 6339,
+ "徬": 6340,
+ "抔": 6341,
+ "攽": 6342,
+ "殂": 6343,
+ "琯": 6344,
+ "痲": 6345,
+ "瞽": 6346,
+ "籴": 6347,
+ "萏": 6348,
+ "萜": 6349,
+ "葶": 6350,
+ "虺": 6351,
+ "蛸": 6352,
+ "跩": 6353,
+ "酤": 6354,
+ "😏": 6355,
+ "唅": 6356,
+ "囝": 6357,
+ "妯": 6358,
+ "姀": 6359,
+ "娌": 6360,
+ "悰": 6361,
+ "昃": 6362,
+ "甙": 6363,
+ "綦": 6364,
+ "芨": 6365,
+ "訾": 6366,
+ "谡": 6367,
+ "谮": 6368,
+ "贳": 6369,
+ "赪": 6370,
+ "铍": 6371,
+ "锳": 6372,
+ "鹋": 6373,
+ "鹩": 6374,
+ "🎂": 6375,
+ "ě": 6376,
+ "Ⅳ": 6377,
+ "搠": 6378,
+ "摽": 6379,
+ "晡": 6380,
+ "毬": 6381,
+ "疔": 6382,
+ "膑": 6383,
+ "舻": 6384,
+ "铟": 6385,
+ "镃": 6386,
+ "鲂": 6387,
+ "😋": 6388,
+ "⼤": 6389,
+ "亻": 6390,
+ "劵": 6391,
+ "缊": 6392,
+ "腘": 6393,
+ "芃": 6394,
+ "钽": 6395,
+ "鹪": 6396,
+ "龉": 6397,
+ "＼": 6398,
+ "🌴": 6399
+ },
+ "unk_token": ""
+ }
+}
\ No newline at end of file
diff --git a/model/char_tokenizer/tokenizer_config.json b/model/char_tokenizer/tokenizer_config.json
new file mode 100644
index 0000000..5289a9a
--- /dev/null
+++ b/model/char_tokenizer/tokenizer_config.json
@@ -0,0 +1,36 @@
+{
+ "added_tokens_decoder": {
+ "0": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "1": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ },
+ "2": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false,
+ "special": true
+ }
+ },
+ "bos_token": "",
+ "clean_up_tokenization_spaces": false,
+ "eos_token": "",
+ "extra_special_tokens": {},
+ "model_max_length": 32768,
+ "pad_token": "",
+ "tokenizer_class": "PreTrainedTokenizerFast",
+ "unk_token": ""
+}
diff --git a/model/dataset.py b/model/dataset.py
index 4336c7e..ff8db5e 100644
--- a/model/dataset.py
+++ b/model/dataset.py
@@ -1,31 +1,87 @@
import json
-import random
-import re
+import subprocess
+from typing import List, Dict
-import pandas as pd
import numpy as np
-from torch.utils.data import Dataset, DataLoader
+from torch.utils.data import Dataset
import torch
-from sklearn.model_selection import train_test_split
import os
-import ast
os.environ["TOKENIZERS_PARALLELISM"] = "false"
+def count_lines(file_name):
+ out = subprocess.getoutput("wc -l %s" % file_name)
+ return int(out.split()[0])
+
+
+def texts_to_bin(input_path, tokenizer, content_key="content"):
+ max_buffered_length = 1 * 1024 * 1024
+ output_path = f'{input_path}.bin'
+ with open(input_path, "r", encoding="utf-8") as reader:
+ with open(output_path, "wb") as writer:
+ buffered_ids = []
+ i = 0
+ while True:
+ line = reader.readline()
+ if not line:
+ break
+ content = json.loads(line).get(content_key, "")
+ if not content:
+ continue
+
+ # 将数据序列化为二进制格式
+ tokenized = tokenizer(content)
+ buffered_ids += tokenized["input_ids"]
+ if len(buffered_ids) >= max_buffered_length:
+ arr = np.array(buffered_ids, dtype=np.uint16)
+ writer.write(arr.tobytes())
+ buffered_ids.clear()
+ i += 1
+ print(f"write {i}m bytes") if i % 100 == 0 else None
+ # 处理最后一段不满max_buffer_length的token序列
+ if len(buffered_ids) > 0:
+ arr = np.array(buffered_ids, dtype=np.uint16)
+ writer.write(arr.tobytes())
+ print(f"write arr: {len(arr)}")
+ return output_path
+
+
+class PretrainBinDataset(Dataset):
+ def __init__(self, data_path, max_tokens):
+ with open(data_path) as f:
+ f.seek(0, 2)
+ self.total_tokens = f.tell() // np.dtype("uint16").itemsize
+ print(f"total_tokens: {self.total_tokens}")
+
+ self.data = np.memmap(data_path, dtype=np.uint16, shape=(self.total_tokens//max_tokens, max_tokens))
+
+ def __len__(self):
+ return self.data.shape[0]
+
+ def __getitem__(self, index):
+ assert isinstance(index, int)
+ item = self.data[index]
+ input = item[:-1].astype(np.int64)
+ target = item[1:].astype(np.int64) # 在计算交叉熵损失时要求目标输出为长整型
+ return torch.from_numpy(input), torch.from_numpy(target)
+
+
class PretrainDataset(Dataset):
- def __init__(self, data_path, tokenizer, max_length=512):
+ def __init__(self, data_paths, tokenizer, max_length=512):
super().__init__()
self.tokenizer = tokenizer
self.max_length = max_length
- self.samples = self.load_data(data_path)
+ self.samples = self.load_data(data_paths)
- def load_data(self, path):
+ @staticmethod
+ def load_data(paths) -> List[Dict]:
samples = []
- with open(path, 'r', encoding='utf-8') as f:
- for line_num, line in enumerate(f, 1):
- data = json.loads(line.strip())
- samples.append(data)
+ for path in paths:
+ with open(path, 'r', encoding='utf-8') as f:
+ for line in f:
+ data = json.loads(line.strip())
+ samples.append(data)
return samples
def __len__(self):
@@ -35,21 +91,16 @@ class PretrainDataset(Dataset):
sample = self.samples[index]
# 构建输入文本
- text = f"{self.tokenizer.bos_token}{str(sample['text'])}{self.tokenizer.eos_token}"
+ text = sample['text']
encoding = self.tokenizer(
text,
max_length=self.max_length,
- padding='max_length',
+ padding=False,
truncation=True,
return_tensors='pt'
)
input_ids = encoding.input_ids.squeeze()
- loss_mask = (input_ids != self.tokenizer.pad_token_id)
-
- X = torch.tensor(input_ids[:-1], dtype=torch.long)
- Y = torch.tensor(input_ids[1:], dtype=torch.long)
- loss_mask = torch.tensor(loss_mask[1:], dtype=torch.long)
- return X, Y, loss_mask
+ return torch.tensor(input_ids, dtype=torch.long)
class SFTDataset(Dataset):
@@ -196,5 +247,6 @@ class DPODataset(Dataset):
return loss_mask
-if __name__ == "__main__":
- pass
+if __name__ == '__main__':
+ c = count_lines("test")
+ print(c)
diff --git a/model/model.py b/model/model.py
index 1dc6478..489ef7b 100644
--- a/model/model.py
+++ b/model/model.py
@@ -12,12 +12,14 @@ from torch import nn
from transformers import PreTrainedModel
from transformers.modeling_outputs import CausalLMOutputWithPast
+from .attention import MLA
+
class RMSNorm(torch.nn.Module):
- def __init__(self, dim: int, eps: float):
+ def __init__(self, dim: int, eps: float, dtype: torch.dtype = torch.float32):
super().__init__()
self.eps = eps
- self.weight = nn.Parameter(torch.ones(dim))
+ self.weight = nn.Parameter(torch.ones(dim, dtype = dtype))
def forward(self, x):
return self.weight * (x.float() * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)).type_as(x)
@@ -68,13 +70,15 @@ class Attention(nn.Module):
self.n_local_kv_heads = self.n_kv_heads
self.n_rep = self.n_local_heads // self.n_local_kv_heads
self.head_dim = args.dim // args.n_heads
- self.wq = nn.Linear(args.dim, args.n_heads * self.head_dim, bias=False)
- self.wk = nn.Linear(args.dim, self.n_kv_heads * self.head_dim, bias=False)
- self.wv = nn.Linear(args.dim, self.n_kv_heads * self.head_dim, bias=False)
- self.wo = nn.Linear(args.n_heads * self.head_dim, args.dim, bias=False)
+ dtype = args.torch_dtype
+ self.wq = nn.Linear(args.dim, args.n_heads * self.head_dim, bias=False, dtype=dtype)
+ self.wk = nn.Linear(args.dim, self.n_kv_heads * self.head_dim, bias=False, dtype=dtype)
+ self.wv = nn.Linear(args.dim, self.n_kv_heads * self.head_dim, bias=False, dtype=dtype)
+ self.wo = nn.Linear(args.n_heads * self.head_dim, args.dim, bias=False, dtype=dtype)
self.attn_dropout = nn.Dropout(args.dropout)
self.resid_dropout = nn.Dropout(args.dropout)
self.dropout = args.dropout
+ self.use_cache = args.use_cache
self.flash = hasattr(torch.nn.functional, 'scaled_dot_product_attention') and args.flash_attn
# print("WARNING: using slow attention. Flash Attention requires PyTorch >= 2.0")
mask = torch.full((1, 1, args.max_seq_len, args.max_seq_len), float("-inf"))
@@ -83,9 +87,10 @@ class Attention(nn.Module):
def forward(self,
x: torch.Tensor,
+ start_pos: int,
pos_cis: torch.Tensor,
past_key_value: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
- use_cache=False):
+ mask: Optional[torch.Tensor] = None):
bsz, seq_len, _ = x.shape
xq, xk, xv = self.wq(x), self.wk(x), self.wv(x)
xq = xq.view(bsz, seq_len, self.n_local_heads, self.head_dim)
@@ -97,7 +102,7 @@ class Attention(nn.Module):
if past_key_value is not None:
xk = torch.cat([past_key_value[0], xk], dim=1)
xv = torch.cat([past_key_value[1], xv], dim=1)
- past_kv = (xk, xv) if use_cache else None
+ past_kv = (xk, xv) if self.use_cache else None
xq, xk, xv = (
xq.transpose(1, 2),
@@ -131,9 +136,10 @@ class FeedForward(nn.Module):
hidden_dim = 4 * config.dim
hidden_dim = int(2 * hidden_dim / 3)
config.hidden_dim = config.multiple_of * ((hidden_dim + config.multiple_of - 1) // config.multiple_of)
- self.w1 = nn.Linear(config.dim, config.hidden_dim, bias=False)
- self.w2 = nn.Linear(config.hidden_dim, config.dim, bias=False)
- self.w3 = nn.Linear(config.dim, config.hidden_dim, bias=False)
+ dtype = config.torch_dtype
+ self.w1 = nn.Linear(config.dim, config.hidden_dim, bias=False, dtype=dtype)
+ self.w2 = nn.Linear(config.hidden_dim, config.dim, bias=False, dtype=dtype)
+ self.w3 = nn.Linear(config.dim, config.hidden_dim, bias=False, dtype=dtype)
self.dropout = nn.Dropout(config.dropout)
def forward(self, x):
@@ -153,7 +159,7 @@ class MoEGate(nn.Module):
self.norm_topk_prob = config.norm_topk_prob
self.gating_dim = config.dim
- self.weight = nn.Parameter(torch.empty((self.n_routed_experts, self.gating_dim)))
+ self.weight = nn.Parameter(torch.empty((self.n_routed_experts, self.gating_dim), dtype=config.torch_dtype))
self.reset_parameters()
def reset_parameters(self) -> None:
@@ -181,7 +187,7 @@ class MoEGate(nn.Module):
topk_idx_for_aux_loss = topk_idx.view(bsz, -1)
if self.seq_aux:
scores_for_seq_aux = scores_for_aux.view(bsz, seq_len, -1)
- ce = torch.zeros(bsz, self.n_routed_experts, device=hidden_states.device)
+ ce = torch.zeros(bsz, self.n_routed_experts, device=hidden_states.device, dtype=self.config.torch_dtype)
ce.scatter_add_(1, topk_idx_for_aux_loss,
torch.ones(bsz, seq_len * aux_topk, device=hidden_states.device)).div_(
seq_len * aux_topk / self.n_routed_experts)
@@ -235,7 +241,7 @@ class MOEFeedForward(nn.Module):
@torch.no_grad()
def moe_infer(self, x, flat_expert_indices, flat_expert_weights):
- expert_cache = torch.zeros_like(x)
+ expert_cache = torch.zeros_like(x, dtype=self.config.torch_dtype)
idxs = flat_expert_indices.argsort()
tokens_per_expert = flat_expert_indices.bincount().cpu().numpy().cumsum(0)
token_idxs = idxs // self.config.num_experts_per_tok
@@ -263,19 +269,20 @@ class MiniMindBlock(nn.Module):
self.n_heads = config.n_heads
self.dim = config.dim
self.head_dim = config.dim // config.n_heads
- self.attention = Attention(config)
+ self.attention = MLA(config) if config.use_mla else Attention(config)
self.layer_id = layer_id
- self.attention_norm = RMSNorm(config.dim, eps=config.norm_eps)
- self.ffn_norm = RMSNorm(config.dim, eps=config.norm_eps)
+ self.attention_norm = RMSNorm(config.dim, eps=config.norm_eps, dtype=config.torch_dtype)
+ self.ffn_norm = RMSNorm(config.dim, eps=config.norm_eps, dtype=config.torch_dtype)
self.feed_forward = FeedForward(config) if not config.use_moe else MOEFeedForward(config)
- def forward(self, x, pos_cis, past_key_value=None, use_cache=False):
+ def forward(self, x, start_pos, pos_cis, past_key_value=None, mask=None):
h_attn, past_kv = self.attention(
self.attention_norm(x),
+ start_pos,
pos_cis,
past_key_value=past_key_value,
- use_cache=use_cache
+ mask=mask
)
h = x + h_attn
out = h + self.feed_forward(self.ffn_norm(h))
@@ -289,32 +296,42 @@ class MiniMindLM(PreTrainedModel):
self.params = params or LMConfig()
super().__init__(self.params)
self.vocab_size, self.n_layers = params.vocab_size, params.n_layers
- self.tok_embeddings = nn.Embedding(params.vocab_size, params.dim)
+ dtype = params.torch_dtype
+ self.tok_embeddings = nn.Embedding(params.vocab_size, params.dim, dtype=dtype)
self.dropout = nn.Dropout(params.dropout)
self.layers = nn.ModuleList([MiniMindBlock(l, params) for l in range(self.n_layers)])
- self.norm = RMSNorm(params.dim, eps=params.norm_eps)
- self.output = nn.Linear(params.dim, params.vocab_size, bias=False)
+ self.norm = RMSNorm(params.dim, eps=params.norm_eps, dtype=dtype)
+ self.output = nn.Linear(params.dim, params.vocab_size, bias=False, dtype=dtype)
self.tok_embeddings.weight = self.output.weight
+ self.use_cache = params.use_cache
self.register_buffer("pos_cis",
- precompute_pos_cis(dim=params.dim // params.n_heads, theta=params.rope_theta),
+ precompute_pos_cis(
+ dim=params.qk_rope_head_dim if params.use_mla else params.dim // params.n_heads,
+ theta=params.rope_theta
+ ),
persistent=False)
self.OUT = CausalLMOutputWithPast()
def forward(self,
input_ids: Optional[torch.Tensor] = None,
past_key_values: Optional[List[Tuple[torch.Tensor, torch.Tensor]]] = None,
- use_cache: bool = False,
**args):
+ seqlen = input_ids.size(1)
past_key_values = past_key_values or [None] * len(self.layers)
start_pos = args.get('start_pos', 0)
h = self.dropout(self.tok_embeddings(input_ids))
- pos_cis = self.pos_cis[start_pos:start_pos + input_ids.size(1)]
+ pos_cis = self.pos_cis[start_pos:start_pos + seqlen]
past_kvs = []
+ mask = None
+ if seqlen > 1:
+ mask = torch.full((seqlen, seqlen), float("-inf"), device=input_ids.device, dtype=self.params.torch_dtype).triu_(1)
for l, layer in enumerate(self.layers):
h, past_kv = layer(
- h, pos_cis,
+ h,
+ start_pos=start_pos,
+ pos_cis=pos_cis,
past_key_value=past_key_values[l],
- use_cache=use_cache
+ mask=mask
)
past_kvs.append(past_kv)
logits = self.output(self.norm(h))
@@ -326,16 +343,16 @@ class MiniMindLM(PreTrainedModel):
@torch.inference_mode()
def generate(self, input_ids, eos_token_id=2, max_new_tokens=1024, temperature=0.75, top_p=0.90,
- stream=False, rp=1., use_cache=True, pad_token_id=0, **args):
+ stream=False, rp=1., pad_token_id=0, **args):
# 流式生成
if stream:
- return self._stream(input_ids, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache, **args)
+ return self._stream(input_ids, eos_token_id, max_new_tokens, temperature, top_p, rp, **args)
# 直接生成
generated = []
for i in range(input_ids.size(0)):
non_pad = input_ids[i][input_ids[i] != pad_token_id].unsqueeze(0)
- out = self._stream(non_pad, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache, **args)
+ out = self._stream(non_pad, eos_token_id, max_new_tokens, temperature, top_p, rp, **args)
tokens_list = [tokens[:, -1:] for tokens in out]
gen = torch.cat(tokens_list, dim=-1) if tokens_list else non_pad
full_sequence = torch.cat([non_pad, gen], dim=-1)
@@ -349,14 +366,13 @@ class MiniMindLM(PreTrainedModel):
]
return torch.cat(generated, dim=0)
- def _stream(self, input_ids, eos_token_id, max_new_tokens, temperature, top_p, rp, use_cache, **args):
+ def _stream(self, input_ids, eos_token_id, max_new_tokens, temperature, top_p, rp, **args):
start, first_seq, past_kvs = input_ids.shape[1], True, None
while input_ids.shape[1] < max_new_tokens - 1:
- if first_seq or not use_cache:
- out, first_seq = self(input_ids, past_key_values=past_kvs, use_cache=use_cache, **args), False
+ if first_seq or not self.use_cache:
+ out, first_seq = self(input_ids, past_key_values=past_kvs, **args), False
else:
- out = self(input_ids[:, -1:], past_key_values=past_kvs, use_cache=use_cache,
- start_pos=input_ids.shape[1] - 1, **args)
+ out = self(input_ids[:, -1:], past_key_values=past_kvs, start_pos=input_ids.shape[1] - 1, **args)
logits, past_kvs = out.logits[:, -1, :], out.past_key_values
logits[:, list(set(input_ids.tolist()[0]))] /= rp
logits /= (temperature + 1e-9)
diff --git a/requirements.txt b/requirements.txt
index 31cd465..177d496 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -19,12 +19,15 @@ scikit_learn==1.5.1
sentence_transformers==2.3.1
simhash==2.1.2
tiktoken==0.5.1
-transformers==4.48.0
+transformers==4.48.3
jinja2==3.1.2
jsonlines==4.0.0
trl==0.13.0
-ujson==5.1.0
+ujson==5.4.0
wandb==0.18.3
streamlit==1.30.0
-torch==2.2.2
-torchvision==0.17.2
\ No newline at end of file
+torch==2.5.1
+requests~=2.32.3
+langchain==0.3.19
+opencc==1.1.9
+tokenizers~=0.21.0
diff --git a/scripts/plot.py b/scripts/plot.py
new file mode 100644
index 0000000..634426f
--- /dev/null
+++ b/scripts/plot.py
@@ -0,0 +1 @@
+# TODO loss图生成
\ No newline at end of file
diff --git a/scripts/train_tokenizer.py b/scripts/train_tokenizer.py
index 868099a..5ad569d 100644
--- a/scripts/train_tokenizer.py
+++ b/scripts/train_tokenizer.py
@@ -16,6 +16,12 @@ import os
random.seed(42)
+base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+
+print(base_dir)
+
+tokenizer_dir = f"{base_dir}/model/my_tokenizer"
+
def train_tokenizer():
# 读取JSONL文件并提取文本数据
@@ -25,7 +31,7 @@ def train_tokenizer():
data = json.loads(line)
yield data['text']
- data_path = '../dataset/pretrain_hq.jsonl'
+ data_path = f'{base_dir}/dataset/pretrain_hq_s.jsonl'
# 初始化tokenizer
tokenizer = Tokenizer(models.BPE())
@@ -57,10 +63,9 @@ def train_tokenizer():
assert tokenizer.token_to_id("") == 2
# 保存tokenizer
- tokenizer_dir = "../model/minimind_tokenizer"
os.makedirs(tokenizer_dir, exist_ok=True)
tokenizer.save(os.path.join(tokenizer_dir, "tokenizer.json"))
- tokenizer.model.save("../model/minimind_tokenizer")
+ tokenizer.model.save(tokenizer_dir)
# 手动创建配置文件
config = {
@@ -118,7 +123,7 @@ def eval_tokenizer():
from transformers import AutoTokenizer
# 加载预训练的tokenizer
- tokenizer = AutoTokenizer.from_pretrained("../model/minimind_tokenizer")
+ tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir)
messages = [
{"role": "system", "content": "你是一个优秀的聊天机器人，总是给我正确的回应！"},
diff --git a/train_pretrain.py b/train_pretrain.py
index a4465b6..af1b871 100644
--- a/train_pretrain.py
+++ b/train_pretrain.py
@@ -1,6 +1,9 @@
+import io
import os
+import sys
import platform
import argparse
+import threading
import time
import math
import warnings
@@ -8,13 +11,16 @@ import pandas as pd
import torch
import torch.distributed as dist
from torch import optim, nn
+from torch._C._profiler import ActiveProfilerType
from torch.nn.parallel import DistributedDataParallel
+from torch.nn.utils.rnn import pad_sequence
from torch.optim.lr_scheduler import CosineAnnealingLR
from torch.utils.data import DataLoader, DistributedSampler
from contextlib import nullcontext
-from transformers import AutoTokenizer
+from transformers import AutoTokenizer, PreTrainedTokenizerFast
+from mem_utils import log_memory_snapshot, ActivationMemoryTracker
from model.model import MiniMindLM
from model.LMConfig import LMConfig
from model.dataset import PretrainDataset
@@ -27,22 +33,43 @@ def Logger(content):
print(content)
+def get_dynamic_lr(cur_step, warmup_steps, decay_steps, lr):
+ min_lr = lr / 10
+ if cur_step < warmup_steps:
+ return lr * (cur_step / warmup_steps)
+ if cur_step > decay_steps:
+ return min_lr
+
+ step_ratio = (cur_step - warmup_steps)/(decay_steps-warmup_steps)
+ cos_scope = 0.5 * (1 + math.cos(math.pi * step_ratio))
+ return min_lr + (lr - min_lr) * cos_scope
+
+
def get_lr(current_step, total_steps, lr):
return lr / 10 + 0.5 * lr * (1 + math.cos(math.pi * current_step / total_steps))
def train_epoch(epoch, wandb):
+ # 初始化跟踪器
+ # tracker = ActivationMemoryTracker()
+ # tracker.track_activations(model)
+
loss_fct = nn.CrossEntropyLoss(reduction='none')
+ step_base = epoch * iter_per_epoch
+ total_steps = iter_per_epoch * args.epochs
+ warmup_steps = int(total_steps * 0.005)
+ decay_steps = int(total_steps * 0.9)
+
+ if epoch == 0:
+ for param_group in optimizer.param_groups: # 在此处设置学习率
+ param_group['lr'] = 5e-6
+
start_time = time.time()
for step, (X, Y, loss_mask) in enumerate(train_loader):
X = X.to(args.device)
Y = Y.to(args.device)
loss_mask = loss_mask.to(args.device)
- lr = get_lr(epoch * iter_per_epoch + step, args.epochs * iter_per_epoch, args.learning_rate)
- for param_group in optimizer.param_groups:
- param_group['lr'] = lr
-
with ctx:
res = model(X)
loss = loss_fct(
@@ -56,6 +83,11 @@ def train_epoch(epoch, wandb):
scaler.scale(loss).backward()
if (step + 1) % args.accumulation_steps == 0:
+ current_step = step_base + step # 计算当前总步数
+ lr = get_dynamic_lr(current_step, warmup_steps, decay_steps, args.learning_rate)
+ for param_group in optimizer.param_groups: # 在此处设置学习率
+ param_group['lr'] = lr
+
scaler.unscale_(optimizer)
torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip)
@@ -67,19 +99,20 @@ def train_epoch(epoch, wandb):
if step % args.log_interval == 0:
spend_time = time.time() - start_time
Logger(
- 'Epoch:[{}/{}]({}/{}) loss:{:.3f} lr:{:.12f} epoch_Time:{}min:'.format(
+ 'Epoch:[{}/{}]({}/{}) loss:{:.3f} lr:{:.12f} spend:{}min remain:{}min'.format(
epoch + 1,
args.epochs,
step,
iter_per_epoch,
loss.item() * args.accumulation_steps,
optimizer.param_groups[-1]['lr'],
+ spend_time // 60,
spend_time / (step + 1) * iter_per_epoch // 60 - spend_time // 60))
if (wandb is not None) and (not ddp or dist.get_rank() == 0):
wandb.log({"loss": loss.item() * args.accumulation_steps,
"lr": optimizer.param_groups[-1]['lr'],
- "epoch_Time": spend_time / (step + 1) * iter_per_epoch // 60 - spend_time // 60})
+ "remain": spend_time / (step + 1) * iter_per_epoch // 60 - spend_time // 60})
if (step + 1) % args.save_interval == 0 and (not ddp or dist.get_rank() == 0):
model.eval()
@@ -91,15 +124,26 @@ def train_epoch(epoch, wandb):
else:
state_dict = model.state_dict()
- torch.save(state_dict, ckp)
+ buffer = io.BytesIO()
+ torch.save(state_dict, buffer)
+ threading.Thread(target=save_checkpoint, args=(buffer, ckp)).start()
model.train()
+ # 内存调试
+ # log_memory_snapshot(model, optimizer, step)
+ # # 输出激活值显存统计
+ # print("中间激活值显存占用：")
+ # total = 0
+ # for name, info in tracker.activation_info.items():
+ # print(f"Layer: {name} | {info}")
+ # total += info["size"]
+ # print(f"中间激活值显存总占用: {total:.2f} MB")
+ # print(torch.cuda.memory_summary())
+ # return
-def init_model(lm_config):
- tokenizer = AutoTokenizer.from_pretrained('./model/minimind_tokenizer')
- model = MiniMindLM(lm_config).to(args.device)
- Logger(f'LLM总参数量：{sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e6:.3f} 百万')
- return model, tokenizer
+def save_checkpoint(buffer: io.BytesIO, path):
+ with open(path, 'wb') as f:
+ f.write(buffer.getvalue())
def init_distributed_mode():
@@ -116,12 +160,21 @@ def init_distributed_mode():
# torchrun --nproc_per_node 2 1-pretrain.py
if __name__ == "__main__":
+ # 启用动态显存段扩展
+ os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
+
parser = argparse.ArgumentParser(description="MiniMind Pretraining")
- parser.add_argument("--out_dir", type=str, default="out")
+ parser.add_argument("--save_dir", type=str, default="out")
# 若要以最快速度实现zero则epochs设置为1轮；否则应当利用有限的数据训练2~6个epochs。
parser.add_argument("--epochs", type=int, default=1)
parser.add_argument("--batch_size", type=int, default=32)
parser.add_argument("--learning_rate", type=float, default=5e-4)
+ parser.add_argument('--dim', default=512, type=int)
+ parser.add_argument('--n_layers', default=8, type=int)
+ parser.add_argument('--n_heads', default=8, type=int)
+ parser.add_argument('--n_kv_heads', default=2, type=int)
+ parser.add_argument('--max_seq_len', default=512, type=int)
+ parser.add_argument('--use_mla', default=False, action='store_true')
parser.add_argument("--device", type=str, default="cuda:0" if torch.cuda.is_available() else "cpu")
parser.add_argument("--dtype", type=str, default="bfloat16")
parser.add_argument("--use_wandb", action="store_true")
@@ -131,27 +184,30 @@ if __name__ == "__main__":
parser.add_argument("--accumulation_steps", type=int, default=8)
parser.add_argument("--grad_clip", type=float, default=1.0)
parser.add_argument("--warmup_iters", type=int, default=0)
- parser.add_argument("--log_interval", type=int, default=100)
- parser.add_argument("--save_interval", type=int, default=100)
+ parser.add_argument("--log_interval", type=int, default=500)
+ parser.add_argument("--save_interval", type=int, default=200)
parser.add_argument('--local_rank', type=int, default=-1)
- parser.add_argument('--dim', default=512, type=int)
- parser.add_argument('--n_layers', default=8, type=int)
- parser.add_argument('--max_seq_len', default=512, type=int)
+ parser.add_argument('--vocab', default='minimind', type=str)
parser.add_argument('--use_moe', default=False, type=bool)
- parser.add_argument("--data_path", type=str, default="./dataset/pretrain_hq.jsonl")
+ parser.add_argument("--data_paths", type=str, nargs="+", default=["./dataset/pretrain_hq.jsonl"])
+ parser.add_argument("--ckpt_path", type=str, default=None)
args = parser.parse_args()
- lm_config = LMConfig(dim=args.dim, n_layers=args.n_layers, max_seq_len=args.max_seq_len, use_moe=args.use_moe)
- args.save_dir = os.path.join(args.out_dir)
+
+ args.log_interval = args.save_interval
os.makedirs(args.save_dir, exist_ok=True)
- os.makedirs(args.out_dir, exist_ok=True)
- tokens_per_iter = args.batch_size * lm_config.max_seq_len
+
torch.manual_seed(1337)
device_type = "cuda" if "cuda" in args.device else "cpu"
args.wandb_run_name = f"MiniMind-Pretrain-Epoch-{args.epochs}-BatchSize-{args.batch_size}-LearningRate-{args.learning_rate}"
- ctx = nullcontext() if device_type == "cpu" else torch.cuda.amp.autocast()
+ if device_type == "cpu":
+ ctx = nullcontext()
+ else:
+ # 映射dtype字符串到torch类型
+ amp_dtype = torch.bfloat16 if args.dtype == 'bfloat16' else torch.float16
+ ctx = torch.cuda.amp.autocast(dtype=amp_dtype)
ddp = int(os.environ.get("RANK", -1)) != -1 # is this a ddp run?
ddp_local_rank, DEVICE = 0, "cuda:0"
@@ -167,20 +223,70 @@ if __name__ == "__main__":
else:
wandb = None
- model, tokenizer = init_model(lm_config)
- train_ds = PretrainDataset(args.data_path, tokenizer, max_length=lm_config.max_seq_len)
+ lm_config = LMConfig(
+ dim=args.dim,
+ n_layers=args.n_layers,
+ n_heads=args.n_heads,
+ n_kv_heads=args.n_kv_heads,
+ max_seq_len=args.max_seq_len,
+ use_moe=args.use_moe,
+ use_mla=args.use_mla,
+ torch_dtype=args.dtype,
+ )
+ tokenizer = AutoTokenizer.from_pretrained(f'./model/{args.vocab}_tokenizer')
+ lm_config.vocab_size = tokenizer.vocab_size
+ model = MiniMindLM(lm_config)
+ Logger(f'LLM总参数量：{sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e6:.3f} 百万')
+
+ if args.ckpt_path is not None:
+ # 接着训练
+ state_dict = torch.load(args.ckpt_path, map_location=args.device)
+ model.load_state_dict({k: v for k, v in state_dict.items() if 'mask' not in k}, strict=True)
+
+ model.to(args.device)
+
+ train_ds = PretrainDataset(args.data_paths, tokenizer, max_length=args.max_seq_len)
+
+ def collate_fn(batch):
+ # 对 batch 中的每个样本进行 padding，动态padding到每个batch的最大长度
+ batch = pad_sequence(batch, batch_first=True, padding_value=tokenizer.pad_token_id)
+ # 以下是padding到最大长度，方便调试显存占用情况
+ # max_len = args.max_seq_len
+ # processed = []
+ # for input_ids in batch:
+ # # 截断超过 max_len 的部分
+ # if len(input_ids) > max_len:
+ # processed.append(input_ids[:max_len])
+ # else:
+ # # 不足则填充到 max_len
+ # padded = torch.full(
+ # (max_len,),
+ # tokenizer.pad_token_id,
+ # dtype=input_ids.dtype
+ # )
+ # padded[:len(input_ids)] = input_ids
+ # processed.append(padded)
+ # batch = torch.stack(processed) # 直接堆叠已等长的样本
+ # 直接通过切片生成输入和目标序列
+ bx = batch[:, :-1] # 输入序列（去掉最后一个token）
+ by = batch[:, 1:] # 目标序列（去掉第一个token）
+ # 生成attention mask（去掉第一个token的位置）
+ bl = (batch != tokenizer.pad_token_id)[:, 1:]
+ return bx, by, bl
+
train_sampler = DistributedSampler(train_ds) if ddp else None
train_loader = DataLoader(
train_ds,
batch_size=args.batch_size,
pin_memory=True,
drop_last=False,
- shuffle=False,
+ shuffle=True,
num_workers=args.num_workers,
- sampler=train_sampler
+ sampler=train_sampler,
+ collate_fn=collate_fn,
)
- scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype in ['float16', 'bfloat16']))
+ scaler = torch.cuda.amp.GradScaler(enabled=(args.dtype == 'float16'))
optimizer = optim.AdamW(model.parameters(), lr=args.learning_rate)
if ddp: