Compare commits

...

8 Commits

Author SHA1 Message Date
1Dreamer666 65fa8d0701
Merge 743420daac into cc312c1cc6 2026-09-19 09:03:44 +08:00
jingyaogong cc312c1cc6 [fix] unify Ctrl+C handling 2026-09-18 16:33:09 +08:00
jingyaogong e2ee878519
Merge pull request #857 from wbbeyourself/fix-ctrl-c-exception
fix: eval_llm.py Ctrl+C 退出异常修复
2026-09-18 15:54:43 +08:00
jingyaogong b63a9565e3 [fix] restore top-1 router gradient #858 2026-09-18 15:28:50 +08:00
jingyaogong 1aae25911b
Merge pull request #859 from basil-k-aji-dev/fix/moe-router-gradient-topk1
[fix] keep the MoE router trainable at num_experts_per_tok=1
2026-09-18 15:03:36 +08:00
basil-k-aji-dev 1c876435ef [fix] keep the MoE router trainable at num_experts_per_tok=1
Normalising a single top-k weight by its own sum gives exactly 1.0 for every
token, so the gate leaves the graph entirely: topk_idx is a discrete index and
the weight is now a constant, which leaves the LM loss no differentiable path
to the router. It then trains on aux_loss alone, whose optimum is a uniform
distribution, and the experts never specialise.

Measured on MOEFeedForward with router_aux_loss_coef=0, so only the LM path can
contribute a gradient:

  num_experts_per_tok=1, norm_topk_prob=True   7.02e-10   <- current default
  num_experts_per_tok=1, norm_topk_prob=False  1.02e-02
  num_experts_per_tok=2, norm_topk_prob=True   1.67e-02

Only normalise when there is more than one weight to normalise.

Note this changes the forward pass for existing num_experts_per_tok=1 MoE
checkpoints: the expert output was scaled by a constant 1.0 and is now scaled
by the gate probability.

Closes #858.
2026-09-17 21:31:41 +05:30
wbbeyourself 9fa620e2e5 fix: eval_llm.py Ctrl+C 退出异常修复 2026-09-16 17:47:55 +08:00
Toward 743420daac [fix] 更新.gitignore文件,添加忽略目录;修改web_demo.py,优化样式和功能 2026-05-10 23:09:26 +08:00
5 changed files with 81 additions and 13 deletions

6
.gitignore vendored
View File

@ -4,3 +4,9 @@ __pycache__/
out
website/
docs-minimind/
checkpoints/
dataset/
minimind-3/
minimind-3-moe/

View File

@ -91,4 +91,7 @@ def main():
print(f'\n[Speed]: {gen_tokens / (time.time() - st):.2f} tokens/s\n\n') if args.show_speed else print('\n\n')
if __name__ == "__main__":
try:
main()
except (EOFError, KeyboardInterrupt):
print()

View File

@ -158,7 +158,12 @@ class MOEFeedForward(nn.Module):
x_flat = x.view(-1, hidden_dim)
scores = F.softmax(self.gate(x_flat), dim=-1)
topk_weight, topk_idx = torch.topk(scores, k=self.config.num_experts_per_tok, dim=-1, sorted=False)
if self.config.norm_topk_prob: topk_weight = topk_weight / (topk_weight.sum(dim=-1, keepdim=True) + 1e-20)
if self.config.norm_topk_prob:
if self.config.num_experts_per_tok > 1:
topk_weight = topk_weight / (topk_weight.sum(dim=-1, keepdim=True) + 1e-20)
else:
top1 = torch.topk(F.softmax(self.gate(x_flat.detach()), dim=-1), k=1, dim=-1, sorted=False)[0]
topk_weight = top1 - top1.detach() + 1.0 # k=1: 1.0 forward on purpose, gradient via straight-through
y = torch.zeros_like(x_flat)
for i, expert in enumerate(self.experts):
mask = (topk_idx == i)

5
scripts/.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
# .gitignore 忽略下列文件和目录
#比如minimind\scripts\minimind-3和minimind\scripts\minimind-3-moe
minimind-3/
minimind-3-moe/

View File

@ -68,6 +68,28 @@ st.markdown("""
margin: 0 2px !important; /* 调整这里的 margin 值 */
}
/* 单独恢复侧边栏按钮(新建对话)的正常横向长方形外观,避免文字被挤成竖排 */
[data-testid="stSidebar"] .stButton > button {
border-radius: 8px !important; /* 恢复稍微圆角的矩形 */
width: 100% !important; /* 撑满侧边栏宽度 */
height: auto !important;
min-width: 100% !important; /* 覆盖掉 18px 的限制 */
min-height: 40px !important;
max-width: 100% !important;
max-height: none !important;
padding: 8px !important; /* 恢复内边距 */
background-color: #f4f6f8 !important; /* 增加一点按钮底色以免透明看不清 */
border: 1px solid #e0e0e0 !important;
font-size: 15px !important;
color: #333 !important;
margin: 10px 0 !important;
}
[data-testid="stSidebar"] .stButton > button:hover {
border-color: #999 !important;
background-color: #e4e8f0 !important;
color: #000 !important;
}
</style>
""", unsafe_allow_html=True)
@ -87,6 +109,7 @@ LANG_TEXTS = {
'disclaimer': 'AI 生成内容可能存在错误,请仔细核实',
'think_tip': '自适应思考,目前多轮对话或Tool Call共存时思考不稳定',
'tool_select': '工具选择(最多4个)',
'new_chat': '新建对话', # 新增
},
'en': {
'settings': 'Model Settings',
@ -100,6 +123,7 @@ LANG_TEXTS = {
'disclaimer': 'AI-generated content may be inaccurate, please verify',
'think_tip': 'Adaptive thinking; may be unstable with multi-turn or Tool Call',
'tool_select': 'Tool Selection (max 4)',
'new_chat': 'New Chat', # 新增
}
}
@ -241,15 +265,20 @@ def regenerate_answer(index):
# 动态扫描模型目录
script_dir = os.path.dirname(os.path.abspath(__file__))
MODEL_PATHS = {}
for d in sorted(os.listdir(script_dir), reverse=True):
@st.cache_data(ttl=3600) # 增加缓存(有效期1小时),避免每次刷新页面都去高频读取硬盘文件树
def get_model_paths():
script_dir = os.path.dirname(os.path.abspath(__file__))
paths = {}
for d in sorted(os.listdir(script_dir), reverse=True):
full_path = os.path.join(script_dir, d)
if os.path.isdir(full_path) and not d.startswith('.') and not d.startswith('_'):
if any(f.endswith(('.bin', '.safetensors', '.pt')) or os.path.exists(os.path.join(full_path, 'model.safetensors.index.json')) for f in os.listdir(full_path) if os.path.isfile(os.path.join(full_path, f))):
MODEL_PATHS[d] = [d, d]
if not MODEL_PATHS:
MODEL_PATHS = {"No models found": ["", "No models"]}
paths[d] =[d, d]
if not paths:
paths = {"No models found": ["", "No models"]}
return paths
MODEL_PATHS = get_model_paths()
# 模型选择
selected_model = st.sidebar.selectbox('Model', list(MODEL_PATHS.keys()), index=0)
@ -289,7 +318,27 @@ with st.sidebar.expander(get_text('tools')):
if checked and len(st.session_state.selected_tools) < 4:
st.session_state.selected_tools.append(name)
image_url = "https://www.modelscope.cn/api/v1/studio/gongjy/MiniMind/repo?Revision=master&FilePath=images%2Flogo2.png&View=true"
# 新建对话
st.sidebar.markdown('<hr style="margin: 12px 0 16px 0;">', unsafe_allow_html=True)
# 新建对话
if st.sidebar.button(get_text('new_chat'), key="new_chat", use_container_width=True):
clear_chat_messages()
st.rerun()
# 将外部网络图片改为读取本地图片转 base64 渲染
import base64
@st.cache_data
def get_image_base64(image_path):
try:
with open(image_path, "rb") as img_file:
return f"data:image/png;base64,{base64.b64encode(img_file.read()).decode()}"
except:
# 如果本地图片不存在,再退回使用网络URL
return "https://www.modelscope.cn/api/v1/studio/gongjy/MiniMind/repo?Revision=master&FilePath=images%2Flogo2.png&View=true"
local_logo_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "images", "logo2.png")
image_url = get_image_base64(local_logo_path)
st.markdown(
f'<div style="display: flex; flex-direction: column; align-items: center; text-align: center; margin: 0; padding: 0;">'