From a10fa6c148ed274d66f96dc119689e93e01be823 Mon Sep 17 00:00:00 2001 From: jingyaogong Date: Sun, 28 Jun 2026 18:25:56 +0800 Subject: [PATCH] [fix] gradio audio devices --- README.md | 2 ++ README_en.md | 2 ++ scripts/web_demo_omni.py | 55 ++++++++++++++++++++++++++++++++-------- 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index bbe3810..863b894 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,8 @@ cp -r minimind-3o ./scripts/minimind-3o cd scripts && python web_demo_omni.py ``` +注：`scripts/web_demo_omni.py` 是 Gradio 非实时演示，上传/录音音频会在后端按实际采样率重采样到 16k；实时语音通话请使用 `webui/web_demo.py`。 + ## Ⅱ 🛠️ 模型训练
diff --git a/README_en.md b/README_en.md index 43a1d8d..a7b3532 100644 --- a/README_en.md +++ b/README_en.md @@ -188,6 +188,8 @@ cp -r minimind-3o ./scripts/minimind-3o cd scripts && python web_demo_omni.py ``` +Note: `scripts/web_demo_omni.py` is the non-realtime Gradio demo. Uploaded or recorded audio is resampled on the backend from its actual sample rate to 16 kHz; use `webui/web_demo.py` for realtime voice calls. + ## Ⅱ 🛠️ Training
diff --git a/scripts/web_demo_omni.py b/scripts/web_demo_omni.py index 2abdffd..81f5dcf 100644 --- a/scripts/web_demo_omni.py +++ b/scripts/web_demo_omni.py @@ -37,6 +37,40 @@ voices_data = {} builtin_voices, clone_voices = set(), set() +def default_device(): + if torch.cuda.is_available(): + return 'cuda' + if hasattr(torch.backends, 'mps') and torch.backends.mps.is_available(): + return 'mps' + return 'cpu' + + +def resolve_aux_device(value, fallback): + if value and value != 'auto': + return value + return fallback + + +def get_device_type(value): + return torch.device(value).type + + +def normalize_audio(samples, sr, target_sr=16000): + if len(samples.shape) > 1: + samples = samples.mean(axis=1) + if np.issubdtype(samples.dtype, np.integer): + max_value = max(abs(np.iinfo(samples.dtype).min), np.iinfo(samples.dtype).max) + samples = samples.astype(np.float32) / max_value + else: + samples = samples.astype(np.float32) + peak = np.abs(samples).max() if samples.size else 0 + if peak > 1.0: + samples = samples / peak + if sr != target_sr: + samples = librosa.resample(samples.astype(float), orig_sr=sr, target_sr=target_sr).astype(np.float32) + return np.ascontiguousarray(samples, dtype=np.float32) + + def frames_to_mimi(frames): codes = [f for f in frames if f and len(f) == 8] if not codes: @@ -111,12 +145,7 @@ def chat_stream(prompt, audio_input=None, image_input=None, voice_name="default" if audio_input is not None: sr, samples = audio_input - if len(samples.shape) > 1: - samples = samples.mean(axis=1) - if samples.dtype != np.float32: - samples = samples.astype(np.float32) / max(np.abs(samples).max(), 1) - if sr != 16000: - samples = librosa.resample(samples.astype(float), orig_sr=sr, target_sr=16000).astype(np.float32) + samples = normalize_audio(samples, sr) inputs = model.audio_processor(samples, sampling_rate=16000, return_tensors="pt", return_attention_mask=True) mel = inputs.input_features.squeeze(0) valid_len = inputs.attention_mask.sum().item() @@ -266,13 +295,17 @@ if __name__ == '__main__': parser.add_argument('--audio_encoder', default='../model/SenseVoiceSmall', type=str) parser.add_argument('--vision_model', default='../model/siglip2-base-p32-256-ve', type=str) parser.add_argument('--mimi_path', default='../model/mimi', type=str) - parser.add_argument('--device', default='cuda' if torch.cuda.is_available() else 'cpu', type=str) + parser.add_argument('--device', default=default_device(), type=str) + parser.add_argument('--asr_device', default='auto', type=str, help='ASR设备；auto 在 MPS 主设备下使用 cpu，其它情况跟随 --device') + parser.add_argument('--mimi_device', default='auto', type=str, help='Mimi 解码设备；auto 跟随 --device') parser.add_argument('--open_thinking', default=0, type=int, choices=[0, 1]) parser.add_argument('--top_p', default=0.85, type=float) parser.add_argument('--port', default=8888, type=int) args = parser.parse_args() device = args.device + asr_device = resolve_aux_device(args.asr_device, 'cpu' if get_device_type(device) == 'mps' else device) + mimi_device = resolve_aux_device(args.mimi_device, device) model_dict = scan_hf_models(args.load_from) if not model_dict: print(f"未在 {os.path.abspath(args.load_from)} 找到 transformers 模型") @@ -281,14 +314,16 @@ if __name__ == '__main__': load_hf_model(model_dict[current_model_name]) try: - mimi_model = MimiModel.from_pretrained(args.mimi_path).eval() - print('Mimi model loaded') + mimi_model = MimiModel.from_pretrained(args.mimi_path).eval().to(mimi_device) + if get_device_type(mimi_device) != 'cpu': + mimi_model = mimi_model.half() + print(f'Mimi model loaded on {mimi_device}') except Exception: mimi_model = None print('Mimi model not found, audio output disabled') with contextlib.redirect_stdout(io.StringIO()): - asr_model = AutoModel(model=args.audio_encoder, trust_remote_code=True, device=device, disable_update=True) + asr_model = AutoModel(model=args.audio_encoder, trust_remote_code=True, device=asr_device, disable_update=True) load_voices() print(f'Voices loaded: {list(voices_data.keys()) or "none"}') launch_gradio(server_port=args.port)