From 3108ee24b546a53dcdc4e8c64486e664254534cb Mon Sep 17 00:00:00 2001
From: swaylq <Fabulouswby@alumni.com>
Date: Wed, 17 Jun 2026 04:43:13 +0800
Subject: [PATCH] fix: don't char-slice the prompt by max_new_tokens /
 max_tokens
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

web_demo.py rendered the full user message in the chat bubble but stored
prompt[-max_new_tokens:] in the message history, so the model received the
user's input front-truncated by *character* count using the generation-length
slider — the UI and the model saw different text, and the beginning of long
questions was silently dropped.

serve_openai_api.py applied the same [-max_tokens:] slice to the rendered
chat template, which corrupts the <|im_start|> structure (e.g. drops the
system block / cuts mid special-token) whenever a client sends a small
max_tokens. max_tokens is the *output* budget and has nothing to do with
trimming the input.

Store the full prompt instead. Input context is already bounded at the token
level by tokenizer(..., truncation=True) (model_max_length=131072).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 scripts/serve_openai_api.py | 4 ++--
 scripts/web_demo.py         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/serve_openai_api.py b/scripts/serve_openai_api.py
index 0550d4a..1e5d3ec 100644
--- a/scripts/serve_openai_api.py
+++ b/scripts/serve_openai_api.py
@@ -104,7 +104,7 @@ def parse_response(text):
 
 def generate_stream_response(messages, temperature, top_p, max_tokens, tools=None, open_thinking=False):
     try:
-        new_prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, tools=tools or None, open_thinking=open_thinking)[-max_tokens:]
+        new_prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, tools=tools or None, open_thinking=open_thinking)
         inputs = tokenizer(new_prompt, return_tensors="pt", truncation=True).to(device)
 
         queue = Queue()
@@ -190,7 +190,7 @@ async def chat_completions(request: ChatRequest):
                 add_generation_prompt=True,
                 tools=request.tools or None,
                 open_thinking=request.get_open_thinking()
-            )[-request.max_tokens:]
+            )
             inputs = tokenizer(new_prompt, return_tensors="pt", truncation=True).to(device)
             with torch.no_grad():
                 generated_ids = model.generate(
diff --git a/scripts/web_demo.py b/scripts/web_demo.py
index b29030d..7debdcc 100644
--- a/scripts/web_demo.py
+++ b/scripts/web_demo.py
@@ -339,8 +339,8 @@ def main():
         st.markdown(
             f'<div style="display: flex; justify-content: flex-end;"><div style="display: inline-block; margin: 10px 0; padding: 8px 12px 8px 12px; background-color: #3d4450; border-radius: 22px; color: white;">{prompt}</div></div>',
             unsafe_allow_html=True)
-        messages.append({"role": "user", "content": prompt[-st.session_state.max_new_tokens:]})
-        st.session_state.chat_messages.append({"role": "user", "content": prompt[-st.session_state.max_new_tokens:]})
+        messages.append({"role": "user", "content": prompt})
+        st.session_state.chat_messages.append({"role": "user", "content": prompt})
 
         placeholder = st.empty()