diff --git a/scripts/serve_openai_api.py b/scripts/serve_openai_api.py index 0550d4a..1e5d3ec 100644 --- a/scripts/serve_openai_api.py +++ b/scripts/serve_openai_api.py @@ -104,7 +104,7 @@ def parse_response(text): def generate_stream_response(messages, temperature, top_p, max_tokens, tools=None, open_thinking=False): try: - new_prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, tools=tools or None, open_thinking=open_thinking)[-max_tokens:] + new_prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, tools=tools or None, open_thinking=open_thinking) inputs = tokenizer(new_prompt, return_tensors="pt", truncation=True).to(device) queue = Queue() @@ -190,7 +190,7 @@ async def chat_completions(request: ChatRequest): add_generation_prompt=True, tools=request.tools or None, open_thinking=request.get_open_thinking() - )[-request.max_tokens:] + ) inputs = tokenizer(new_prompt, return_tensors="pt", truncation=True).to(device) with torch.no_grad(): generated_ids = model.generate( diff --git a/scripts/web_demo.py b/scripts/web_demo.py index b29030d..7debdcc 100644 --- a/scripts/web_demo.py +++ b/scripts/web_demo.py @@ -339,8 +339,8 @@ def main(): st.markdown( f'