From cf2329fab09c9594edf8767a51435901ccca37d8 Mon Sep 17 00:00:00 2001 From: RileyChen23 <2017009663@qq.com> Date: Tue, 14 Jul 2026 18:42:22 +0800 Subject: [PATCH] feat: add reproducible MBPP code evaluation --- code_agent/backends.py | 21 +++-- docs/code-agent.md | 24 +++++- scripts/generate_codegen.py | 82 ++++++++++++++++++ scripts/prepare_mbpp.py | 149 +++++++++++++++++++++++++++++++++ scripts/serve_openai_api.py | 14 +++- tests/test_code_backend.py | 5 ++ tests/test_generate_codegen.py | 25 ++++++ tests/test_prepare_mbpp.py | 50 +++++++++++ 8 files changed, 358 insertions(+), 12 deletions(-) create mode 100644 scripts/generate_codegen.py create mode 100644 scripts/prepare_mbpp.py create mode 100644 tests/test_generate_codegen.py create mode 100644 tests/test_prepare_mbpp.py diff --git a/code_agent/backends.py b/code_agent/backends.py index e6aa34f..fcac0c5 100644 --- a/code_agent/backends.py +++ b/code_agent/backends.py @@ -17,6 +17,7 @@ class OpenAICompatibleGenerator: temperature: float = 0.7, max_tokens: int = 1024, open_thinking: bool = False, + seed: int | None = None, client: Any = None, ) -> None: if client is None: @@ -30,15 +31,23 @@ class OpenAICompatibleGenerator: self.temperature = temperature self.max_tokens = max_tokens self.open_thinking = open_thinking + self.seed = seed + self._request_index = 0 def __call__(self, prompt: str) -> str: + request = { + "model": self.model, + "messages": [{"role": "user", "content": prompt}], + "temperature": self.temperature, + "max_tokens": self.max_tokens, + "stream": False, + "extra_body": {"open_thinking": self.open_thinking}, + } + if self.seed is not None: + request["seed"] = self.seed + self._request_index + self._request_index += 1 response = self.client.chat.completions.create( - model=self.model, - messages=[{"role": "user", "content": prompt}], - temperature=self.temperature, - max_tokens=self.max_tokens, - stream=False, - extra_body={"open_thinking": self.open_thinking}, + **request, ) if not response.choices or response.choices[0].message is None: raise RuntimeError("Model API returned no completion choice.") diff --git a/docs/code-agent.md b/docs/code-agent.md index f11b988..7d0f4df 100644 --- a/docs/code-agent.md +++ b/docs/code-agent.md @@ -37,13 +37,33 @@ Each JSONL record contains a prompt, a required function name, and JSON-serializ `dataset/code_rl_mini.jsonl` contains four smoke-test tasks. It is deliberately too small for a meaningful training claim; use it to validate the pipeline before preparing a larger train/validation/test split. +For a standard public benchmark, convert the compatible subset of MBPP-sanitized while preserving its official train/validation/test split: + +```bash +python scripts/prepare_mbpp.py --output-dir out/mbpp_sanitized +``` + +The converter accepts direct, JSON-serializable function assertions, validates every converted task against the published reference solution, and reports all skipped records. It also emits `train_sft.jsonl` from the official training references, enabling an SFT → execution-reward GRPO ablation without leaking validation or test solutions. Generated benchmark files remain under the ignored `out/` directory. + ## Evaluate saved candidates +Generate multiple candidates from a running OpenAI-compatible model service with deterministic per-request seeds: + +```bash +python scripts/generate_codegen.py \ + --tasks dataset/code_rl_mini.jsonl \ + --samples-per-task 10 \ + --seed 42 \ + --output out/code_predictions.jsonl +``` + +Then execute and aggregate them: + ```bash python scripts/eval_codegen.py \ --tasks dataset/code_rl_mini.jsonl \ - --predictions tests/fixtures/code_rl_predictions.jsonl \ - --ks 1 2 + --predictions out/code_predictions.jsonl \ + --ks 1 5 10 ``` The report separates: diff --git a/scripts/generate_codegen.py b/scripts/generate_codegen.py new file mode 100644 index 0000000..a8b548a --- /dev/null +++ b/scripts/generate_codegen.py @@ -0,0 +1,82 @@ +"""Generate reproducible multi-sample candidates for execution-based evaluation.""" + +import argparse +import json +import os +import sys +from pathlib import Path + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from code_agent import CodeTask, OpenAICompatibleGenerator + + +def read_tasks(path): + tasks = [] + with Path(path).open("r", encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + if not line.strip(): + continue + try: + tasks.append(CodeTask.from_dict(json.loads(line))) + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise ValueError(f"Invalid task at {path}:{line_number}: {exc}") from exc + return tasks + + +def build_prompt(task): + return ( + "You are solving a Python algorithm task. Return exactly one ```python``` code block.\n" + f"Required entry point: {task.entry_point}\n" + f"Task:\n{task.prompt}\n" + "Do not read files, access the network, spawn processes, or call unsafe builtins." + ) + + +def generate_predictions(tasks, generator, samples_per_task): + if samples_per_task < 1: + raise ValueError("samples_per_task must be at least 1") + return [ + { + "task_id": task.task_id, + "candidates": [generator(build_prompt(task)) for _ in range(samples_per_task)], + } + for task in tasks + ] + + +def main(): + parser = argparse.ArgumentParser(description="Generate code candidates through an OpenAI-compatible API") + parser.add_argument("--tasks", required=True, help="CodeTask JSONL file") + parser.add_argument("--output", required=True, help="Prediction JSONL output path") + parser.add_argument("--samples-per-task", type=int, default=10) + parser.add_argument("--base-url", default="http://localhost:8998/v1") + parser.add_argument("--api-key", default="not-needed") + parser.add_argument("--model", default="minimind") + parser.add_argument("--temperature", type=float, default=0.8) + parser.add_argument("--max-tokens", type=int, default=512) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--open-thinking", action="store_true") + args = parser.parse_args() + + tasks = read_tasks(args.tasks) + generator = OpenAICompatibleGenerator( + base_url=args.base_url, + api_key=args.api_key, + model=args.model, + temperature=args.temperature, + max_tokens=args.max_tokens, + open_thinking=args.open_thinking, + seed=args.seed, + ) + records = generate_predictions(tasks, generator, args.samples_per_task) + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("w", encoding="utf-8") as handle: + for record in records: + handle.write(json.dumps(record, ensure_ascii=False) + "\n") + print(f"Generated {len(tasks) * args.samples_per_task} candidates for {len(tasks)} tasks -> {output_path}") + + +if __name__ == "__main__": + main() diff --git a/scripts/prepare_mbpp.py b/scripts/prepare_mbpp.py new file mode 100644 index 0000000..f9f6256 --- /dev/null +++ b/scripts/prepare_mbpp.py @@ -0,0 +1,149 @@ +"""Convert compatible MBPP-sanitized assertions into MiniMind CodeTask splits.""" + +import argparse +import ast +import json +import os +import sys +from collections import Counter +from pathlib import Path + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from code_agent import CodeTask, VerificationStatus, verify_code + + +def _literal(node): + value = ast.literal_eval(node) + json.dumps(value, ensure_ascii=False) + return value + + +def parse_assertion(source): + statement = ast.parse(source.strip()).body + if len(statement) != 1 or not isinstance(statement[0], ast.Assert): + raise ValueError("test is not a single assert statement") + expression = statement[0].test + + if isinstance(expression, ast.Call): + call, expected = expression, True + elif isinstance(expression, ast.UnaryOp) and isinstance(expression.op, ast.Not) and isinstance(expression.operand, ast.Call): + call, expected = expression.operand, False + elif ( + isinstance(expression, ast.Compare) + and len(expression.ops) == 1 + and isinstance(expression.ops[0], ast.Eq) + and len(expression.comparators) == 1 + ): + left, right = expression.left, expression.comparators[0] + if isinstance(left, ast.Call): + call, expected = left, _literal(right) + elif isinstance(right, ast.Call): + call, expected = right, _literal(left) + else: + raise ValueError("equality does not compare a function call") + else: + raise ValueError("unsupported assertion shape") + + if not isinstance(call.func, ast.Name): + raise ValueError("entry point is not a direct function call") + args = [_literal(argument) for argument in call.args] + kwargs = {} + for keyword in call.keywords: + if keyword.arg is None: + raise ValueError("expanded keyword arguments are unsupported") + kwargs[keyword.arg] = _literal(keyword.value) + return call.func.id, {"args": args, "kwargs": kwargs, "expected": expected} + + +def convert_record(record): + parsed = [parse_assertion(test) for test in record["test_list"]] + entry_points = {entry_point for entry_point, _ in parsed} + if len(entry_points) != 1: + raise ValueError("tests use multiple entry points") + payload = { + "task_id": f"mbpp-{record['task_id']}", + "prompt": record["prompt"].strip(), + "entry_point": entry_points.pop(), + "tests": [test for _, test in parsed], + } + task = CodeTask.from_dict(payload) + verification = verify_code(record["code"], task) + if verification.status != VerificationStatus.PASSED: + raise ValueError(f"reference solution failed verifier: {verification.status.value}") + return task.to_dict() + + +def build_sft_record(record, task): + return { + "conversations": [ + { + "role": "user", + "content": ( + "Solve the Python algorithm task below. Return one complete ```python``` code block.\n" + f"Required entry point: {task['entry_point']}\n" + f"Task:\n{task['prompt']}" + ), + }, + {"role": "assistant", "content": f"```python\n{record['code'].strip()}\n```"}, + ] + } + + +def convert_split(records): + converted = [] + sft_records = [] + skipped = Counter() + for record in records: + try: + task = convert_record(record) + converted.append(task) + sft_records.append(build_sft_record(record, task)) + except (KeyError, SyntaxError, TypeError, ValueError) as exc: + skipped[str(exc)] += 1 + return converted, sft_records, skipped + + +def main(): + parser = argparse.ArgumentParser(description="Prepare verifier-compatible MBPP sanitized splits") + parser.add_argument("--dataset", default="google-research-datasets/mbpp") + parser.add_argument("--config", default="sanitized") + parser.add_argument("--splits", nargs="+", default=["train", "validation", "test"]) + parser.add_argument("--output-dir", default="out/mbpp_sanitized") + args = parser.parse_args() + + from datasets import load_dataset + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + summary = {} + for split in args.splits: + records = load_dataset(args.dataset, args.config, split=split) + converted, sft_records, skipped = convert_split(records) + output_path = output_dir / f"{split}.jsonl" + with output_path.open("w", encoding="utf-8") as handle: + for record in converted: + handle.write(json.dumps(record, ensure_ascii=False) + "\n") + sft_output = None + if split == "train": + sft_output = output_dir / "train_sft.jsonl" + with sft_output.open("w", encoding="utf-8") as handle: + for record in sft_records: + handle.write(json.dumps(record, ensure_ascii=False) + "\n") + summary[split] = { + "source_count": len(records), + "converted_count": len(converted), + "skipped_count": len(records) - len(converted), + "top_skip_reasons": skipped.most_common(10), + "output": str(output_path), + "sft_output": str(sft_output) if sft_output else None, + } + print(f"{split}: converted {len(converted)}/{len(records)} -> {output_path}") + (output_dir / "summary.json").write_text( + json.dumps(summary, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/serve_openai_api.py b/scripts/serve_openai_api.py index adf2212..1be9b3a 100644 --- a/scripts/serve_openai_api.py +++ b/scripts/serve_openai_api.py @@ -56,6 +56,7 @@ class ChatRequest(BaseModel): stream: bool = True tools: list = Field(default_factory=list) open_thinking: bool = False + seed: int | None = None chat_template_kwargs: dict = None def get_open_thinking(self) -> bool: @@ -102,7 +103,7 @@ def parse_response(text): return text.strip(), reasoning_content, tool_calls or None -def generate_stream_response(messages, temperature, top_p, max_tokens, tools=None, open_thinking=False): +def generate_stream_response(messages, temperature, top_p, max_tokens, tools=None, open_thinking=False, seed=None): try: new_prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, tools=tools or None, open_thinking=open_thinking) inputs = tokenizer(new_prompt, return_tensors="pt", truncation=True).to(device) @@ -112,6 +113,7 @@ def generate_stream_response(messages, temperature, top_p, max_tokens, tools=Non def _generate(): try: + generator = torch.Generator(device=inputs.input_ids.device).manual_seed(seed) if seed is not None else None model.generate( inputs.input_ids, max_new_tokens=max_tokens, @@ -121,7 +123,8 @@ def generate_stream_response(messages, temperature, top_p, max_tokens, tools=Non attention_mask=inputs.attention_mask, pad_token_id=tokenizer.pad_token_id, eos_token_id=tokenizer.eos_token_id, - streamer=streamer + streamer=streamer, + generator=generator ) except Exception as e: queue.put({"error": str(e)}) @@ -186,7 +189,8 @@ async def chat_completions(request: ChatRequest): top_p=request.top_p, max_tokens=request.max_tokens, tools=request.tools, - open_thinking=request.get_open_thinking() + open_thinking=request.get_open_thinking(), + seed=request.seed )), media_type="text/event-stream" ) @@ -200,6 +204,7 @@ async def chat_completions(request: ChatRequest): ) inputs = tokenizer(new_prompt, return_tensors="pt", truncation=True).to(device) with torch.no_grad(): + generator = torch.Generator(device=inputs["input_ids"].device).manual_seed(request.seed) if request.seed is not None else None generated_ids = model.generate( inputs["input_ids"], max_length=inputs["input_ids"].shape[1] + request.max_tokens, @@ -208,7 +213,8 @@ async def chat_completions(request: ChatRequest): pad_token_id=tokenizer.pad_token_id, eos_token_id=tokenizer.eos_token_id, top_p=request.top_p, - temperature=request.temperature + temperature=request.temperature, + generator=generator ) answer = tokenizer.decode(generated_ids[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True) content, reasoning_content, tool_calls = parse_response(answer) diff --git a/tests/test_code_backend.py b/tests/test_code_backend.py index 3ccacfe..05ce7d1 100644 --- a/tests/test_code_backend.py +++ b/tests/test_code_backend.py @@ -25,12 +25,17 @@ def test_openai_backend_builds_non_streaming_request(): temperature=0.2, max_tokens=128, open_thinking=True, + seed=17, ) assert generator("solve it").startswith("```python") assert completions.kwargs["messages"][0]["content"] == "solve it" assert completions.kwargs["stream"] is False assert completions.kwargs["extra_body"] == {"open_thinking": True} + assert completions.kwargs["seed"] == 17 + + generator("solve another") + assert completions.kwargs["seed"] == 18 def test_openai_backend_rejects_empty_completion(): diff --git a/tests/test_generate_codegen.py b/tests/test_generate_codegen.py new file mode 100644 index 0000000..f6df16f --- /dev/null +++ b/tests/test_generate_codegen.py @@ -0,0 +1,25 @@ +from types import SimpleNamespace + +import pytest + +from scripts.generate_codegen import build_prompt, generate_predictions + + +def test_generate_predictions_groups_samples_by_task(): + task = SimpleNamespace(task_id="add", entry_point="add", prompt="Implement add(a, b).") + prompts = [] + + def generate(prompt): + prompts.append(prompt) + return f"candidate-{len(prompts)}" + + records = generate_predictions([task], generate, samples_per_task=3) + + assert records == [{"task_id": "add", "candidates": ["candidate-1", "candidate-2", "candidate-3"]}] + assert all("Required entry point: add" in prompt for prompt in prompts) + assert "exactly one ```python``` code block" in build_prompt(task) + + +def test_generate_predictions_rejects_empty_sample_count(): + with pytest.raises(ValueError, match="at least 1"): + generate_predictions([], lambda prompt: prompt, samples_per_task=0) diff --git a/tests/test_prepare_mbpp.py b/tests/test_prepare_mbpp.py new file mode 100644 index 0000000..7ad94fb --- /dev/null +++ b/tests/test_prepare_mbpp.py @@ -0,0 +1,50 @@ +import pytest + +from scripts.prepare_mbpp import build_sft_record, convert_record, convert_split, parse_assertion + + +def test_parse_mbpp_equality_and_boolean_assertions(): + assert parse_assertion("assert add(1, 2) == 3") == ( + "add", + {"args": [1, 2], "kwargs": {}, "expected": 3}, + ) + assert parse_assertion("assert is_even(4)")[-1]["expected"] is True + assert parse_assertion("assert not is_even(3)")[-1]["expected"] is False + + +def test_convert_record_validates_reference_solution(): + record = { + "task_id": 1, + "prompt": "Write a function that adds two integers.", + "code": "def add(a, b):\n return a + b", + "test_list": ["assert add(1, 2) == 3", "assert add(-1, 1) == 0"], + } + + converted = convert_record(record) + + assert converted["task_id"] == "mbpp-1" + assert converted["entry_point"] == "add" + assert len(converted["tests"]) == 2 + + sft = build_sft_record(record, converted) + assert "Required entry point: add" in sft["conversations"][0]["content"] + assert "def add(a, b)" in sft["conversations"][1]["content"] + + +def test_convert_split_keeps_tasks_and_sft_records_aligned(): + record = { + "task_id": 2, + "prompt": "Return the input integer.", + "code": "def identity(value):\n return value", + "test_list": ["assert identity(7) == 7"], + } + + tasks, sft_records, skipped = convert_split([record]) + + assert len(tasks) == len(sft_records) == 1 + assert not skipped + + +def test_parse_mbpp_rejects_nonliteral_expectation(): + with pytest.raises(ValueError): + parse_assertion("assert add(1, 2) == expected")