83 lines
3.0 KiB
Python
83 lines
3.0 KiB
Python
"""Generate reproducible multi-sample candidates for execution-based evaluation."""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
|
|
|
from code_agent import CodeTask, OpenAICompatibleGenerator
|
|
|
|
|
|
def read_tasks(path):
|
|
tasks = []
|
|
with Path(path).open("r", encoding="utf-8") as handle:
|
|
for line_number, line in enumerate(handle, start=1):
|
|
if not line.strip():
|
|
continue
|
|
try:
|
|
tasks.append(CodeTask.from_dict(json.loads(line)))
|
|
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
|
raise ValueError(f"Invalid task at {path}:{line_number}: {exc}") from exc
|
|
return tasks
|
|
|
|
|
|
def build_prompt(task):
|
|
return (
|
|
"You are solving a Python algorithm task. Return exactly one ```python``` code block.\n"
|
|
f"Required entry point: {task.entry_point}\n"
|
|
f"Task:\n{task.prompt}\n"
|
|
"Do not read files, access the network, spawn processes, or call unsafe builtins."
|
|
)
|
|
|
|
|
|
def generate_predictions(tasks, generator, samples_per_task):
|
|
if samples_per_task < 1:
|
|
raise ValueError("samples_per_task must be at least 1")
|
|
return [
|
|
{
|
|
"task_id": task.task_id,
|
|
"candidates": [generator(build_prompt(task)) for _ in range(samples_per_task)],
|
|
}
|
|
for task in tasks
|
|
]
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Generate code candidates through an OpenAI-compatible API")
|
|
parser.add_argument("--tasks", required=True, help="CodeTask JSONL file")
|
|
parser.add_argument("--output", required=True, help="Prediction JSONL output path")
|
|
parser.add_argument("--samples-per-task", type=int, default=10)
|
|
parser.add_argument("--base-url", default="http://localhost:8998/v1")
|
|
parser.add_argument("--api-key", default="not-needed")
|
|
parser.add_argument("--model", default="minimind")
|
|
parser.add_argument("--temperature", type=float, default=0.8)
|
|
parser.add_argument("--max-tokens", type=int, default=512)
|
|
parser.add_argument("--seed", type=int, default=42)
|
|
parser.add_argument("--open-thinking", action="store_true")
|
|
args = parser.parse_args()
|
|
|
|
tasks = read_tasks(args.tasks)
|
|
generator = OpenAICompatibleGenerator(
|
|
base_url=args.base_url,
|
|
api_key=args.api_key,
|
|
model=args.model,
|
|
temperature=args.temperature,
|
|
max_tokens=args.max_tokens,
|
|
open_thinking=args.open_thinking,
|
|
seed=args.seed,
|
|
)
|
|
records = generate_predictions(tasks, generator, args.samples_per_task)
|
|
output_path = Path(args.output)
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with output_path.open("w", encoding="utf-8") as handle:
|
|
for record in records:
|
|
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
|
print(f"Generated {len(tasks) * args.samples_per_task} candidates for {len(tasks)} tasks -> {output_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|