feat: add code agent model service runner
RileyChen23
parent
125b49dfc1
commit
65def67cee
|
|
@ -1,6 +1,7 @@
|
|||
"""Utilities for execution-feedback code generation experiments."""
|
||||
|
||||
from .agent import AgentResult, AttemptRecord, ExecutionFeedbackAgent
|
||||
from .backends import OpenAICompatibleGenerator
|
||||
from .dataset import CodeRLDataset, collate_code_rl
|
||||
from .reward import CodeReward, score_code_batch, score_verification
|
||||
from .evaluation import (
|
||||
|
|
@ -26,6 +27,7 @@ __all__ = [
|
|||
"CodeTask",
|
||||
"EvaluationReport",
|
||||
"ExecutionFeedbackAgent",
|
||||
"OpenAICompatibleGenerator",
|
||||
"TestCase",
|
||||
"VerificationResult",
|
||||
"VerificationStatus",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
"""Inference adapters for the execution-feedback agent."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
class OpenAICompatibleGenerator:
|
||||
"""Callable adapter for MiniMind, vLLM, SGLang, or other OpenAI-style APIs."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_url: str = "http://localhost:8998/v1",
|
||||
api_key: str = "not-needed",
|
||||
model: str = "minimind",
|
||||
temperature: float = 0.7,
|
||||
max_tokens: int = 1024,
|
||||
open_thinking: bool = False,
|
||||
client: Any = None,
|
||||
) -> None:
|
||||
if client is None:
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("Install the project 'openai' dependency to use this backend.") from exc
|
||||
client = OpenAI(api_key=api_key, base_url=base_url)
|
||||
self.client = client
|
||||
self.model = model
|
||||
self.temperature = temperature
|
||||
self.max_tokens = max_tokens
|
||||
self.open_thinking = open_thinking
|
||||
|
||||
def __call__(self, prompt: str) -> str:
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=self.temperature,
|
||||
max_tokens=self.max_tokens,
|
||||
stream=False,
|
||||
extra_body={"open_thinking": self.open_thinking},
|
||||
)
|
||||
if not response.choices or response.choices[0].message is None:
|
||||
raise RuntimeError("Model API returned no completion choice.")
|
||||
content = response.choices[0].message.content
|
||||
if not content:
|
||||
raise RuntimeError("Model API returned an empty completion.")
|
||||
return content
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{"task_id":"reverse-string","prompt":"Implement reverse_string(text), returning the characters in reverse order.","entry_point":"reverse_string","tests":[{"args":["hello"],"expected":"olleh"},{"args":[""],"expected":""},{"args":["a b"],"expected":"b a"}]}
|
||||
{"task_id":"valid-parentheses","prompt":"Implement valid_parentheses(text). The input contains only ()[]{}. Return whether every bracket is correctly matched and nested.","entry_point":"valid_parentheses","tests":[{"args":["()[]{}"],"expected":true},{"args":["([{}])"],"expected":true},{"args":["(]"],"expected":false},{"args":["([)]"],"expected":false}]}
|
||||
{"task_id":"binary-search","prompt":"Implement binary_search(values, target) for an ascending list of distinct integers. Return the target index or -1 when absent.","entry_point":"binary_search","tests":[{"args":[[1,3,5,7],5],"expected":2},{"args":[[1,3,5,7],2],"expected":-1},{"args":[[],4],"expected":-1}]}
|
||||
{"task_id":"max-subarray","prompt":"Implement max_subarray(values), returning the maximum sum of a non-empty contiguous subarray.","entry_point":"max_subarray","tests":[{"args":[[-2,1,-3,4,-1,2,1,-5,4]],"expected":6},{"args":[[-5,-2,-8]],"expected":-2},{"args":[[7]],"expected":7}]}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
"""Run the execution-feedback repair agent against an OpenAI-compatible model."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
|
||||
from code_agent import CodeTask, ExecutionFeedbackAgent, OpenAICompatibleGenerator
|
||||
|
||||
|
||||
def read_tasks(path):
|
||||
tasks = []
|
||||
with Path(path).open("r", encoding="utf-8") as handle:
|
||||
for line_number, line in enumerate(handle, start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
tasks.append(CodeTask.from_dict(json.loads(line)))
|
||||
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
|
||||
raise ValueError(f"Invalid task at {path}:{line_number}: {exc}") from exc
|
||||
return tasks
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Generate, execute, and repair Python algorithm solutions")
|
||||
parser.add_argument("--tasks", required=True, help="CodeTask JSONL file")
|
||||
parser.add_argument("--task-id", help="Run only one task id")
|
||||
parser.add_argument("--output", help="Optional JSON output path")
|
||||
parser.add_argument("--base-url", default="http://localhost:8998/v1")
|
||||
parser.add_argument("--api-key", default="not-needed")
|
||||
parser.add_argument("--model", default="minimind")
|
||||
parser.add_argument("--temperature", type=float, default=0.7)
|
||||
parser.add_argument("--max-tokens", type=int, default=1024)
|
||||
parser.add_argument("--max-attempts", type=int, default=3)
|
||||
parser.add_argument("--timeout", type=float, default=2.0)
|
||||
parser.add_argument("--memory-mb", type=int, default=256)
|
||||
parser.add_argument("--open-thinking", action="store_true")
|
||||
parser.add_argument("--reveal-test-details", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
tasks = read_tasks(args.tasks)
|
||||
if args.task_id:
|
||||
tasks = [task for task in tasks if task.task_id == args.task_id]
|
||||
if not tasks:
|
||||
raise ValueError(f"Unknown task id: {args.task_id}")
|
||||
|
||||
generator = OpenAICompatibleGenerator(
|
||||
base_url=args.base_url,
|
||||
api_key=args.api_key,
|
||||
model=args.model,
|
||||
temperature=args.temperature,
|
||||
max_tokens=args.max_tokens,
|
||||
open_thinking=args.open_thinking,
|
||||
)
|
||||
agent = ExecutionFeedbackAgent(
|
||||
generator,
|
||||
max_attempts=args.max_attempts,
|
||||
timeout_seconds=args.timeout,
|
||||
memory_mb=args.memory_mb,
|
||||
reveal_test_details=args.reveal_test_details,
|
||||
)
|
||||
results = [agent.run(task).to_dict() for task in tasks]
|
||||
output = {
|
||||
"task_count": len(results),
|
||||
"success_count": sum(result["success"] for result in results),
|
||||
"results": results,
|
||||
}
|
||||
rendered = json.dumps(output, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).write_text(rendered + "\n", encoding="utf-8")
|
||||
print(rendered)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{"task_id":"reverse-string","candidates":["def reverse_string(text):\n return text[::-1]","def reverse_string(text):\n return text"]}
|
||||
{"task_id":"valid-parentheses","candidates":["def valid_parentheses(text):\n pairs = {')': '(', ']': '[', '}': '{'}\n stack = []\n for char in text:\n if char in '([{':\n stack.append(char)\n elif not stack or stack.pop() != pairs[char]:\n return False\n return not stack","def valid_parentheses(text):\n return True"]}
|
||||
{"task_id":"binary-search","candidates":["def binary_search(values, target):\n left, right = 0, len(values) - 1\n while left <= right:\n mid = (left + right) // 2\n if values[mid] == target:\n return mid\n if values[mid] < target:\n left = mid + 1\n else:\n right = mid - 1\n return -1","def binary_search(values, target):\n return -1"]}
|
||||
{"task_id":"max-subarray","candidates":["def max_subarray(values):\n best = current = values[0]\n for value in values[1:]:\n current = max(value, current + value)\n best = max(best, current)\n return best","def max_subarray(values):\n return sum(values)"]}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from code_agent import OpenAICompatibleGenerator
|
||||
|
||||
|
||||
class FakeCompletions:
|
||||
def __init__(self, content):
|
||||
self.content = content
|
||||
self.kwargs = None
|
||||
|
||||
def create(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
message = SimpleNamespace(content=self.content)
|
||||
return SimpleNamespace(choices=[SimpleNamespace(message=message)])
|
||||
|
||||
|
||||
def test_openai_backend_builds_non_streaming_request():
|
||||
completions = FakeCompletions("```python\ndef solve():\n return 1\n```")
|
||||
client = SimpleNamespace(chat=SimpleNamespace(completions=completions))
|
||||
generator = OpenAICompatibleGenerator(
|
||||
client=client,
|
||||
model="local-minimind",
|
||||
temperature=0.2,
|
||||
max_tokens=128,
|
||||
open_thinking=True,
|
||||
)
|
||||
|
||||
assert generator("solve it").startswith("```python")
|
||||
assert completions.kwargs["messages"][0]["content"] == "solve it"
|
||||
assert completions.kwargs["stream"] is False
|
||||
assert completions.kwargs["extra_body"] == {"open_thinking": True}
|
||||
|
||||
|
||||
def test_openai_backend_rejects_empty_completion():
|
||||
completions = FakeCompletions("")
|
||||
client = SimpleNamespace(chat=SimpleNamespace(completions=completions))
|
||||
generator = OpenAICompatibleGenerator(client=client)
|
||||
|
||||
with pytest.raises(RuntimeError, match="empty completion"):
|
||||
generator("solve it")
|
||||
Loading…
Reference in New Issue