import ast import io import re import unittest from contextlib import redirect_stdout import swe_bench_toolkit as S import coding_agents_toolkit as C TASKS = S.curated_tasks() BY = {t.id: t for t in TASKS} SYN, _ = S.synthesize_tasks(limit=40) FILES = {"pkg/__init__.py": "", "pkg/a.py": "def f(x):\n return x + 1\n\n\ndef g(x):\n return f(x) * 2\n", "tests/test_a.py": "from pkg.a import f, g\n\n\ndef test_f():\n assert f(1) == 2\n\n\ndef test_g():\n assert g(1) == 4\n"} def ws(**kw): return C.Workspace(FILES, **kw) class TestLedger(unittest.TestCase): def test_turn_accumulates_history(self): led = C.Ledger("mid", system_tokens=100) led.turn(10, 20) self.assertEqual((led.tin, led.tout, led.context, led.turns), (100, 10, 130, 1)) led.turn(5, 0) self.assertEqual((led.tin, led.tout, led.context, led.turns), (230, 15, 135, 2)) def test_input_grows_faster_than_turns(self): led = C.Ledger() seen = [] for _ in range(30): led.turn(50, 200) seen.append(led.tin) self.assertGreater(seen[29] / seen[9], 5) def test_call_has_no_history(self): led = C.Ledger() led.call(100, 10) led.call(100, 10) self.assertEqual((led.tin, led.tout, led.turns, led.context), (200, 20, 2, 600)) def test_cost_uses_tier_and_scale(self): led = C.Ledger("mid") led.tin, led.tout = 1000000, 500000 self.assertAlmostEqual(led.cost(scale=1), 1.0 * 1 + 4.0 * 0.5) self.assertAlmostEqual(led.cost(scale=2), 6.0) led.tier = "large" self.assertAlmostEqual(led.cost(scale=1), 5.0 + 10.0) def test_cost_with_cache(self): led = C.Ledger("mid") led.tin, led.tout = 1000000, 0 self.assertAlmostEqual(led.cost(scale=1, cached_frac=0.5, cache_price=0.1), 0.55) self.assertAlmostEqual(led.cost(scale=1, cached_frac=1.0, cache_price=0.1), 0.1) def test_result_carries_counts(self): led = C.Ledger() led.turn(10, 10) r = led.result("p", ["x"]) self.assertEqual((r.patch, r.turns, r.log), ("p", 1, ["x"])) self.assertEqual(r.tokens_in, 600 * C.SCALE) self.assertEqual(r.tokens_out, 10 * C.SCALE) self.assertEqual(r.peak_context, 620 * C.SCALE) class TestLint(unittest.TestCase): def test_modes(self): self.assertIsNone(C.lint_source("x = (", "none")) self.assertIn("SyntaxError", C.lint_source("x = (", "syntax")) self.assertIsNone(C.lint_source("def f():\n return y\n", "syntax")) self.assertIn("undefined name y", C.lint_source("def f():\n return y\n", "names")) self.assertIsNone(C.lint_source("def f(y):\n return y\n", "names")) with self.assertRaises(ValueError): C.lint_source("x", "bogus") def test_syntax_message_has_line(self): self.assertRegex(C.lint_source("a = 1\nb = (\n", "syntax"), r"line \d") def test_undefined_names_scoping_is_coarse_but_sane(self): src = "import os\nfrom math import sqrt as rt\nfor i in range(3):\n pass\nwith open('f') as h:\n pass\n" \ "try:\n pass\nexcept ValueError as e:\n print(e)\nz = [k for k in range(2)]\nlam = lambda q: q + i\nclass K:\n pass\n" \ "print(os, rt, h, z, lam, K)\n" self.assertEqual(C.undefined_names(ast.parse(src)), []) self.assertEqual(C.undefined_names(ast.parse("a = b + c\n")), [("b", 1), ("c", 1)]) def test_undefined_names_sorted_by_line(self): self.assertEqual(C.undefined_names(ast.parse("x = 1\ny = q\nz = p\n")), [("q", 2), ("p", 3)]) def test_lines_helpers(self): self.assertEqual(C._lines("a\nb\n"), ["a", "b"]) self.assertEqual(C._lines("a\nb"), ["a", "b"]) self.assertEqual(C._unlines(["a", "b"]), "a\nb\n") self.assertEqual(C._unlines(["a", "b"], False), "a\nb") self.assertEqual(C._unlines([]), "") class TestViewer(unittest.TestCase): def test_open_shows_header_and_numbers(self): w = ws(window=4) out = w.open("pkg/a.py") self.assertTrue(out.startswith("[File: pkg/a.py (6 lines total)]\n(0 more lines above)\n1: def f(x):")) self.assertIn("4: ", out) self.assertNotIn("5: ", out) self.assertTrue(out.endswith("(2 more lines below)")) def test_window_is_a_cap(self): w = ws(window=3) self.assertEqual(w.open("pkg/a.py").count("\n"), 2 + 3 + 0) def test_open_centers_on_line(self): w = ws(window=2) out = w.open("pkg/a.py", 4) self.assertIn("3: ", out) self.assertIn("4: ", out) self.assertIn("(2 more lines above)", out) def test_open_clamps_at_end(self): w = ws(window=4) out = w.open("pkg/a.py", 6) self.assertIn("6: return f(x) * 2", out) self.assertIn("(0 more lines below)", out) def test_open_missing(self): self.assertEqual(ws().open("nope.py"), "Error: file not found: nope.py") def test_goto_scroll(self): w = ws(window=2) self.assertIn("no file is open", w.goto(1)) self.assertIn("no file is open", w.scroll_down()) self.assertIn("no file is open", w.scroll_up()) w.open("pkg/a.py") out = w.goto(5) self.assertIn("4: ", out) self.assertIn("5: def g(x):", out) self.assertIn("(3 more lines above)", out) self.assertIn("outside the file", w.goto(0)) self.assertIn("outside the file", w.goto(7)) up = w.scroll_up() self.assertIn("(1 more lines above)", up) self.assertIn("2: return x + 1", up) self.assertIn("3: ", up) down = w.scroll_down() self.assertIn("(3 more lines above)", down) for _ in range(5): last = w.scroll_down() self.assertIn("(0 more lines below)", last) self.assertIn("6:", last) def test_scroll_up_stops_at_top(self): w = ws(window=2) w.open("pkg/a.py") self.assertIn("1: def f(x):", w.scroll_up()) class TestSearch(unittest.TestCase): def test_find_file(self): w = ws() self.assertEqual(w.find_file("a.py"), "Found 2 file(s):\npkg/a.py\ntests/test_a.py") self.assertEqual(w.find_file("zzz"), 'No files named "zzz".') self.assertIn("More than 1 files", ws(search_cap=1).find_file("a.py")) def test_search_dir_hits_have_path_and_line(self): out = ws().search_dir("return") self.assertIn('Found 2 match(es) for "return":', out) self.assertIn("pkg/a.py:2: return x + 1", out) self.assertIn("pkg/a.py:6: return f(x) * 2", out) def test_search_cap_suppresses_listing(self): out = ws(search_cap=1).search_dir("return") self.assertEqual(out, 'More than 1 matches for "return". Please narrow your search.') self.assertIn("Found", ws(search_cap=2).search_dir("return")) def test_search_none(self): self.assertEqual(ws().search_dir("qqq"), 'No matches found for "qqq".') def test_search_file(self): w = ws() self.assertIn("Error: file not found", w.search_file("f")) w.open("pkg/a.py") self.assertIn("pkg/a.py:1: def f(x):", w.search_file("def f")) self.assertIn("tests/test_a.py:5", w.search_file("f(1)", "tests/test_a.py")) self.assertIn("Error", w.search_file("f", "zzz.py")) class TestEdit(unittest.TestCase): def test_edit_replaces_range(self): w = ws(lint="syntax") w.open("pkg/a.py") out = w.edit(2, 2, " return x + 10") self.assertTrue(out.startswith("File updated.")) self.assertEqual(w.files["pkg/a.py"].split("\n")[1], " return x + 10") self.assertTrue(w.files["pkg/a.py"].endswith("\n")) self.assertEqual(w.stats["edits"], 1) def test_edit_can_grow_and_shrink(self): w = ws() w.open("pkg/a.py") w.edit(2, 2, " y = x\n return y + 1\n") self.assertEqual(len(C._lines(w.files["pkg/a.py"])), 7) w.edit(2, 3, " return x + 1") self.assertEqual(w.files["pkg/a.py"], FILES["pkg/a.py"]) def test_edit_rejects_syntax_error_and_leaves_file(self): w = ws(lint="syntax") w.open("pkg/a.py") out = w.edit(2, 2, " return (") self.assertTrue(out.startswith("Your edit was not applied (SyntaxError")) self.assertIn("This is how it would have looked:", out) self.assertIn("2: return (", out) self.assertEqual(w.files["pkg/a.py"], FILES["pkg/a.py"]) self.assertEqual(w.stats["rejected"], 1) self.assertEqual(w.stats["edits"], 0) def test_lint_none_lets_bad_edit_in(self): w = ws(lint="none") w.open("pkg/a.py") self.assertTrue(w.edit(2, 2, " return (").startswith("File updated")) self.assertIn("return (", w.files["pkg/a.py"]) def test_lint_syntax_misses_names_but_names_catches(self): a = ws(lint="syntax") a.open("pkg/a.py") self.assertTrue(a.edit(2, 2, " return y + 1").startswith("File updated")) b = ws(lint="names") b.open("pkg/a.py") self.assertIn("undefined name y", b.edit(2, 2, " return y + 1")) def test_edit_errors(self): w = ws() self.assertIn("no file is open", w.edit(1, 1, "x")) w.open("pkg/a.py") self.assertIn("bad line range 0 to 1", w.edit(0, 1, "x")) self.assertIn("bad line range 3 to 2", w.edit(3, 2, "x")) self.assertIn("bad line range 1 to 7 (the file has 6 lines)", w.edit(1, 7, "x")) def test_deny_rule(self): w = ws(deny=("tests/",)) w.open("tests/test_a.py") out = w.edit(1, 1, "pass") self.assertIn("protected path", out) self.assertEqual(w.files["tests/test_a.py"], FILES["tests/test_a.py"]) self.assertEqual(w.stats["blocked"], 1) def test_non_python_file_not_linted(self): w = C.Workspace({"notes.txt": "a (\n"}, lint="syntax") w.open("notes.txt") self.assertTrue(w.edit(1, 1, "b (").startswith("File updated")) def test_view_after_edit_shows_new_text(self): w = ws(window=10) w.open("pkg/a.py") self.assertIn("2: return x + 7", w.edit(2, 2, " return x + 7")) class TestRunAndSubmit(unittest.TestCase): def test_run_tests_summary(self): w = ws() self.assertEqual(w.run_tests(), "2 passed, 0 not passing") w.open("pkg/a.py") w.edit(2, 2, " return x + 5") out = w.run_tests() self.assertTrue(out.startswith("0 passed, 2 not passing") or out.startswith("1 passed, 1 not passing")) self.assertIn("tests/test_a.py::test_f", out) self.assertTrue(all(v != "PASS" for k, v in w.last_tests.items() if k.endswith("test_f"))) def test_python_needs_open_python_file(self): w = ws() self.assertEqual(w.python("1"), "Error: open a python file first.") w.open("pkg/a.py") self.assertEqual(w.python("f(2)"), "=> 3") self.assertEqual(w.python("f(z)", "z = 4"), "=> 5") self.assertEqual(w.python("1 / 0"), "=> raises ZeroDivisionError") def test_revert_and_diff(self): w = ws() w.open("pkg/a.py") self.assertEqual(w.diff(), "") w.edit(2, 2, " return x + 9") d = w.diff() self.assertIn("- return x + 1", d) self.assertIn("+ return x + 9", d) self.assertEqual(w.submit(), d) self.assertEqual(w.revert("pkg/a.py"), "Reverted.") self.assertEqual(w.diff(), "") w.edit(2, 2, " return x + 9") w.revert() self.assertEqual(w.files, FILES) def test_revert_removes_new_file(self): w = ws() w.files["new.py"] = "x = 1\n" w.revert("new.py") self.assertNotIn("new.py", w.files) def test_act_charges_ledger_and_logs(self): w = ws() obs = w.act("open", "pkg/a.py") self.assertEqual(w.ledger.turns, 1) self.assertEqual(w.ledger.tin, 600) self.assertGreater(w.ledger.tout, 30) self.assertEqual(w.ledger.context, 600 + w.ledger.tout + S.est_tokens(obs)) self.assertEqual(w.history[0][0], "open") w.act("goto", 2) self.assertGreater(w.ledger.tin, 1200) def test_act_rejects_unknown_command(self): with self.assertRaises(ValueError): ws().act("rm_rf", "/") with self.assertRaises(ValueError): ws().act("act") class TestSlip(unittest.TestCase): def test_paren_slip_is_syntax_error(self): out = C.slip_edit("def f(x):\n return x + 1", "paren") with self.assertRaises(SyntaxError): ast.parse(out) def test_name_slip_parses_but_is_undefined(self): out = C.slip_edit("def f(x):\n y = x\n return x + 1", "name") ast.parse(out) self.assertTrue(C.undefined_names(ast.parse(out))) self.assertIn("y = x", out) self.assertIn("return x_ + 1", out) def test_name_slip_skips_self(self): out = C.slip_edit(" def add(self, name):\n self.n = name\n", "name") self.assertIn("self.n = name_", out) def test_name_slip_falls_back_to_paren(self): out = C.slip_edit("def f():\n return 1", "name") self.assertTrue(out.endswith(" (")) class TestAciAgent(unittest.TestCase): def test_solves_easy_task(self): t = BY["textkit-3"] r = C.AciAgent("a").solve(t.view()) self.assertTrue(S.evaluate(t, r.patch).resolved) self.assertGreater(r.turns, 3) self.assertTrue(any("submit" in x for x in r.log)) self.assertGreater(r.peak_context, 0) def test_agent_never_changes_tests(self): for t in TASKS[:8]: r = C.AciAgent("a").solve(t.view()) self.assertNotIn("tests/", r.patch) def test_curated_rate_and_pareto(self): runs = [S.evaluate(t, C.AciAgent("a").solve(t.view()).patch).resolved for t in TASKS] self.assertEqual(sum(runs), 14) def test_vague_task_has_no_example_to_check(self): r = C.AciAgent("a").solve(BY["textkit-1"].view()) self.assertNotEqual(r.patch, "") self.assertIsNone(S.parse_repro(BY["textkit-1"].issue)) self.assertTrue(any("after 1 tries" in x for x in r.log)) def test_turn_budget(self): t = BY["configkit-2"] r = C.AciAgent("a", max_turns=3).solve(t.view()) self.assertEqual(r.patch, "") self.assertIn("turn budget used up; gave up", r.log) self.assertLessEqual(r.turns, 4) def test_slip_with_lint_costs_turns_not_results(self): t = BY["textkit-3"] clean = C.AciAgent("s", slip=0.0).solve(t.view()) slipped = C.AciAgent("s", slip=1.0, lint="syntax") r = slipped.solve(t.view()) self.assertGreater(r.turns, clean.turns) kinds = [x for x in r.log if x.startswith("slip")] self.assertTrue(kinds) def test_slip_without_lint_hurts(self): ok_lint = sum(S.evaluate(t, C.AciAgent("s", slip=0.5, lint="syntax").solve(t.view()).patch).resolved for t in TASKS) ok_none = sum(S.evaluate(t, C.AciAgent("s", slip=0.5, lint="none").solve(t.view()).patch).resolved for t in TASKS) self.assertGreater(ok_lint, ok_none) def test_names_lint_recovers_name_slips(self): ok_syn = sum(S.evaluate(t, C.AciAgent("s", slip=0.5, lint="syntax").solve(t.view()).patch).resolved for t in TASKS) ok_names = sum(S.evaluate(t, C.AciAgent("s", slip=0.5, lint="names").solve(t.view()).patch).resolved for t in TASKS) self.assertGreaterEqual(ok_names, ok_syn) self.assertEqual(ok_names, 14) def test_deterministic(self): t = BY["mathkit-4"] a = C.AciAgent("d", slip=0.3).solve(t.view()) b = C.AciAgent("d", slip=0.3).solve(t.view()) self.assertEqual((a.patch, a.turns, a.tokens_in), (b.patch, b.turns, b.tokens_in)) def test_window_changes_cost_not_outcome(self): t = BY["listkit-2"] small = C.AciAgent("w", window=3).solve(t.view()) big = C.AciAgent("w", window=100).solve(t.view()) self.assertEqual(small.patch, big.patch) self.assertLess(small.tokens_in, big.tokens_in) def test_cost_through_swe_bench_helper(self): a = C.AciAgent("c", tier="large") r = a.solve(BY["textkit-3"].view()) self.assertGreater(S.cost_of(a, r), S.cost_of(C.AciAgent("c", tier="small"), r)) def test_without_tests_still_submits(self): r = C.AciAgent("nt", use_tests=False).solve(BY["textkit-3"].view()) self.assertNotEqual(r.patch, "") class TestRepro(unittest.TestCase): def test_script_is_valid_python_with_three_labels(self): repro = S.parse_repro(BY["textkit-3"].issue) src = C.repro_script(repro, "") ast.parse(src) for label in ("Issue reproduced", "Issue resolved", "Other issues"): self.assertIn(label, src) def test_script_runs_and_matches_run_repro(self): t = BY["textkit-3"] repro = S.parse_repro(t.issue) gold, _ = S.apply_patch(t.base, t.gold_patch) for files, want in ((t.base, "Issue reproduced"), (gold, "Issue resolved")): ns = dict(vars(S.Sandbox(files).load(S.path_to_module(t.module)))) buf = io.StringIO() with redirect_stdout(buf): exec(C.repro_script(repro, ""), ns) self.assertEqual(buf.getvalue().strip(), want) self.assertEqual(C.run_repro(files, t.module, repro), want) def test_script_with_setup_lines(self): src = C.repro_script(("c = Cart()\nc.add('a', 1.0)", "c.total()", "1.0"), "") self.assertIn(" c = Cart()\n c.add('a', 1.0)\n got = c.total()", src) def test_run_repro_other_issues(self): t = BY["textkit-3"] self.assertEqual(C.run_repro(t.base, t.module, ("", "nope()", "1")), "Other issues") def test_add_repro_test_appends_failing_test(self): t = BY["textkit-3"] repro = S.parse_repro(t.issue) files = C.add_repro_test(t.base, t.test_path, repro) self.assertIn("def test_issue_repro():", files[t.test_path]) res = S.run_tests(files) self.assertEqual(res[t.test_path + "::test_issue_repro"], "FAIL") gold, _ = S.apply_patch(files, t.gold_patch) self.assertEqual(S.run_tests(gold)[t.test_path + "::test_issue_repro"], "PASS") def test_add_repro_test_keeps_other_files(self): t = BY["textkit-3"] files = C.add_repro_test(t.base, t.test_path, S.parse_repro(t.issue)) self.assertEqual({k for k in files}, {k for k in t.base}) self.assertEqual(files[t.module], t.base[t.module]) def test_repro_quality(self): q = C.repro_quality(TASKS) self.assertEqual(q["textkit-1"], "no repro") self.assertEqual(q["mathkit-3"], "no repro") self.assertEqual(sum(1 for v in q.values() if v == "valid"), 18) def test_repro_quality_detects_a_test_that_passes_before(self): t = BY["textkit-3"] class Fake: id, issue, base, test_path, gold_patch = t.id, "x\n>>> startswith_any('foo', ['o'])\nExpected: True", t.base, t.test_path, t.gold_patch self.assertEqual(C.repro_quality([Fake])[t.id], "passes before") def test_repro_quality_detects_wrong_expectation(self): t = BY["textkit-3"] class Fake: id, issue, base, test_path, gold_patch = t.id, "x\n>>> startswith_any('foo', ['f'])\nExpected: 'x'", t.base, t.test_path, t.gold_patch self.assertEqual(C.repro_quality([Fake])[t.id], "fails after gold") class TestPipelinePieces(unittest.TestCase): def test_skeleton(self): out = C.skeleton("pkg/a.py", FILES["pkg/a.py"]) self.assertEqual(out, "pkg/a.py:\n def f(x):\n def g(x):") def test_normalized_key_ignores_format_and_comments(self): a = "def f(x):\n return x+1\n" b = "def f(x): # note\n return x + 1\n" self.assertEqual(C.normalized_key(a), C.normalized_key(b)) self.assertNotEqual(C.normalized_key(a), C.normalized_key("def f(x):\n return x + 2\n")) self.assertTrue(C.normalized_key("def (").startswith("syntax-error:")) def test_localize_finds_the_function(self): for t in TASKS: if t.id in ("textkit-1", "mathkit-3"): continue p = C.AgentlessPipeline("l", n_units=3) files, ranked, units = p.localize(t.view()) self.assertIn(t.module, files) self.assertLessEqual(len(files), 2) self.assertLessEqual(len(ranked), 3) t = BY["textkit-3"] files, ranked, units = C.AgentlessPipeline("l").localize(t.view()) self.assertEqual(ranked[0][1], "startswith_any") def test_localize_skips_tests(self): t = BY["textkit-3"] files, ranked, units = C.AgentlessPipeline("l").localize(t.view()) self.assertFalse([f for f in files if "test_" in f]) self.assertFalse([k for k in units if "test_" in k[0]]) def test_pool_candidates_differ_from_base(self): t = BY["textkit-3"] p = C.AgentlessPipeline("p") files, ranked, units = p.localize(t.view()) pool = p.pool(t.view(), ranked, units) self.assertTrue(pool) self.assertTrue(all(c.new_src != t.base[c.path] for c in pool)) self.assertEqual([c.rank for c in pool], list(range(len(pool)))) def test_sample_greedy_first_and_deterministic(self): t = BY["textkit-3"] p = C.AgentlessPipeline("p", n_samples=6) files, ranked, units = p.localize(t.view()) pool = p.pool(t.view(), ranked, units) d1, d2 = p.sample(t.view(), pool, 0), p.sample(t.view(), pool, 0) self.assertEqual(len(d1), 6) self.assertIs(d1[0], pool[0]) self.assertEqual([c.rank for c in d1], [c.rank for c in d2]) self.assertNotEqual([c.rank for c in d1], [c.rank for c in p.sample(t.view(), pool, 1)]) self.assertEqual(p.sample(t.view(), [], 0), []) def test_sample_skew_favours_early(self): t = BY["textkit-3"] files, ranked, units = C.AgentlessPipeline("p").localize(t.view()) flat = C.AgentlessPipeline("p", n_samples=200, skew=1.0) steep = C.AgentlessPipeline("p", n_samples=200, skew=3.0) pool = flat.pool(t.view(), ranked, units) mf = S.mean(c.rank for c in flat.sample(t.view(), pool)) ms = S.mean(c.rank for c in steep.sample(t.view(), pool)) self.assertLess(ms, mf) def test_validate_flags_bad_candidates(self): t = BY["textkit-3"] p = C.AgentlessPipeline("p") view = t.view() gold, _ = S.apply_patch(t.base, t.gold_patch) good = C.Cand(t.module, "startswith_any", "x", gold[t.module], 0) same = C.Cand(t.module, "startswith_any", "x", t.base[t.module], 1) broken = C.Cand(t.module, "startswith_any", "x", t.base[t.module].replace("s.endswith(p)", "s.endswith(p) and False"), 2) v = p.validate(view, [good, same, broken]) self.assertEqual(v[good.key], (True, True)) self.assertEqual(v[same.key], (True, False)) self.assertEqual(v[broken.key], (False, False)) worse = C.Cand(t.module, "x", "x", t.base[t.module].replace("return s\n", "return s + 'x'\n"), 3) self.assertEqual(p.validate(view, [worse])[worse.key][0], False) def test_validate_without_repro_passes_repro_check(self): t = BY["textkit-1"] p = C.AgentlessPipeline("p") c = C.Cand(t.module, "slugify", "x", t.base[t.module], 0) self.assertEqual(p.validate(t.view(), [c])[c.key], (True, True)) def test_validate_dedupes(self): t = BY["textkit-3"] p = C.AgentlessPipeline("p") c1 = C.Cand(t.module, "a", "x", t.base[t.module], 0) c2 = C.Cand(t.module, "a", "x", t.base[t.module] + "\n", 1) self.assertEqual(len(p.validate(t.view(), [c1, c2, c1])), 1) def mk(self, srcs): return [C.Cand("m.py", "f", "x", s, i) for i, s in enumerate(srcs)] def test_choose_modes(self): p = C.AgentlessPipeline("p") c = self.mk(["a = 1\n", "a = 2\n", "a = 2\n", "a = 3\n"]) valid = {c[0].key: (True, False), c[1].key: (True, True), c[3].key: (True, True)} self.assertIs(p.choose(c, valid, "none"), c[0]) self.assertIs(p.choose(c, valid, "first"), c[1]) self.assertIs(p.choose(c, valid, "vote"), c[1]) c2 = self.mk(["a = 1\n", "a = 2\n", "a = 3\n", "a = 3\n"]) v2 = {c2[0].key: (True, True), c2[1].key: (True, True), c2[2].key: (True, True)} self.assertIs(p.choose(c2, v2, "first"), c2[0]) self.assertIs(p.choose(c2, v2, "vote"), c2[2]) v3 = {k: (True, False) for k in v2} self.assertIsNone(p.choose(c2, v3, "vote")) self.assertIsNone(p.choose(c2, v3, "first")) self.assertIsNone(p.choose([], {}, "vote")) self.assertIs(C.AgentlessPipeline("q", select="first").choose(c2, v2), c2[0]) def test_choose_requires_both_checks(self): p = C.AgentlessPipeline("p") c = self.mk(["a = 1\n", "a = 2\n"]) valid = {c[0].key: (False, True), c[1].key: (True, True)} self.assertIs(p.choose(c, valid, "first"), c[1]) def test_vote_tie_goes_to_earliest(self): p = C.AgentlessPipeline("p") c = self.mk(["a = 1\n", "a = 2\n"]) valid = {x.key: (True, True) for x in c} self.assertIs(p.choose(c, valid, "vote"), c[0]) class TestPipelineEndToEnd(unittest.TestCase): def run_all(self, **kw): a = C.AgentlessPipeline("e", **kw) return sum(S.evaluate(t, a.solve(t.view()).patch).resolved for t in TASKS) def test_validation_beats_no_validation(self): self.assertGreater(self.run_all(select="first"), self.run_all(select="none")) def test_counts(self): self.assertEqual(self.run_all(select="none"), 6) self.assertEqual(self.run_all(select="first"), 9) def test_more_samples_help(self): self.assertGreaterEqual(self.run_all(n_samples=16), self.run_all(n_samples=1)) def test_cost_is_far_below_agent_loop(self): t = BY["textkit-3"] a, b = C.AgentlessPipeline("c"), C.AciAgent("c") self.assertLess(S.cost_of(a, a.solve(t.view())) * 3, S.cost_of(b, b.solve(t.view()))) def test_turns_are_model_calls(self): t = BY["textkit-3"] self.assertEqual(C.AgentlessPipeline("c").solve(t.view()).turns, 5) self.assertEqual(C.AgentlessPipeline("c", select="none").solve(t.view()).turns, 4) self.assertEqual(C.AgentlessPipeline("c").solve(BY["textkit-1"].view()).turns, 4) def test_nothing_validates_submits_nothing(self): r = C.AgentlessPipeline("c", repertoire=["method"]).solve(BY["textkit-2"].view()) self.assertEqual(r.patch, "") self.assertIn("nothing validated; submitted nothing", r.log) def test_log_mentions_counts(self): r = C.AgentlessPipeline("c", n_samples=5).solve(BY["textkit-3"].view()) self.assertRegex(r.log[0], r"1 files|2 files") self.assertIn("5 samples", r.log[0]) def test_patch_only_touches_source(self): for t in TASKS[:6]: self.assertNotIn("tests/", C.AgentlessPipeline("c").solve(t.view()).patch) class TestSelectionStudy(unittest.TestCase): @classmethod def setUpClass(cls): cls.out = C.selection_study(TASKS[:8], n_samples=8, seeds=2) def test_shape(self): self.assertEqual(set(self.out), {"greedy", "random", "visible-tests first", "visible-tests vote", "repro+tests first", "repro+tests vote", "oracle"}) self.assertTrue(all(len(v) == 8 for v in self.out.values())) self.assertTrue(all(0 <= x <= 1 for v in self.out.values() for x in v)) def test_oracle_is_upper_bound(self): for k, v in self.out.items(): for a, b in zip(v, self.out["oracle"]): self.assertLessEqual(a, b + 1e-9) def test_tests_beat_random(self): self.assertGreater(sum(self.out["repro+tests first"]), sum(self.out["random"])) def test_cache_is_reused(self): cache = {} C.selection_study(TASKS[:2], n_samples=4, seeds=1, hidden_cache=cache) n = len(cache) self.assertGreater(n, 0) C.selection_study(TASKS[:2], n_samples=4, seeds=1, hidden_cache=cache) self.assertEqual(len(cache), n) class TestRepoMap(unittest.TestCase): def test_symbol_index(self): idx = C.symbol_index(S.synth_repo("cartkit")) syms = idx["cartkit/cart.py"] self.assertIn(("class", "Cart", "class Cart", 1, None), syms) self.assertTrue(any(s[1] == "add" and s[4] == "Cart" and s[2] == "def add(self, name, price, qty=1)" for s in syms)) def test_symbol_index_skips_bad_and_non_py(self): idx = C.symbol_index({"a.py": "def (", "b.txt": "def f(): pass", "c.py": "def f(x, *a, k=2): pass\n"}) self.assertEqual(list(idx), ["c.py"]) self.assertEqual(idx["c.py"][0][2], "def f(x, *a, k=2)") def test_reference_counts(self): refs = C.reference_counts({"a.py": "from m import f\nx = f(1) + f(2)\ny = o.attr\n"}) self.assertEqual(refs["a.py"]["f"], 3) self.assertEqual(refs["a.py"]["attr"], 1) self.assertEqual(C.reference_counts({"a.py": "def ("}), {}) def test_file_graph_edges(self): files = {"a.py": "def fa(): return 1\n", "b.py": "from a import fa\ndef fb(): return fa() + fa()\n", "c.py": "def fc(): return 2\n"} edges = C.file_graph(files) self.assertEqual(edges, {("b.py", "a.py"): 3}) def test_file_graph_ignores_own_definitions(self): files = {"a.py": "def f(): return f\n", "b.py": "def f(): return 1\n"} self.assertEqual(C.file_graph(files), {}) def test_pagerank_sums_to_one_and_favours_hub(self): r = C.pagerank(["a", "b", "c"], {("a", "c"): 1, ("b", "c"): 1}) self.assertAlmostEqual(sum(r.values()), 1.0, places=6) self.assertGreater(r["c"], r["a"]) self.assertEqual(C.pagerank([], {}), {}) self.assertAlmostEqual(r["a"], r["b"]) def test_pagerank_personalization(self): edges = {("a", "b"): 1, ("b", "a"): 1} r = C.pagerank(["a", "b", "c"], edges, personal={"c": 10.0, "a": 1.0, "b": 1.0}) self.assertGreater(r["c"], 0.3) plain = C.pagerank(["a", "b", "c"], edges) self.assertGreater(r["c"], plain["c"]) def test_pagerank_weights_matter(self): r = C.pagerank(["a", "b", "c"], {("a", "b"): 9, ("a", "c"): 1}) self.assertGreater(r["b"], r["c"]) def test_map_respects_budget(self): files = C.synthetic_repo() for b in (20, 60, 150, 1000): text, info = C.repo_map(files, budget=b) self.assertLessEqual(S.est_tokens(text) if text else 0, b) self.assertEqual(info["tokens"], S.est_tokens(text) if text else 1) def test_map_grows_with_budget(self): files = C.synthetic_repo() sizes = [C.repo_map(files, budget=b)[1]["symbols"] for b in (20, 60, 150, 1000)] self.assertEqual(sizes, sorted(sizes)) self.assertEqual(sizes[-1], C.repo_map(files, budget=1000)[1]["symbols_total"]) self.assertGreater(sizes[-1], sizes[0]) def test_hub_listed_first(self): text, info = C.repo_map(C.synthetic_repo(), budget=60) self.assertTrue(text.startswith("pkg/m00.py:")) top = max(info["rank"], key=info["rank"].get) self.assertEqual(top, "pkg/m00.py") def test_map_is_smaller_than_the_code(self): _, info = C.repo_map(C.synthetic_repo(), budget=1000) self.assertLess(info["tokens"], info["full_tokens"]) def test_focus_moves_a_file_up(self): files = C.synthetic_repo() _, plain = C.repo_map(files, budget=1000) _, focused = C.repo_map(files, budget=1000, focus=("pkg/m09.py",)) self.assertGreater(focused["rank"]["pkg/m09.py"], plain["rank"]["pkg/m09.py"]) t1, _ = C.repo_map(files, budget=60, focus=("pkg/m09.py",)) self.assertIn("pkg/m09.py:", t1) def test_map_lines_nest_methods_under_class(self): text, _ = C.repo_map(S.synth_repo("cartkit"), budget=1000) self.assertIn(" class Cart\n def __init__(self, discount=0.0)", text) def test_method_pulls_in_its_class_header(self): files = {"a.py": "class K:\n def m(self):\n return 1\n", "b.py": "x = K().m()\n"} text, info = C.repo_map(files, budget=1000) self.assertIn("class K", text) self.assertIn("def m(self)", text) def test_zero_budget(self): text, info = C.repo_map(C.synthetic_repo(), budget=0) self.assertEqual(text, "") self.assertEqual(info["symbols"], 0) def test_synthetic_repo_is_valid_and_deterministic(self): a, b = C.synthetic_repo(8, 2, 3), C.synthetic_repo(8, 2, 3) self.assertEqual(a, b) self.assertNotEqual(a, C.synthetic_repo(8, 2, 4)) for p, src in a.items(): ast.parse(src) self.assertEqual(len(a), 9) self.assertEqual(S.run_tests(dict(a)), {}) mod = S.Sandbox(a).load("pkg.m05") self.assertIsInstance(mod.f5_0(1), int) def test_localize_study_bodies_help_when_names_are_missing(self): named = C.localize_study(TASKS, False) masked = C.localize_study(TASKS, True) self.assertEqual(named["bodies file@1"], 0.95) self.assertEqual(named["signatures function@3"], 0.95) self.assertEqual(masked["bodies function@3"], 0.45) self.assertGreater(masked["bodies function@3"], masked["signatures function@3"]) self.assertLess(masked["bodies file@1"], named["bodies file@1"]) class TestHarness(unittest.TestCase): def test_project_files(self): for repo in S.REPOS: files, feats = C.project_files(repo) self.assertEqual(len(feats), 4) self.assertFalse(any(f["passes"] for f in feats)) self.assertTrue(C._visible_pass(files, feats)) def test_project_features_pass_when_source_is_fixed(self): files, feats = C.project_files("textkit") files = dict(files) files[S.REPOS["textkit"]["module"]] = S.synth_repo("textkit")[S.REPOS["textkit"]["module"]] res = S.run_tests(files) self.assertTrue(all(res[f["test"]] == "PASS" for f in feats)) def test_visible_pass_fails_on_broken_source(self): files, feats = C.project_files("textkit") files = dict(files) files["textkit/text.py"] = "def (" self.assertFalse(C._visible_pass(files, feats)) def test_config_defaults(self): c = C.HarnessConfig() self.assertEqual((c.feature_list, c.progress, c.commits, c.crash, c.max_sessions, c.max_attempts), (True, True, True, 0.3, 8, 2)) def test_bare_run_declares_done(self): r = C.run_long("textkit", C.HarnessConfig(feature_list=False, progress=False, commits=False)) self.assertTrue(r["declared_done"]) self.assertEqual((r["passing"], r["sessions"]), (0, 1)) self.assertTrue(r["healthy"]) def test_full_harness_without_crashes_fixes_most(self): r = C.run_long("textkit", C.HarnessConfig(crash=0.0)) self.assertEqual(r["passing"], 3) self.assertTrue(r["healthy"]) self.assertGreater(r["cost"], 0) self.assertEqual(r["started_passing"], 0) def test_full_harness_survives_crashes(self): rows = [C.run_long(repo, C.HarnessConfig(crash=0.3), seed=s) for repo in ("textkit", "listkit", "cartkit") for s in range(3)] self.assertTrue(all(r["healthy"] for r in rows)) self.assertGreater(sum(r["passing"] for r in rows), 15) def test_crashes_without_commits_leave_damage(self): cfg = C.HarnessConfig(feature_list=True, progress=True, commits=False, crash=0.5) rows = [C.run_long(repo, cfg, seed=s) for repo in ("textkit", "listkit", "cartkit") for s in range(3)] self.assertTrue(any(not r["healthy"] for r in rows)) full = [C.run_long(repo, C.HarnessConfig(crash=0.5), seed=s) for repo in ("textkit", "listkit", "cartkit") for s in range(3)] self.assertGreater(sum(r["passing"] for r in full), sum(r["passing"] for r in rows)) def test_progress_notes_reduce_sessions(self): with_notes = C.run_long("mathkit", C.HarnessConfig(feature_list=True, progress=True, commits=True, crash=0.0)) without = C.run_long("mathkit", C.HarnessConfig(feature_list=True, progress=False, commits=True, crash=0.0)) self.assertLessEqual(with_notes["sessions"], without["sessions"]) self.assertLessEqual(with_notes["cost"], without["cost"]) def test_run_is_deterministic(self): a = C.run_long("listkit", C.HarnessConfig(), seed=2) b = C.run_long("listkit", C.HarnessConfig(), seed=2) self.assertEqual({k: v for k, v in a.items()}, {k: v for k, v in b.items()}) def test_session_context_limits_attempts(self): tiny = C.HarnessConfig(session_context=1, crash=0.0, max_sessions=3) r = C.run_long("textkit", tiny) self.assertEqual(r["sessions"], 3) self.assertLessEqual(r["passing"], 3) self.assertEqual(sum(1 for x in r["log"] if "cut off" in x), 0) def test_max_sessions_cap(self): r = C.run_long("textkit", C.HarnessConfig(crash=1.0, max_sessions=2)) self.assertLessEqual(r["sessions"], 2) self.assertEqual(r["passing"], 0) self.assertTrue(any("cut off" in x for x in r["log"])) def test_attempt_cap_gives_up_on_unfixable_feature(self): r = C.run_long("textkit", C.HarnessConfig(crash=0.0, max_attempts=1)) self.assertTrue(any("nothing left to try" in x for x in r["log"])) def test_custom_agent_is_used(self): class Nothing: tier = "mid" def solve(self, view, hint=None): res = S.AgentResult("", 10, 1, 1, []) res.peak_context = 5 return res r = C.run_long("textkit", C.HarnessConfig(crash=0.0), agent=Nothing()) self.assertEqual(r["passing"], 0) self.assertGreater(r["sessions"], 1) class TestHelpers(unittest.TestCase): def test_four_tuple(self): t = BY["textkit-3"] ft = C.four_tuple(t) self.assertEqual(list(ft), ["request", "environment", "stopping", "scorer"]) self.assertEqual(ft["request"], "startswith_any() looks at the end of the string") self.assertIn("3 files", ft["environment"]) self.assertEqual(ft["scorer"], "%d FAIL_TO_PASS and %d PASS_TO_PASS tests must all pass" % (len(t.f2p), len(t.p2p))) def test_parse_agents_md(self): md = "# T\nintro\n## Setup\n- Run `make x`\n- Other\n## Style\n - Short\n- orphan `y`\n" p = C.parse_agents_md(md) self.assertEqual(p["Setup"], ["Run `make x`", "Other"]) self.assertEqual(p["Style"], ["Short", "orphan `y`"]) self.assertEqual(p["T"], []) self.assertEqual(p["_commands"], ["make x", "y"]) self.assertEqual(C.parse_agents_md("- before any heading\n")["_commands"], []) self.assertEqual(list(C.parse_agents_md("- before any heading\n")), ["_commands"]) class TestOnSynthesized(unittest.TestCase): def test_loop_and_pipeline_on_synthesized(self): loop = sum(S.evaluate(t, C.AciAgent("s").solve(t.view()).patch).resolved for t in SYN) pipe = sum(S.evaluate(t, C.AgentlessPipeline("s").solve(t.view()).patch).resolved for t in SYN) self.assertEqual(len(SYN), 40) self.assertGreater(loop, 15) self.assertGreater(pipe, 10) def test_demo_runs_and_is_clean(self): buf = io.StringIO() with redirect_stdout(buf): C.demo() out = buf.getvalue() for h in ("== 1.", "== 5.", "== 8.", "== 10."): self.assertIn(h, out) self.assertNotIn("\u2014", out) self.assertNotIn("\u2013", out) out.encode("ascii") def test_source_is_clean(self): with open(C.__file__, encoding="utf-8") as fh: src = fh.read() src.encode("ascii") self.assertNotIn("\u2014", src) self.assertNotIn("\u2013", src) class TestPinned(unittest.TestCase): """Tests added after the first mutation run: exact values that a one-character change would move.""" def test_pagerank_divides_by_total_weight(self): r = C.pagerank(["a", "b", "c"], {("a", "b"): 3, ("a", "c"): 1}) self.assertAlmostEqual(sum(r.values()), 1.0, places=6) self.assertGreater(r["b"], r["c"]) self.assertLess(r["b"], 1.0) def test_pipeline_token_counts_are_pinned(self): r = C.AgentlessPipeline("p").solve(BY["textkit-3"].view()) self.assertEqual((r.tokens_in // C.SCALE, r.tokens_out // C.SCALE, r.turns), (339, 570, 5)) def test_agent_token_counts_are_pinned(self): r = C.AciAgent("a").solve(BY["textkit-3"].view()) self.assertEqual((r.tokens_in // C.SCALE, r.tokens_out // C.SCALE, r.turns), (7411, 328, 8)) def test_pool_candidates_change_only_the_unit(self): t = BY["textkit-3"] p = C.AgentlessPipeline("p") files, ranked, units = p.localize(t.view()) for c in p.pool(t.view(), ranked, units): d = S.diff_stats(S.make_diff({c.path: t.base[c.path]}, {c.path: c.new_src})) self.assertLessEqual(d["added"], 3) self.assertLessEqual(d["removed"], 3) self.assertEqual(len(c.new_src.split("\n")), len(t.base[c.path].split("\n")) + d["added"] - d["removed"]) ast.parse(c.new_src) def test_act_charges_exact_output_tokens(self): w = C.Workspace({"pkg/a.py": "x\n"}) w.act("open", "pkg/a.py") self.assertEqual(w.ledger.tout, 30 + S.est_tokens("open pkg/a.py")) w.act("goto", 1, think=7) self.assertEqual(w.ledger.tout, 30 + S.est_tokens("open pkg/a.py") + 7 + S.est_tokens("goto 1")) def test_default_window_is_one_hundred_lines(self): w = C.Workspace({"big.py": "\n".join("x = %d" % i for i in range(150)) + "\n"}) out = w.open("big.py") self.assertEqual(out.count("\n"), 2 + 100) self.assertIn("(50 more lines below)", out) self.assertIn("100: x = 99", out) def test_run_repro_uses_setup(self): files = {"m/__init__.py": "", "m/c.py": "class K:\n def __init__(self):\n self.v = 1\n"} self.assertEqual(C.run_repro(files, "m/c.py", ("k = K()", "k.v", "1")), "Issue resolved") self.assertEqual(C.run_repro(files, "m/c.py", ("k = K()", "k.v", "2")), "Issue reproduced") def test_method_header_belongs_to_its_own_class(self): files = {"a.py": "class A:\n def x(self): pass\n\n\nclass B:\n def m(self): pass\n", "b.py": "obj.m()\nobj.m()\n"} self.assertEqual(C.repo_map(files, budget=8)[0], "a.py:\n class B\n def m(self)") def test_symbol_use_counts_only_other_files(self): files = {"a.py": "def f(): return 1\ndef g(): return 2\ndef h(): return f() + f() + f() + f()\n", "b.py": "x = g()\n"} self.assertEqual(C.repo_map(files, budget=4)[0], "a.py:\n def g()") def test_synthetic_repo_defaults_and_values(self): self.assertEqual(C.synthetic_repo(), C.synthetic_repo(12, 3, 0)) self.assertEqual(len(C.synthetic_repo()), 13) mod = S.Sandbox(C.synthetic_repo()).load("pkg.m00") self.assertEqual([mod.f0_0(5), mod.f0_1(5), mod.f0_2(5)], [5, 6, 7]) def test_undefined_names_tie_break_by_name(self): self.assertEqual(C.undefined_names(ast.parse("a = c + b\n")), [("b", 1), ("c", 1)]) def test_find_file_cap_is_inclusive(self): w = ws(search_cap=2) self.assertTrue(w.find_file("a.py").startswith("Found 2 file(s)")) self.assertTrue(ws(search_cap=1).find_file("a.py").startswith("More than")) self.assertTrue(ws(search_cap=2).search_dir("return").startswith("Found 2")) def test_row_format(self): row = C._row("x", [(True, 4, 0.5), (False, 6, 1.5)]) self.assertEqual(row, "x 1/2 (95% CI 9 to 91%) 5.0 1.000 2.000") self.assertTrue(C._row("y", [(False, 1, 0.0)]).endswith(" n/a")) def test_harness_without_notes_reads_the_whole_repo_each_session(self): class Nothing: tier = "mid" def solve(self, view, hint=None): res = S.AgentResult("", 0, 0, 1, []) res.peak_context = 0 return res files, _ = C.project_files("textkit") orient = sum(S.est_tokens(v) for v in files.values()) * C.SCALE r = C.run_long("textkit", C.HarnessConfig(progress=False, commits=True, crash=0.0), agent=Nothing()) self.assertEqual(r["sessions"], 8) self.assertEqual(r["tokens"], 8 * orient) def test_rejected_edit_message_is_pinned(self): w = C.Workspace({"m.py": "a = 1\nb = 2\nc = 3\nd = 4\ne = 5\n"}) w.act("open", "m.py") self.assertEqual(w.act("edit", 3, 3, "c = ("), "Your edit was not applied (SyntaxError: '(' was never closed (line 3)).\n" "This is how it would have looked:\n1: a = 1\n2: b = 2\n3: c = (\n4: d = 4\n5: e = 5\n" "Fix the edit and try again.") self.assertEqual(w.act("edit", 5, 5, "e = (").split("\n")[2:5], ["3: c = 3", "4: d = 4", "5: e = ("]) def test_goto_first_line_is_allowed(self): w = C.Workspace({"m.py": "a = 1\nb = 2\n"}) w.act("open", "m.py") self.assertTrue(w.act("goto", 1).startswith("[File: m.py")) self.assertTrue(w.act("goto", 0).startswith("Error")) def test_pagerank_single_node(self): self.assertAlmostEqual(C.pagerank(["a"], {})["a"], 1.0) def test_attempt_limit_is_exact(self): class Nothing: tier = "mid" def solve(self, view, hint=None): res = S.AgentResult("", 0, 0, 1, []) res.peak_context = 0 return res n = len(C.project_files("textkit")[1]) for limit in (1, 2): r = C.run_long("textkit", C.HarnessConfig(crash=0.0, max_attempts=limit), agent=Nothing()) self.assertEqual(r["sessions"], limit + 1) self.assertEqual(sum(1 for l in r["log"] if "nothing left" in l), 1) self.assertGreater(n, 0) def test_harness_cost_uses_the_tier_price(self): class Nothing: tier = "small" def solve(self, view, hint=None): res = S.AgentResult("", 0, 5000, 1, []) res.peak_context = 0 return res r = C.run_long("textkit", C.HarnessConfig(progress=False, commits=True, crash=0.0), agent=Nothing()) orient = sum(S.est_tokens(v) for v in C.project_files("textkit")[0].values()) * C.SCALE pin, pout = S.TIERS["small"] self.assertAlmostEqual(r["cost"], (8 * orient * pin + 8 * len(C.project_files("textkit")[1]) * 5000 * pout) / 1e6) def test_localization_study_is_pinned(self): tasks = S.curated_tasks() named = C.localize_study(tasks) self.assertEqual({k: round(v, 2) for k, v in named.items()}, {"bodies file@1": 0.95, "bodies function@3": 0.95, "signatures file@1": 0.95, "signatures function@3": 0.95}) masked = C.localize_study(tasks, mask_names=True) self.assertEqual({k: round(v, 2) for k, v in masked.items()}, {"bodies file@1": 0.55, "bodies function@3": 0.45, "signatures file@1": 0.45, "signatures function@3": 0.35}) def test_row_interval_is_in_percent(self): row = C._row("y", [(True, 2, 1.0)] * 30 + [(False, 2, 1.0)] * 30) self.assertEqual(row, "y 30/60 (95% CI 38 to 62%) 2.0 1.000 2.000") def test_documented_defaults(self): import inspect self.assertEqual(C.AciAgent("a").max_turns, 60) self.assertEqual(C.HarnessConfig().session_context, 400000) sig = inspect.signature(C.repo_map).parameters self.assertEqual((sig["budget"].default, sig["boost"].default), (300, 5.0)) self.assertEqual(inspect.signature(C.selection_study).parameters["n_samples"].default, 8) def test_window_larger_than_the_file_starts_at_line_one(self): w = C.Workspace({"m.py": "\n".join("x = %d" % i for i in range(60)) + "\n"}) out = w.open("m.py", 55) self.assertIn("(0 more lines above)", out) self.assertIn("1: x = 0", out) if __name__ == "__main__": unittest.main()