import io import re import sys import unittest from contextlib import redirect_stdout import swe_bench_toolkit as T TASKS = T.curated_tasks() BY = {t.id: t for t in TASKS} SYNTH, SYNTH_STATS = T.synthesize_tasks(limit=40) A = {"x.py": "def f():\n return 1\n\n\ndef g():\n return 2\n"} B = {"x.py": "def f():\n return 1\n\n\ndef g():\n return 3\n", "y.py": "new\n"} class TestHelpers(unittest.TestCase): def test_mean(self): self.assertEqual(T.mean([]), 0.0) self.assertAlmostEqual(T.mean(iter([1, 2, 6])), 3.0) def test_rng_for_is_order_independent(self): a = T.rng_for("x", 1).random() T.rng_for("y").random() self.assertEqual(a, T.rng_for("x", 1).random()) self.assertNotEqual(a, T.rng_for("x", 2).random()) def test_wilson(self): self.assertEqual(T.wilson(0, 0), (0.0, 1.0)) lo, hi = T.wilson(5, 10) self.assertLess(lo, 0.5) self.assertGreater(hi, 0.5) self.assertAlmostEqual(lo + hi, 1.0, places=6) lo0, hi0 = T.wilson(0, 20) self.assertEqual(lo0, 0.0) self.assertGreater(hi0, 0.1) self.assertLessEqual(T.wilson(20, 20)[1], 1.0) self.assertLess(T.wilson(50, 100)[1] - T.wilson(50, 100)[0], T.wilson(5, 10)[1] - T.wilson(5, 10)[0]) def test_bootstrap_ci(self): lo, hi = T.bootstrap_ci([1] * 10) self.assertEqual((lo, hi), (1.0, 1.0)) lo, hi = T.bootstrap_ci([0, 1] * 20) self.assertLess(lo, 0.5) self.assertGreater(hi, 0.5) self.assertEqual(T.bootstrap_ci([]), (0.0, 1.0)) self.assertEqual(T.bootstrap_ci([0, 1, 1], seed=3), T.bootstrap_ci([0, 1, 1], seed=3)) self.assertEqual(T.bootstrap_ci([0, 1] * 10, seed=1), (0.3, 0.7)) self.assertEqual(T.bootstrap_ci([0] * 30 + [1] * 10, seed=2), (0.125, 0.375)) def test_tokens_split_camel_and_snake(self): self.assertEqual(T.tokens("fooBar"), ["foo", "bar"]) t = T.tokens("is_palindrome") self.assertIn("is_palindrome", t) self.assertIn("palindrome", t) def test_content_tokens_drop_stopwords_and_single_letters(self): self.assertEqual(T.content_tokens("the a x cart"), ["cart"]) def test_jaccard(self): self.assertEqual(T.jaccard([], []), 1.0) self.assertAlmostEqual(T.jaccard("ab", "bc"), 1 / 3) def test_est_tokens(self): self.assertEqual(T.est_tokens(""), 1) self.assertEqual(T.est_tokens("x" * 40), 10) class TestDiff(unittest.TestCase): def test_round_trip_two_files(self): d = T.make_diff(A, B) new, rep = T.apply_patch(A, d) self.assertEqual(new, B) self.assertEqual(rep["files"], ["x.py", "y.py"]) self.assertEqual(rep["hunks"], 2) def test_identical_trees_give_empty_diff(self): self.assertEqual(T.make_diff(A, dict(A)), "") def test_delete_file(self): d = T.make_diff(B, {"x.py": B["x.py"]}) new, _ = T.apply_patch(B, d) self.assertNotIn("y.py", new) def test_new_file_must_not_exist(self): d = T.make_diff({}, {"y.py": "new\n"}) with self.assertRaises(T.PatchError): T.apply_patch({"y.py": "old\n"}, d) def test_missing_file(self): d = T.make_diff(A, {"x.py": "z\n"}).replace("x.py", "nope.py") with self.assertRaisesRegex(T.PatchError, "file not found"): T.apply_patch(A, d) def test_context_mismatch(self): d = T.make_diff(A, B).replace("- return 2", "- return 99") with self.assertRaisesRegex(T.PatchError, "context does not match"): T.apply_patch(A, d) def test_hunk_applies_at_offset(self): d = T.make_diff(A, B).replace("@@ -3,4 +3,4 @@", "@@ -1,4 +1,4 @@") new, rep = T.apply_patch(A, d) self.assertEqual(new["x.py"], B["x.py"]) self.assertEqual(rep["offsets"], 1) def test_offset_limit(self): padded = {"x.py": "\n" * 30 + A["x.py"]} d = T.make_diff(A, B) with self.assertRaises(T.PatchError): T.apply_patch(padded, d) new, _ = T.apply_patch(padded, d, max_offset=40) self.assertIn("return 3", new["x.py"]) def test_failed_patch_changes_nothing(self): d = T.make_diff(A, B).replace("y.py", "x.py", 2) before = dict(A) with self.assertRaises(T.PatchError): T.apply_patch(A, d) self.assertEqual(A, before) def test_strict_counts_reject_long_and_short_headers(self): d = T.make_diff(A, {"x.py": A["x.py"].replace("2", "3")}) longer = d.replace("@@ -3,4 +3,4 @@", "@@ -3,4 +3,5 @@") shorter = d.replace("@@ -3,4 +3,4 @@", "@@ -3,4 +3,3 @@") with self.assertRaisesRegex(T.PatchError, "counts"): T.parse_diff(longer) with self.assertRaisesRegex(T.PatchError, "longer"): T.parse_diff(shorter) def test_lenient_parse_and_count_repair(self): d = T.make_diff(A, {"x.py": A["x.py"].replace("2", "3")}) bad = d.replace("@@ -3,4 +3,4 @@", "@@ -3,4 +3,9 @@") new, _ = T.apply_patch(A, bad, strict_counts=False) self.assertIn("return 3", new["x.py"]) fixed = T.fix_hunk_counts(bad) self.assertIn("@@ -3,4 +3,4 @@", fixed) self.assertEqual(T.apply_patch(A, fixed)[0]["x.py"], new["x.py"]) def test_fix_hunk_counts_leaves_good_patch_alone(self): d = T.make_diff(A, {"x.py": B["x.py"]}) self.assertEqual(T.fix_hunk_counts(d), d) def test_fix_hunk_counts_handles_two_hunks(self): src = {"a.py": "\n".join("line%d" % i for i in range(40)) + "\n"} dst = {"a.py": src["a.py"].replace("line2\n", "LINE2\n").replace("line35\n", "LINE35\n")} d = T.make_diff(src, dst) self.assertEqual(d.count("@@ -"), 2) bad = re.sub(r"@@ -(\d+),(\d+) \+(\d+),(\d+)", lambda m: "@@ -%s,%d +%s,%d" % (m.group(1), int(m.group(2)) + 1, m.group(3), int(m.group(4)) + 2), d) self.assertEqual(T.apply_patch(src, T.fix_hunk_counts(bad))[0], dst) def test_parse_errors(self): for bad in ("", "just prose\n", "--- a/x.py\n", "--- a/x.py\n+++ b/x.py\n", "--- a/x.py\n+++ b/x.py\n@@ nonsense\n"): with self.assertRaises(T.PatchError, msg=repr(bad)): T.parse_diff(bad) def test_header_error_messages(self): with self.assertRaisesRegex(T.PatchError, "without '\\+\\+\\+'"): T.parse_diff("--- a/x.py\n+++ b/x.py\n".replace("+++ b/x.py\n", "")) with self.assertRaisesRegex(T.PatchError, "without hunks"): T.parse_diff("--- a/x.py\n+++ b/x.py\n") with self.assertRaisesRegex(T.PatchError, "malformed hunk header"): T.parse_diff("--- a/x.py\n+++ b/x.py\n@@ nonsense\n") def test_lenient_parse_stops_at_the_next_file_header(self): d = T.make_diff(A, B).replace("@@ -3,4 +3,4 @@", "@@ -3,4 +3,9 @@").replace("@@ -0,0 +1 @@", "@@ -0,0 +1,7 @@") new, rep = T.apply_patch(A, d, strict_counts=False) self.assertEqual(new, B) self.assertEqual(rep["hunks"], 2) def test_single_prose_line_before_header_is_skipped(self): d = "Here is my patch:\n" + T.make_diff(A, {"x.py": A["x.py"].replace("2", "3")}) self.assertEqual(T.apply_patch(A, d)[0]["x.py"].count("return 3"), 1) def test_second_hunk_is_placed_after_a_large_first_hunk(self): src = {"a.py": "\n".join("l%d" % i for i in range(60)) + "\n"} dst = {"a.py": src["a.py"].replace("l2\n", "".join("n%d\n" % i for i in range(30)), 1).replace("l50\n", "X50\n")} d = T.make_diff(src, dst) self.assertEqual(d.count("@@ -"), 2) new, rep = T.apply_patch(src, d) self.assertEqual(new, dst) self.assertEqual(rep["offsets"], 0) def test_pure_insertion_hunks_follow_earlier_size_changes(self): src = {"a.py": "\n".join("l%d" % i for i in range(60)) + "\n"} dst = {"a.py": src["a.py"].replace("l5\n", "l5\n" + "".join("n%d\n" % i for i in range(20)), 1).replace("l40\n", "l40\nZ\n")} d = T.make_diff(src, dst, context=0) self.assertEqual(d.count("@@ -"), 2) new, rep = T.apply_patch(src, d) self.assertEqual(new, dst) self.assertEqual(rep["offsets"], 0) def test_git_header_lines_are_skipped(self): d = "diff --git a/x.py b/x.py\nindex 1..2 100644\n" + T.make_diff(A, B) self.assertEqual(len(T.parse_diff(d)), 2) def test_no_newline_marker_ignored(self): d = T.make_diff(A, {"x.py": A["x.py"].replace("2", "3")}) + "" d = d.replace("+ return 3\n", "+ return 3\n\\ No newline at end of file\n") self.assertEqual(T.apply_patch(A, d)[0]["x.py"].count("return 3"), 1) def test_file_patch_flags(self): fps = T.parse_diff(T.make_diff(B, {"x.py": B["x.py"]})) self.assertTrue(fps[0].is_delete) self.assertFalse(fps[0].is_new) self.assertEqual(fps[0].path, "y.py") fps = T.parse_diff(T.make_diff({}, {"z.py": "q\n"})) self.assertTrue(fps[0].is_new) self.assertEqual(fps[0].path, "z.py") def test_hunk_old_new_lines(self): h = T.parse_diff(T.make_diff(A, B))[0].hunks[0] self.assertIn(" return 2", h.old_lines()) self.assertIn(" return 3", h.new_lines()) self.assertNotIn(" return 3", h.old_lines()) def test_diff_stats(self): self.assertEqual(T.diff_stats(T.make_diff(A, B)), {"files": 2, "added": 2, "removed": 1, "hunks": 2}) self.assertEqual(T.diff_stats("garbage"), {"files": 0, "added": 0, "removed": 0, "hunks": 0}) def test_blank_lines_survive(self): src = {"a.py": "a\n\n\nb\n"} dst = {"a.py": "a\n\n\nc\n"} self.assertEqual(T.apply_patch(src, T.make_diff(src, dst))[0], dst) def test_insert_at_top_and_empty_old_side(self): src = {"a.py": "b\n"} dst = {"a.py": "a\nb\n"} self.assertEqual(T.apply_patch(src, T.make_diff(src, dst))[0], dst) class TestRunner(unittest.TestCase): FILES = {"pk/__init__.py": "", "pk/m.py": "import math\n\n\ndef spin():\n while True:\n pass\n\n\ndef two():\n return 2\n", "tests/test_m.py": "import pk.m\nfrom pk.m import spin, two\n\n\ndef test_ok():\n assert two() == 2\n\n\ndef test_bad():\n assert two() == 3\n\n\ndef test_err():\n raise ValueError\n\n\ndef test_spin():\n spin()\n\n\ndef test_attr():\n assert pk.m.two() == 2\n"} def test_statuses(self): r = T.run_tests(self.FILES, step_limit=2000) self.assertEqual(r["tests/test_m.py::test_ok"], "PASS") self.assertEqual(r["tests/test_m.py::test_bad"], "FAIL") self.assertEqual(r["tests/test_m.py::test_err"], "ERROR") self.assertEqual(r["tests/test_m.py::test_spin"], "TIMEOUT") self.assertEqual(r["tests/test_m.py::test_attr"], "PASS") def test_only_filter(self): r = T.run_tests(self.FILES, only={"tests/test_m.py::test_ok"}) self.assertEqual(list(r), ["tests/test_m.py::test_ok"]) def test_import_error_marks_all_tests_error(self): f = dict(self.FILES) f["tests/test_m.py"] = "import os\n\n\ndef test_a():\n assert True\n" self.assertEqual(T.run_tests(f), {"tests/test_m.py::test_a": "ERROR"}) def test_missing_name_is_error(self): f = dict(self.FILES) f["tests/test_m.py"] = "from pk.m import nope\n\n\ndef test_a():\n assert True\n" self.assertEqual(T.run_tests(f), {"tests/test_m.py::test_a": "ERROR"}) def test_syntax_error_in_test_file(self): f = dict(self.FILES) f["tests/test_m.py"] = "def test_a(:\n" self.assertEqual(T.run_tests(f), {"tests/test_m.py::": "ERROR"}) def test_module_level_loop_times_out(self): f = {"pk/__init__.py": "", "tests/test_m.py": "while True:\n pass\n\n\ndef test_a():\n pass\n"} self.assertEqual(T.run_tests(f, step_limit=500), {"tests/test_m.py::test_a": "TIMEOUT"}) def test_state_does_not_leak_between_runs(self): f = {"pk/__init__.py": "", "pk/m.py": "COUNT = []\n", "tests/test_m.py": "from pk.m import COUNT\n\n\ndef test_a():\n COUNT.append(1)\n assert len(COUNT) == 1\n"} self.assertEqual(T.run_tests(f)["tests/test_m.py::test_a"], "PASS") self.assertEqual(T.run_tests(f)["tests/test_m.py::test_a"], "PASS") def test_trace_function_is_restored(self): before = sys.gettrace() T.run_tests(self.FILES, step_limit=300) self.assertIs(sys.gettrace(), before) def test_open_is_not_available_to_package_code(self): f = {"pk/__init__.py": "", "pk/m.py": "def r():\n return open('/etc/hostname')\n", "tests/test_m.py": "from pk.m import r\n\n\ndef test_a():\n r()\n"} self.assertEqual(T.run_tests(f)["tests/test_m.py::test_a"], "ERROR") def test_whitelisted_stdlib_import_works(self): self.assertEqual(T.eval_snippet(self.FILES, "pk/m.py", "", "math.floor(2.5)"), "2") def test_eval_snippet(self): self.assertEqual(T.eval_snippet(self.FILES, "pk/m.py", "x = 40", "two() + x"), "42") self.assertEqual(T.eval_snippet(self.FILES, "pk/m.py", "", "1/0"), "raises ZeroDivisionError") self.assertEqual(T.eval_snippet(self.FILES, "pk/m.py", "", "spin()", step_limit=300), "raises StepLimit") self.assertEqual(T.eval_snippet(self.FILES, "pk/missing.py", "", "1"), "raises ImportError") def test_step_limit_threshold_is_pinned(self): f = {"pk/__init__.py": "", "pk/m.py": "def f():\n x = 0\n for i in range(10):\n x += 1\n return x\n", "tests/test_m.py": "from pk.m import f\n\n\ndef test_a():\n assert f() == 10\n"} self.assertEqual(T.run_tests(f, step_limit=24)["tests/test_m.py::test_a"], "PASS") self.assertEqual(T.run_tests(f, step_limit=23)["tests/test_m.py::test_a"], "TIMEOUT") def test_package_init_runs_and_star_import_works(self): f = {"pk/__init__.py": "VERSION = 3\n", "pk/m.py": "def two():\n return 2\n", "tests/test_m.py": "import pk.m\nfrom pk.m import *\n\n\ndef test_a():\n assert pk.VERSION == 3\n assert two() == 2\n"} self.assertEqual(T.run_tests(f), {"tests/test_m.py::test_a": "PASS"}) def test_from_package_import_submodule(self): f = {"pk/__init__.py": "", "pk/m.py": "def two():\n return 2\n", "tests/test_m.py": "from pk import m\n\n\ndef test_a():\n assert m.two() == 2\n"} self.assertEqual(T.run_tests(f), {"tests/test_m.py::test_a": "PASS"}) def test_helpers(self): self.assertEqual(T.path_to_module("a/b/c.py"), "a.b.c") self.assertEqual(T.test_files({"tests/test_x.py": "", "pk/m.py": "", "test_y.py": "", "tests/helper.py": ""}), ["test_y.py", "tests/test_x.py"]) self.assertEqual(T.test_names("def test_a():\n pass\n\ndef helper():\n pass\n"), ["test_a"]) self.assertIsNone(T.test_names("def (")) class TestTasks(unittest.TestCase): def test_counts_and_ids(self): self.assertEqual(len(TASKS), 20) self.assertEqual(len({t.id for t in TASKS}), 20) flaws = [t.flaw for t in TASKS] self.assertEqual((flaws.count("narrow"), flaws.count("wide"), flaws.count("vague")), (3, 2, 2)) def test_every_task_validates(self): for t in TASKS: v = T.validate_task(t) self.assertTrue(v["ok"], (t.id, v)) def test_f2p_p2p_shape(self): for t in TASKS: self.assertGreaterEqual(len(t.f2p), 1, t.id) self.assertEqual(len(t.p2p), 4, t.id) self.assertFalse(set(t.f2p) & set(t.p2p)) self.assertEqual(len(BY["listkit-1"].f2p), 2) self.assertEqual(len(BY["cartkit-2"].f2p), 2) def test_f2p_fail_on_base_and_pass_with_gold(self): for t in TASKS: before = T.run_tests(t.tests_files()) for name in t.f2p: self.assertNotEqual(before[name], "PASS", (t.id, name)) for name in t.p2p: self.assertEqual(before[name], "PASS", (t.id, name)) def test_gold_resolves_and_empty_does_not(self): for t in TASKS: self.assertTrue(T.evaluate(t, t.gold_patch).resolved, t.id) self.assertEqual(T.evaluate(t, "").status, "empty") def test_gold_patch_touches_no_tests(self): for t in TASKS: for fp in T.parse_diff(t.gold_patch): self.assertNotIn("test_", fp.path) for fp in T.parse_diff(t.test_patch): self.assertIn("test_", fp.path) def test_view_hides_test_patch_and_gold(self): t = BY["cartkit-3"] v = t.view() blob = "\n".join(v.files.values()) self.assertNotIn("test_can_checkout_regress", blob) self.assertIn("test_can_checkout_basic", blob) v.files["cartkit/cart.py"] = "mutated" self.assertNotEqual(t.base["cartkit/cart.py"], "mutated") def test_alt_patches_follow_planted_flaws(self): for t in TASKS: fails = bool(T.alt_solution_fails(t)) self.assertEqual(fails, t.flaw in ("narrow", "wide"), t.id) def test_narrow_tests_import_names_the_issue_never_mentions(self): for tid, name in (("textkit-2", "ELLIPSIS"), ("listkit-2", "_is_seq"), ("mathkit-1", "_extent")): self.assertIn(name, BY[tid].test_patch) self.assertNotIn(name, BY[tid].issue) def test_wide_tasks_need_two_fixes(self): for tid in ("listkit-1", "cartkit-2"): t = BY[tid] self.assertEqual(sum(1 for ln in t.gold_patch.split("\n") if ln.startswith("+") and not ln.startswith("+++")), 2) def test_bug_substitutions_apply_exactly_once(self): with self.assertRaises(ValueError): T._sub_once("aa", "a", "b") with self.assertRaises(ValueError): T._sub_once("aa", "z", "b") self.assertEqual(T._sub_once("abc", "b", "x"), "axc") def test_created_days_are_stable(self): self.assertEqual([t.created for t in TASKS], [T.rng_for("day", t.id).randrange(200) for t in TASKS]) self.assertTrue(all(0 <= t.created < 200 for t in TASKS)) def test_validate_flags_broken_tasks(self): t = BY["textkit-3"] noop = T.make_diff({t.module: t.base[t.module]}, {t.module: t.base[t.module] + "# harmless\n"}) broken = T.Task("x", t.repo, t.fn, t.issue, t.base, t.test_patch, noop, t.alts) v = T.validate_task(broken) self.assertFalse(v["ok"]) self.assertTrue(any("no FAIL_TO_PASS" in r for r in v["reasons"])) self.assertTrue(any("still fail" in r for r in v["reasons"])) def test_validate_flags_gold_that_breaks_a_passing_test(self): t = BY["textkit-3"] bad_gold = T.make_diff({t.module: t.base[t.module]}, {t.module: t.base[t.module].replace("s.endswith(p)", "False")}) broken = T.Task("x", t.repo, t.fn, t.issue, t.base, t.test_patch, bad_gold, t.alts) v = T.validate_task(broken) self.assertFalse(v["ok"]) self.assertTrue(any("breaks" in r for r in v["reasons"])) def test_curated_tasks_are_cached(self): self.assertIs(T.curated_tasks(), T.curated_tasks()) def test_repo_files_layout(self): f = T.repo_files("cartkit") self.assertEqual(sorted(f), ["cartkit/__init__.py", "cartkit/cart.py", "tests/test_cart.py"]) self.assertEqual(sum(1 for v in T.run_tests(f).values() if v == "PASS"), 4) class TestHarness(unittest.TestCase): def test_invalid_versus_apply_fail(self): t = BY["textkit-4"] self.assertEqual(T.evaluate(t, "prose only\n").status, "invalid_patch") self.assertEqual(T.evaluate(t, t.gold_patch.replace("def is_palindrome", "def is_palindrom")).status, "apply_fail") self.assertEqual(T.evaluate(t, t.gold_patch.replace("textkit/text.py", "textkit/nope.py")).status, "apply_fail") def test_whitespace_only_patch_is_empty(self): self.assertEqual(T.evaluate(BY["textkit-4"], " \n\n").status, "empty") def test_miscounted_header_needs_repair(self): t = BY["textkit-4"] bad = re.sub(r"^(@@ -\d+,)(\d+)", lambda m: m.group(1) + str(int(m.group(2)) + 1), t.gold_patch, count=1, flags=re.M) self.assertEqual(T.evaluate(t, bad).status, "invalid_patch") self.assertTrue(T.evaluate(t, bad, fix_counts=True).resolved) def test_editing_tests_does_not_help(self): t = BY["textkit-4"] src = t.base[t.test_path] p = T.make_diff({t.test_path: src}, {t.test_path: src + "\n\ndef test_is_palindrome_regress():\n assert True\n"}) ev = T.evaluate(t, p) self.assertEqual(ev.status, "unresolved") self.assertEqual((ev.f2p_pass, ev.f2p_total), (0, 1)) def test_new_test_file_is_deleted_before_grading(self): t = BY["textkit-4"] p = T.make_diff({}, {"tests/test_extra.py": "def test_x():\n assert True\n"}) + t.gold_patch ev = T.evaluate(t, p) self.assertTrue(ev.resolved) self.assertEqual(ev.p2p_total, 4) def test_resolution_needs_every_p2p_test(self): t = BY["textkit-3"] src = t.base[t.module] fixed = src.replace("s.endswith(p)", "s.startswith(p) and p != 'abc'") ev = T.evaluate(t, T.make_diff({t.module: src}, {t.module: fixed})) self.assertEqual(ev.status, "unresolved") self.assertEqual(ev.f2p_pass, 1) self.assertEqual(ev.p2p_pass, 3) self.assertEqual(len(ev.broken), 1) def test_resolution_needs_every_f2p_test(self): ev = T.evaluate(BY["listkit-1"], BY["listkit-1"].alts[0]) self.assertEqual((ev.status, ev.f2p_pass, ev.f2p_total), ("unresolved", 1, 2)) def test_infinite_loop_patch_times_out(self): t = BY["mathkit-4"] src = t.base[t.module] evil = src.replace("return a", "while True:\n pass\n return a") ev = T.evaluate(t, T.make_diff({t.module: src}, {t.module: evil}), step_limit=1500) self.assertEqual(ev.status, "unresolved") self.assertEqual(ev.detail, "timeout") def test_eval_result_defaults(self): r = T.EvalResult("empty") self.assertEqual((r.f2p_pass, r.f2p_total, r.p2p_pass, r.p2p_total, r.broken, r.detail), (0, 0, 0, 0, [], "")) def test_repr(self): r = repr(T.EvalResult("resolved", "", 1, 1, 4, 4)) self.assertIn("resolved", r) self.assertIn("F2P 1/1", r) self.assertTrue(T.EvalResult("resolved").resolved) self.assertFalse(T.EvalResult("empty").resolved) def test_cache_modes(self): t = BY["textkit-4"] c = T.EvalCache("content") self.assertEqual(c.evaluate("r", t, "").status, "empty") self.assertTrue(c.evaluate("r", t, t.gold_patch).resolved) self.assertTrue(c.evaluate("r", t, t.gold_patch).resolved) self.assertEqual((c.hits, c.misses), (1, 2)) s = T.EvalCache("id_only") s.evaluate("r", t, "") self.assertEqual(s.evaluate("r", t, t.gold_patch).status, "empty") self.assertEqual(s.evaluate("r2", t, t.gold_patch).status, "resolved") self.assertEqual(s.hits, 1) class TestAgents(unittest.TestCase): def test_parse_repro_matches_specs(self): for s in T.SPECS: got = T.parse_repro(s["issue"]) if s["repro"] is None: self.assertIsNone(got, s["id"]) else: self.assertEqual(got, s["repro"], s["id"]) def test_parse_repro_edge_cases(self): self.assertIsNone(T.parse_repro(">>> f()\n1")) self.assertIsNone(T.parse_repro("Expected: 1")) self.assertEqual(T.parse_repro("t\n>>> a = 1\n>>> a\nExpected: 1 (because)"), ("a = 1", "a", "1")) def test_code_units(self): u = T.code_units("m.py", "def a():\n return 1\n\n\nclass C:\n def m(self):\n return 2\n") self.assertEqual(sorted(u), ["C.m", "a"]) self.assertEqual(u["a"][:2], (1, 2)) self.assertEqual(u["C.m"][2].strip().split("\n")[0], "def m(self):") self.assertEqual(T.code_units("m.py", "def ("), {}) def test_bm25_prefers_matching_doc(self): r = T.bm25_rank(["cart", "total"], {"a": ["cart", "total", "x"], "b": ["list"], "c": ["cart"]}) self.assertEqual([n for _, n in r], ["a", "c", "b"]) self.assertEqual(r[2][0], 0.0) self.assertEqual([n for _, n in T.bm25_rank(["zzz"], {"p": ["a"], "q": ["b"]})], ["p", "q"]) def test_candidates_each_family(self): got = lambda fam, src, **kw: [c for f, c in T.candidates_for_unit(src, [fam], **kw)] self.assertIn(" return a <= b", got("cmp", " return a < b")) self.assertIn(" return a < b", got("cmp", " return a <= b")) self.assertIn(" return s.startswith(p)", got("method", " return s.endswith(p)")) self.assertIn(" return a - b", got("arith", " return a + b")) self.assertIn(" return y - x", got("operands", " return x - y")) self.assertIn(" return max(xs) - min(xs)", got("operands", " return min(xs) - max(xs)")) self.assertIn(" k = n + 1", got("const", " k = n + 0")) self.assertIn(" self.items[n] += q", got("augassign", " self.items[n] = q")) self.assertIn(" return a and b", got("boolop", " return a or b")) self.assertIn(" s = sorted(xs)", got("wrap", " s = xs")) self.assertIn(" return abs(a)", got("wrap", " return a")) def test_guard_candidate(self): unit = "def rot(xs, k):\n k = k % len(xs)\n return xs" got = [c for _, c in T.candidates_for_unit(unit, ["guard"], expected="[]")] self.assertEqual(got, ["def rot(xs, k):\n if not xs:\n return []\n k = k % len(xs)\n return xs"]) self.assertIn("return None", [c for _, c in T.candidates_for_unit(unit, ["guard"])][0]) self.assertEqual(list(T.candidates_for_unit("def f(x):\n return x", ["guard"])), []) def test_template_candidate(self): t = BY["configkit-3"] unit = T.code_units(t.module, t.base[t.module])["merge"][2] fams = [f for f, _ in T.candidates_for_unit(unit, ["template"])] self.assertEqual(fams, ["template"]) def test_agents_are_deterministic(self): t = BY["cartkit-1"] a = T.ScriptedAgent("a", T.STD_OPS) r1, r2 = a.solve(t.view()), a.solve(t.view()) self.assertEqual((r1.patch, r1.tokens_in, r1.tokens_out), (r2.patch, r2.tokens_in, r2.tokens_out)) def test_repro_check_picks_the_right_edit(self): t = BY["cartkit-1"] blind = T.ScriptedAgent("b", T.STD_OPS, k_units=3, checks="none").solve(t.view()) checked = T.ScriptedAgent("c", T.STD_OPS, k_units=3, checks="repro").solve(t.view()) self.assertFalse(T.evaluate(t, blind.patch).resolved) self.assertTrue(T.evaluate(t, checked.patch).resolved) self.assertGreater(checked.tokens_in, blind.tokens_in) def test_tests_check_rejects_edit_that_breaks_visible_test(self): t = BY["textkit-3"] a = T.ScriptedAgent("x", T.STD_OPS, k_units=1, checks="repro+tests") self.assertTrue(T.evaluate(t, a.solve(t.view()).patch).resolved) def test_agent_gives_up_without_a_passing_candidate(self): t = BY["configkit-1"] r = T.ScriptedAgent("weak", T.BASIC_OPS, k_units=3, checks="repro").solve(t.view()) self.assertEqual(r.patch, "") self.assertIn("gave up", r.log[-1]) def test_max_cands_caps_tries(self): t = BY["configkit-1"] r = T.ScriptedAgent("cap", T.STD_OPS, k_units=4, checks="repro", max_cands=2).solve(t.view()) self.assertIn("after 2 tries", r.log[-1]) def test_token_accounting_is_pinned(self): t = BY["cartkit-1"] r = T.ScriptedAgent("c", T.STD_OPS, k_units=3, checks="repro").solve(t.view()) self.assertEqual((r.tokens_in, r.tokens_out, r.turns), (23880, 2670, 3)) r = T.ScriptedAgent("c", T.STD_OPS, k_units=3, checks="repro+tests").solve(t.view()) self.assertEqual((r.tokens_in, r.tokens_out, r.turns), (38200, 2670, 5)) m = T.make_memorizer([t]).solve(t.view()) self.assertEqual((m.tokens_in, m.tokens_out, m.turns), (1200, 730, 1)) def test_max_cands_counts_candidates_actually_tried(self): t = BY["cartkit-1"] one = T.ScriptedAgent("x", T.STD_OPS, k_units=3, checks="repro", max_cands=1).solve(t.view()) two = T.ScriptedAgent("x", T.STD_OPS, k_units=3, checks="repro", max_cands=2).solve(t.view()) self.assertEqual(one.patch, "") self.assertIn("after 1 tries", one.log[-1]) self.assertTrue(T.evaluate(t, two.patch).resolved) self.assertIn("after 2 tries", two.log[-1]) def test_default_agent_reads_three_units(self): r = T.ScriptedAgent("d", T.STD_OPS).solve(BY["cartkit-1"].view()) self.assertEqual(r.log[0].count(","), 2) self.assertEqual(T.ScriptedAgent("d", []).max_cands, 14) def test_bm25_scores_are_pinned(self): r = T.bm25_rank(["cart", "total"], {"a": ["cart", "total", "x"], "b": ["cart", "total", "x", "y", "z", "w"], "c": ["cart"]}) self.assertEqual([n for _, n in r], ["a", "b", "c"]) self.assertAlmostEqual(r[0][0], 0.6319738, places=6) self.assertAlmostEqual(r[1][0], 0.4437758, places=6) self.assertAlmostEqual(r[2][0], 0.1949363, places=6) def test_null_agent(self): r = T.NullAgent().solve(BY["textkit-1"].view()) self.assertEqual(r.patch, "") self.assertEqual((r.tokens_in, r.tokens_out, r.turns), (1680, 10, 1)) def test_sloppy_agent_breaks_headers_and_repair_recovers(self): ag = T.ScriptedAgent("s", T.FULL_OPS, k_units=3, sloppy=1.0) t = BY["textkit-4"] r = ag.solve(t.view()) self.assertEqual(T.evaluate(t, r.patch).status, "invalid_patch") self.assertTrue(T.evaluate(t, r.patch, fix_counts=True).resolved) self.assertIn("miscounted", " ".join(r.log)) ok = T.ScriptedAgent("s", T.FULL_OPS, sloppy=0.0).solve(t.view()) self.assertTrue(T.evaluate(t, ok.patch).resolved) def test_sloppy_header_gets_exactly_one_extra_old_line(self): t = BY["textkit-4"] good = T.ScriptedAgent("g", T.FULL_OPS, k_units=3).solve(t.view()).patch bad = T.ScriptedAgent("s", T.FULL_OPS, k_units=3, sloppy=1.0).solve(t.view()).patch g = re.search(r"@@ -\d+,(\d+) \+\d+,(\d+) @@", good) b = re.search(r"@@ -\d+,(\d+) \+\d+,(\d+) @@", bad) self.assertEqual(int(b.group(1)), int(g.group(1)) + 1) self.assertEqual(b.group(2), g.group(2)) def test_oracle_localizer_uses_hint(self): t = BY["mathkit-3"] ag = T.ScriptedAgent("o", T.FULL_OPS, k_units=1, localizer="oracle") self.assertIn("mean", ag.solve(t.view(), hint="mean").log[0]) rand = T.ScriptedAgent("r", T.FULL_OPS, k_units=1, localizer="random") self.assertEqual(rand.solve(t.view()).log[0], rand.solve(t.view()).log[0]) def test_bm25_localization_accuracy(self): self.assertGreaterEqual(T.localization_accuracy(TASKS, 1), 0.85) self.assertEqual(T.localization_accuracy(TASKS, 3), 1.0) self.assertGreaterEqual(T.localization_accuracy(TASKS, 3), T.localization_accuracy(TASKS, 1)) self.assertEqual(T.localization_accuracy([], 1), 0.0) def test_roster(self): names = [a.name for a in T.roster()] self.assertEqual(len(names), 9) self.assertEqual(len(set(names)), 9) self.assertEqual(names[0], "null") def test_memorizer_recalls_only_seen_source(self): seen = TASKS[:3] m = T.make_memorizer(seen) for t in seen: self.assertEqual(m.solve(t.view()).patch, t.gold_patch) self.assertTrue(m.recalls(t.base)) self.assertFalse(m.recalls(TASKS[5].base)) self.assertNotEqual(m.solve(TASKS[5].view()).patch, TASKS[5].gold_patch) self.assertLess(m.solve(seen[0].view()).tokens_out, T.ScriptedAgent("x", T.STD_OPS).solve(TASKS[5].view()).tokens_in) def test_fingerprint_ignores_tests_and_issue(self): t = BY["cartkit-3"] f2 = dict(t.base) f2[t.test_path] = "changed" self.assertEqual(T.buggy_fingerprint(t.base), T.buggy_fingerprint(f2)) f2[t.module] += "# x\n" self.assertNotEqual(T.buggy_fingerprint(t.base), T.buggy_fingerprint(f2)) class TestScoring(unittest.TestCase): @classmethod def setUpClass(cls): cls.res = T.run_benchmark(T.roster(), TASKS) def test_shapes(self): self.assertEqual(len(self.res), 9) for recs in self.res.values(): self.assertEqual([r.task_id for r in recs], [t.id for t in TASKS]) def test_null_scores_zero_and_cost_positive(self): self.assertEqual(T.resolved_rate(self.res["null"]), 0.0) for recs in self.res.values(): self.assertTrue(all(r.cost > 0 for r in recs)) def test_resolved_rate_and_cap(self): recs = self.res["full-repertoire"] base = T.resolved_rate(recs) self.assertEqual(base, 0.7) self.assertEqual(T.resolved_rate(recs, cap=0.0), 0.0) self.assertEqual(T.resolved_rate(recs, cap=1e9), base) self.assertEqual(T.resolved_rate([]), 0.0) def test_expected_ranking_facts(self): rate = {k: T.resolved_rate(v) for k, v in self.res.items()} self.assertGreater(rate["full-repertoire"], rate["repro-std"]) self.assertGreater(rate["repro-std"], rate["repro-basic"]) self.assertGreater(rate["full-repertoire"], rate["full-sloppy-format"]) self.assertGreater(rate["naive-first"], rate["repro-basic"]) cost = {k: sum(r.cost for r in v) for k, v in self.res.items()} self.assertGreater(cost["full-repertoire"], cost["repro-tests-std"]) self.assertGreater(cost["repro-tests-std"], cost["repro-std"]) def test_no_agent_resolves_the_narrow_or_wide_tasks(self): flawed = {t.id for t in TASKS if t.flaw in ("narrow", "wide")} for recs in self.res.values(): self.assertFalse([r for r in recs if r.task_id in flawed and r.resolved]) def test_fix_counts_recovers_sloppy_agent(self): ag = [a for a in T.roster() if a.name == "full-sloppy-format"] raw = T.run_benchmark(ag, TASKS)["full-sloppy-format"] fixed = T.run_benchmark(ag, TASKS, fix_counts=True)["full-sloppy-format"] self.assertGreater(T.status_counts(raw).get("invalid_patch", 0), 0) self.assertEqual(T.status_counts(fixed).get("invalid_patch", 0), 0) self.assertEqual(T.resolved_rate(fixed), T.resolved_rate(self.res["full-repertoire"])) def test_leaderboard_order_and_fields(self): rows = T.leaderboard(self.res) rates = [r["rate"] for r in rows] self.assertEqual(rates, sorted(rates, reverse=True)) for r in rows: self.assertLessEqual(r["lo"], r["rate"]) self.assertGreaterEqual(r["hi"], r["rate"]) self.assertEqual(r["n"], 20) null = [r for r in rows if r["agent"] == "null"][0] self.assertEqual(null["cost_per_resolved"], float("inf")) sloppy = [r for r in rows if r["agent"] == "full-sloppy-format"][0] self.assertEqual(sloppy["invalid"], 9) def test_cap_boundary_is_inclusive(self): recs = [T.Record("a", "1", T.EvalResult("resolved"), None, 0.5)] self.assertEqual(T.resolved_rate(recs, cap=0.5), 1.0) self.assertEqual(T.resolved_rate(recs, cap=0.49), 0.0) def test_pareto_flags(self): rows = {r["agent"]: r for r in T.leaderboard(self.res)} self.assertTrue(rows["oracle-localizer"]["pareto"]) self.assertFalse(rows["full-repertoire"]["pareto"]) self.assertTrue(rows["naive-first"]["pareto"]) self.assertFalse(rows["repro-tests-std"]["pareto"]) def test_leaderboard_cost_arithmetic(self): recs = [T.Record("a", str(i), T.EvalResult("resolved" if i < 2 else "unresolved"), None, 0.5) for i in range(8)] row = T.leaderboard({"a": recs})[0] self.assertAlmostEqual(row["mean_cost"], 0.5) self.assertAlmostEqual(row["cost_per_resolved"], 2.0) self.assertEqual((row["resolved"], row["rate"]), (2, 0.25)) def test_pareto_on_hand_data(self): def recs(k, cost): return [T.Record("a", str(i), T.EvalResult("resolved" if i < k else "unresolved"), None, cost) for i in range(10)] rows = {r["agent"]: r["pareto"] for r in T.leaderboard({"cheap_low": recs(2, 1), "dear_high": recs(8, 5), "dear_low": recs(2, 5), "tie": recs(2, 1)})} self.assertTrue(rows["cheap_low"] and rows["tie"] and rows["dear_high"]) self.assertFalse(rows["dear_low"]) def test_leaderboard_cap_lowers_rates(self): rows = {r["agent"]: r for r in T.leaderboard(self.res, cap=0.2)} self.assertLess(rows["full-repertoire"]["rate"], 0.7) def test_paired_gap(self): gap, (lo, hi) = T.paired_gap(self.res["full-repertoire"], self.res["null"]) self.assertEqual(gap, 0.7) self.assertGreater(lo, 0.4) gap0, _ = T.paired_gap(self.res["null"], self.res["null"]) self.assertEqual(gap0, 0.0) def test_cost_tiers(self): r = T.AgentResult("", 1_000_000, 1_000_000) small, large = T.ScriptedAgent("a", [], tier="small"), T.ScriptedAgent("b", [], tier="large") self.assertAlmostEqual(T.cost_of(small, r), 0.15 + 0.60) self.assertAlmostEqual(T.cost_of(large, r), 25.0) def test_status_counts(self): self.assertEqual(T.status_counts(self.res["null"]), {"empty": 20}) class TestFilters(unittest.TestCase): def test_severity(self): self.assertEqual(T.issue_severity(BY["textkit-3"]), 0) self.assertEqual(T.issue_severity(BY["textkit-1"]), 3) self.assertEqual(T.issue_severity(BY["mathkit-3"]), 3) t = BY["textkit-3"] no_example = T.Task("x", t.repo, t.fn, "startswith_any is broken", t.base, t.test_patch, t.gold_patch, t.alts) self.assertEqual(T.issue_severity(no_example), 2) def test_annotators_take_max_and_stay_in_range(self): for t in TASKS: a = T.annotate(t) self.assertEqual(len(a), 3) self.assertTrue(all(0 <= x <= 3 for x in a)) self.assertEqual(T.annotate(BY["textkit-1"], noise=0.0), [3, 3, 3]) self.assertEqual(T.annotate(BY["textkit-1"]), T.annotate(BY["textkit-1"])) self.assertEqual(T.annotate(BY["textkit-3"], annotators=5, noise=0.0), [0] * 5) def test_unmentioned_symbols(self): self.assertEqual(T.unmentioned_symbols(BY["listkit-2"]), ["_is_seq"]) self.assertEqual(T.unmentioned_symbols(BY["cartkit-2"]), ["discount_amount"]) self.assertEqual(T.unmentioned_symbols(BY["textkit-3"]), []) self.assertEqual(T.unmentioned_symbols(BY["textkit-2"]), []) def test_static_check_misses_the_constant_but_the_dynamic_check_catches_it(self): t = BY["textkit-2"] self.assertEqual(T.unmentioned_symbols(t), []) self.assertEqual(T.alt_solution_fails(t), [0]) def test_verified_filter_matches_planted_flaws(self): d = T.verified_filter(TASKS) for t in TASKS: self.assertEqual(d[t.id]["kept"], t.flaw is None, t.id) self.assertEqual(bool(d[t.id]["reasons"]), t.flaw is not None, t.id) self.assertTrue(any("underspecified" in r for r in d["textkit-1"]["reasons"])) self.assertTrue(any("alternative" in r for r in d["mathkit-1"]["reasons"])) def test_lite_filter_keeps_the_curated_tasks_including_the_flawed_ones(self): for t in TASKS: self.assertEqual(T.lite_filter(t), [], t.id) def test_lite_filter_rules(self): t = BY["textkit-3"] mk = lambda **kw: self._variant(t, **kw) self.assertEqual(T.lite_filter(mk(issue="too short")), ["short issue"]) self.assertEqual(T.lite_filter(mk(issue="too short"), min_words=2), []) self.assertIn("links, commit ids, other issues or images", T.lite_filter(mk(issue=t.issue + " see https://example.org"))) self.assertIn("links, commit ids, other issues or images", T.lite_filter(mk(issue=t.issue + " see #123"))) self.assertIn("links, commit ids, other issues or images", T.lite_filter(mk(issue=t.issue + " ![img](x.png)"))) self.assertIn("links, commit ids, other issues or images", T.lite_filter(mk(issue=t.issue + " commit " + "a1" * 20))) two_files = T.make_diff({"a.py": "x\n", "b.py": "y\n"}, {"a.py": "x2\n", "b.py": "y2\n"}) self.assertEqual(T.lite_filter(mk(gold_patch=two_files)), ["edits more than one file"]) new_file = T.make_diff({}, {"c.py": "z\n"}) self.assertEqual(T.lite_filter(mk(gold_patch=new_file)), ["creates or deletes a file"]) gone = T.make_diff({"c.py": "z\n"}, {}) self.assertEqual(T.lite_filter(mk(gold_patch=gone)), ["creates or deletes a file"]) src = "\n".join("l%d" % i for i in range(60)) + "\n" four = src for i in (2, 15, 28, 41): four = four.replace("l%d\n" % i, "L%d\n" % i) self.assertEqual(T.lite_filter(mk(gold_patch=T.make_diff({"a.py": src}, {"a.py": four}))), ["more than three hunks"]) three = src for i in (2, 15, 28): three = three.replace("l%d\n" % i, "L%d\n" % i) self.assertEqual(T.lite_filter(mk(gold_patch=T.make_diff({"a.py": src}, {"a.py": three}))), []) self.assertEqual(T.lite_filter(mk(test_patch=t.test_patch + "+ with pytest.raises(ValueError, match='x'):\n")), ["test checks an error message"]) @staticmethod def _variant(t, **kw): v = T.Task.__new__(T.Task) v.__dict__.update(t.__dict__) v.__dict__.update(kw) return v def test_unmentioned_symbols_with_top_level_import(self): t = BY["textkit-3"] v = self._variant(t, test_patch="--- a/x\n+++ b/x\n@@ -1,1 +1,2 @@\n x\n+from textkit.text import _secret, truncate\n") self.assertEqual(T.unmentioned_symbols(v), ["_secret", "truncate"]) def test_default_annotator_noise_is_pinned(self): slips = sum(1 for t in TASKS for a in T.annotate(t) if a != T.issue_severity(t)) self.assertEqual(slips, 8) def test_filter_score_arithmetic(self): a = T.Task.__new__(T.Task) ts = [] for i, fl in enumerate(["narrow", "narrow", None, None]): t = T.Task.__new__(T.Task) t.id, t.flaw = "t%d" % i, fl ts.append(t) dec = {"t0": {"kept": False}, "t1": {"kept": True}, "t2": {"kept": False}, "t3": {"kept": True}} s = T.filter_scores(ts, dec) self.assertEqual((s["tp"], s["fp"], s["fn"]), (1, 1, 1)) self.assertEqual((s["precision"], s["recall"]), (0.5, 0.5)) self.assertEqual(T.filter_scores([], {})["precision"], 1.0) def test_filter_raises_resolved_rate_of_full_agent(self): d = T.verified_filter(TASKS) kept = {t.id for t in TASKS if d[t.id]["kept"]} res = T.run_benchmark([T.ScriptedAgent("f", T.FULL_OPS, k_units=4)], TASKS)["f"] before = T.resolved_rate(res) after = T.resolved_rate([r for r in res if r.task_id in kept]) self.assertGreater(after, before) def test_sev_cut_changes_outcome(self): d3 = T.verified_filter(TASKS, sev_cut=4) self.assertTrue(d3["textkit-1"]["kept"] is False) # still dropped by the other checks self.assertFalse(any("underspecified" in r for r in d3["textkit-1"]["reasons"])) class TestProbes(unittest.TestCase): @classmethod def setUpClass(cls): cls.pool = TASKS + SYNTH cls.cs = T.contamination_study(cls.pool) def test_guess_path(self): self.assertEqual(T.guess_path("the cart total is off", ["textkit/text.py", "cartkit/cart.py"]), "cartkit/cart.py") def test_reword_keeps_examples(self): t = BY["listkit-1"] r = T.reword_task(t) self.assertEqual([l for l in r.issue.split("\n") if l.startswith((">>>", "Expected"))], [l for l in t.issue.split("\n") if l.startswith((">>>", "Expected"))]) self.assertNotEqual(r.issue, t.issue) self.assertEqual(r.gold_patch, t.gold_patch) self.assertNotEqual(t.issue, BY["listkit-1"].__class__ and r.issue) self.assertIn("listkit", T.reword_task(BY["textkit-1"]).issue + "listkit") def test_rename_task_stays_valid_and_changes_source(self): for tid in ("textkit-3", "cartkit-1", "cartkit-2", "textkit-1"): t = BY[tid] r = T.rename_task(t) self.assertTrue(T.validate_task(r)["ok"], tid) self.assertTrue(T.evaluate(r, r.gold_patch).resolved, tid) self.assertNotEqual(T.buggy_fingerprint(r.base), T.buggy_fingerprint(t.base)) self.assertTrue(r.fn.endswith("_v2")) self.assertIn("startswith_any_v2", T.rename_task(BY["textkit-3"]).issue) def test_rename_does_not_change_an_honest_agent(self): ag = T.ScriptedAgent("h", T.STD_OPS, k_units=3, checks="repro") orig = T.resolved_rate(T.run_benchmark([ag], TASKS)["h"]) ren = T.resolved_rate(T.run_benchmark([ag], [T.rename_task(t) for t in TASKS])["h"]) self.assertEqual(orig, ren) def test_date_split_counts(self): recs = T.run_benchmark([T.NullAgent()], TASKS)["null"] (a, n1), (b, n2) = T.date_split(recs, TASKS, 100) self.assertEqual(n1 + n2, 20) self.assertEqual(n1, sum(1 for t in TASKS if t.created < 100)) self.assertEqual((a, b), (0, 0)) self.assertEqual(T.date_split(recs, TASKS, 0)[0], (0, 0)) def test_file_path_and_prefix_probes(self): seen = [t for t in self.pool if t.created < 100] m = T.make_memorizer(seen) paths = sorted({t.module for t in self.pool}) self.assertEqual(T.file_path_probe(m, seen, paths), (len(seen), len(seen))) self.assertEqual(T.prefix_probe(m, seen), (len(seen), len(seen))) honest = T.ScriptedAgent("h", T.STD_OPS) self.assertEqual(T.prefix_probe(honest, seen)[0], 0) self.assertLess(T.file_path_probe(honest, seen, paths)[0], len(seen) // 2) def test_prefix_probe_with_one_added_line_uses_empty_prefix(self): t = BY["textkit-3"] m = T.make_memorizer([t]) self.assertEqual(T.prefix_probe(m, [t]), (1, 1)) self.assertEqual(T._added_lines(t.gold_patch), [" return any(s.startswith(p) for p in prefixes)"]) def test_study_date_split_shows_the_gap(self): c = self.cs self.assertEqual(c["memorizer"]["pre"][0], c["memorizer"]["pre"][1]) self.assertLess(c["memorizer"]["post"][0] / c["memorizer"]["post"][1], 0.6) h_pre = c["repro-std"]["pre"][0] / c["repro-std"]["pre"][1] h_post = c["repro-std"]["post"][0] / c["repro-std"]["post"][1] self.assertLess(abs(h_pre - h_post), 0.25) def test_study_probes_separate_memorizer_from_honest(self): c = self.cs self.assertEqual(c["memorizer"]["prefix_seen"][0], c["n_seen"]) self.assertEqual(c["repro-std"]["prefix_seen"][0], 0) self.assertGreater(c["memorizer"]["path_seen"][0], c["repro-std"]["path_seen"][0]) self.assertLess(c["memorizer"]["prefix_unseen"][0], 5) def test_study_perturbation_levels(self): lv = self.cs["memorizer"]["levels"] self.assertEqual(lv["original"], lv["reworded"]) self.assertLess(lv["renamed"][0], lv["original"][0]) h = self.cs["repro-std"]["levels"] self.assertEqual(h["original"], h["renamed"]) self.assertEqual(lv["renamed"], h["renamed"]) def test_study_counts_duplicate_issues(self): self.assertEqual(self.cs["n_seen"] + self.cs["n_unseen"], len(self.pool)) self.assertEqual(self.cs["dup_issues"], 3) class TestSynthesis(unittest.TestCase): def test_stats_add_up(self): s = SYNTH_STATS self.assertEqual(s["candidates"], s.get("no_change", 0) + s.get("syntax_error", 0) + s.get("timeout", 0) + s.get("no_test_fails", 0) + s["kept"]) self.assertEqual(s["tasks"], len(SYNTH)) self.assertGreater(s["no_test_fails"], 0) self.assertGreater(s["syntax_error"], 0) self.assertLessEqual(s["tasks"], s["kept"]) def test_synthetic_repos_pass_their_own_tests(self): for name in T.REPOS: self.assertTrue(all(v == "PASS" for v in T.run_tests(T.synth_repo(name)).values()), name) def test_every_synthetic_task_is_valid_and_solvable_by_gold(self): for t in SYNTH: self.assertTrue(T.validate_task(t)["ok"], t.id) self.assertTrue(T.evaluate(t, t.gold_patch).resolved, t.id) self.assertEqual(t.test_patch, "") self.assertGreaterEqual(len(t.f2p), 1) self.assertIsNone(t.flaw) def test_gold_is_the_reverse_of_a_single_edit(self): for t in SYNTH: st = T.diff_stats(t.gold_patch) self.assertEqual(st["files"], 1, t.id) self.assertLessEqual(st["added"] + st["removed"], 6, t.id) def test_issue_has_repro_from_failing_test(self): withrepro = [t for t in SYNTH if T.parse_repro(t.issue)] self.assertGreaterEqual(len(withrepro), len(SYNTH) * 0.9) t = withrepro[0] setup, expr, exp = T.parse_repro(t.issue) self.assertNotEqual(T.eval_snippet(t.base, t.module, setup, expr), exp) def test_repro_from_test_directly(self): f = T.synth_repo("textkit") broken = dict(f) broken["textkit/text.py"] = broken["textkit/text.py"].replace("s.startswith(p)", "s.endswith(p)") rep = T.repro_from_test(broken, "textkit/text.py", broken["tests/test_text.py"], "test_startswith_any_regress") self.assertEqual(rep, ("", "startswith_any('foobar', ['foo'])", "True", "False")) self.assertIsNone(T.repro_from_test(f, "textkit/text.py", f["tests/test_text.py"], "test_startswith_any_regress")) self.assertIsNone(T.repro_from_test(f, "textkit/text.py", f["tests/test_text.py"], "test_nope")) self.assertIsNone(T.repro_from_test(f, "textkit/text.py", "def (", "test_x")) def test_deterministic_and_seeded(self): a, _ = T.synthesize_tasks(limit=8) b, _ = T.synthesize_tasks(limit=8) self.assertEqual([t.id for t in a], [t.id for t in b]) c, _ = T.synthesize_tasks(limit=8, seed="other") self.assertNotEqual([t.id for t in a], [t.id for t in c]) def test_repo_filter_and_limit(self): ts, s = T.synthesize_tasks(limit=100, repos=["mathkit"]) self.assertTrue(all(t.repo == "mathkit" for t in ts)) self.assertEqual(len(ts), s["tasks"]) self.assertLess(len(ts), 20) def test_no_synthetic_task_needs_a_test_patch_and_none_vague(self): self.assertTrue(all(T.issue_severity(t) <= 1 for t in SYNTH)) def test_agents_solve_some_but_not_all(self): res = T.run_benchmark([T.ScriptedAgent("f", T.FULL_OPS, k_units=4)], SYNTH)["f"] r = T.resolved_rate(res) self.assertGreater(r, 0.3) self.assertLess(r, 0.9) def test_injection_operators_compile_on_some_site(self): self.assertGreaterEqual(len(T.INJECT_OPS), 20) self.assertEqual(len({o[0] for o in T.INJECT_OPS}), len(T.INJECT_OPS)) class TestDemo(unittest.TestCase): def test_demo_runs_and_has_no_dashes(self): buf = io.StringIO() with redirect_stdout(buf): T.demo() out = buf.getvalue() for n in range(1, 8): self.assertIn("== %d." % n, out) self.assertNotIn("\u2014", out) self.assertNotIn("\u2013", out) self.assertIn("20 of 20 tasks pass", out) self.assertIn("14/20 (70%, 95% CI 48 to 85%)", out) self.assertIn("Lite-style shape filter keeps 20 of 20, including 7 of the 7 planted flaws", out) def test_source_has_no_dashes(self): with open(T.__file__, encoding="utf-8") as fh: src = fh.read() self.assertNotIn("\u2014", src) self.assertNotIn("\u2013", src) if __name__ == "__main__": unittest.main()