"""swe_bench_toolkit.py: a toy lab for Week 9a, SWE-bench (guest lecture by C. Jimenez and A. Wettig).

Everything here is made up and runs offline with the standard library. There is no language
model and no Docker. The "repos" are five tiny Python packages that live in memory, the "issues"
are written by hand, and the "agents" are scripted programs that edit source text. A task is
resolved when its FAIL_TO_PASS and PASS_TO_PASS tests all pass after a patch is applied, which is
the rule that the SWE-bench sources describe. Every number the demo prints is about this toy
and about no real model or benchmark. The in-process test runner executes the code that it is
given, so it is NOT a sandbox: do not feed it patches that you did not write.

Parts: 0 helpers, 1 unified diff (parse, apply, make, fix counts), 2 toy repos and task specs,
3 in-process test runner, 4 task build and validation, 5 evaluation harness, 6 scripted agents,
7 scorer, cost and leaderboard, 8 task-quality filters, 9 contamination probes, 10 task
synthesis, 11 demo.
"""
import ast
import builtins
import copy
import difflib
import hashlib
import math
import random
import re
import sys
import types
from collections import Counter, defaultdict


# ---------------------------------------------------------------- Part 0: helpers
def mean(xs):
    xs = list(xs)
    return sum(xs) / len(xs) if xs else 0.0


def rng_for(*parts):
    """A random generator whose stream depends only on its name, never on call order."""
    return random.Random("|".join(str(p) for p in parts))


def wilson(k, n, z=1.96):
    if n == 0:
        return (0.0, 1.0)
    p = k / n
    d = 1 + z * z / n
    c = p + z * z / (2 * n)
    r = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n))
    return (max(0.0, (c - r) / d), min(1.0, (c + r) / d))


def bootstrap_ci(values, n_boot=400, seed=0, alpha=0.05):
    """Percentile bootstrap interval for the mean of per-task values (resampling tasks)."""
    values = list(values)
    if not values:
        return (0.0, 1.0)
    rng = rng_for("boot", seed, len(values))
    ms = sorted(mean(rng.choice(values) for _ in values) for _ in range(n_boot))
    return (ms[int(alpha / 2 * n_boot)], ms[min(n_boot - 1, int((1 - alpha / 2) * n_boot))])


STOP = set("the a an of to in is it and or for on with when that this be as are was at by from but not "
           "can i if so do does should would returns return get gets there than then its has have".split())


def tokens(text):
    """Lower-case word tokens; identifiers are split on underscores and camel case."""
    out = []
    for w in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", text):
        parts = re.sub(r"([a-z])([A-Z])", r"\1_\2", w).lower().split("_")
        out.extend(p for p in parts if p)
        if "_" in w:
            out.append(w.lower())
    return out


def content_tokens(text):
    return [t for t in tokens(text) if t not in STOP and len(t) > 1]


def jaccard(a, b):
    a, b = set(a), set(b)
    return len(a & b) / len(a | b) if (a | b) else 1.0


def est_tokens(text):
    """A rough count of model tokens, four characters each. My own rule of thumb, not a tokenizer."""
    return max(1, len(text) // 4)


# ---------------------------------------------------------------- Part 1: unified diff
class PatchError(Exception):
    """A patch that cannot be parsed or applied. The text starts with the reason."""


class Hunk:
    def __init__(self, old_start, old_len, new_start, new_len, lines):
        self.old_start, self.old_len, self.new_start, self.new_len = old_start, old_len, new_start, new_len
        self.lines = lines  # list of (tag, text) with tag in " ", "-", "+"

    def old_lines(self):
        return [t for tag, t in self.lines if tag in " -"]

    def new_lines(self):
        return [t for tag, t in self.lines if tag in " +"]


class FilePatch:
    def __init__(self, old_path, new_path, hunks):
        self.old_path, self.new_path, self.hunks = old_path, new_path, hunks

    @property
    def path(self):
        return self.new_path if self.new_path != "/dev/null" else self.old_path

    @property
    def is_new(self):
        return self.old_path == "/dev/null"

    @property
    def is_delete(self):
        return self.new_path == "/dev/null"


HUNK_RE = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@")


def _strip_prefix(p):
    p = p.split("\t")[0].strip()
    if p == "/dev/null":
        return p
    return p[2:] if p[:2] in ("a/", "b/") else p


def parse_diff(text, strict_counts=True):
    """Parse a unified diff into FilePatch objects. With strict_counts a hunk is read until its header
    counts are met and then must end (as the patch program requires); without it the hunk is whatever
    run of diff lines follows. Raises PatchError."""
    lines = text.split("\n")
    if lines and lines[-1] == "":
        lines.pop()
    out, i = [], 0
    while i < len(lines):
        if not lines[i].startswith("--- "):
            i += 1  # "diff --git", "index ..." and prose lines are skipped
            continue
        if i + 1 >= len(lines) or not lines[i + 1].startswith("+++ "):
            raise PatchError("malformed header: '---' line without '+++' line")
        old_p, new_p = _strip_prefix(lines[i][4:]), _strip_prefix(lines[i + 1][4:])
        i += 2
        hunks = []
        while i < len(lines) and lines[i].startswith("@@"):
            m = HUNK_RE.match(lines[i])
            if not m:
                raise PatchError("malformed hunk header: %r" % lines[i][:40])
            os_, ol, ns, nl = int(m.group(1)), m.group(2), int(m.group(3)), m.group(4)
            ol = 1 if ol is None else int(ol)
            nl = 1 if nl is None else int(nl)
            i += 1
            body, so, sn = [], 0, 0
            while i < len(lines):
                ln = lines[i]
                if strict_counts and so >= ol and sn >= nl:
                    break
                if ln.startswith("\\"):
                    i += 1
                    continue
                tag = ln[:1] if ln else " "
                if tag not in (" ", "-", "+") or (ln.startswith(("--- ", "+++ ")) and
                                                   i + 1 < len(lines) and lines[i + 1].startswith(("+++ ", "@@"))):
                    break
                body.append((tag, ln[1:]))
                so += tag in " -"
                sn += tag in " +"
                i += 1
            h = Hunk(os_, ol, ns, nl, body)
            if strict_counts and (so != ol or sn != nl):
                raise PatchError("hunk header counts %d,%d but body has %d,%d lines" % (ol, nl, so, sn))
            hunks.append(h)
            if strict_counts and i < len(lines) and lines[i][:1] in (" ", "-", "+") \
                    and not lines[i].startswith(("--- ", "+++ ")):
                raise PatchError("hunk body is longer than its header says")
        if not hunks:
            raise PatchError("file header without hunks: %s" % new_p)
        out.append(FilePatch(old_p, new_p, hunks))
    if not out:
        raise PatchError("no file patches found")
    return out


def _split(text):
    return text.split("\n") if text else []


def _join(lines):
    return "\n".join(lines) + ("\n" if lines else "")


def apply_patch(files, diff, strict_counts=True, max_offset=8):
    """Apply a unified diff to a dict of {path: text}. Returns (new_files, report). Context lines must
    match exactly; a hunk may sit up to max_offset lines away from where its header says (as the patch
    program allows). Nothing is changed if any hunk fails. Raises PatchError."""
    fps = parse_diff(diff, strict_counts)
    new = dict(files)
    report = {"files": [], "offsets": 0, "hunks": 0}
    for fp in fps:
        if fp.is_new:
            if fp.path in new:
                raise PatchError("file already exists: " + fp.path)
            new[fp.path] = _join([t for h in fp.hunks for t in h.new_lines()])
            report["files"].append(fp.path)
            report["hunks"] += len(fp.hunks)
            continue
        if fp.old_path not in new:
            raise PatchError("file not found: " + fp.old_path)
        src = _split(new[fp.old_path].rstrip("\n")) if new[fp.old_path] else []
        shift = 0
        for h in fp.hunks:
            old = h.old_lines()
            want = h.old_start - 1 + shift if old else h.old_start + shift
            placed = None
            for off in sorted(range(-max_offset, max_offset + 1), key=lambda d: (abs(d), d)):
                s = want + off
                if 0 <= s <= len(src) and src[s:s + len(old)] == old:
                    placed = s
                    if off:
                        report["offsets"] += 1
                    break
            if placed is None:
                raise PatchError("context does not match in %s near line %d" % (fp.old_path, h.old_start))
            src[placed:placed + len(old)] = h.new_lines()
            shift += len(h.new_lines()) - len(old)
            report["hunks"] += 1
        if fp.is_delete:
            del new[fp.old_path]
        else:
            new[fp.new_path] = _join(src)
        report["files"].append(fp.path)
    return new, report


def make_diff(old_files, new_files, context=3):
    """A unified diff that turns old_files into new_files (sorted by path)."""
    out = []
    for p in sorted(set(old_files) | set(new_files)):
        a, b = old_files.get(p), new_files.get(p)
        if a == b:
            continue
        al = a.split("\n") if a else []
        bl = b.split("\n") if b else []
        if a is not None and a.endswith("\n"):
            al = al[:-1]
        if b is not None and b.endswith("\n"):
            bl = bl[:-1]
        fa = "a/" + p if a is not None else "/dev/null"
        fb = "b/" + p if b is not None else "/dev/null"
        body = list(difflib.unified_diff(al, bl, fa, fb, lineterm="", n=context))
        out.extend(body)
    return "\n".join(out) + "\n" if out else ""


def fix_hunk_counts(diff):
    """Recompute the line counts in each hunk header from the hunk body (the kind of repair that a harness
    can make for a model that miscounts). Leaves the start lines alone."""
    lines = diff.rstrip("\n").split("\n")
    out, i = [], 0
    while i < len(lines):
        m = HUNK_RE.match(lines[i])
        if not m:
            out.append(lines[i])
            i += 1
            continue
        j = i + 1
        while j < len(lines) and lines[j][:1] in (" ", "-", "+", "\\") and not lines[j].startswith(("--- ", "+++ ")):
            j += 1
        body = lines[i + 1:j]
        ol = sum(1 for b in body if b[:1] in (" ", "-"))
        nl = sum(1 for b in body if b[:1] in (" ", "+"))
        out.append("@@ -%s,%d +%s,%d @@%s" % (m.group(1), ol, m.group(3), nl, lines[i][m.end():]))
        out.extend(body)
        i = j
    return "\n".join(out) + "\n"


def diff_stats(diff):
    """Files touched, lines added and removed, hunks."""
    try:
        fps = parse_diff(diff, strict_counts=False)
    except PatchError:
        return {"files": 0, "added": 0, "removed": 0, "hunks": 0}
    add = sum(1 for fp in fps for h in fp.hunks for t, _ in h.lines if t == "+")
    rem = sum(1 for fp in fps for h in fp.hunks for t, _ in h.lines if t == "-")
    return {"files": len(fps), "added": add, "removed": rem, "hunks": sum(len(fp.hunks) for fp in fps)}


# ---------------------------------------------------------------- Part 2: toy repos and task specs
_TEXT = '''def slugify(s):
    words = s.lower().split()
    return "-".join(words)


def truncate(s, n):
    if len(s) <= n:
        return s
    return s[: n - 3] + "..."


def startswith_any(s, prefixes):
    return any(s.startswith(p) for p in prefixes)


def is_palindrome(s):
    chars = [c.lower() for c in s if c.isalnum()]
    return chars == chars[::-1]
'''
_LISTS = '''def chunk(xs, n):
    return [xs[i : i + n] for i in range(0, len(xs), n)]


def flatten(xs):
    out = []
    for x in xs:
        if isinstance(x, list):
            out.extend(x)
        else:
            out.append(x)
    return out


def rotate(xs, k):
    if not xs:
        return xs
    k = k % len(xs)
    return xs[k:] + xs[:k]


def window_sums(xs, k):
    return [sum(xs[i : i + k]) for i in range(len(xs) - k + 1)]
'''
_STATS = '''def spread(xs):
    return max(xs) - min(xs)


def median(xs):
    s = sorted(xs)
    n = len(s)
    mid = n // 2
    if n % 2:
        return s[mid]
    return (s[mid - 1] + s[mid]) / 2


def mean(xs):
    if not xs:
        return 0.0
    return sum(xs) / len(xs)


def gcd(a, b):
    while b:
        a, b = b, a % b
    return abs(a)
'''
_CART = '''class Cart:
    def __init__(self, discount=0.0):
        self.items = {}
        self.prices = {}
        self.discount = discount

    def add(self, name, price, qty=1):
        self.prices[name] = price
        if name in self.items:
            self.items[name] += qty
        else:
            self.items[name] = qty

    def subtotal(self):
        return sum(self.prices[n] * q for n, q in self.items.items())

    def total(self):
        return round(self.subtotal() * (1 - self.discount), 2)

    def discount_amount(self):
        return self.subtotal() - self.total()

    def can_checkout(self):
        return bool(self.items) and self.total() > 0
'''
_CONFIG = '''def parse_kv(line):
    key, _, value = line.partition("=")
    return key.strip(), value.strip()


def get_path(d, path, default=None):
    cur = d
    for part in path.split("."):
        if not isinstance(cur, dict) or part not in cur:
            return default
        cur = cur[part]
    return cur


def merge(a, b):
    out = dict(a)
    for k, v in b.items():
        if isinstance(v, dict) and isinstance(out.get(k), dict):
            out[k] = merge(out[k], v)
        else:
            out[k] = v
    return out


def to_bool(s):
    return s.strip().lower() in ("1", "true", "yes")
'''
REPOS = {
    "textkit": {"module": "textkit/text.py", "test": "tests/test_text.py", "src": _TEXT,
                "header": "from textkit.text import slugify, truncate, startswith_any, is_palindrome\n"},
    "listkit": {"module": "listkit/lists.py", "test": "tests/test_lists.py", "src": _LISTS,
                "header": "from listkit.lists import chunk, flatten, rotate, window_sums\n"},
    "mathkit": {"module": "mathkit/stats.py", "test": "tests/test_stats.py", "src": _STATS,
                "header": "from mathkit.stats import spread, median, mean, gcd\n"},
    "cartkit": {"module": "cartkit/cart.py", "test": "tests/test_cart.py", "src": _CART,
                "header": "from cartkit.cart import Cart\n"},
    "configkit": {"module": "configkit/config.py", "test": "tests/test_config.py", "src": _CONFIG,
                  "header": "from configkit.config import parse_kv, get_path, merge, to_bool\n"},
}
# Existing tests: each passes on the buggy code too (they are the PASS_TO_PASS tests).
BASIC = {
    "slugify": 'assert slugify("hello world") == "hello-world"',
    "truncate": 'assert truncate("hi", 5) == "hi"\n    assert truncate("hello world", 8) == "hello..."',
    "startswith_any": 'assert startswith_any("abc", []) is False\n    assert startswith_any("abc", ["abc"]) is True',
    "is_palindrome": 'assert is_palindrome("abba") is True\n    assert is_palindrome("ab") is False',
    "chunk": "assert chunk([1, 2, 3, 4], 2) == [[1, 2], [3, 4]]\n    assert chunk([], 2) == []",
    "flatten": "assert flatten([1, 2]) == [1, 2]\n    assert flatten([]) == []",
    "rotate": "assert rotate([1, 2, 3], 1) == [2, 3, 1]",
    "window_sums": "assert window_sums([], 2) == []\n    assert window_sums([1, 2, 3], 4) == []",
    "spread": "assert spread([2]) == 0",
    "median": "assert median([1, 2, 3]) == 2",
    "mean": "assert mean([1, 2, 3]) == 2.0",
    "gcd": "assert gcd(12, 18) == 6\n    assert gcd(7, 0) == 7",
    "add": 'c = Cart()\n    c.add("a", 2.0)\n    assert c.items == {"a": 1}',
    "total": 'c = Cart()\n    c.add("a", 3.0, 2)\n    assert c.total() == 6.0',
    "can_checkout": 'assert Cart().can_checkout() is False\n    c = Cart()\n    c.add("a", 2.0)\n    assert c.can_checkout() is True',
    "discount_amount": 'c = Cart()\n    c.add("a", 2.0)\n    assert c.discount_amount() == 0.0',
    "parse_kv": 'assert parse_kv("a=1") == ("a", "1")',
    "get_path": 'assert get_path({"a": {"b": 1}}, "a.b") == 1',
    "merge": 'assert merge({"a": 1}, {"b": 2}) == {"a": 1, "b": 2}',
    "to_bool": 'assert to_bool("yes") is True\n    assert to_bool("no") is False',
}
# New tests added by the "pull request" (the FAIL_TO_PASS tests).
REGRESS = {
    "slugify": 'assert slugify("Hello World") == "hello-world"',
    "truncate": 'from textkit.text import ELLIPSIS\n    assert truncate("hello", 5) == "hello"\n    assert truncate("hello world", 8).endswith(ELLIPSIS)',
    "startswith_any": 'assert startswith_any("foobar", ["foo"]) is True',
    "is_palindrome": 'assert is_palindrome("a1b2a") is False',
    "chunk": "assert chunk([1, 2, 3], 2) == [[1, 2], [3]]",
    "flatten": "from listkit.lists import _is_seq\n    assert _is_seq([1]) and not _is_seq(1)\n    assert flatten([1, [2, 3]]) == [1, 2, 3]",
    "rotate": "assert rotate([], 3) == []",
    "window_sums": "assert window_sums([1, 2, 3, 4], 2) == [3, 5, 7]",
    "spread": "from mathkit.stats import _extent\n    assert _extent([1, 4]) == (1, 4)\n    assert spread([1, 4, 2]) == 3",
    "median": "assert median([3, 1, 2]) == 2",
    "mean": "assert mean([]) == 0.0",
    "gcd": "assert gcd(4, -6) == 2",
    "add": 'c = Cart()\n    c.add("a", 2.0)\n    c.add("a", 2.0, 2)\n    assert c.items["a"] == 3',
    "total": 'c = Cart(0.1)\n    c.add("a", 3.33, 3)\n    assert c.total() == 8.99',
    "can_checkout": 'c = Cart()\n    c.add("free", 0.0)\n    assert c.can_checkout() is False',
    "discount_amount": 'c = Cart(0.1)\n    c.add("a", 5.0)\n    assert c.discount_amount() == 0.5',
    "parse_kv": 'assert parse_kv("a=b=c") == ("a", "b=c")',
    "get_path": 'assert get_path({"a": 1}, "a.b", 5) == 5',
    "merge": 'assert merge({"a": {"x": 1}}, {"a": {"y": 2}}) == {"a": {"x": 1, "y": 2}}',
    "to_bool": 'assert to_bool(" yes ") is True',
}
_PARSE_KV = ('    key, _, value = line.partition("=")\n    return key.strip(), value.strip()',
             '    parts = line.split("=")\n    return parts[0].strip(), parts[1].strip()')
_GET_PATH = ('        if not isinstance(cur, dict) or part not in cur:\n            return default\n        cur = cur[part]',
             '        cur = cur[part]')
_MERGE = ('        if isinstance(v, dict) and isinstance(out.get(k), dict):\n            out[k] = merge(out[k], v)\n        else:\n            out[k] = v',
          '        out[k] = v')


def _REPRO(setup, expr, exp):
    return (setup, expr, exp)


# repo, function, bug (old, new) substitutions on the fixed module, flaw, extra fix sites, issue, repro
SPECS = [
    dict(id="textkit-1", repo="textkit", fn="slugify", bug=[("s.lower().split()", "s.split()")], flaw="vague", family="wrap",
         issue="Text helper gives odd results\n\nAfter the last release some of our page addresses look wrong. I think something is off with the text utilities but I could not pin it down.",
         repro=None),
    dict(id="textkit-2", repo="textkit", fn="truncate", bug=[("len(s) <= n", "len(s) < n")], flaw="narrow", family="boundary",
         fixed_extra=[("def truncate(s, n):", 'ELLIPSIS = "..."\n\n\ndef truncate(s, n):'), ('s[: n - 3] + "..."', "s[: n - len(ELLIPSIS)] + ELLIPSIS")],
         issue="truncate() cuts a string that already fits\n\ntruncate('hello', 5) should return 'hello' because the string is exactly 5 characters long, but I get 'he...'.\n\n>>> truncate('hello', 5)\n'he...'\nExpected: 'hello'",
         repro=_REPRO("", "truncate('hello', 5)", "'hello'")),
    dict(id="textkit-3", repo="textkit", fn="startswith_any", bug=[("s.startswith(p)", "s.endswith(p)")], flaw=None, family="method",
         issue="startswith_any() looks at the end of the string\n\n>>> startswith_any('foobar', ['foo'])\nFalse\nExpected: True",
         repro=_REPRO("", "startswith_any('foobar', ['foo'])", "True")),
    dict(id="textkit-4", repo="textkit", fn="is_palindrome", bug=[("c.isalnum()", "c.isalpha()")], flaw=None, family="method",
         issue="is_palindrome() ignores digits\n\n>>> is_palindrome('a1b2a')\nTrue\nExpected: False (the digits 1 and 2 are different)",
         repro=_REPRO("", "is_palindrome('a1b2a')", "False")),
    dict(id="listkit-1", repo="listkit", fn="chunk", bug=[("range(0, len(xs), n)", "range(0, len(xs) - 1, n)")], flaw="wide", family="const",
         wide=("window_sums", [("len(xs) - k + 1", "len(xs) - k + 0")]),
         issue="chunk() drops the last partial piece\n\n>>> chunk([1, 2, 3], 2)\n[[1, 2]]\nExpected: [[1, 2], [3]]",
         repro=_REPRO("", "chunk([1, 2, 3], 2)", "[[1, 2], [3]]")),
    dict(id="listkit-2", repo="listkit", fn="flatten", bug=[("out.extend(x)", "out.append(x)")], flaw="narrow", family="method",
         fixed_extra=[("def flatten(xs):", "def _is_seq(x):\n    return isinstance(x, list)\n\n\ndef flatten(xs):"), ("if isinstance(x, list):", "if _is_seq(x):")],
         issue="flatten() nests lists instead of joining them\n\n>>> flatten([1, [2, 3]])\n[1, [2, 3]]\nExpected: [1, 2, 3]",
         repro=_REPRO("", "flatten([1, [2, 3]])", "[1, 2, 3]")),
    dict(id="listkit-3", repo="listkit", fn="rotate", bug=[("    if not xs:\n        return xs\n    k = k % len(xs)", "    k = k % len(xs)")], flaw=None, family="guard",
         issue="rotate() crashes on an empty list\n\n>>> rotate([], 3)\nZeroDivisionError: integer division or modulo by zero\nExpected: []",
         repro=_REPRO("", "rotate([], 3)", "[]")),
    dict(id="listkit-4", repo="listkit", fn="window_sums", bug=[("len(xs) - k + 1", "len(xs) - k + 0")], flaw=None, family="const",
         issue="window_sums() is missing the last window\n\n>>> window_sums([1, 2, 3, 4], 2)\n[3, 5]\nExpected: [3, 5, 7]",
         repro=_REPRO("", "window_sums([1, 2, 3, 4], 2)", "[3, 5, 7]")),
    dict(id="mathkit-1", repo="mathkit", fn="spread", bug=[("max(xs) - min(xs)", "min(xs) - max(xs)")], flaw="narrow", family="minmax",
         fixed_extra=[("def spread(xs):", "def _extent(xs):\n    return min(xs), max(xs)\n\n\ndef spread(xs):"), ("return max(xs) - min(xs)", "lo, hi = _extent(xs)\n    return hi - lo")],
         issue="spread() is negative\n\n>>> spread([1, 4, 2])\n-3\nExpected: 3",
         repro=_REPRO("", "spread([1, 4, 2])", "3")),
    dict(id="mathkit-2", repo="mathkit", fn="median", bug=[("s = sorted(xs)", "s = xs")], flaw=None, family="wrap",
         issue="median() is wrong when the input is not sorted\n\n>>> median([3, 1, 2])\n1\nExpected: 2",
         repro=_REPRO("", "median([3, 1, 2])", "2")),
    dict(id="mathkit-3", repo="mathkit", fn="mean", bug=[("    if not xs:\n        return 0.0\n    return sum(xs) / len(xs)", "    return sum(xs) / len(xs)")], flaw="vague", family="guard",
         issue="Statistics helper crashes on some input\n\nOur nightly report failed again with a division error. It only happens on days when no data came in.",
         repro=None),
    dict(id="mathkit-4", repo="mathkit", fn="gcd", bug=[("return abs(a)", "return a")], flaw=None, family="wrap",
         issue="gcd() can return a negative number\n\n>>> gcd(4, -6)\n-2\nExpected: 2",
         repro=_REPRO("", "gcd(4, -6)", "2")),
    dict(id="cartkit-1", repo="cartkit", fn="add", bug=[("self.items[name] += qty", "self.items[name] = qty")], flaw=None, family="augassign",
         issue="Cart.add() forgets the earlier quantity\n\n>>> c = Cart(); c.add('a', 2.0); c.add('a', 2.0, 2)\n>>> c.items['a']\n2\nExpected: 3",
         repro=_REPRO("c = Cart(); c.add('a', 2.0); c.add('a', 2.0, 2)", "c.items['a']", "3")),
    dict(id="cartkit-2", repo="cartkit", fn="total", bug=[("(1 - self.discount), 2)", "(1 - self.discount), 1)")], flaw="wide", family="const",
         wide=("discount_amount", [("self.subtotal() - self.total()", "self.total() - self.subtotal()")]),
         issue="Cart.total() rounds to one decimal\n\n>>> c = Cart(0.1); c.add('a', 3.33, 3)\n>>> c.total()\n9.0\nExpected: 8.99",
         repro=_REPRO("c = Cart(0.1); c.add('a', 3.33, 3)", "c.total()", "8.99")),
    dict(id="cartkit-3", repo="cartkit", fn="can_checkout", bug=[("bool(self.items) and self.total() > 0", "bool(self.items) or self.total() > 0")], flaw=None, family="boolop",
         issue="can_checkout() is True for a cart of free items\n\n>>> c = Cart(); c.add('free', 0.0)\n>>> c.can_checkout()\nTrue\nExpected: False",
         repro=_REPRO("c = Cart(); c.add('free', 0.0)", "c.can_checkout()", "False")),
    dict(id="cartkit-4", repo="cartkit", fn="discount_amount", bug=[("self.subtotal() - self.total()", "self.total() - self.subtotal()")], flaw=None, family="operands",
         issue="discount_amount() has the wrong sign\n\n>>> c = Cart(0.1); c.add('a', 5.0)\n>>> c.discount_amount()\n-0.5\nExpected: 0.5",
         repro=_REPRO("c = Cart(0.1); c.add('a', 5.0)", "c.discount_amount()", "0.5")),
    dict(id="configkit-1", repo="configkit", fn="parse_kv", bug=[_PARSE_KV], flaw=None, family="multi",
         issue="parse_kv() cuts values that contain '='\n\n>>> parse_kv('a=b=c')\n('a', 'b')\nExpected: ('a', 'b=c')",
         repro=_REPRO("", "parse_kv('a=b=c')", "('a', 'b=c')")),
    dict(id="configkit-2", repo="configkit", fn="get_path", bug=[_GET_PATH], flaw=None, family="multi",
         issue="get_path() raises instead of returning the default\n\n>>> get_path({'a': 1}, 'a.b', 5)\nTypeError: 'int' object is not subscriptable\nExpected: 5",
         repro=_REPRO("", "get_path({'a': 1}, 'a.b', 5)", "5")),
    dict(id="configkit-3", repo="configkit", fn="merge", bug=[_MERGE], flaw=None, family="multi",
         issue="merge() overwrites nested dicts\n\n>>> merge({'a': {'x': 1}}, {'a': {'y': 2}})\n{'a': {'y': 2}}\nExpected: {'a': {'x': 1, 'y': 2}}",
         repro=_REPRO("", "merge({'a': {'x': 1}}, {'a': {'y': 2}})", "{'a': {'x': 1, 'y': 2}}")),
    dict(id="configkit-4", repo="configkit", fn="to_bool", bug=[("s.strip().lower()", "s.lower()")], flaw=None, family="wrap",
         issue="to_bool() fails on padded input\n\n>>> to_bool(' yes ')\nFalse\nExpected: True",
         repro=_REPRO("", "to_bool(' yes ')", "True")),
]


# ---------------------------------------------------------------- Part 3: in-process test runner
class StepLimit(Exception):
    """Raised inside patched code that runs too long (a trace-based line budget)."""


_SAFE_STD = {"math", "re", "collections", "itertools", "functools", "string", "json", "operator", "copy", "typing"}


class _Tracer:
    def __init__(self, limit):
        self.limit, self.steps = limit, 0

    def glob(self, frame, event, arg):
        return self.local if frame.f_code.co_filename.startswith("<mem:") else None

    def local(self, frame, event, arg):
        if event == "line":
            self.steps += 1
            if self.steps > self.limit:
                raise StepLimit("step limit of %d lines exceeded" % self.limit)
        return self.local


class _Tracing:
    def __init__(self, limit):
        self.t = _Tracer(limit)

    def __enter__(self):
        self.old = sys.gettrace()
        sys.settrace(self.t.glob)
        return self.t

    def __exit__(self, *exc):
        sys.settrace(self.old)
        return False


class Sandbox:
    """Loads modules from an in-memory file dict. Not a security sandbox: it only keeps the toy
    packages apart from the real ones and removes a few builtins."""

    def __init__(self, files):
        self.files, self.mods = files, {}
        b = dict(vars(builtins))
        for name in ("open", "exec", "eval", "compile", "input", "breakpoint"):
            b.pop(name, None)
        b["__import__"] = self._import
        self.builtins = b

    def _path(self, name):
        base = name.replace(".", "/")
        for cand in (base + ".py", base + "/__init__.py"):
            if cand in self.files:
                return cand
        return None

    def _load(self, name):
        if name in self.mods:
            return self.mods[name]
        path = self._path(name)
        if path is None:
            raise ImportError("No module named %r" % name)
        mod = types.ModuleType(name)
        mod.__file__ = "<mem:%s>" % path
        mod.__dict__["__builtins__"] = self.builtins
        self.mods[name] = mod
        try:
            exec(compile(self.files[path], mod.__file__, "exec"), mod.__dict__)
        except BaseException:
            del self.mods[name]
            raise
        parent, _, child = name.rpartition(".")
        if parent in self.mods:
            setattr(self.mods[parent], child, mod)
        return mod

    def _import(self, name, globals=None, locals=None, fromlist=(), level=0):
        if name.split(".")[0] in _SAFE_STD:
            return builtins.__import__(name, globals, locals, fromlist, level)
        parts = name.split(".")
        for i in range(1, len(parts) + 1):
            self._load(".".join(parts[:i]))
        mod = self.mods[name]
        for f in fromlist or ():
            if f != "*" and not hasattr(mod, f) and self._path(name + "." + f):
                setattr(mod, f, self._load(name + "." + f))
        return mod if fromlist else self.mods[parts[0]]

    def load(self, name):
        parts = name.split(".")
        for i in range(1, len(parts) + 1):
            self._load(".".join(parts[:i]))
        return self.mods[name]


def path_to_module(path):
    p = path[:-3] if path.endswith(".py") else path
    return p.replace("/", ".")


def test_files(files):
    return sorted(p for p in files if p.endswith(".py") and re.search(r"(^|/)test_[^/]*\.py$", p))


def test_names(src):
    try:
        tree = ast.parse(src)
    except SyntaxError:
        return None
    return [n.name for n in tree.body if isinstance(n, ast.FunctionDef) and n.name.startswith("test_")]


def run_tests(files, step_limit=20000, only=None):
    """Run every test function in tests/test_*.py. Returns {"path::name": "PASS"|"FAIL"|"ERROR"|"TIMEOUT"}.
    A syntax error or import error in a test file marks every test in it ERROR (or the file itself when its
    names cannot be read). Package code is imported fresh for each call."""
    sb = Sandbox(files)
    results = {}
    for tf in test_files(files):
        names = test_names(files[tf])
        if names is None:
            results[tf + "::<file>"] = "ERROR"
            continue
        wanted = [n for n in names if only is None or (tf + "::" + n) in only]
        if not wanted:
            continue
        sb.mods.pop(path_to_module(tf), None)
        try:
            with _Tracing(step_limit):
                mod = sb._load(path_to_module(tf))
        except StepLimit:
            for n in wanted:
                results[tf + "::" + n] = "TIMEOUT"
            continue
        except BaseException:
            for n in wanted:
                results[tf + "::" + n] = "ERROR"
            continue
        for n in wanted:
            fn = getattr(mod, n)
            try:
                with _Tracing(step_limit):
                    fn()
                results[tf + "::" + n] = "PASS"
            except AssertionError:
                results[tf + "::" + n] = "FAIL"
            except StepLimit:
                results[tf + "::" + n] = "TIMEOUT"
            except BaseException:
                results[tf + "::" + n] = "ERROR"
    return results


def eval_snippet(files, module_path, setup, expr, step_limit=20000):
    """Run `setup` then evaluate `expr` inside the named module's namespace. Returns repr(value) or
    'raises ExcName'. Used for the reproduction check that an agent can run before it submits."""
    sb = Sandbox(files)
    try:
        with _Tracing(step_limit):
            mod = sb.load(path_to_module(module_path))
            ns = dict(vars(mod))
            if setup:
                exec(compile(setup, "<mem:setup>", "exec"), ns)
            return repr(eval(compile(expr, "<mem:expr>", "eval"), ns))
    except StepLimit:
        return "raises StepLimit"
    except BaseException as e:
        return "raises " + type(e).__name__


# ---------------------------------------------------------------- Part 4: task build and validation
def _sub_once(text, old, new):
    if text.count(old) != 1:
        raise ValueError("pattern must occur exactly once: %r (found %d)" % (old[:40], text.count(old)))
    return text.replace(old, new)


def _subs(text, pairs):
    for old, new in pairs:
        text = _sub_once(text, old, new)
    return text


def repo_files(name, fns=None):
    """The fixed repo: package files plus the tests that exist before any pull request."""
    r = REPOS[name]
    fns = fns or [s["fn"] for s in SPECS if s["repo"] == name]
    body = "\n\n".join("def test_%s_basic():\n    %s\n" % (f, BASIC[f]) for f in fns)
    pkg = r["module"].split("/")[0]
    return {pkg + "/__init__.py": "", r["module"]: r["src"], r["test"]: r["header"] + "\n\n" + body}


class Task:
    """One SWE-bench-style instance: issue text, base repo, test patch, gold patch, and the two test lists."""

    def __init__(self, id, repo, fn, issue, base, test_patch, gold_patch, alts, flaw=None, family="", created=0,
                 module=None, test_path=None):
        self.id, self.repo, self.fn, self.issue = id, repo, fn, issue
        self.base, self.test_patch, self.gold_patch, self.alts = base, test_patch, gold_patch, alts
        self.flaw, self.family, self.created = flaw, family, created
        self.module = module or REPOS[repo]["module"]
        self.test_path = test_path or REPOS[repo]["test"]
        self.f2p, self.p2p = [], []
        self.derive()

    def tests_files(self):
        """Base files with the test patch applied (what the harness runs against)."""
        return apply_patch(self.base, self.test_patch)[0] if self.test_patch else dict(self.base)

    def derive(self):
        before = run_tests(self.tests_files())
        after_files, _ = apply_patch(self.tests_files(), self.gold_patch)
        after = run_tests(after_files)
        self.f2p = sorted(t for t, s in after.items() if s == "PASS" and before.get(t) != "PASS")
        self.p2p = sorted(t for t, s in after.items() if s == "PASS" and before.get(t) == "PASS")
        self.after_failing = sorted(t for t, s in after.items() if s != "PASS")
        self.p2f = sorted(t for t, s in before.items() if s == "PASS" and after.get(t) != "PASS")

    def view(self):
        """What an agent may see: the issue and the repo at the base commit. No test patch, no gold patch."""
        return TaskView(self.id, self.issue, dict(self.base))


class TaskView:
    def __init__(self, id, issue, files):
        self.id, self.issue, self.files = id, issue, files


def build_task(spec):
    r = REPOS[spec["repo"]]
    fns = [s["fn"] for s in SPECS if s["repo"] == spec["repo"]]
    base_files = repo_files(spec["repo"], fns)
    fixed_plain = r["src"]
    buggy = _subs(fixed_plain, spec["bug"])
    if spec.get("wide"):
        fn2, bug2 = spec["wide"]
        buggy = _subs(buggy, bug2)
        # a patch that follows the issue literally fixes only the named function
        only_issue = _subs(fixed_plain, bug2)
    gold_src = _subs(fixed_plain, spec.get("fixed_extra", []))
    base = dict(base_files)
    base[r["module"]] = buggy
    tests_before = base[r["test"]]
    extra = "\n\ndef test_%s_regress():\n    %s\n" % (spec["fn"], REGRESS[spec["fn"]])
    if spec.get("wide"):
        extra += "\n\ndef test_%s_regress():\n    %s\n" % (spec["wide"][0], REGRESS[spec["wide"][0]])
    test_patch = make_diff({r["test"]: tests_before}, {r["test"]: tests_before.rstrip("\n") + "\n" + extra})
    gold = make_diff({r["module"]: buggy}, {r["module"]: gold_src})
    if spec.get("wide"):
        alts = [make_diff({r["module"]: buggy}, {r["module"]: only_issue})]
    elif spec["flaw"] == "narrow":
        alts = [make_diff({r["module"]: buggy}, {r["module"]: fixed_plain})]
    else:
        alts = [make_diff({r["module"]: buggy}, {r["module"]: fixed_plain})]
    created = rng_for("day", spec["id"]).randrange(200)
    return Task(spec["id"], spec["repo"], spec["fn"], spec["issue"], base, test_patch, gold, alts,
                flaw=spec["flaw"], family=spec["family"], created=created)


def validate_task(task):
    """Stage-III style checks: the gold patch applies, at least one test goes fail to pass, no test that passed
    before fails after, and every selected test passes after. Returns a dict."""
    reasons = []
    if not task.f2p:
        reasons.append("no FAIL_TO_PASS test")
    if task.p2f:
        reasons.append("gold patch breaks %d passing test(s)" % len(task.p2f))
    if task.after_failing:
        reasons.append("%d test(s) still fail with the gold patch" % len(task.after_failing))
    if not task.p2p:
        reasons.append("no PASS_TO_PASS test")
    return {"id": task.id, "ok": not reasons, "reasons": reasons, "f2p": len(task.f2p), "p2p": len(task.p2p)}


_TASKS = None


def curated_tasks():
    """The 20 hand-written tasks (built once, then reused)."""
    global _TASKS
    if _TASKS is None:
        _TASKS = [build_task(s) for s in SPECS]
    return _TASKS


# ---------------------------------------------------------------- Part 5: evaluation harness
class EvalResult:
    def __init__(self, status, detail="", f2p_pass=0, f2p_total=0, p2p_pass=0, p2p_total=0, broken=()):
        self.status, self.detail = status, detail
        self.f2p_pass, self.f2p_total, self.p2p_pass, self.p2p_total = f2p_pass, f2p_total, p2p_pass, p2p_total
        self.broken = list(broken)

    @property
    def resolved(self):
        return self.status == "resolved"

    def __repr__(self):
        return "EvalResult(%s, F2P %d/%d, P2P %d/%d%s)" % (
            self.status, self.f2p_pass, self.f2p_total, self.p2p_pass, self.p2p_total,
            ", " + self.detail if self.detail else "")


def evaluate(task, model_patch, step_limit=20000, fix_counts=False):
    """The grading rule. Apply the model patch to the base repo, reset any test files the patch touched, apply
    the test patch, run the FAIL_TO_PASS and PASS_TO_PASS tests, and call the task resolved only if all pass.
    (The paper's Appendix A.4 lists the test patch first and the model patch second; this order is my choice.)
    Statuses: empty, invalid_patch, apply_fail, error, unresolved, resolved."""
    if not model_patch or not model_patch.strip():
        return EvalResult("empty")
    if fix_counts:
        model_patch = fix_hunk_counts(model_patch)
    try:
        patched, rep = apply_patch(task.base, model_patch)
    except PatchError as e:
        kind = "apply_fail" if str(e).startswith(("context", "file")) else "invalid_patch"
        return EvalResult(kind, str(e))
    for p in list(patched):  # my design: a model patch must not change tests, so test files go back to base first
        if p in task.base and re.search(r"(^|/)test_[^/]*\.py$", p):
            patched[p] = task.base[p]
        elif p not in task.base and re.search(r"(^|/)test_[^/]*\.py$", p):
            del patched[p]
    try:
        final, _ = apply_patch(patched, task.test_patch) if task.test_patch else (patched, None)
    except PatchError as e:
        return EvalResult("apply_fail", "test patch conflict: " + str(e))
    want = set(task.f2p) | set(task.p2p)
    res = run_tests(final, step_limit=step_limit, only=want)
    f_ok = sum(1 for t in task.f2p if res.get(t) == "PASS")
    p_ok = sum(1 for t in task.p2p if res.get(t) == "PASS")
    broken = [t for t in task.p2p if res.get(t) != "PASS"]
    ok = f_ok == len(task.f2p) and p_ok == len(task.p2p)
    detail = "" if ok else ("timeout" if any(v == "TIMEOUT" for v in res.values()) else "")
    return EvalResult("resolved" if ok else "unresolved", detail, f_ok, len(task.f2p), p_ok, len(task.p2p), broken)


class EvalCache:
    """A result cache. key_mode='content' keys on (run, instance, patch hash). key_mode='id_only' mimics the
    flaw reported for the real harness, where the cache key ignores the diff: a changed patch under the same
    run and instance id returns the stale result."""

    def __init__(self, key_mode="content"):
        self.key_mode, self.store, self.hits, self.misses = key_mode, {}, 0, 0

    def key(self, run_id, task, patch):
        if self.key_mode == "id_only":
            return (run_id, task.id)
        return (run_id, task.id, hashlib.sha256(patch.encode()).hexdigest())

    def evaluate(self, run_id, task, patch, **kw):
        k = self.key(run_id, task, patch)
        if k in self.store:
            self.hits += 1
            return self.store[k]
        self.misses += 1
        self.store[k] = evaluate(task, patch, **kw)
        return self.store[k]


# ---------------------------------------------------------------- Part 6: scripted agents
CONTEXT_SCALE = 40  # token counts are multiplied by this to mimic a larger repo (my own made-up factor)


def parse_repro(issue):
    """Read a doctest-style reproduction out of the issue text. Returns (setup, expr, expected) or None.
    The last '>>>' line is the expression; earlier '>>>' lines are setup. 'Expected: X' gives the target."""
    lines = issue.split("\n")
    code = [ln[4:] for ln in lines if ln.startswith(">>> ")]
    exp = None
    for ln in lines:
        if ln.startswith("Expected: "):
            exp = ln[len("Expected: "):].split(" (")[0].strip()
    if not code or exp is None:
        return None
    return ("\n".join(code[:-1]), code[-1], exp)


def code_units(path, src):
    """Top-level functions and class methods of one module: {qualified name: (start, end, text)} (1-based lines)."""
    try:
        tree = ast.parse(src)
    except SyntaxError:
        return {}
    lines = src.split("\n")
    out = {}

    def add(name, node):
        out[name] = (node.lineno, node.end_lineno, "\n".join(lines[node.lineno - 1:node.end_lineno]))

    for n in tree.body:
        if isinstance(n, ast.FunctionDef):
            add(n.name, n)
        elif isinstance(n, ast.ClassDef):
            for m in n.body:
                if isinstance(m, ast.FunctionDef):
                    add(n.name + "." + m.name, m)
    return out


def bm25_rank(query_tokens, docs, k1=1.5, b=0.75):
    """Rank {name: [tokens]} by BM25. Returns [(score, name)] best first; ties keep the input order."""
    names = list(docs)
    n = len(names)
    avg = mean(len(docs[x]) for x in names) or 1.0
    df = Counter(t for x in names for t in set(docs[x]))
    scored = []
    for idx, name in enumerate(names):
        tf = Counter(docs[name])
        s = 0.0
        for t in set(query_tokens):
            if t in tf:
                idf = math.log(1 + (n - df[t] + 0.5) / (df[t] + 0.5))
                s += idf * tf[t] * (k1 + 1) / (tf[t] + k1 * (1 - b + b * len(docs[name]) / avg))
        scored.append((-s, idx, name))
    scored.sort()
    return [(-s, name) for s, _, name in scored]


_TERM = r"[\w.]+(?:\([^()]*\))?"
_M = re.M
REPAIR_OPS = {
    "cmp": [(r"(?<![<>=!-])<(?!=)", "<="), (r"(?<![<>=!-])>(?!=)", ">="), (r"<=", "<"), (r">=", ">")],
    "method": [(r"\.endswith\(", ".startswith("), (r"\.startswith\(", ".endswith("), (r"\.isalpha\(", ".isalnum("),
               (r"\.isalnum\(", ".isalpha("), (r"\.append\(", ".extend("), (r"\.extend\(", ".append(")],
    "arith": [(r" \+ ", " - "), (r" - ", " + ")],
    "operands": [("(%s) - (%s)" % (_TERM, _TERM), r"\2 - \1")],
    "const": [(r" \+ 0\b", " + 1"), (r", 1\)", ", 2)"), (r" - 1\b", "")],
    "augassign": [(r"^(\s*)(self\.\w+\[\w+\]) = (\w+)$", r"\1\2 += \3")],
    "boolop": [(r"\bor\b", "and"), (r"\band\b", "or")],
    "wrap": [(r"= ([a-z_]+)$", r"= sorted(\1)"), (r"\breturn ([a-z_]+)$", r"return abs(\1)"),
             (r"\b(\w+)\.lower\(\)", r"\1.strip().lower()"), (r"\b(\w+)\.split\(\)", r"\1.lower().split()")],
}
TEMPLATES = [(_PARSE_KV[1], _PARSE_KV[0]), (_GET_PATH[1], _GET_PATH[0]), (_MERGE[1], _MERGE[0])]
BASIC_OPS = ["cmp", "method"]
STD_OPS = BASIC_OPS + ["arith", "operands", "const", "augassign", "boolop", "wrap"]
FULL_OPS = STD_OPS + ["guard", "template"]


def candidates_for_unit(unit_src, repertoire, expected=None):
    """Yield (family, new_unit_text) for every single edit that the repertoire can make to one function."""
    for fam in repertoire:
        if fam == "guard":
            if re.search(r"[%/] len\(", unit_src):
                lines = unit_src.split("\n")
                m = re.match(r"\s*def \w+\((?:self, )?(\w+)", lines[0])
                if m:
                    ind = re.match(r"\s*", lines[1]).group(0)
                    ret = expected if expected is not None else "None"
                    guard = ["%sif not %s:" % (ind, m.group(1)), "%s    return %s" % (ind, ret)]
                    yield fam, "\n".join([lines[0]] + guard + lines[1:])
        elif fam == "template":
            for old, new in TEMPLATES:
                if old in unit_src:
                    yield fam, unit_src.replace(old, new)
        else:
            for pat, rep in REPAIR_OPS[fam]:
                for m in re.finditer(pat, unit_src, _M):
                    yield fam, unit_src[:m.start()] + m.expand(rep) + unit_src[m.end():]


class AgentResult:
    def __init__(self, patch, tokens_in=0, tokens_out=0, turns=0, log=None):
        self.patch, self.tokens_in, self.tokens_out, self.turns = patch, tokens_in, tokens_out, turns
        self.log = log or []


TIERS = {"small": (0.15, 0.60), "mid": (1.0, 4.0), "large": (5.0, 20.0)}  # made-up dollars per million tokens


class NullAgent:
    name, tier = "null", "small"

    def solve(self, view, hint=None):
        return AgentResult("", est_tokens(view.issue) * CONTEXT_SCALE, 10, 1, ["gave up"])


class ScriptedAgent:
    """Localize with BM25, try single edits from a fixed repertoire, check each against the issue's example and
    (optionally) the visible tests, submit the first edit that passes. A program, not a language model."""

    def __init__(self, name, repertoire, k_units=3, checks="repro+tests", max_cands=14, tier="mid",
                 localizer="bm25", sloppy=0.0):
        self.name, self.repertoire, self.k_units, self.checks = name, list(repertoire), k_units, checks
        self.max_cands, self.tier, self.localizer, self.sloppy = max_cands, tier, localizer, sloppy

    def _units(self, view):
        out = {}
        for p, src in view.files.items():
            if p.endswith(".py") and not re.search(r"(^|/)test_", p) and not p.endswith("__init__.py"):
                for n, (a, b, text) in code_units(p, src).items():
                    out[(p, n)] = (a, b, text)
        return out

    def _rank(self, view, units, hint):
        if self.localizer == "oracle" and hint:
            first = [k for k in units if k[1].split(".")[-1] == hint]
            return first + [k for k in units if k not in first]
        if self.localizer == "random":
            keys = list(units)
            rng_for("loc", self.name, view.id).shuffle(keys)
            return keys
        q = content_tokens(view.issue)
        docs = {k: content_tokens(v[2]) + 3 * [t for t in tokens(k[1]) if t not in STOP] for k, v in units.items()}
        return [k for _, k in bm25_rank(q, docs)]

    def solve(self, view, hint=None):
        units = self._units(view)
        ranked = self._rank(view, units, hint)[:self.k_units]
        repro = parse_repro(view.issue)
        expected = repro[2] if repro else None
        ctx = est_tokens(view.issue) + sum(est_tokens(units[k][2]) for k in ranked)
        tin, tout, turns, log = ctx, 40, 1, ["read %d unit(s): %s" % (len(ranked), ", ".join(k[1] for k in ranked))]
        base_pass = None
        if self.checks == "repro+tests":
            base_pass = {t for t, s in run_tests(view.files).items() if s == "PASS"}
            tin += ctx
            turns += 1
        tried, chosen = 0, None
        for key in ranked:
            path, name = key
            a, b, text = units[key]
            for fam, new_unit in candidates_for_unit(text, self.repertoire, expected):
                if tried >= self.max_cands:
                    break
                tried += 1
                lines = view.files[path].split("\n")
                new_src = "\n".join(lines[:a - 1] + new_unit.split("\n") + lines[b:])
                cand = dict(view.files)
                cand[path] = new_src
                ok = True
                if self.checks in ("repro", "repro+tests") and repro:
                    got = eval_snippet(cand, path, repro[0], repro[1])
                    ok = got == repro[2]
                    tin += ctx + 120
                    tout += est_tokens(new_unit) + 30
                    turns += 1
                if ok and self.checks == "repro+tests":
                    res = run_tests(cand)
                    ok = all(res.get(t) == "PASS" for t in base_pass)
                    tin += ctx + 120
                    turns += 1
                if ok:
                    chosen = (path, new_src, fam, name)
                    break
            if chosen or tried >= self.max_cands:
                break
        if chosen is None and self.checks == "none":
            chosen = None
        if chosen is None:
            log.append("no candidate passed after %d tries; gave up" % tried)
            return AgentResult("", tin * CONTEXT_SCALE, tout * CONTEXT_SCALE // 4, turns, log)
        path, new_src, fam, name = chosen
        patch = make_diff({path: view.files[path]}, {path: new_src})
        tout += est_tokens(patch)
        if self.sloppy and rng_for("sloppy", self.name, view.id).random() < self.sloppy:
            patch = re.sub(r"^(@@ -\d+,)(\d+)( \+\d+,)(\d+)( @@)", lambda m: m.group(1) + str(int(m.group(2)) + 1) + m.group(3) + m.group(4) + m.group(5), patch, count=1, flags=re.M)
            log.append("miscounted a hunk header")
        log.append("submit %s edit in %s after %d tries" % (fam, name, tried))
        return AgentResult(patch, tin * CONTEXT_SCALE, tout * CONTEXT_SCALE // 4, turns, log)


def buggy_fingerprint(files):
    """What a memorizer recognizes: a hash of the package source files (not of the issue text)."""
    h = hashlib.sha256()
    for p in sorted(files):
        if p.endswith(".py") and not re.search(r"(^|/)test_", p):
            h.update(p.encode() + b"\0" + files[p].encode())
    return h.hexdigest()


class MemorizerAgent:
    """Stands in for a model that saw the benchmark in training: when it recognizes the buggy source it writes
    the gold patch at tiny cost; otherwise it falls back to a normal scripted agent. This recall is built in by
    construction, so the lab can show what contamination probes do and do not detect."""

    def __init__(self, name, seen_tasks, fallback, tier="mid"):
        self.name, self.tier, self.fallback = name, tier, fallback
        self.memory = {buggy_fingerprint(t.base): t.gold_patch for t in seen_tasks}
        self.seen_ids = {t.id for t in seen_tasks}
        self.issue_memory = {}

    def recalls(self, files):
        return buggy_fingerprint(files) in self.memory

    def solve(self, view, hint=None):
        fp = buggy_fingerprint(view.files)
        if fp in self.memory:
            p = self.memory[fp]
            return AgentResult(p, est_tokens(view.issue) * CONTEXT_SCALE, est_tokens(p) * CONTEXT_SCALE // 4, 1, ["recalled a patch"])
        return self.fallback.solve(view, hint)


def roster(seen_tasks=()):
    """The nine toy agents of the lab."""
    full = ScriptedAgent("full-repertoire", FULL_OPS, k_units=4, tier="large")
    agents = [
        NullAgent(),
        ScriptedAgent("naive-first", STD_OPS, k_units=1, checks="none", tier="small"),
        ScriptedAgent("repro-basic", BASIC_OPS, k_units=3, checks="repro", tier="small"),
        ScriptedAgent("repro-std", STD_OPS, k_units=3, checks="repro", tier="mid"),
        ScriptedAgent("repro-tests-std", STD_OPS, k_units=3, checks="repro+tests", tier="mid"),
        full,
        ScriptedAgent("full-sloppy-format", FULL_OPS, k_units=4, tier="large", sloppy=0.5),
        ScriptedAgent("oracle-localizer", FULL_OPS, k_units=1, tier="large", localizer="oracle"),
    ]
    agents.append(make_memorizer(seen_tasks))
    return agents


# ---------------------------------------------------------------- Part 7: scorer, cost, leaderboard
class Record:
    def __init__(self, agent, task_id, ev, res, cost):
        self.agent, self.task_id, self.ev, self.res, self.cost = agent, task_id, ev, res, cost

    @property
    def resolved(self):
        return self.ev.resolved


def cost_of(agent, res):
    pin, pout = TIERS[agent.tier]
    return (res.tokens_in * pin + res.tokens_out * pout) / 1e6


def run_benchmark(agents, tasks, fix_counts=False, cache=None, run_id="run", step_limit=20000, oracle_hint=True):
    """Run every agent on every task and grade the patches. Returns {agent name: [Record]}."""
    out = {}
    for a in agents:
        recs = []
        for t in tasks:
            res = a.solve(t.view(), hint=t.fn if oracle_hint else None)
            if cache is not None:
                ev = cache.evaluate(run_id + ":" + a.name, t, res.patch, fix_counts=fix_counts, step_limit=step_limit)
            else:
                ev = evaluate(t, res.patch, fix_counts=fix_counts, step_limit=step_limit)
            recs.append(Record(a.name, t.id, ev, res, cost_of(a, res)))
        out[a.name] = recs
    return out


def resolved_rate(recs, cap=None):
    """Share of tasks resolved. With cap, a task whose cost is above the cap counts as unresolved."""
    if not recs:
        return 0.0
    return sum(1 for r in recs if r.resolved and (cap is None or r.cost <= cap)) / len(recs)


def status_counts(recs):
    return dict(Counter(r.ev.status for r in recs))


def leaderboard(results, cap=None):
    """Rows sorted by resolved rate, then cost. Each row has Wilson interval, mean cost, cost per resolved task,
    invalid-patch count and a Pareto flag (no other row is at least as good on both axes and better on one)."""
    rows = []
    for name, recs in results.items():
        n = len(recs)
        k = sum(1 for r in recs if r.resolved and (cap is None or r.cost <= cap))
        lo, hi = wilson(k, n)
        total = sum(r.cost for r in recs)
        st = status_counts(recs)
        rows.append({"agent": name, "n": n, "resolved": k, "rate": k / n if n else 0.0, "lo": lo, "hi": hi,
                     "mean_cost": total / n if n else 0.0, "cost_per_resolved": (total / k) if k else float("inf"),
                     "invalid": st.get("invalid_patch", 0) + st.get("apply_fail", 0), "empty": st.get("empty", 0)})
    for r in rows:
        r["pareto"] = not any(o["rate"] >= r["rate"] and o["mean_cost"] <= r["mean_cost"] and
                              (o["rate"] > r["rate"] or o["mean_cost"] < r["mean_cost"]) for o in rows)
    rows.sort(key=lambda r: (-r["rate"], r["mean_cost"], r["agent"]))
    return rows


def paired_gap(recs_a, recs_b, n_boot=400):
    """Difference in resolved rate (a minus b) over the same tasks, with a paired bootstrap interval."""
    d = [int(x.resolved) - int(y.resolved) for x, y in zip(recs_a, recs_b)]
    return mean(d), bootstrap_ci(d, n_boot=n_boot, seed="gap")


def localization_accuracy(tasks, k=1):
    """How often BM25 over function bodies puts the function that the gold patch changes in the top k."""
    probe = ScriptedAgent("probe", STD_OPS)
    hit = 0
    for t in tasks:
        v = t.view()
        top = probe._rank(v, probe._units(v), None)[:k]
        hit += any(x[1].split(".")[-1] == t.fn for x in top)
    return hit / len(tasks) if tasks else 0.0


# ---------------------------------------------------------------- Part 8: task-quality filters
def issue_severity(task):
    """A rule-based stand-in for an annotator who rates how well specified the issue is, 0 (clear) to 3 (cannot
    be solved from the text). One point is lost for each missing item: an example, an expected value, and a
    name for the function that is broken."""
    repro = parse_repro(task.issue)
    has_example = any(ln.startswith(">>> ") for ln in task.issue.split("\n"))
    has_expected = repro is not None
    names = set()
    for p, src in task.base.items():
        if p.endswith(".py") and not re.search(r"(^|/)test_", p):
            names |= {n.split(".")[-1] for n in code_units(p, src)}
    named = any(re.search(r"\b%s\b" % re.escape(n), task.issue) for n in names if len(n) > 3)
    return 3 - (has_example + has_expected + named)


def annotate(task, annotators=3, noise=0.2):
    """Simulated annotators (rule plus random +/-1 slips). Verified-style: the final severity is the maximum."""
    base = issue_severity(task)
    sev = []
    for i in range(annotators):
        r = rng_for("ann", task.id, i)
        s = base + (r.choice((-1, 1)) if r.random() < noise else 0)
        sev.append(min(3, max(0, s)))
    return sev


def unmentioned_symbols(task):
    """Names that the added tests use but the issue never mentions: imports and calls to repo functions."""
    added = "\n".join(ln[1:] for ln in task.test_patch.split("\n") if ln.startswith("+") and not ln.startswith("+++"))
    unit_names = set()
    for p, src in task.base.items():
        if p.endswith(".py") and not re.search(r"(^|/)test_", p):
            unit_names |= {n.split(".")[-1] for n in code_units(p, src)}
            unit_names |= set(re.findall(r"^([A-Z_][A-Z_0-9]*) = ", src, re.M))
    flat = set(re.findall(r"\b([A-Za-z_]\w*)\(", added))
    for ln in added.split("\n"):
        m = re.match(r"\s*from [\w.]+ import (.+)", ln)
        if m:
            flat |= {x.strip() for x in m.group(1).split(",")}
    flat |= set(re.findall(r"\b([A-Z_]{3,})\b", added))
    cands = {n for n in flat if n in unit_names or n.startswith("_")}
    return sorted(n for n in cands if not re.search(r"\b%s\b" % re.escape(n), task.issue))


def alt_solution_fails(task):
    """Run each reasonable alternative fix (one a careful engineer could write from the issue alone)."""
    return [i for i, p in enumerate(task.alts) if not evaluate(task, p).resolved]


def verified_filter(tasks, sev_cut=2):
    """Keep a task only if the issue is clear enough (max annotator severity below sev_cut) and no static or
    dynamic sign of over-specific tests appears. Returns {id: {'kept': bool, 'reasons': [...]}}."""
    out = {}
    for t in tasks:
        reasons = []
        sev = max(annotate(t))
        if sev >= sev_cut:
            reasons.append("underspecified (max severity %d)" % sev)
        syms = unmentioned_symbols(t)
        if syms:
            reasons.append("tests use names the issue never mentions: " + ", ".join(syms))
        if alt_solution_fails(t):
            reasons.append("a valid alternative fix fails the tests")
        out[t.id] = {"kept": not reasons, "reasons": reasons}
    return out


def lite_filter(task, min_words=10):
    """Lite-style selection rules, from the list on the SWE-bench Lite page: drop issues with images, links, commit
    ids or references to other issues; very short issues; patches that touch more than one file or more than three
    hunks; patches that create or delete files; tests that check error messages. min_words is 10 here, scaled down
    from 40 because the toy issues are short. Returns the list of reasons to drop (empty means keep)."""
    reasons = []
    if len(task.issue.split()) < min_words:
        reasons.append("short issue")
    if re.search(r"https?://|\b[0-9a-f]{40}\b|#\d+|!\[", task.issue):
        reasons.append("links, commit ids, other issues or images")
    fps = parse_diff(task.gold_patch, strict_counts=False)
    if len(fps) > 1:
        reasons.append("edits more than one file")
    if sum(len(fp.hunks) for fp in fps) > 3:
        reasons.append("more than three hunks")
    if any(fp.is_new or fp.is_delete for fp in fps):
        reasons.append("creates or deletes a file")
    added = "\n".join(ln for ln in task.test_patch.split("\n") if ln.startswith("+"))
    if re.search(r"match=|str\(e|\.message", added):
        reasons.append("test checks an error message")
    return reasons


def filter_scores(tasks, decisions):
    """Compare what the filter dropped with the planted flaw labels. Precision and recall of 'dropped = flawed'."""
    tp = sum(1 for t in tasks if t.flaw and not decisions[t.id]["kept"])
    fp = sum(1 for t in tasks if not t.flaw and not decisions[t.id]["kept"])
    fn = sum(1 for t in tasks if t.flaw and decisions[t.id]["kept"])
    prec = tp / (tp + fp) if tp + fp else 1.0
    rec = tp / (tp + fn) if tp + fn else 1.0
    return {"tp": tp, "fp": fp, "fn": fn, "precision": prec, "recall": rec}


# ---------------------------------------------------------------- Part 9: contamination probes
def _ihash(text):
    return hashlib.sha256(text.encode()).hexdigest()[:16]


def guess_path(issue, candidate_paths):
    """What an agent with no repo access can do from the issue alone: BM25 of the issue words against file paths."""
    docs = {p: tokens(p.replace("/", " ").replace(".py", "")) for p in candidate_paths}
    return bm25_rank(content_tokens(issue), docs)[0][1]


def _agent_guess_path(agent, issue, candidate_paths):
    if isinstance(agent, MemorizerAgent):
        h = _ihash(agent_issue_key(issue))
        if h in agent.issue_memory:
            return agent.issue_memory[h][0]
        return _agent_guess_path(agent.fallback, issue, candidate_paths)
    return guess_path(issue, candidate_paths)


def _agent_complete(agent, issue, prefix):
    """Given the issue and the first lines of the gold patch's additions, write the rest. Only a model that
    memorized the patch can do this from nothing but text."""
    if isinstance(agent, MemorizerAgent):
        h = _ihash(agent_issue_key(issue))
        if h in agent.issue_memory:
            return agent.issue_memory[h][1][len(prefix):]
    return []


def agent_issue_key(issue):
    """A memorizer recognizes an issue by its full text."""
    return issue.strip()


def _added_lines(patch):
    return [ln[1:] for ln in patch.split("\n") if ln.startswith("+") and not ln.startswith("+++")]


def remember_issues(agent, seen_tasks):
    """Give a memorizer its issue-to-patch memory (a second index next to the code fingerprint)."""
    agent.issue_memory = {_ihash(agent_issue_key(t.issue)): (t.module, _added_lines(t.gold_patch)) for t in seen_tasks}
    return agent


def file_path_probe(agent, tasks, all_paths):
    """Issue text only, no repo: does the model name the file that the gold patch edits? Compare seen with unseen tasks."""
    hits = sum(1 for t in tasks if _agent_guess_path(agent, t.issue, all_paths) == t.module)
    return hits, len(tasks)


def prefix_probe(agent, tasks, frac=0.5):
    """Give the issue and the first half of the gold patch's added lines; count tasks whose remaining added lines
    are reproduced exactly. A model that has not seen the patch should score near zero."""
    hit = n = 0
    for t in tasks:
        gold = _added_lines(t.gold_patch)
        k = int(len(gold) * frac)
        n += 1
        hit += _agent_complete(agent, t.issue, gold[:k]) == gold[k:]
    return hit, n


REWORD = {"wrong": "incorrect", "crashes": "fails", "forgets": "loses", "drops": "skips", "cuts": "trims", "looks": "seems",
          "instead of": "rather than", "odd": "strange", "gives": "produces", "returns": "gives back"}


def reword_task(task):
    """Prose-only change: swap common words in the non-code lines of the issue. Identifiers and examples stay."""
    out = []
    for ln in task.issue.split("\n"):
        if not ln.startswith((">>>", "Expected")) and ln and not ln[0] in "'[{(-0123456789TZ":
            for a, b in REWORD.items():
                ln = ln.replace(a, b)
        out.append(ln)
    t = copy.copy(task)
    t.issue = "\n".join(out)
    return t


def rename_task(task, suffix="_v2"):
    """Rename the broken function everywhere: repo, tests, patches, issue. A solver is unaffected; a model that
    recognizes the source text is not."""
    pat = re.compile(r"\b%s\b" % re.escape(task.fn))
    sub = lambda s: pat.sub(task.fn + suffix, s)
    base = {p: sub(s) for p, s in task.base.items()}
    return Task(task.id, task.repo, task.fn + suffix, sub(task.issue), base, sub(task.test_patch), sub(task.gold_patch),
                [sub(a) for a in task.alts], flaw=task.flaw, family=task.family, created=task.created,
                module=task.module, test_path=task.test_path)


def date_split(results, tasks, cutoff):
    """Resolved counts before and after a date cutoff (tasks carry a made-up 'created' day, 0 to 199)."""
    by = {t.id: t for t in tasks}
    pre = [r for r in results if by[r.task_id].created < cutoff]
    post = [r for r in results if by[r.task_id].created >= cutoff]
    return (sum(r.resolved for r in pre), len(pre)), (sum(r.resolved for r in post), len(post))


# ---------------------------------------------------------------- Part 10: SWE-smith style task synthesis
INJECT_OPS = [
    ("lt_to_le", r"(?<![<>=!-])<(?!=)", "<="), ("le_to_lt", r"<=", "<"), ("gt_to_ge", r"(?<![-<>=!])>(?!=)", ">="), ("ge_to_gt", r">=", ">"),
    ("plus_to_minus", r" \+ ", " - "), ("minus_to_plus", r" - ", " + "), ("eq_to_ne", r"==", "!="),
    ("and_to_or", r"\band\b", "or"), ("or_to_and", r"\bor\b", "and"),
    ("startswith_to_endswith", r"\.startswith\(", ".endswith("), ("drop_strip", r"\.strip\(\)", ""), ("drop_lower", r"\.lower\(\)", ""),
    ("drop_abs", r"abs\(([^()]*)\)", r"\1"), ("drop_plus_one", r" \+ 1\b", ""), ("zero_to_one", r"\b0\b", "1"),
    ("one_to_two", r"\b1\b", "2"), ("two_to_three", r"\b2\b", "3"), ("drop_not", r"\bnot ", ""), ("in_to_not_in", r" in ", " not in "),
    ("extend_to_append", r"\.extend\(", ".append("), ("isalnum_to_isalpha", r"\.isalnum\(", ".isalpha("),
    ("drop_sorted", r"sorted\((\w+)\)", r"\1"), ("floor_to_true_div", r" // ", " / "), ("mod_to_floor", r" % ", " // "),
]


def synth_repo(name):
    """A repo whose tests all pass: the narrow tasks' fixed code (with helpers) plus all basic and regression tests."""
    r = REPOS[name]
    fns = [s["fn"] for s in SPECS if s["repo"] == name]
    src = r["src"]
    for s in SPECS:
        if s["repo"] == name and s.get("fixed_extra"):
            src = _subs(src, s["fixed_extra"])
    files = repo_files(name, fns)
    body = files[r["test"]] + "".join("\n\ndef test_%s_regress():\n    %s\n" % (f, REGRESS[f]) for f in fns)
    files[r["module"]] = src
    files[r["test"]] = body
    return files


def repro_from_test(files, module, test_src, test_name):
    """Turn the first failing assertion of a test into a doctest-style issue snippet (a stand-in for the
    language-model step that SWE-smith uses to write an issue from a failing test)."""
    try:
        tree = ast.parse(test_src)
    except SyntaxError:
        return None
    fn = next((n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == test_name), None)
    if fn is None:
        return None
    setup = []
    for st in fn.body:
        if isinstance(st, ast.Assert) and isinstance(st.test, ast.Compare) and len(st.test.ops) == 1 \
                and isinstance(st.test.ops[0], (ast.Eq, ast.Is)):
            left, right = ast.unparse(st.test.left), ast.unparse(st.test.comparators[0])
            got = eval_snippet(files, module, "\n".join(setup), left)
            try:
                same = got == repr(eval(right, {}))
            except Exception:
                same = False
            if not same:
                return ("\n".join(setup), left, right, got)
        elif not isinstance(st, ast.Assert):
            setup.append(ast.unparse(st))
    return None


def synthesize_tasks(limit=40, seed="synth", repos=None):
    """Inject one bug at a time into each function of each repo, keep a mutant only if at least one existing test
    fails (and none times out). Returns (tasks, stats)."""
    stats = Counter()
    pool = []
    for name in (repos or sorted(REPOS)):
        files = synth_repo(name)
        r = REPOS[name]
        base_res = run_tests(files)
        assert all(v == "PASS" for v in base_res.values()), "synthetic base repo must pass its own tests"
        src = files[r["module"]]
        for uname, (a, b, text) in code_units(r["module"], src).items():
            for op, pat, rep in INJECT_OPS:
                for m in re.finditer(pat, text, re.M):
                    new_unit = text[:m.start()] + m.expand(rep) + text[m.end():]
                    stats["candidates"] += 1
                    if new_unit == text:
                        stats["no_change"] += 1
                        continue
                    try:
                        ast.parse("class _X:\n" + "\n".join("    " + l for l in new_unit.split("\n")) if "." in uname else new_unit)
                    except SyntaxError:
                        stats["syntax_error"] += 1
                        continue
                    lines = src.split("\n")
                    mutated = "\n".join(lines[:a - 1] + new_unit.split("\n") + lines[b:])
                    mf = dict(files)
                    mf[r["module"]] = mutated
                    res = run_tests(mf, step_limit=4000)
                    if any(v == "TIMEOUT" for v in res.values()):
                        stats["timeout"] += 1
                        continue
                    failing = sorted(t for t, v in res.items() if v != "PASS")
                    if not failing:
                        stats["no_test_fails"] += 1
                        continue
                    stats["kept"] += 1
                    pool.append((name, uname, op, m.start(), mutated, failing, mf))
    rng = rng_for(seed, "pick")
    rng.shuffle(pool)
    tasks = []
    for name, uname, op, pos, mutated, failing, mf in pool:
        if len(tasks) >= limit:
            break
        r = REPOS[name]
        fn_short = uname.split(".")[-1]
        tname = failing[0].split("::")[1]
        rep = repro_from_test(mf, r["module"], mf[r["test"]], tname)
        title = "%s returns a wrong result" % (uname + "()")
        if rep:
            lines = [title, "", "A test that used to pass now fails: %s." % tname, ""]
            lines += [">>> " + l for l in (rep[0].split("\n") if rep[0] else [])] + [">>> " + rep[1], rep[3], "Expected: " + rep[2]]
            issue = "\n".join(lines)
        else:
            issue = "%s\n\nA test that used to pass now fails: %s." % (title, tname)
        fixed = synth_repo(name)
        gold = make_diff({r["module"]: mutated}, {r["module"]: fixed[r["module"]]})
        tid = "synth-%s-%s-%s-%d" % (name, fn_short, op, pos)
        t = Task(tid, name, fn_short, issue, mf, "", gold, [gold], flaw=None, family=op,
                 created=rng_for("day", tid).randrange(200), module=r["module"], test_path=r["test"])
        if not validate_task(t)["ok"]:
            stats["failed_validation"] += 1
            continue
        tasks.append(t)
    stats["tasks"] = len(tasks)
    return tasks, dict(stats)


# ---------------------------------------------------------------- Part 11: study and demo
def make_memorizer(seen_tasks):
    fb = ScriptedAgent("memorizer-fallback", STD_OPS, k_units=3, checks="repro", tier="mid")
    return remember_issues(MemorizerAgent("memorizer", list(seen_tasks), fb), seen_tasks)


def contamination_study(pool, cutoff=100):
    """Everything created before `cutoff` is in the memorizer's training data. Compare it with an honest solver
    on three diagnostics: the date split, the issue-only file-path probe and the prefix probe, then show how
    resolved rate moves when the seen tasks are reworded and renamed."""
    seen = [t for t in pool if t.created < cutoff]
    unseen = [t for t in pool if t.created >= cutoff]
    mem = make_memorizer(seen)
    honest = ScriptedAgent("repro-std", STD_OPS, k_units=3, checks="repro")
    paths = sorted({t.module for t in pool})
    seen_issues = {t.issue for t in seen}
    out = {"n_seen": len(seen), "n_unseen": len(unseen), "cutoff": cutoff,
           "dup_issues": sum(1 for t in unseen if t.issue in seen_issues)}
    for ag in (mem, honest):
        recs = run_benchmark([ag], pool)[ag.name]
        pre, post = date_split(recs, pool, cutoff)
        out[ag.name] = {
            "pre": pre, "post": post,
            "path_seen": file_path_probe(ag, seen, paths), "path_unseen": file_path_probe(ag, unseen, paths),
            "prefix_seen": prefix_probe(ag, seen), "prefix_unseen": prefix_probe(ag, unseen),
        }
        levels = {}
        for lvl, fn in (("original", lambda t: t), ("reworded", reword_task), ("renamed", rename_task)):
            ts = [fn(t) for t in seen]
            r = run_benchmark([ag], ts)[ag.name]
            levels[lvl] = (sum(x.resolved for x in r), len(r))
        out[ag.name]["levels"] = levels
    return out


def _fmt_ci(k, n):
    lo, hi = wilson(k, n)
    return "%d/%d (%.0f%%, 95%% CI %.0f to %.0f%%)" % (k, n, 100 * k / n if n else 0, 100 * lo, 100 * hi)


def demo():
    tasks = curated_tasks()
    print("== 1. Task table (20 hand-written toy tasks; flaw column is planted by me)")
    print("%-12s %-7s %-10s %4s %4s %5s" % ("task", "flaw", "family", "F2P", "P2P", "gold+-"))
    for t in tasks:
        st = diff_stats(t.gold_patch)
        print("%-12s %-7s %-10s %4d %4d %3d/%-2d" % (t.id, t.flaw or "-", t.family, len(t.f2p), len(t.p2p), st["added"], st["removed"]))
    bad = [validate_task(t) for t in tasks if not validate_task(t)["ok"]]
    print("validation: %d of %d tasks pass the Stage III style checks" % (len(tasks) - len(bad), len(tasks)))

    print("\n== 2. Patch validation on one task (%s)" % tasks[3].id)
    t = tasks[3]
    good = t.gold_patch
    miscount = re.sub(r"^(@@ -\d+,)(\d+)", lambda m: m.group(1) + str(int(m.group(2)) + 1), good, count=1, flags=re.M)
    wrong_ctx = good.replace("def is_palindrome", "def is_palindrom")
    tests_edit = make_diff({t.test_path: t.base[t.test_path]}, {t.test_path: t.base[t.test_path] + "\ndef test_is_palindrome_regress():\n    assert True\n"})
    cases = [("gold patch", good, False), ("empty patch", "", False), ("hunk count off by one", miscount, False),
             ("same, with count repair", miscount, True), ("context does not match", wrong_ctx, False),
             ("only edits the test file", tests_edit, False), ("prose, no diff", "I fixed it, trust me.\n", False)]
    for label, p, fx in cases:
        print("  %-26s -> %r" % (label, evaluate(t, p, fix_counts=fx)))

    print("\n== 3. Leaderboard, 20 curated tasks (cost is made up: dollars from token counts times a toy price)")
    seen0 = [t for t in tasks if t.created < 100]
    res = run_benchmark(roster(seen0), tasks)
    print("   (the memorizer has seen the %d tasks created before day 100)" % len(seen0))
    print("%-20s %-28s %9s %9s %5s" % ("agent", "resolved (Wilson CI)", "cost/task", "$/solved", "pareto"))
    for r in leaderboard(res):
        cps = "n/a" if r["cost_per_resolved"] == float("inf") else "%.3f" % r["cost_per_resolved"]
        print("%-20s %-28s %9.4f %9s %5s" % (r["agent"], _fmt_ci(r["resolved"], r["n"]), r["mean_cost"], cps, "yes" if r["pareto"] else ""))
    st = {a: status_counts(rs) for a, rs in res.items()}
    print("status counts (full-sloppy-format):", st["full-sloppy-format"])
    res_fix = run_benchmark([a for a in roster(seen0) if a.name == "full-sloppy-format"], tasks, fix_counts=True)
    print("same agent when the harness repairs hunk counts: %s" % _fmt_ci(sum(r.resolved for r in res_fix["full-sloppy-format"]), len(tasks)))
    print("BM25 localization, top-1 %.0f%%, top-3 %.0f%%" % (100 * localization_accuracy(tasks, 1), 100 * localization_accuracy(tasks, 3)))

    print("\n== 4. Task-quality filters (Verified-style)")
    dec = verified_filter(tasks)
    for t in tasks:
        if not dec[t.id]["kept"]:
            print("  drop %-12s planted=%-6s %s" % (t.id, t.flaw or "-", "; ".join(dec[t.id]["reasons"])))
    fs = filter_scores(tasks, dec)
    print("  vs planted flaws: caught %d, missed %d, wrongly dropped %d" % (fs["tp"], fs["fn"], fs["fp"]))
    kept = [t for t in tasks if dec[t.id]["kept"]]
    res_k = {a: [r for r in rs if r.task_id in {t.id for t in kept}] for a, rs in res.items()}
    print("  leaderboard on the %d kept tasks (resolved before -> after):" % len(kept))
    for r in leaderboard(res_k):
        before = sum(x.resolved for x in res[r["agent"]])
        print("    %-20s %2d/%d -> %2d/%d" % (r["agent"], before, len(tasks), r["resolved"], r["n"]))

    lite_keep = [t for t in tasks if not lite_filter(t)]
    print("  Lite-style shape filter keeps %d of %d, including %d of the %d planted flaws" % (
        len(lite_keep), len(tasks), sum(1 for t in lite_keep if t.flaw), sum(1 for t in tasks if t.flaw)))

    print("\n== 5. Synthesizing tasks (SWE-smith style)")
    synth, ss = synthesize_tasks(limit=40)
    print("  ", ss)
    res_s = run_benchmark(roster(), synth)  # memorizer sees none of these
    print("   resolved on %d synthetic tasks:" % len(synth), ", ".join("%s %d" % (a, sum(r.resolved for r in rs)) for a, rs in res_s.items()))

    print("\n== 6. Contamination diagnostics (memorizer is built to recall; made-up cutoff day 100)")
    pool = tasks + synth
    cs = contamination_study(pool)
    print("   pool %d tasks: %d before the cutoff (seen), %d after" % (len(pool), cs["n_seen"], cs["n_unseen"]))
    print("   %d unseen task(s) have issue text identical to a seen one (same symptom, different bug)" % cs["dup_issues"])
    for name in ("memorizer", "repro-std"):
        c = cs[name]
        print("   %-10s resolved pre %s | post %s" % (name, _fmt_ci(*c["pre"]), _fmt_ci(*c["post"])))
        print("   %-10s path-from-issue seen %d/%d, unseen %d/%d | prefix completion seen %d/%d, unseen %d/%d" % (
            "", c["path_seen"][0], c["path_seen"][1], c["path_unseen"][0], c["path_unseen"][1],
            c["prefix_seen"][0], c["prefix_seen"][1], c["prefix_unseen"][0], c["prefix_unseen"][1]))
        print("   %-10s on seen tasks: original %d/%d, reworded %d/%d, renamed %d/%d" % ("", *c["levels"]["original"], *c["levels"]["reworded"], *c["levels"]["renamed"]))

    print("\n== 7. Result cache keyed without the patch")
    t = tasks[3]
    for mode in ("content", "id_only"):
        cache = EvalCache(mode)
        a = cache.evaluate("r1", t, "", step_limit=20000)
        b = cache.evaluate("r1", t, t.gold_patch, step_limit=20000)
        print("   key=%-8s first (empty patch): %s, second (gold patch): %s, hits %d" % (mode, a.status, b.status, cache.hits))
if __name__ == "__main__":
    demo()