"""Tests for judge_infra_toolkit.py. Run: python3 test_judge_infra_toolkit.py -v""" import contextlib import io import json import math import os import random import tempfile import unittest import judge_infra_toolkit as T W = T.make_world(0) R = W["responses"] PAIRS = T.pair_list(W) class TestHelpers(unittest.TestCase): def test_mean_empty(self): self.assertEqual(T.mean([]), 0.0) def test_mean(self): self.assertAlmostEqual(T.mean([1, 2, 6]), 3.0) def test_stdev(self): self.assertAlmostEqual(T.stdev([2, 4, 4, 4, 5, 5, 7, 9], ddof=0), 2.0) def test_stdev_short(self): self.assertEqual(T.stdev([3]), 0.0) def test_sigmoid_symmetry(self): self.assertAlmostEqual(T.sigmoid(2) + T.sigmoid(-2), 1.0) def test_sigmoid_extremes(self): self.assertEqual(T.sigmoid(800), 1.0) self.assertAlmostEqual(T.sigmoid(-800), 0.0) def test_clip(self): self.assertEqual(T.clip(5, 0, 1), 1) self.assertEqual(T.clip(-5, 0, 1), 0) self.assertEqual(T.clip(0.5, 0, 1), 0.5) def test_normalize(self): self.assertEqual(T.normalize(" The ANSWER, is: 42! "), "the answer is 42") self.assertEqual(T.normalize("42."), "42") def test_ranks_ties(self): self.assertEqual(T.ranks([10, 20, 20, 30]), [1.0, 2.5, 2.5, 4.0]) def test_pearson_perfect(self): self.assertAlmostEqual(T.pearson([1, 2, 3], [2, 4, 6]), 1.0) self.assertAlmostEqual(T.pearson([1, 2, 3], [6, 4, 2]), -1.0) def test_pearson_constant(self): self.assertEqual(T.pearson([1, 1, 1], [1, 2, 3]), 0.0) def test_spearman_monotone(self): self.assertAlmostEqual(T.spearman([1, 2, 3, 4], [1, 8, 27, 64]), 1.0) def test_kendall_perfect_and_reverse(self): self.assertAlmostEqual(T.kendall_tau_b([1, 2, 3], [1, 2, 3]), 1.0) self.assertAlmostEqual(T.kendall_tau_b([1, 2, 3], [3, 2, 1]), -1.0) def test_kendall_ties_known_value(self): # x = 1,2,3,4 ; y = 1,1,2,2: concordant 4, tied-y 2, tau-b = 4 / sqrt(6 * 4) self.assertAlmostEqual(T.kendall_tau_b([1, 2, 3, 4], [1, 1, 2, 2]), 4 / math.sqrt(24)) def test_kendall_all_tied(self): self.assertEqual(T.kendall_tau_b([1, 1, 1], [1, 2, 3]), 0.0) def test_comb_edges(self): self.assertEqual(T.comb(5, 2), 10) self.assertEqual(T.comb(5, 6), 0) self.assertEqual(T.comb(5, -1), 0) def test_pass_at_k_known(self): self.assertAlmostEqual(T.pass_at_k(10, 3, 1), 0.3) self.assertAlmostEqual(T.pass_at_k(4, 1, 2), 0.5) self.assertEqual(T.pass_at_k(5, 0, 3), 0.0) self.assertEqual(T.pass_at_k(5, 5, 3), 1.0) def test_pass_hat_k_known(self): self.assertAlmostEqual(T.pass_hat_k(10, 3, 1), 0.3) self.assertAlmostEqual(T.pass_hat_k(4, 3, 2), 0.5) self.assertEqual(T.pass_hat_k(5, 2, 3), 0.0) def test_k_larger_than_n_raises(self): with self.assertRaises(ValueError): T.pass_at_k(3, 1, 4) with self.assertRaises(ValueError): T.pass_hat_k(3, 1, 4) def test_pass_hat_never_above_pass_at(self): for n in (5, 8): for c in range(n + 1): for k in range(1, n + 1): self.assertLessEqual(T.pass_hat_k(n, c, k), T.pass_at_k(n, c, k) + 1e-12) def test_wilson(self): lo, hi = T.wilson(5, 10) self.assertLess(lo, 0.5) self.assertGreater(hi, 0.5) self.assertEqual(T.wilson(0, 0), (0.0, 1.0)) self.assertEqual(T.wilson(0, 10)[0], 0.0) class TestCodeGraders(unittest.TestCase): def test_exact_match(self): self.assertTrue(T.exact_match("The Answer: 42.", "the answer 42")) self.assertFalse(T.exact_match("43", "42")) def test_contains_all(self): self.assertTrue(T.contains_all("Paris is the capital of France", ["paris", "FRANCE"])) self.assertFalse(T.contains_all("Paris", ["paris", "france"])) def test_contains_all_empty(self): self.assertTrue(T.contains_all("anything", [])) def test_regex(self): self.assertTrue(T.regex_match("order #123", r"#\d+")) self.assertFalse(T.regex_match("order", r"#\d+")) def test_first_number(self): self.assertEqual(T.first_number("cost is -3.5 dollars"), -3.5) self.assertIsNone(T.first_number("none")) def test_number_close(self): self.assertTrue(T.number_close("about 3.0001", 3.0, tol=0.001)) self.assertFalse(T.number_close("about 3.1", 3.0, tol=0.001)) self.assertFalse(T.number_close("no number", 3.0)) def test_json_fields(self): self.assertTrue(T.json_fields('{"a": 1, "b": 2}', {"a": 1})) self.assertFalse(T.json_fields('{"a": 2}', {"a": 1})) self.assertFalse(T.json_fields("not json", {"a": 1})) self.assertFalse(T.json_fields("[1, 2]", {"a": 1})) self.assertFalse(T.json_fields(None, {"a": 1})) def test_tool_calls_subsequence(self): self.assertTrue(T.tool_calls_ok(["search", "read", "write"], ["search", "write"])) self.assertFalse(T.tool_calls_ok(["write", "search"], ["search", "write"])) def test_tool_calls_forbidden(self): self.assertFalse(T.tool_calls_ok(["search", "delete"], ["search"], forbidden=["delete"])) def test_tool_calls_tuples(self): self.assertTrue(T.tool_calls_ok([("search", {"q": 1})], ["search"])) def test_run_assertions(self): share, ok, res = T.run_assertions("The answer is 42.", [("number", 42), ("contains", ["answer"]), ("exact", "x")]) self.assertEqual(res, [True, True, False]) self.assertFalse(ok) self.assertAlmostEqual(share, 2 / 3) def test_run_assertions_unknown(self): with self.assertRaises(ValueError): T.run_assertions("x", [("nope", 1)]) def test_run_assertions_empty(self): self.assertEqual(T.run_assertions("x", [])[:2], (0.0, True)) class TestWorld(unittest.TestCase): def test_size(self): self.assertEqual(len(R), 300) self.assertEqual(len(W["questions"]), 60) self.assertEqual(len(PAIRS), 600) def test_deterministic(self): W2 = T.make_world(0) self.assertEqual([r["text"] for r in W2["responses"]], [r["text"] for r in R]) self.assertEqual([r["h"] for r in W2["responses"]], [r["h"] for r in R]) def test_other_seed_differs(self): self.assertNotEqual(T.make_world(1)["responses"][0]["h"], R[0]["h"]) def test_unique_ids(self): self.assertEqual(len({r["id"] for r in R}), 300) def test_text_encodes_the_correct_trait(self): gold = {q["qid"]: q["gold"] for q in W["questions"]} for r in R: ans = int(r["text"].split("The answer is ")[1].split(".")[0]) self.assertEqual(ans == gold[r["qid"]], r["traits"]["correct"]) def test_words_count(self): for r in R[:20]: self.assertEqual(r["words"], len(r["text"].split())) def test_zlen_standardized(self): z = [r["zlen"] for r in R] self.assertAlmostEqual(T.mean(z), 0.0, places=9) self.assertAlmostEqual(T.stdev(z), 1.0, places=9) def test_system_quality_order(self): q = T.true_system_quality(W) self.assertEqual(max(q, key=q.get), "alpha") self.assertEqual(min(q, key=q.get), "epsilon") def test_epsilon_is_longest(self): by = {s: T.mean(r["words"] for r in R if r["system"] == s) for s in W["systems"]} self.assertEqual(max(by, key=by.get), "epsilon") def test_rating_of(self): self.assertEqual([T.rating_of(x) for x in (-1, 0.2, 0.6, 1.0, 1.4, 3)], [1, 2, 3, 4, 5, 5]) def test_human_pass_threshold(self): self.assertTrue(T.human_pass({"h": 0.9})) self.assertFalse(T.human_pass({"h": 0.89})) def test_pad_response(self): p = T.pad_response(W, R[0]) self.assertEqual(p["h"], R[0]["h"]) self.assertGreater(p["words"], R[0]["words"]) self.assertGreater(p["zlen"], R[0]["zlen"]) self.assertTrue(p["id"].endswith("+pad")) self.assertEqual(R[0]["id"] + "+pad", p["id"]) def test_pad_does_not_change_original(self): w = R[0]["words"] T.pad_response(W, R[0]) self.assertEqual(R[0]["words"], w) def test_human_prefers_deterministic(self): a, b = PAIRS[0] self.assertEqual(T.human_prefers(a, b), T.human_prefers(a, b)) def test_human_prefers_big_gap(self): hi = max(R, key=lambda r: r["h"]) lo = min(R, key=lambda r: r["h"]) self.assertEqual(T.human_prefers(hi, lo), "A") self.assertEqual(T.human_prefers(lo, hi), "B") def test_pair_list_sampling(self): s = T.pair_list(W, n_pairs=25, seed=3) self.assertEqual(len(s), 25) self.assertEqual(s, T.pair_list(W, n_pairs=25, seed=3)) def test_pairs_same_question_different_system(self): for a, b in PAIRS[:50]: self.assertEqual(a["qid"], b["qid"]) self.assertNotEqual(a["system"], b["system"]) class TestMockJudge(unittest.TestCase): def test_deterministic(self): j = T.MockJudge("x", seed=1) self.assertEqual(j.pointwise(R[0]), j.pointwise(R[0])) self.assertEqual(j.pairwise(R[0], R[1]), j.pairwise(R[0], R[1])) def test_k_changes_call_noise(self): j = T.MockJudge("x", sigma=1.0, seed=1) vals = {j.perceived(R[0], k) for k in range(5)} self.assertEqual(len(vals), 5) def test_zero_noise_matches_human(self): j = T.MockJudge("x", sigma=0, item_noise=0, seed=1) for r in R[:40]: self.assertEqual(j.pointwise(r), T.human_rating(r)) self.assertEqual(j.pass_label(r), T.human_pass(r)) def test_zero_noise_pairwise(self): j = T.MockJudge("x", sigma=0, item_noise=0, tie_margin=0.0) hi = max(R, key=lambda r: r["h"]) lo = min(R, key=lambda r: r["h"]) self.assertEqual(j.pairwise(hi, lo), "A") self.assertEqual(j.pairwise(lo, hi), "B") def test_tie_margin(self): j = T.MockJudge("x", sigma=0, item_noise=0, tie_margin=10.0) self.assertEqual(j.pairwise(R[0], R[1]), "tie") def test_pos_bias_flips_equal_pair(self): j = T.MockJudge("x", sigma=0, item_noise=0, pos_bias=0.5, tie_margin=0.0) r = R[0] self.assertEqual(j.pairwise(r, r), "A") def test_pos_bias_is_added_to_first_only(self): j = T.MockJudge("x", sigma=0, item_noise=0, pos_bias=0.5, tie_margin=0.0) a, b = R[0], R[1] d = a["h"] - b["h"] self.assertEqual(j.pairwise(a, b), "A" if d + 0.5 > 0 else "B") self.assertEqual(j.pairwise(b, a), "A" if -d + 0.5 > 0 else "B") def test_len_bias_raises_long(self): j = T.MockJudge("x", sigma=0, item_noise=0, len_bias=0.5) longest = max(R, key=lambda r: r["zlen"]) self.assertGreater(j.perceived(longest), longest["h"]) def test_self_bias_only_for_own_system(self): j = T.MockJudge("x", sigma=0, item_noise=0, self_bias=0.4, self_system="alpha") a = next(r for r in R if r["system"] == "alpha") b = next(r for r in R if r["system"] == "beta") self.assertAlmostEqual(j.perceived(a), a["h"] + 0.4) self.assertAlmostEqual(j.perceived(b), b["h"]) def test_pass_confidence_range_and_order(self): j = T.MockJudge("x", sigma=0, item_noise=0) hi = max(R, key=lambda r: r["h"]) lo = min(R, key=lambda r: r["h"]) self.assertGreater(j.pass_confidence(hi), 0.9) self.assertLess(j.pass_confidence(lo), 0.1) def test_taste_judge_ignores_h(self): j = T.MockJudge("t", sigma=0, item_noise=0, taste={"correct": 1.0}) for r in R[:30]: self.assertEqual(j.perceived(r), 1.0 if r["traits"]["correct"] else 0.0) def test_pointwise_range(self): j = T.MockJudge("x", sigma=2.0, item_noise=2.0) self.assertTrue(all(1 <= j.pointwise(r) <= 5 for r in R)) def test_different_names_differ(self): a, b = T.MockJudge("a"), T.MockJudge("b") self.assertNotEqual(a.perceived(R[0]), b.perceived(R[0])) class TestBiasMeasurement(unittest.TestCase): def test_perfect_judge_consistent(self): j = T.MockJudge("p", sigma=0, item_noise=0) r = T.position_report(j, PAIRS) self.assertEqual(r["consistency"], 1.0) self.assertAlmostEqual(r["first_rate"], 0.5, delta=0.05) def test_position_bias_lowers_consistency(self): base = T.position_report(T.MockJudge("p", seed=1), PAIRS) bias = T.position_report(T.MockJudge("p", pos_bias=0.5, seed=1), PAIRS) self.assertLess(bias["consistency"], base["consistency"]) self.assertGreater(bias["first_rate"], 0.6) def test_position_empty(self): self.assertEqual(T.position_report(T.MockJudge("p"), [])["consistency"], 0.0) def test_verbosity_unbiased_perfect_judge(self): j = T.MockJudge("p", sigma=0, item_noise=0, tie_margin=0.05) self.assertEqual(T.verbosity_report(j, W, R[:30])["padded_share"], 0.5) def test_verbosity_bias_detected(self): j = T.MockJudge("p", sigma=0, item_noise=0, len_bias=0.5, tie_margin=0.05) self.assertEqual(T.verbosity_report(j, W, R[:30])["padded_share"], 1.0) def test_verbosity_n(self): self.assertEqual(T.verbosity_report(T.MockJudge("p"), W, R[:10])["n"], 20) def test_self_pref_none_without_system(self): r = T.self_preference_report(T.MockJudge("p"), W) self.assertEqual(r["n"], 0) def test_self_pref_bias_raises_gap(self): a = T.self_preference_report(T.MockJudge("p", self_system="epsilon", seed=2), W) b = T.self_preference_report(T.MockJudge("p", self_system="epsilon", self_bias=0.5, seed=2), W) self.assertGreater(b["gap"], a["gap"] + 0.1) self.assertEqual(a["human_winrate"], b["human_winrate"]) def test_self_pref_n(self): r = T.self_preference_report(T.MockJudge("p", self_system="alpha"), W) self.assertEqual(r["n"], 2 * 4 * 60) def test_length_correlation_sign(self): j = T.MockJudge("p", len_bias=0.6, seed=1) c = T.length_correlation(j, R) self.assertGreater(c["judge"], c["human"]) class TestMitigations(unittest.TestCase): def test_swapped_mirror(self): j = T.MockJudge("p", pos_bias=0.4, seed=1) flip = {"a": "b", "b": "a", "tie": "tie"} for a, b in PAIRS[:120]: self.assertEqual(T.swapped_verdict(j, a, b), flip[T.swapped_verdict(j, b, a)]) def test_swapped_pos_bias_only_ties(self): j = T.MockJudge("p", sigma=0, item_noise=0, pos_bias=10.0, tie_margin=0.0) self.assertTrue(all(T.swapped_verdict(j, a, b) == "tie" for a, b in PAIRS[:50])) def test_swapped_fewer_wrong(self): j = T.MockJudge("p", pos_bias=0.4, seed=2) truth = ["a" if a["h"] > b["h"] else "b" for a, b in PAIRS] raw = [{"A": "a", "B": "b", "tie": "tie"}[j.pairwise(a, b)] for a, b in PAIRS] sw = [T.swapped_verdict(j, a, b) for a, b in PAIRS] wrong = lambda v: sum(x not in (t, "tie") for x, t in zip(v, truth)) self.assertLess(wrong(sw), wrong(raw) / 3) self.assertGreater(sw.count("tie"), raw.count("tie")) def test_random_order_unbiased_on_average(self): j = T.MockJudge("p", sigma=0, item_noise=0, pos_bias=10.0, tie_margin=0.0) rng = random.Random(0) v = [T.random_order_verdict(j, a, b, rng) for a, b in PAIRS[:400]] self.assertAlmostEqual(v.count("a") / len(v), 0.5, delta=0.1) self.assertEqual(v.count("tie"), 0) def test_random_order_deterministic_with_seed(self): j = T.MockJudge("p", seed=3) v1 = [T.random_order_verdict(j, a, b, random.Random(7)) for a, b in PAIRS[:30]] v2 = [T.random_order_verdict(j, a, b, random.Random(7)) for a, b in PAIRS[:30]] self.assertEqual(v1, v2) def test_panel_excludes_own(self): own = T.MockJudge("o", self_system="alpha", self_bias=5.0, sigma=0, item_noise=0) fair = T.MockJudge("f", sigma=0, item_noise=0, tie_margin=0.0) a = next(r for r in R if r["system"] == "alpha") b = next(r for r in R if r["system"] == "epsilon" and r["qid"] == a["qid"]) self.assertEqual(T.panel_verdict([own], a, b), "tie") # no voters self.assertEqual(T.panel_verdict([own, fair], a, b), "a" if a["h"] > b["h"] else "b") def test_panel_includes_own_when_asked(self): own = T.MockJudge("o", self_system="epsilon", self_bias=5.0, sigma=0, item_noise=0, tie_margin=0.0) a = next(r for r in R if r["system"] == "alpha") b = next(r for r in R if r["system"] == "epsilon" and r["qid"] == a["qid"]) self.assertEqual(T.panel_verdict([own], a, b, exclude_own=False, swap=False), "b") def test_panel_tie_on_split(self): j1 = T.MockJudge("1", sigma=0, item_noise=0, len_bias=0, tie_margin=0.0) j2 = T.MockJudge("2", sigma=0, item_noise=0, tie_margin=0.0, taste={"correct": -1.0}) hi = max(R, key=lambda r: r["h"]) lo = min(R, key=lambda r: r["h"] if not r["traits"]["correct"] else 9) self.assertTrue(hi["traits"]["correct"] and not lo["traits"]["correct"]) self.assertEqual(T.panel_verdict([j1], hi, lo, swap=False), "a") self.assertEqual(T.panel_verdict([j2], hi, lo, swap=False), "b") self.assertEqual(T.panel_verdict([j1, j2], hi, lo, swap=False), "tie") def test_fit_logistic_recovers(self): rng = random.Random(0) X = [[rng.gauss(0, 1)] for _ in range(2000)] y = [1.0 if rng.random() < T.sigmoid(0.5 + 1.5 * x[0]) else 0.0 for x in X] w = T.fit_logistic(X, y, iters=400, lr=1.0) self.assertAlmostEqual(w[0], 0.5, delta=0.2) self.assertAlmostEqual(w[1], 1.5, delta=0.3) def test_lc_no_length_effect(self): rows = [{"len_diff": d, "pref": p} for d, p in [(-1, 1), (1, 0), (-1, 0), (1, 1)] * 10] r = T.length_controlled_winrate(rows) self.assertAlmostEqual(r["controlled"], 0.5, delta=0.05) def test_lc_removes_length_effect(self): rng = random.Random(1) rows = [] for _ in range(1000): d = rng.choice([-1.0, 1.0]) rows.append({"len_diff": d, "pref": 1.0 if rng.random() < T.sigmoid(1.5 * d) else 0.0}) r = T.length_controlled_winrate(rows, iters=400, lr=1.0) self.assertAlmostEqual(r["controlled"], 0.5, delta=0.08) self.assertGreater(r["length_coef"], 1.0) def test_lc_wordy_judge_demo(self): wordy = T.MockJudge("wordy", len_bias=0.25, seed=3) rows = T.system_pair_rows(W, wordy, "epsilon", "delta") r = T.length_controlled_winrate(rows) self.assertLess(r["controlled"], r["raw"] - 0.2) def test_system_pair_rows_count(self): rows = T.system_pair_rows(W, T.MockJudge("p"), "alpha", "beta") self.assertEqual(len(rows), 120) self.assertEqual(len(T.system_pair_rows(W, T.MockJudge("p"), "alpha", "beta", both_orders=False)), 60) class TestRanking(unittest.TestCase): systems = ["a", "b", "c"] def test_bt_order(self): comps = [("a", "b", 1.0)] * 8 + [("a", "b", 0.0)] * 2 + [("b", "c", 1.0)] * 8 + [("b", "c", 0.0)] * 2 + [("a", "c", 1.0)] * 9 + [("a", "c", 0.0)] bt = T.bradley_terry(comps, self.systems) self.assertGreater(bt["a"], bt["b"]) self.assertGreater(bt["b"], bt["c"]) def test_bt_mean_zero(self): comps = [("a", "b", 1.0), ("b", "c", 0.0), ("a", "c", 0.5)] * 5 bt = T.bradley_terry(comps, self.systems) self.assertAlmostEqual(sum(bt.values()), 0.0, places=6) def test_bt_two_equal_systems(self): comps = [("a", "b", 1.0), ("a", "b", 0.0)] * 20 bt = T.bradley_terry(comps, ["a", "b"]) self.assertAlmostEqual(bt["a"], bt["b"], places=6) def test_bt_known_odds(self): comps = [("a", "b", 1.0)] * 75 + [("a", "b", 0.0)] * 25 bt = T.bradley_terry(comps, ["a", "b"], prior=0.0) self.assertAlmostEqual(bt["a"] - bt["b"], math.log(3), places=4) def test_bt_finite_with_no_losses(self): comps = [("a", "b", 1.0)] * 10 bt = T.bradley_terry(comps, ["a", "b"]) self.assertTrue(all(math.isfinite(v) for v in bt.values())) def test_bt_ties_count_half(self): comps = [("a", "b", 0.5)] * 10 bt = T.bradley_terry(comps, ["a", "b"]) self.assertAlmostEqual(bt["a"], bt["b"], places=6) def test_to_elo_scale(self): e = T.to_elo({"a": math.log(10), "b": 0.0}) self.assertAlmostEqual(e["a"] - e["b"], 400.0) def test_elo_updates_zero_sum(self): e = T.elo_online([("a", "b", 1.0)] * 5) self.assertAlmostEqual(e["a"] + e["b"], 2000.0) self.assertGreater(e["a"], e["b"]) def test_elo_order_dependence(self): comps = [("a", "b", 1.0), ("b", "c", 1.0), ("a", "c", 0.0), ("c", "b", 1.0)] * 3 vals = {round(T.elo_online(comps, seed=s)["a"], 3) for s in range(6)} self.assertGreater(len(vals), 1) def test_elo_seed_repeatable(self): comps = [("a", "b", 1.0), ("b", "c", 0.0)] * 5 self.assertEqual(T.elo_online(comps, seed=2), T.elo_online(comps, seed=2)) def test_comparisons_from(self): comps = T.comparisons_from(W, lambda a, b: "a") self.assertEqual(len(comps), 600) self.assertTrue(all(o == 1.0 for _, _, o in comps)) comps = T.comparisons_from(W, lambda a, b: "tie") self.assertTrue(all(o == 0.5 for _, _, o in comps)) def test_bt_on_world_matches_truth_with_good_judge(self): j = T.MockJudge("p", sigma=0.1, item_noise=0.05, seed=1) comps = T.comparisons_from(W, lambda a, b: T.swapped_verdict(j, a, b)) bt = T.bradley_terry(comps, W["systems"]) order = sorted(bt, key=bt.get) self.assertEqual((order[0], order[-1]), ("epsilon", "alpha")) self.assertEqual(order.index("beta"), 3) def test_bootstrap_interval_contains_point(self): comps = T.comparisons_from(W, lambda a, b: T.swapped_verdict(T.MockJudge("p", seed=1), a, b)) bt = T.bradley_terry(comps, W["systems"]) b = T.bootstrap_bt(comps, W["systems"], n_boot=30) for s in W["systems"]: lo, hi = b["ci"][s] self.assertLessEqual(lo, hi) self.assertLessEqual(lo - 0.1, bt[s]) self.assertGreaterEqual(hi + 0.1, bt[s]) self.assertEqual(sum(b["rank_counts"][s].values()), 30) def test_pointwise_system_scores(self): s = T.pointwise_system_scores(W, T.MockJudge("p", seed=1)) self.assertGreater(s["alpha"], s["epsilon"]) def test_ranking_agreement(self): r = T.ranking_agreement({"a": 3, "b": 2, "c": 1}, {"a": 9, "b": 5, "c": 1}) self.assertEqual(r["kendall"], 1.0) r = T.ranking_agreement({"a": 1, "b": 2, "c": 3}, {"a": 9, "b": 5, "c": 1}) self.assertEqual(r["kendall"], -1.0) def test_wordy_judge_flips_close_systems(self): w = T.MockJudge("wordy", len_bias=0.25, seed=3) pw = T.pointwise_system_scores(W, w) tq = T.true_system_quality(W) self.assertGreater(pw["epsilon"], pw["delta"]) self.assertLess(tq["epsilon"], tq["delta"]) class TestAgreement(unittest.TestCase): def test_percent_agreement(self): self.assertEqual(T.percent_agreement([1, 2, 3, 4], [1, 2, 0, 0]), 0.5) def test_kappa_perfect_and_chance(self): self.assertEqual(T.cohen_kappa([1, 0, 1, 0], [1, 0, 1, 0]), 1.0) self.assertAlmostEqual(T.cohen_kappa([1, 1, 0, 0], [1, 0, 1, 0]), 0.0) def test_kappa_known_value(self): # 2x2: both yes 20, both no 15, a yes b no 5, a no b yes 10 (n=50). po = .7, pe = .5*.6+.5*.4 = .5? compute below a = [1] * 20 + [0] * 15 + [1] * 5 + [0] * 10 b = [1] * 20 + [0] * 15 + [0] * 5 + [1] * 10 po = 35 / 50 pe = (25 / 50) * (30 / 50) + (25 / 50) * (20 / 50) self.assertAlmostEqual(T.cohen_kappa(a, b), (po - pe) / (1 - pe)) def test_kappa_always_pass_is_zero(self): human = [True] * 95 + [False] * 5 judge = [True] * 100 self.assertAlmostEqual(T.percent_agreement(judge, human), 0.95) self.assertEqual(T.cohen_kappa(judge, human), 0.0) def test_kappa_constant_both(self): self.assertEqual(T.cohen_kappa([1, 1], [1, 1]), 0.0) def test_weighted_kappa_perfect(self): self.assertAlmostEqual(T.weighted_kappa([1, 2, 3, 4, 5], [1, 2, 3, 4, 5]), 1.0) def test_weighted_kappa_near_miss_beats_far_miss(self): a = [1, 2, 3, 4, 5, 3] near = [2, 3, 4, 5, 4, 3] far = [5, 4, 1, 1, 1, 3] self.assertGreater(T.weighted_kappa(a, near), T.weighted_kappa(a, far)) def test_confusion(self): j = [False, False, True, True] h = [False, True, False, True] self.assertEqual(T.confusion(j, h), (1, 1, 1, 1)) def test_tpr_tnr(self): j = [False, False, True, True, True] h = [False, True, True, True, False] r = T.tpr_tnr(j, h) self.assertAlmostEqual(r["tpr"], 0.5) self.assertAlmostEqual(r["tnr"], 2 / 3) def test_tpr_tnr_empty_class(self): self.assertEqual(T.tpr_tnr([True], [True]), {"tpr": 0.0, "tnr": 1.0}) def test_reliability_bins_sum(self): confs = [0.05, 0.15, 0.95, 0.99, 1.0] b = T.reliability_bins(confs, [0, 0, 1, 1, 1]) self.assertEqual(sum(x["n"] for x in b), 5) self.assertEqual(b[9]["n"], 3) def test_ece_perfect(self): confs = [1.0] * 10 + [0.0] * 10 labels = [True] * 10 + [False] * 10 self.assertEqual(T.expected_calibration_error(confs, labels), 0.0) def test_ece_overconfident(self): confs = [0.9] * 10 labels = [True] * 5 + [False] * 5 self.assertAlmostEqual(T.expected_calibration_error(confs, labels), 0.4) def test_platt_improves(self): rng = random.Random(0) scores = [rng.random() for _ in range(600)] labels = [rng.random() < T.sigmoid(8 * (s - 0.5)) * 0.5 + 0.0 for s in scores] # a judge whose confidence is too high before = T.expected_calibration_error(scores, labels) a, b = T.platt_fit(scores, labels, iters=3000, lr=2.0) after = T.expected_calibration_error([T.sigmoid(a + b * s) for s in scores], labels) self.assertLess(after, before) def test_score_to_pass_rate(self): r = T.score_to_pass_rate([1, 1, 2, 2], [False, False, True, False]) self.assertEqual(r[1], (0.0, 2)) self.assertEqual(r[2], (0.5, 2)) def test_imbalanced_threshold(self): thr = T.imbalanced_labels(W, 0.05) share = sum(r["h"] < thr for r in R) / len(R) self.assertAlmostEqual(share, 0.05, delta=0.01) def test_fair_judge_has_decent_agreement(self): j = T.MockJudge("fair", seed=1) jl = [j.pass_label(r) for r in R] hl = [T.human_pass(r) for r in R] self.assertGreater(T.cohen_kappa(jl, hl), 0.7) def test_code_grader_vs_human(self): gold = {q["qid"]: q["gold"] for q in W["questions"]} right = [T.number_close(r["text"].split("The answer is ")[1], gold[r["qid"]]) for r in R] self.assertEqual(right, [r["traits"]["correct"] for r in R]) self.assertLess(T.percent_agreement(right, [T.human_pass(r) for r in R]), 1.0) class TestPassK(unittest.TestCase): def test_graded_runs_shape(self): t, j = T.graded_runs([0.5] * 7, 6, seed=1) self.assertEqual((len(t), len(t[0])), (7, 6)) self.assertEqual((len(j), len(j[0])), (7, 6)) def test_perfect_grader_equal(self): t, j = T.graded_runs([0.3, 0.8] * 10, 8, tpr=1.0, tnr=1.0, seed=2) self.assertEqual(t, j) def test_deterministic(self): self.assertEqual(T.graded_runs([0.5] * 5, 4, 0.9, 0.9, seed=1), T.graded_runs([0.5] * 5, 4, 0.9, 0.9, seed=1)) def test_curves_monotone(self): t, _ = T.graded_runs([0.2, 0.5, 0.9] * 10, 12, seed=3) c = T.curves_from(t, [1, 2, 4, 8, 12]) self.assertTrue(all(x <= y + 1e-12 for x, y in zip(c["pass_at_k"], c["pass_at_k"][1:]))) self.assertTrue(all(x >= y - 1e-12 for x, y in zip(c["pass_hat_k"], c["pass_hat_k"][1:]))) def test_noisy_grader_lowers_pass_hat(self): t, j = T.graded_runs([0.95] * 40, 16, tpr=0.92, tnr=0.9, seed=1) ct, cj = T.curves_from(t, [8]), T.curves_from(j, [8]) self.assertLess(cj["pass_hat_k"][0], ct["pass_hat_k"][0]) def test_noisy_grader_raises_pass_at_for_weak_tasks(self): t, j = T.graded_runs([0.05] * 60, 16, tpr=0.9, tnr=0.9, seed=1) ct, cj = T.curves_from(t, [4]), T.curves_from(j, [4]) self.assertGreater(cj["pass_at_k"][0], ct["pass_at_k"][0]) def test_curve_k1_equal(self): t, _ = T.graded_runs([0.4] * 30, 10, seed=4) c = T.curves_from(t, [1]) self.assertAlmostEqual(c["pass_at_k"][0], c["pass_hat_k"][0]) def test_judge_rerun_zero_noise(self): j = T.MockJudge("p", sigma=0, item_noise=0) r = T.judge_rerun_agreement(j, R[:50]) self.assertEqual(r, {"all_same": 1.0, "flip_rate": 0.0}) def test_judge_rerun_noisy(self): j = T.MockJudge("p", sigma=0.4) r = T.judge_rerun_agreement(j, R[:100]) self.assertLess(r["all_same"], 1.0) self.assertGreater(r["flip_rate"], 0.0) class TestInduction(unittest.TestCase): def test_solve_linear(self): x = T.solve_linear([[2, 1], [1, 3]], [5, 10]) self.assertAlmostEqual(x[0], 1.0) self.assertAlmostEqual(x[1], 3.0) def test_solve_singular(self): with self.assertRaises(ValueError): T.solve_linear([[1, 2], [2, 4]], [1, 2]) def test_zscore(self): Z, mu, sd = T.zscore_cols([[1, 10], [3, 10], [5, 10]]) self.assertAlmostEqual(T.mean([r[0] for r in Z]), 0.0) self.assertEqual([r[1] for r in Z], [0.0, 0.0, 0.0]) # a constant column does not blow up def test_ridge_recovers(self): rng = random.Random(0) Z = [[rng.gauss(0, 1), rng.gauss(0, 1)] for _ in range(500)] y = [3 * z[0] - 1 * z[1] + 5 + rng.gauss(0, 0.1) for z in Z] c, my = T.ridge_fit(Z, y, lam=0.01) self.assertAlmostEqual(c[0], 3, delta=0.05) self.assertAlmostEqual(c[1], -1, delta=0.05) self.assertAlmostEqual(my, 5, delta=0.2) def test_ridge_shrinks(self): rng = random.Random(0) Z = [[rng.gauss(0, 1)] for _ in range(100)] y = [2 * z[0] for z in Z] self.assertLess(abs(T.ridge_fit(Z, y, 1000.0)[0][0]), abs(T.ridge_fit(Z, y, 0.1)[0][0])) def test_criterion_scores_shape_and_determinism(self): X = T.criterion_scores(R[:10]) self.assertEqual((len(X), len(X[0])), (10, len(T.CRITERIA))) self.assertEqual(X, T.criterion_scores(R[:10])) self.assertNotEqual(X, T.criterion_scores(R[:10], seed=1)) def test_criteria_look_at_text(self): r = next(r for r in R if r["traits"]["steps"]) self.assertEqual(T.CRITERIA["shows_steps"][0](r), 1.0) r = next(r for r in R if not r["traits"]["steps"]) self.assertEqual(T.CRITERIA["shows_steps"][0](r), 0.0) def test_induce_selects_known_drivers(self): tr = R[:200] m = T.induce_metric(tr, [T.human_rating(r) for r in tr], n_keep=4) self.assertIn("answer_looks_right", m["names"]) self.assertEqual(len(m["names"]), len(m["weights"])) def test_induce_predict_length(self): tr = R[:100] m = T.induce_metric(tr, [T.human_rating(r) for r in tr]) self.assertEqual(len(m["predict"](R[100:130])), 30) def test_induce_beats_random(self): tr, te = R[:150], R[150:] m = T.induce_metric(tr, [T.human_rating(r) for r in tr]) self.assertGreater(T.kendall_tau_b(m["predict"](te), [T.human_rating(r) for r in te]), 0.4) def test_drop_negative(self): tr = R[:200] y = [T.human_rating(r) for r in tr] m = T.induce_metric(tr, y, n_keep=8, drop_negative=True) self.assertTrue(all(w >= 0 for w in m["weights"])) def test_induction_curve_shape(self): rows = T.induction_curve(W, sizes=(20, 80), repeats=3) self.assertEqual([r["n"] for r in rows], [20, 80]) self.assertTrue(all(set(r) == {"n", "induced", "baseline", "taste"} for r in rows)) def test_induced_beats_mismatched_judge_with_labels(self): rows = T.induction_curve(W, sizes=(80,), repeats=5) self.assertGreater(rows[0]["induced"], rows[0]["taste"]) def test_more_labels_help(self): rows = T.induction_curve(W, sizes=(10, 160), repeats=8) self.assertGreater(rows[1]["induced"], rows[0]["induced"]) class TestHarness(unittest.TestCase): tasks = [{"id": f"t{i}", "input": i} for i in range(12)] @staticmethod def fn(task, rng): return round(rng.random() + task["input"], 6) def test_run_counts(self): h = T.EvalHarness(self.fn) out = h.run(self.tasks, trials=3) self.assertEqual(len(out), 36) self.assertTrue(all(o["status"] == "ok" for o in out)) def test_trials_differ_seeds_same(self): h = T.EvalHarness(self.fn, seed=1) a = [o["result"] for o in h.run(self.tasks[:1], trials=3)] self.assertEqual(len(set(a)), 3) h2 = T.EvalHarness(self.fn, seed=1) self.assertEqual(a, [o["result"] for o in h2.run(self.tasks[:1], trials=3)]) def test_order_independence(self): h1 = T.EvalHarness(self.fn, seed=2) h2 = T.EvalHarness(self.fn, seed=2) r1 = {(o["task"], o["trial"]): o["result"] for o in h1.run(self.tasks, 2)} r2 = {(o["task"], o["trial"]): o["result"] for o in h2.run(list(reversed(self.tasks)), 2)} self.assertEqual(r1, r2) def test_seed_changes_results(self): a = T.EvalHarness(self.fn, seed=1).run(self.tasks[:3]) b = T.EvalHarness(self.fn, seed=2).run(self.tasks[:3]) self.assertNotEqual([o["result"] for o in a], [o["result"] for o in b]) def test_cache_hit(self): calls = [] h = T.EvalHarness(lambda t, r: calls.append(1) or 1) h.run(self.tasks, 1) h.run(self.tasks, 1) self.assertEqual(len(calls), 12) self.assertEqual(h.summary()["cached"], 12) def test_version_busts_cache(self): calls = [] fn = lambda t, r: calls.append(1) or 1 h = T.EvalHarness(fn, version="v1") h.run(self.tasks, 1) T.EvalHarness(fn, version="v2", cache=h.cache).run(self.tasks, 1) self.assertEqual(len(calls), 24) def test_config_busts_cache(self): h = T.EvalHarness(self.fn, config={"temp": 0}) h2 = T.EvalHarness(self.fn, config={"temp": 1}) self.assertNotEqual(h.key(self.tasks[0], 0), h2.key(self.tasks[0], 0)) def test_key_includes_input_and_trial(self): h = T.EvalHarness(self.fn) self.assertNotEqual(h.key({"id": "a", "input": 1}, 0), h.key({"id": "a", "input": 2}, 0)) self.assertNotEqual(h.key(self.tasks[0], 0), h.key(self.tasks[0], 1)) def test_retry_then_success(self): state = {"n": 0} def fn(task, rng): state["n"] += 1 if state["n"] < 3: raise T.TransientError("x") return 5 h = T.EvalHarness(fn, max_retries=3, backoff=1.0) out = h.run_one({"id": "a"}) self.assertEqual(out["status"], "ok") s = h.summary() self.assertEqual(s["retries"], 2) self.assertEqual(s["slept"], 3.0) # 1 + 2 def test_retry_gives_up(self): def fn(task, rng): raise T.TransientError("x") h = T.EvalHarness(fn, max_retries=2, backoff=1.0) out = h.run_one({"id": "a"}) self.assertEqual(out["status"], "error") s = h.summary() self.assertEqual((s["retries"], s["errors"], s["slept"]), (2, 1, 3.0)) self.assertEqual(h.cache, {}) def test_other_errors_propagate(self): def fn(task, rng): raise KeyError("bug") with self.assertRaises(KeyError): T.EvalHarness(fn).run_one({"id": "a"}) def test_errors_not_cached_and_rerun_succeeds(self): flaky = T.FlakyBackend(self.fn, fail_rate=1.0) h = T.EvalHarness(flaky, max_retries=1) self.assertEqual(h.run_one(self.tasks[0])["status"], "error") flaky.fail_rate = 0.0 self.assertEqual(h.run_one(self.tasks[0])["status"], "ok") def test_sleep_injected(self): waits = [] h = T.EvalHarness(T.FlakyBackend(self.fn, fail_rate=0.6, seed=1), sleep=waits.append, max_retries=5) h.run(self.tasks, 1) self.assertEqual(sum(waits), h.summary()["slept"]) self.assertGreater(len(waits), 0) def test_flaky_does_not_consume_the_generator(self): r1, r2 = random.Random(3), random.Random(3) T.FlakyBackend(lambda t, r: 1, fail_rate=0.0)({"id": "a"}, r1) self.assertEqual(r1.random(), r2.random()) def test_flaky_deterministic(self): def run(): h = T.EvalHarness(T.FlakyBackend(self.fn, fail_rate=0.3, seed=5), seed=1) h.run(self.tasks, 2) return h.summary() self.assertEqual(run(), run()) def test_flaky_results_equal_clean_results(self): clean = T.EvalHarness(self.fn, seed=4).run(self.tasks, 2) flaky = T.EvalHarness(T.FlakyBackend(self.fn, fail_rate=0.3, seed=5), seed=4, max_retries=8).run(self.tasks, 2) self.assertEqual([o["result"] for o in clean], [o["result"] for o in flaky]) def test_budget(self): h = T.EvalHarness(self.fn, budget=5.0) out = h.run(self.tasks, 1) self.assertEqual([o["status"] for o in out].count("ok"), 5) self.assertEqual([o["status"] for o in out].count("skipped"), 7) self.assertEqual(h.summary()["spent"], 5.0) def test_cost_fn(self): h = T.EvalHarness(self.fn, budget=1.0, cost_fn=lambda t, r: 10.0) out = h.run(self.tasks[:3]) self.assertEqual([o["status"] for o in out], ["ok", "skipped", "skipped"]) def test_cached_runs_cost_nothing(self): h = T.EvalHarness(self.fn, budget=12.0) h.run(self.tasks, 1) h.run(self.tasks, 1) self.assertEqual(h.summary()["spent"], 12.0) def test_log_records(self): h = T.EvalHarness(T.FlakyBackend(self.fn, fail_rate=0.5, seed=2), max_retries=6) h.run(self.tasks[:4], 1) self.assertEqual([r["step"] for r in h.log], list(range(1, len(h.log) + 1))) self.assertTrue(all({"task", "trial", "key", "version", "status", "attempt"} <= set(r) for r in h.log)) def test_log_file_jsonl(self): with tempfile.TemporaryDirectory() as d: p = os.path.join(d, "log.jsonl") h = T.EvalHarness(self.fn, log_path=p) h.run(self.tasks[:3], 2) with open(p) as f: rows = [json.loads(line) for line in f] self.assertEqual(len(rows), 6) self.assertEqual(rows, [json.loads(json.dumps(r, default=str)) for r in h.log]) def test_summary_zero(self): self.assertEqual(T.EvalHarness(self.fn).summary()["ok"], 0) class TestDefaultsAndEdges(unittest.TestCase): """Added after a light mutation run: defaults, empty cases and pinned constants.""" def test_pass_k_with_k_equal_n(self): self.assertEqual(T.pass_at_k(5, 2, 5), 1.0) self.assertEqual(T.pass_at_k(5, 0, 5), 0.0) self.assertEqual(T.pass_hat_k(5, 5, 5), 1.0) self.assertEqual(T.pass_hat_k(5, 4, 5), 0.0) def test_kendall_joint_ties_are_skipped(self): self.assertAlmostEqual(T.kendall_tau_b([1, 1, 2], [1, 1, 3]), 1.0) def test_judge_defaults_equal_k0(self): j = T.MockJudge("d", seed=2) r = R[3] self.assertEqual(j.pointwise(r), j.pointwise(r, 0)) self.assertEqual(j.perceived(r), j.perceived(r, 0, "")) self.assertEqual(j.pairwise(R[0], R[1]), j.pairwise(R[0], R[1], 0)) self.assertEqual(j.pass_label(r), j.pass_label(r, T.PASS_THRESHOLD, 0)) self.assertEqual(j.pass_confidence(r), j.pass_confidence(r, T.PASS_THRESHOLD, 0.35, 0)) def test_random_order_default_k0(self): j = T.MockJudge("d", seed=2) a = [T.random_order_verdict(j, x, y, random.Random(1)) for x, y in PAIRS[:60]] b = [T.random_order_verdict(j, x, y, random.Random(1), 0) for x, y in PAIRS[:60]] self.assertEqual(a, b) def test_bootstrap_default_seed(self): comps = T.comparisons_from(W, lambda a, b: T.swapped_verdict(T.MockJudge("p", seed=1), a, b)) d = T.bootstrap_bt(comps, W["systems"], n_boot=5) e = T.bootstrap_bt(comps, W["systems"], n_boot=5, seed=0) self.assertEqual(d["ci"], e["ci"]) def test_self_pref_empty_values(self): r = T.self_preference_report(T.MockJudge("p"), W) self.assertEqual((r["judge_winrate"], r["human_winrate"], r["gap"], r["n"]), (0.0, 0.0, 0.0, 0)) def test_reliability_bins_empty_bin(self): b = T.reliability_bins([0.95], [True]) self.assertEqual(b[0], {"n": 0, "conf": 0.0, "freq": 0.0}) def test_graded_runs_default_is_perfect(self): t, j = T.graded_runs([0.5] * 10, 6, seed=3) self.assertEqual(t, j) def test_judge_rerun_flip_rate_exact(self): class Stub: def pass_label(self, r, thr=0.9, k=0): return [True, False, False, False, False][k] out = T.judge_rerun_agreement(Stub(), [{"id": 1}], runs=5) self.assertEqual(out, {"all_same": 0.0, "flip_rate": 1.0}) class Stub2: def pass_label(self, r, thr=0.9, k=0): return [True, True, False, True, True][k] self.assertEqual(T.judge_rerun_agreement(Stub2(), [{"id": 1}], runs=5)["flip_rate"], 0.25) def test_criteria_values_and_noise_levels(self): for nm, (fn, sd) in T.CRITERIA.items(): vals = {fn(r) for r in R} if nm in ("is_long", "sentence_count"): self.assertTrue(all(0.0 <= v <= 1.0 for v in vals)) elif nm == "mentions_answer": self.assertEqual(vals, {1.0}) # every toy response states an answer, so this criterion is constant else: self.assertEqual(vals, {0.0, 1.0}, nm) sds = {nm: sd for nm, (fn, sd) in T.CRITERIA.items()} self.assertEqual(sds, {"shows_steps": 0.20, "cites_a_source": 0.20, "hedges": 0.20, "polite_closing": 0.20, "is_long": 0.10, "has_filler": 0.20, "answer_looks_right": 0.45, "mentions_answer": 0.05, "uses_because": 0.20, "exclamation": 0.10, "sentence_count": 0.10, "starts_with_then": 0.20}) def test_taste_pinned(self): self.assertEqual(T.TASTE, {"correct": 0.6, "steps": 0.1, "source": 0.0, "hedge": 0.0, "polite": 0.4, "zlen": 0.3}) def test_predict_matches_manual_formula(self): tr = R[:120] y = [T.human_rating(r) for r in tr] m = T.induce_metric(tr, y, n_keep=4) Z, _, _ = T.zscore_cols(T.criterion_scores(tr, m["names"])) expect = [T.mean(y) + sum(w * z for w, z in zip(m["weights"], row)) for row in Z] got = m["predict"](tr) for a, b in zip(expect, got): self.assertAlmostEqual(a, b, places=9) def test_drop_negative_removes_exactly_the_negatives(self): tr = R[:200] y = [T.human_rating(r) for r in tr] keep = T.induce_metric(tr, y, n_keep=8) drop = T.induce_metric(tr, y, n_keep=8, drop_negative=True) self.assertTrue(any(w < 0 for w in keep["weights"])) self.assertEqual(len(keep["names"]), 8) self.assertEqual(drop["names"], [n for n, w in zip(keep["names"], keep["weights"]) if w >= 0]) def test_drop_negative_off_when_all_positive(self): tr = R[:200] y = [T.human_rating(r) for r in tr] a = T.induce_metric(tr, y, n_keep=3, drop_negative=True) b = T.induce_metric(tr, y, n_keep=3) self.assertTrue(all(w > 0 for w in b["weights"])) self.assertEqual(a["names"], b["names"]) def test_human_prefers_pinned(self): got = "".join(T.human_prefers(a, b)[0] for a, b in PAIRS[:40]) self.assertEqual(got, "BABBABABBABABAABABBAAAAABtBAABAAAAAAAtBB") def test_human_prefers_tie_zone(self): a = dict(R[0]); b = dict(R[1]) b["h"] = a["h"] v = {T.human_prefers(a, b, noise=0.0)} self.assertEqual(v, {"tie"}) def test_imbalanced_labels_signature(self): self.assertEqual(T.imbalanced_labels(W, 0.0), min(r["h"] for r in R)) def test_fit_logistic_default_pinned(self): w = T.fit_logistic([[0.0], [1.0], [2.0], [3.0]], [0, 0, 1, 1]) self.assertAlmostEqual(w[0], -4.705169887236282, places=9) self.assertAlmostEqual(w[1], 3.3253805705039285, places=9) def test_number_close_boundary_is_inclusive(self): self.assertTrue(T.number_close("5", 5, tol=0)) self.assertTrue(T.number_close("5.5", 5, tol=0.5)) def test_induce_default_keeps_five(self): tr = R[:150] m = T.induce_metric(tr, [T.human_rating(r) for r in tr]) self.assertEqual(len(m["names"]), 5) def test_bootstrap_default_count(self): comps = T.comparisons_from(W, lambda a, b: "a" if a["h"] > b["h"] else "b")[:80] b = T.bootstrap_bt(comps, W["systems"]) self.assertEqual(sum(b["rank_counts"]["alpha"].values()), 100) def test_position_report_all_ties(self): j = T.MockJudge("p", tie_margin=99.0) r = T.position_report(j, PAIRS[:20]) self.assertEqual((r["consistency"], r["first_rate"], r["pairs"]), (1.0, 0.0, 20)) class TestMain(unittest.TestCase): @classmethod def setUpClass(cls): buf = io.StringIO() with contextlib.redirect_stdout(buf): T.main() cls.out = buf.getvalue() def test_ten_sections(self): for i in range(1, 11): self.assertIn(f"== {i}. ", self.out) def test_deterministic(self): buf = io.StringIO() with contextlib.redirect_stdout(buf): T.main() self.assertEqual(buf.getvalue(), self.out) def test_pinned_lines(self): for s in [ "number_close finds the right answer in 189 of 300 responses", "but a human passes 151 of 300", "fair consistency 0.772 first-slot rate 0.501", "first consistency 0.635 first-slot rate 0.652", "fair padded copy gets 0.440 of the credit", "wordy padded copy gets 0.845 of the credit", "selfish judge win rate 0.428 human win rate 0.267 gap +0.161", "first-biased judge, raw order : (473, 82, 45)", "first-biased judge, swapped : (370, 7, 223)", "raw win rate 0.550, length-controlled 0.216, human (from h) 0.300", "fair judge pass/fail: agreement 0.900, kappa 0.800", "a judge that always says pass has agreement 0.953, kappa 0.000", "'alpha': 1103, 'beta': 1023, 'gamma': 987, 'delta': 994, 'epsilon': 893", "judged pass^k [0.7, 0.518, 0.313, 0.145, 0.067]", "first run : {'ok': 60, 'cached': 0, 'retries': 10, 'errors': 0, 'slept': 5.0}", "results identical on rerun: True", "{'ok': 10, 'skipped': 20, 'spent': 10.0}", "labels 10 induced metric tau 0.427 holistic judge 0.713 holistic judge with its own taste 0.330", "labels 160 induced metric tau 0.638 holistic judge 0.718 holistic judge with its own taste 0.338", "criteria kept from 80 labels: {'answer_looks_right': 0.8, 'cites_a_source': 0.4, 'shows_steps': 0.13, 'starts_with_then': 0.31}", "judged pass@k [0.7, 0.882, 0.969, 0.995, 1.0]", "the fair judge asked 5 times on 120 responses: all runs equal on 0.717, mean flip rate 0.115", "after a Platt fit on half: error on the other half 0.084", "bootstrap 95% interval of log-strength: {'alpha': (0.41, 0.78), 'beta': (-0.02, 0.34)", ]: self.assertIn(s, self.out) def test_no_dashes_in_output(self): self.assertNotIn("\u2014", self.out) self.assertNotIn("\u2013", self.out) if __name__ == "__main__": unittest.main()