Trusting Your Judge

# script_id: day_046_judge_validation/agreement_min
# No install, no API key. Ten toy cases: True = pass, False = fail.
human = [True, True, False, True, True, False, True, True, False, True]
judge = [True, True, True, True, True, False, True, True, True, True]

def agreement(human: list[bool], judge: list[bool]) -> dict[str, float]:
    pairs = list(zip(human, judge))
    tp, fn = pairs.count((True, True)), pairs.count((True, False))
    fp, tn = pairs.count((False, True)), pairs.count((False, False))
    print(f"TP={tp} FN={fn} FP={fp} TN={tn}")
    return {"agreement": (tp + tn) / len(pairs),
            "TPR": tp / (tp + fn),       # real passes the judge passed
            "TNR": tn / (tn + fp)}       # real fails the judge caught

for name, verdicts in [("LLM judge", judge), ("always-pass", [True] * 10)]:
    stats = agreement(human, verdicts)
    print(name, " ".join(f"{k}={v:.2f}" for k, v in stats.items()))

TP=7 FN=0 FP=2 TN=1
LLM judge agreement=0.80 TPR=1.00 TNR=0.33
TP=7 FN=0 FP=3 TN=0
always-pass agreement=0.70 TPR=1.00 TNR=0.00

# script_id: day_046_judge_validation/agreement
# atlas/evals/agreement.py: measure a judge against human labels. Stdlib only, no API key.
from __future__ import annotations
from collections import Counter
from dataclasses import dataclass, replace
from typing import Callable, Sequence, TypeVar

T = TypeVar("T")

@dataclass(frozen=True)
class AgreementStats:
    n: int
    agreement: float
    tpr: float      # of human passes, the share the judge passed
    tnr: float      # of human fails, the share the judge failed (bad answers caught)
    tp: int
    fn: int
    fp: int
    tn: int

def _rate(num: int, den: int) -> float:
    return num / den if den else float("nan")   # "no fails labelled" is unknown, not 0

def agreement(human: Sequence[bool], judge: Sequence[bool]) -> AgreementStats:
    if len(human) != len(judge):
        raise ValueError(f"{len(human)} human labels vs {len(judge)} judge verdicts")
    c = Counter(zip(human, judge))
    tp, fn, fp, tn = c[True, True], c[True, False], c[False, True], c[False, False]
    return AgreementStats(len(human), _rate(tp + tn, len(human)),
                          _rate(tp, tp + fn), _rate(tn, tn + fp), tp, fn, fp, tn)

def confusion_table(s: AgreementStats) -> str:
    return ("| | judge pass | judge fail |\n|---|---|---|\n"
            f"| **human pass** | {s.tp} (TP) | {s.fn} (FN) |\n"
            f"| **human fail** | {s.fp} (FP) | {s.tn} (TN) |")

# script_id: day_046_judge_validation/judges
# atlas/evals/judges.py (compact): the Day 43-45 pieces this lesson needs, so it runs on its own.
# Install: uv add "openai>=3.46,<4" pydantic · Env: OPENAI_API_KEY (optional: without it an
# offline keyword stand-in judges, so you can see the mechanics), ATLAS_JUDGE_MODEL, ORBIT_DATA
from __future__ import annotations
import json, os, re
from dataclasses import dataclass
from functools import cache
from pathlib import Path
from typing import Literal, Sequence
from pydantic import BaseModel

DATA_DIR = Path(os.environ.get("ORBIT_DATA", "datasets/orbit"))
JUDGE_MODEL = os.environ.get("ATLAS_JUDGE_MODEL", "gpt-6-luna")
LABELS = Path("labels/grounded_test.jsonl")   # your 30 blind labels: the TEST set

@dataclass(frozen=True)
class Record:              # the Day 43 RunRecord fields this lesson reads
    case_id: str
    question: str
    answer: str
    context: str           # retrieved text the answer must rest on
    planted: bool = False  # demo data only: True when we broke the answer on purpose

def demo_run(n: int = 30) -> list[Record]:
    """No Day 44 run file? qa_gold answers, with a wrong number planted in every third one."""
    out = []
    for i, line in enumerate((DATA_DIR / "eval/qa_gold.jsonl").open(encoding="utf-8")):
        g = json.loads(line)
        ctx = "\n\n".join((DATA_DIR / "docs" / f"{d}.md").read_text(encoding="utf-8")[:4000]
                          for d in g["citations"])
        answer = g["answer"]
        if i % 3 == 2:   # bump a number (multi-digit first): the answer no longer matches the docs
            bump = lambda m: str(int(m.group()) + 1)
            answer = re.sub(r"\d{2,}", bump, answer, count=1)
            if answer == g["answer"]:
                answer = re.sub(r"\d+", bump, answer, count=1)
        out.append(Record(g["id"], g["question"], answer, ctx, answer != g["answer"]))
    return out[:n]

# script_id: day_046_judge_validation/judges
def load_run(path: str | Path = "runs/ask_v1_fix.jsonl") -> list[Record]:
    if not Path(path).exists():
        return demo_run()
    keep = Record.__dataclass_fields__   # rename here if your RunRecord fields differ
    return [Record(**{k: v for k, v in json.loads(line).items() if k in keep})
            for line in Path(path).open(encoding="utf-8")]

def load_labels(records: Sequence[Record], path: Path = LABELS) -> dict[str, bool]:
    """Your blind labels as {case_id: passed}. Demo fallback: the planted flags."""
    if path.exists() and path.stat().st_size:
        return {d["case_id"]: d["passed"] for d in map(json.loads, path.open(encoding="utf-8"))}
    print(f"[demo] no labels in {path}: using planted flags as stand-in labels")
    return {r.case_id: not r.planted for r in records}

# script_id: day_046_judge_validation/label_cases
# Blind labelling CLI. Run in a terminal from your project root: python label_cases.py
# No API key. ~25 min for 30 cases. Writes labels/grounded_test.jsonl (resumable).
import json, random
from pathlib import Path
from judges import LABELS, Record, load_run     # atlas/evals/judges.py in your repo

ANNOTATIONS = Path("annotations.jsonl")   # Day 44 dev notes: used ONLY to choose likely fails

def pick_cases(records: list[Record], n: int = 30, seed: int = 46) -> list[Record]:
    flagged = set()
    if ANNOTATIONS.exists():
        flagged = {a["case_id"] for a in map(json.loads, ANNOTATIONS.open()) if a.get("mode")}
    rng = random.Random(seed)
    likely = [r for r in records if r.case_id in flagged]
    picked = rng.sample(likely, min(len(likely), n // 2))           # up to half likely fails
    rest = [r for r in records if r not in picked]
    picked += rng.sample(rest, min(len(rest), n - len(picked)))
    rng.shuffle(picked)       # so the likely fails don't arrive in a predictable block
    return picked

def label(records: list[Record], out: Path = LABELS) -> int:
    out.parent.mkdir(parents=True, exist_ok=True)
    done = {json.loads(l)["case_id"] for l in out.open()} if out.exists() else set()
    with out.open("a", encoding="utf-8") as f:
        for i, r in enumerate(records, 1):
            if r.case_id in done:
                continue
            print(f"\n[{i}/{len(records)}] {r.case_id}  Q: {r.question}\n"
                  f"--- context (first 1500 chars)\n{r.context[:1500]}\n--- answer\n{r.answer}")
            try:
                key = input("grounded? [p]ass / [f]ail / [s]kip / [q]uit: ").strip().lower()
            except EOFError:   # no terminal (CI, piped stdin): stop cleanly
                print("\nno input available: run this in a terminal")
                break
            if key == "q":
                break
            if key in ("p", "f"):
                note = input("why (one line): ") if key == "f" else ""
                f.write(json.dumps({"case_id": r.case_id, "passed": key == "p", "note": note}) + "\n")
                done.add(r.case_id)
    return len(done)

if __name__ == "__main__":
    print(f"{label(pick_cases(load_run()))} labels in {LABELS}")

[1/30] qa-017  Q: What does orbit dlq replay do to the job ID and attempt count?
--- context (first 1500 chars)
---
id: dead-letter-queue
title: Dead-letter queue
...
--- answer
It puts the job back on its original queue as a new job with a new ID. The arguments and
idempotency key are kept, the attempt count is reset to zero, ...
grounded? [p]ass / [f]ail / [s]kip / [q]uit: p

# script_id: day_046_judge_validation/judges
GROUNDED_V1 = "Pass if the answer is supported by the context. Fail otherwise."  # Day 45 rubric

class Verdict(BaseModel):
    reason: str      # first, so the evidence comes before the verdict
    passed: bool

@cache
def _client():
    from openai import OpenAI   # lazy import: offline runs never need a key
    return OpenAI()

def make_judge(definition: str, *, examples: Sequence[dict] = (), model: str = JUDGE_MODEL):
    """Day 45's binary judge, compact: returns judge(record) -> bool."""
    shots = "".join(f"\n\nExample ({'PASS' if e['passed'] else 'FAIL'})\nContext: {e['context']}"
                    f"\nAnswer: {e['answer']}\nWhy: {e['why']}" for e in examples)
    instructions = f"You grade ONE criterion: grounded.\n{definition}{shots}"
    def judge(rec: Record) -> bool:
        resp = _client().responses.parse(
            model=model, instructions=instructions, text_format=Verdict,
            reasoning={"effort": "low"},   # explicit: reasoning tokens bill as output
            input=f"Context:\n{rec.context}\n\nQuestion: {rec.question}\n\nAnswer: {rec.answer}")
        if resp.output_parsed is None:    # refusal or truncation: an error, never a verdict
            raise RuntimeError(f"judge_error on {rec.case_id}: status={resp.status}")
        return resp.output_parsed.passed
    return judge

def offline_judge(definition: str, examples: Sequence[dict] = ()):
    """No-key stand-in, lenient on purpose: most answer words must appear in the context.
    A definition that talks about numbers also requires every number to appear (the v2 rule)."""
    strict = "number" in definition.lower()
    def judge(rec: Record) -> bool:
        words = set(re.findall(r"[a-z_]{4,}", rec.answer.lower()))
        overlap = len(words & set(re.findall(r"[a-z_]{4,}", rec.context.lower())))
        nums = re.findall(r"\d+(?:\.\d+)*", rec.answer)
        exact = all(re.search(rf"(?<![\d.]){re.escape(n)}(?!\d)", rec.context) for n in nums)
        return overlap >= 0.6 * len(words) and (exact or not strict)
    return judge

def get_judge(definition: str, examples: Sequence[dict] = ()):
    if os.environ.get("OPENAI_API_KEY"):
        return make_judge(definition, examples=examples)
    print("[offline] OPENAI_API_KEY not set: keyword stand-in judge, not an LLM")
    return offline_judge(definition, examples)

# script_id: day_046_judge_validation/judge_v1
# Judge v1 (Day 45's grounded rubric) on your labelled cases, compared with your labels.
# Env: OPENAI_API_KEY (optional) · ~30 judge calls ≈ $0.01-0.02 on gpt-6-luna (as of 2026-11, verify)
import json
from pathlib import Path
from agreement import agreement, confusion_table
from judges import GROUNDED_V1, get_judge, load_labels, load_run

records = load_run()
labels = load_labels(records)
cases = [r for r in records if r.case_id in labels]
judge = get_judge(GROUNDED_V1)
verdicts = {r.case_id: judge(r) for r in cases}

stats = agreement([labels[r.case_id] for r in cases], [verdicts[r.case_id] for r in cases])
print(f"n={stats.n} agreement={stats.agreement:.2f} TPR={stats.tpr:.2f} TNR={stats.tnr:.2f}")
print(confusion_table(stats))
for r in cases:   # read each disagreement next to your own note: this is where fixes come from
    if verdicts[r.case_id] != labels[r.case_id]:
        side = "judge PASS, you FAIL" if verdicts[r.case_id] else "judge FAIL, you PASS"
        print(f"- {r.case_id} ({side}): {r.answer[:80]}")
Path("reports").mkdir(exist_ok=True)
Path("reports/judge_v1_verdicts.json").write_text(json.dumps(verdicts, indent=1))

# offline run (no key, demo data); with your labels and the LLM judge, your numbers will differ
[demo] no labels in labels/grounded_test.jsonl: using planted flags as stand-in labels
[offline] OPENAI_API_KEY not set: keyword stand-in judge, not an LLM
n=30 agreement=0.57 TPR=0.85 TNR=0.00
| | judge pass | judge fail |
|---|---|---|
| **human pass** | 17 (TP) | 3 (FN) |
| **human fail** | 10 (FP) | 0 (TN) |
- qa-003 (judge PASS, you FAIL): The 2.x line reaches end of life on 2027-12-31. 2.9.0, released 2025-11-20, is t
- qa-007 (judge FAIL, you PASS): This is a known issue in 3.2.0 that is still open. Make the proxy send X-Forward
- qa-012 (judge PASS, you FAIL): Keep [worker] prefetch at 101 or below. The default is 10. On 3.2.0 and 3.2.1, p
...  (13 disagreements in total)

# script_id: day_046_judge_validation/agreement
Compare = Callable[[T, str, str], str]   # (case, answer in slot A, answer in slot B) -> "A"|"B"|"tie"

@dataclass(frozen=True)
class SwapResult:
    n: int
    consistency: float          # share of pairs whose winner survived the swap
    slot_wins: dict[str, int]   # how often each label won across both orders

def swap_test(pairs: Sequence[tuple[T, str, str]], compare: Compare) -> SwapResult:
    """pairs: (case, answer_x, answer_y). Runs each pair in both orders."""
    consistent, slots = 0, Counter()
    for case, x, y in pairs:
        first, second = compare(case, x, y), compare(case, y, x)   # x in slot A, then slot B
        slots.update([first, second])
        # The labels stay "A"/"B" while the content swaps, so an unbiased judge's LABEL flips.
        # Compare by answer identity: map each label back to the answer it pointed at.
        winner_1 = {"A": "x", "B": "y"}.get(first, "tie")
        winner_2 = {"A": "y", "B": "x"}.get(second, "tie")
        consistent += winner_1 == winner_2
    return SwapResult(len(pairs), _rate(consistent, len(pairs)), dict(slots))

def order_averaged(compare: Compare) -> Compare:
    """Wrap a pairwise judge: ask in both orders, keep a win only if it survives the swap."""
    def both(case, a: str, b: str) -> str:
        w1, w2 = compare(case, a, b), compare(case, b, a)
        if (w1, w2) == ("A", "B"):
            return "A"
        return "B" if (w1, w2) == ("B", "A") else "tie"
    return both

# script_id: day_046_judge_validation/agreement
def verbosity_test(cases: Sequence[T], judge: Callable[[T], bool],
                   pad: Callable[[T], str]) -> float:
    """Flip rate: share of cases whose verdict changes when pad(case) is appended to the answer.
    `cases` are dataclasses with an `answer` field."""
    flips = sum(judge(c) != judge(replace(c, answer=f"{c.answer} {pad(c)}")) for c in cases)
    return _rate(flips, len(cases))

# script_id: day_046_judge_validation/judges
class PairwiseVerdict(BaseModel):
    reason: str
    winner: Literal["A", "B", "tie"]

def compare(rec: Record, a: str, b: str, *, model: str = JUDGE_MODEL) -> str:
    """Day 45's pairwise judge, compact: which answer is better grounded in rec.context?"""
    if not os.environ.get("OPENAI_API_KEY"):   # offline stand-in: longer wins, ties go to A
        return "A" if len(a) >= len(b) else "B"
    resp = _client().responses.parse(
        model=model, text_format=PairwiseVerdict, reasoning={"effort": "low"},
        instructions="Pick the answer better grounded in the context. Say tie if they are equal.",
        input=f"Context:\n{rec.context}\n\nQuestion: {rec.question}\n\nAnswer A: {a}\n\nAnswer B: {b}")
    if resp.output_parsed is None:
        raise RuntimeError(f"judge_error on {rec.case_id}: status={resp.status}")
    return resp.output_parsed.winner

def load_pairs(old: str = "runs/ask_v1_baseline.jsonl", new: str = "runs/ask_v1_fix.jsonl"):
    """[(record, v1 answer, v1.1 answer)]. Demo mode: (record, correct answer, planted answer)."""
    if Path(old).exists() and Path(new).exists():
        v1 = {r.case_id: r.answer for r in load_run(old)}
        return [(r, v1[r.case_id], r.answer) for r in load_run(new)
                if r.case_id in v1 and v1[r.case_id] != r.answer]
    gold = [json.loads(line)["answer"] for line in (DATA_DIR / "eval/qa_gold.jsonl").open()]
    return [(r, gold[i], r.answer) for i, r in enumerate(demo_run()) if r.planted]

def pad_from_context(rec: Record) -> str:
    """True but irrelevant: the first prose line of the context, so grounding can't change."""
    lines = [l for l in rec.context.splitlines()
             if len(l) > 60 and not l.startswith(("#", "|", "-", "`"))]
    return lines[0] if lines else ""

# script_id: day_046_judge_validation/bias_tests
# Position-swap test on the pairwise judge, padding test on the binary judge.
# Env: OPENAI_API_KEY (optional) · ~180 calls ≈ $0.05-0.08 on gpt-6-luna (as of 2026-11, verify)
import json
from dataclasses import asdict
from pathlib import Path
from agreement import order_averaged, swap_test, verbosity_test
from judges import GROUNDED_V1, compare, get_judge, load_pairs, load_run, pad_from_context

pairs = load_pairs()
raw = swap_test(pairs, compare)
fixed = swap_test(pairs, order_averaged(compare))   # 4 calls per pair: both orders, twice
print(f"swap consistency  raw={raw.consistency:.2f}  slot_wins={raw.slot_wins}")
print(f"swap consistency  order-averaged={fixed.consistency:.2f}  slot_wins={fixed.slot_wins}")

flip = verbosity_test(load_run(), get_judge(GROUNDED_V1), pad_from_context)
print(f"verbosity flip rate (judge v1) = {flip:.2f}")
Path("reports").mkdir(exist_ok=True)
Path("reports/bias.json").write_text(json.dumps({"swap": asdict(fixed), "flip_rate_v1": flip}))

# offline run; your numbers will differ
swap consistency  raw=0.00  slot_wins={'A': 20}
swap consistency  order-averaged=1.00  slot_wins={'tie': 20}
[offline] OPENAI_API_KEY not set: keyword stand-in judge, not an LLM
verbosity flip rate (judge v1) = 0.10

# script_id: day_046_judge_validation/judge_v2
# Judge v2: sharper definition + 2 pass / 2 fail anchors from DEV data; measured on TEST labels.
# Env: OPENAI_API_KEY (optional) · ~90 calls ≈ $0.03-0.05 on gpt-6-luna (as of 2026-11, verify)
# Needs reports/ from judge_v1.py and bias_tests.py.
import json
from pathlib import Path
from agreement import SwapResult, agreement, judge_report, verbosity_test
from judges import LABELS, get_judge, load_labels, load_run, pad_from_context

GROUNDED_V2 = """Pass only if EVERY claim in the answer is stated in the context.
- Every number, version, default, duration and error code must match the context exactly.
- Advice or facts that may be true but are absent from the context make the answer FAIL.
- Extra sentences copied from the context do not change the verdict."""

DEV_ANCHORS = Path("evals/anchors_grounded.jsonl")   # built from Day 44 annotations (DEV)
CTX = ("ORB-1901 BrokerRateLimited. Cause: SQS throttled a request from a worker or producer, "
       "most common on FIFO queues. Fix: lower the number of receive calls (fewer workers, "
       "prefetch <= 10) or spread work across message groups. SQS caps the visibility timeout at 12 hours.")
FALLBACK_ANCHORS = [   # hand-written on Orbit docs, on a question that is NOT in the test set
    {"case_id": "dev-01", "passed": True, "context": CTX, "why": "every claim is in the context",
     "answer": "SQS throttled the request. Use fewer workers or prefetch <= 10."},
    {"case_id": "dev-02", "passed": False, "context": CTX, "why": "the context says <= 10, not 50",
     "answer": "SQS throttled the request. Keep prefetch <= 50."},
    {"case_id": "dev-03", "passed": False, "context": CTX, "why": "raising a quota is not in the context",
     "answer": "SQS throttled the request. Ask AWS to raise your SQS quota."},
    {"case_id": "dev-04", "passed": True, "context": CTX, "why": "the extra sentence is in the context",
     "answer": "Spread work across message groups. SQS caps the visibility timeout at 12 hours."},
]

def load_anchors(test_ids: set[str]) -> list[dict]:
    anchors = ([json.loads(l) for l in DEV_ANCHORS.open()] if DEV_ANCHORS.exists()
               else FALLBACK_ANCHORS)
    leaked = test_ids & {a["case_id"] for a in anchors}
    if leaked:   # tuning on the test set makes every number you report meaningless
        raise ValueError(f"anchors overlap the test labels in {LABELS}: {sorted(leaked)}")
    return anchors

# script_id: day_046_judge_validation/judge_v2
records = load_run()
labels = load_labels(records)
cases = [r for r in records if r.case_id in labels]
human = [labels[r.case_id] for r in cases]
v1_file, bias_file = Path("reports/judge_v1_verdicts.json"), Path("reports/bias.json")
if not (v1_file.exists() and bias_file.exists()):
    raise SystemExit("run judge_v1.py and bias_tests.py first: they write reports/")
v1 = json.loads(v1_file.read_text())
before = agreement(human, [v1[r.case_id] for r in cases])

judge = get_judge(GROUNDED_V2, load_anchors(set(labels)))
v2 = {r.case_id: judge(r) for r in cases}
after = agreement(human, [v2[r.case_id] for r in cases])
for name, s in [("v1", before), ("v2", after)]:
    print(f"judge {name}: agreement={s.agreement:.2f} TPR={s.tpr:.2f} TNR={s.tnr:.2f}")

flip_v2 = verbosity_test(cases, judge, pad_from_context)
swap = SwapResult(**json.loads(bias_file.read_text())["swap"])
disagree = [f"{r.case_id}: judge {'PASS' if v2[r.case_id] else 'FAIL'}, "
            f"human {'PASS' if labels[r.case_id] else 'FAIL'}" for r in cases
            if v2[r.case_id] != labels[r.case_id]]
report = judge_report(after, swap, flip_v2, disagree, before=before, label_file=str(LABELS))
Path("reports/judge_report.md").write_text(report, encoding="utf-8")
print(report.splitlines()[-1])

# offline run; your numbers will differ
judge v1: agreement=0.57 TPR=0.85 TNR=0.00
judge v2: agreement=0.87 TPR=0.85 TNR=0.90
**Fit to gate CI:** yes

# script_id: day_046_judge_validation/agreement
def judge_report(stats: AgreementStats, swap: SwapResult, flip_rate: float,
                 disagreements: Sequence[str], *, before: AgreementStats | None = None,
                 label_file: str = "labels/grounded_test.jsonl", criterion: str = "grounded",
                 bar: float = 0.8, min_fails: int = 8) -> str:
    """Render judge_report.md. `bar` and `min_fails` are team conventions, not standards."""
    fails = stats.tn + stats.fp
    problems = [msg for ok, msg in [
        (stats.tpr >= bar, f"TPR {stats.tpr:.2f} < {bar}"),
        (stats.tnr >= bar, f"TNR {stats.tnr:.2f} < {bar}"),
        (fails >= min_fails, f"only {fails} human fails labelled (want >= {min_fails})"),
        (swap.consistency >= bar, f"swap consistency {swap.consistency:.2f} < {bar}"),
    ] if not ok]   # NaN fails every comparison, so "unknown" never counts as fit
    rows = ([("before", before)] if before else []) + [("this judge", stats)]
    return "\n".join([
        f"# Judge report: {criterion}", "",
        f"Labels: `{label_file}` (held-out, never used in prompts or anchors) · "
        f"n = {stats.n} · positive class = pass", "",
        "| judge | agreement | TPR | TNR |", "|---|---|---|---|",
        *[f"| {name} | {s.agreement:.2f} | {s.tpr:.2f} | {s.tnr:.2f} |" for name, s in rows], "",
        confusion_table(stats), "",
        f"Swap consistency (order-averaged): {swap.consistency:.2f} over {swap.n} pairs",
        f"Verbosity flip rate: {flip_rate:.2f}", "", "## Disagreements",
        *([f"- {d}" for d in disagreements] or ["- none"]), "",
        f"**Fit to gate CI:** {'yes' if not problems else 'no. ' + '; '.join(problems)}"])

# script_id: day_046_judge_validation/agreement
if __name__ == "__main__":
    @dataclass(frozen=True)
    class Toy:
        answer: str
    always_a = lambda case, a, b: "A"              # a judge that only looks at the slot
    print(swap_test([(None, "x", "y")] * 5, always_a))
    print(swap_test([(None, "x", "y")] * 5, order_averaged(always_a)))
    likes_length = lambda c: len(c.answer) > 20    # a judge that rewards padding
    print(verbosity_test([Toy("short"), Toy("a fine answer")], likes_length, lambda c: "padding" * 3))
    s = agreement([True] * 7 + [False] * 3, [True] * 9 + [False])
    print(judge_report(s, SwapResult(5, 1.0, {}), 0.0, ["case 3", "case 9"]).splitlines()[-1])

SwapResult(n=5, consistency=0.0, slot_wins={'A': 10})
SwapResult(n=5, consistency=1.0, slot_wins={'tie': 10})
1.0
**Fit to gate CI:** no. TNR 0.33 < 0.8; only 3 human fails labelled (want >= 8)