Error Analysis

# script_id: day_044_error_analysis/mini_r2
# mini_r2.py: a compact, offline stand-in for your Day 34 R2 "Ask". Save as mini_r2.py.
# Install: none (stdlib only) · Env: ORBIT_DATA (optional) · Services: none
import hashlib
import json
import math
import os
import re
from collections import Counter
from dataclasses import dataclass
from pathlib import Path

DATA_DIR = Path(os.environ.get("ORBIT_DATA", "datasets/orbit"))
WORD = re.compile(r"[a-z0-9_]+(?:[.\-*][a-z0-9_*]+)*")  # keeps orbit.toml, ORB-1202, ORBIT_* whole
STOP = set("a an and are can do does for how i in is it of on or the to what when which why with my".split())


@dataclass
class Chunk:
    doc_id: str
    applies_to: str  # "3.x" | "2.x" | "all", from the page's front matter (its metadata header)
    text: str


def terms(text: str) -> list[str]:
    return [t for t in WORD.findall(text.lower()) if t not in STOP]


def load_chunks() -> list[Chunk]:
    chunks = []
    for path in sorted((DATA_DIR / "docs").glob("*.md")):
        _, front, body = path.read_text(encoding="utf-8").split("---", 2)
        meta = dict(line.split(":", 1) for line in front.strip().splitlines() if ":" in line)
        applies = meta.get("applies_to", "all").strip().strip('"')
        for para in re.split(r"\n\s*\n", body):  # paragraph chunks: Day 25, simplified
            if len(para.split()) >= 12:
                chunks.append(Chunk(path.stem, applies, para.strip()))
    if not chunks:  # fail loudly: an empty corpus would make every check meaningless
        raise SystemExit(f"no docs under {DATA_DIR / 'docs'}; set ORBIT_DATA")
    return chunks

# script_id: day_044_error_analysis/mini_r2


class Index:
    """BM25 keyword search (Day 27's sparse half). `where` is Day 27's metadata filter."""

    def __init__(self, chunks: list[Chunk]):
        self.chunks, self.tfs = chunks, [Counter(terms(c.text)) for c in chunks]
        self.df = Counter(t for tf in self.tfs for t in tf)
        self.avg_len = sum(sum(tf.values()) for tf in self.tfs) / len(chunks)

    def search(self, query: str, k: int = 4, where=None) -> list[tuple[float, Chunk]]:
        n, hits = len(self.chunks), []
        for chunk, tf in zip(self.chunks, self.tfs):
            if where and not where(chunk):
                continue
            norm = 1.2 * (0.25 + 0.75 * sum(tf.values()) / self.avg_len)
            score = sum(math.log(1 + (n - self.df[t] + 0.5) / (self.df[t] + 0.5))
                        * tf[t] * 2.2 / (tf[t] + norm) for t in terms(query) if t in tf)
            if score > 0:
                hits.append((round(score, 1), chunk))
        return sorted(hits, key=lambda h: -h[0])[:k]


def extractive_generate(question: str, hits: list[tuple[float, Chunk]],
                        min_score: float = 6.0) -> tuple[str, list[str]]:
    """No-key 'generator': quote the best-matching sentence. Swap in llm_generate when you have a key."""
    if not hits or hits[0][0] < min_score:  # weak evidence: abstain
        return "I can't find this in the Orbit docs.", []
    best = hits[0][1]
    q = set(terms(question))
    sentence = max(re.split(r"(?<=[.!?])\s+", best.text), key=lambda s: len(q & set(terms(s))))
    cited = [best.doc_id] + [c.doc_id for _, c in hits[1:2] if c.doc_id != best.doc_id]
    return " ".join(sentence.split()), cited

# script_id: day_044_error_analysis/mini_r2


EXTRA_CASES = [  # added because qa_gold has few 2.x or unanswerable questions; Day 43's set has more
    ("x-v-01", "On Orbit 2.9 I changed concurrency in orbit.ini and nothing happened. Why?",
     ["environment-variables-2x"], "2.x"),
    ("x-v-02", "Can I set ORBIT_LEASE=5m on Orbit 2.x?", ["environment-variables-2x"], "2.x"),
    ("x-v-03", "I set ORBIT_PREFETCH=50 but workers still prefetch more jobs. Why?",
     ["environment-variables-2x"], "2.x"),
    ("x-v-04", "My 2.9 workers run jobs twice when a job takes longer than five minutes. "
     "Which setting do I raise?", ["environment-variables-2x", "leases-and-heartbeats"], "2.x"),
    ("x-v-05", "How do I set worker concurrency with an environment variable in Orbit 3.2?",
     ["configuration-reference"], "3.x"),
    ("x-na-01", "Which GPU types does Orbit support for GPU workers?", [], "all"),
    ("x-na-02", "What uptime SLA does the hosted Orbit Cloud offer?", [], "all"),
    ("x-na-03", "How do I enqueue Orbit jobs from a Node.js producer?", [], "all"),
]


def load_cases() -> list[dict]:
    """Day 43's ask_v1 if you built it, else qa_gold plus EXTRA_CASES, in the same dict shape."""
    ask_v1 = Path("evals/datasets/ask_v1.jsonl")
    if ask_v1.exists():
        return [json.loads(line) for line in ask_v1.open(encoding="utf-8")]
    gold = [json.loads(line) for line in (DATA_DIR / "eval/qa_gold.jsonl").open(encoding="utf-8")]
    cases = [{"id": g["id"], "question": g["question"], "expected": g["answer"],
              "must_cite": g["citations"], "applies_to": g["applies_to"]} for g in gold]
    cases += [{"id": i, "question": q, "expected": None, "must_cite": cite, "applies_to": v}
              for i, q, cite, v in EXTRA_CASES]
    return cases

# script_id: day_044_error_analysis/mini_r2


def run_set(cases: list[dict], *, label: str = "baseline", where_for=None,
            generate=extractive_generate, out: Path | None = None) -> list[dict]:
    """Answer every case and write one trace-shaped record per case (Day 43's run-file idea)."""
    index, records = Index(load_chunks()), []
    for case in cases:
        where = where_for(case["question"]) if where_for else None
        hits = index.search(case["question"], where=where)
        answer, cited = generate(case["question"], hits)
        gold = set(case["must_cite"])
        passed = bool(gold & set(cited)) if gold else not cited  # unanswerable: must abstain
        records.append({
            "case_id": case["id"], "context": "\n\n".join(c.text for _, c in hits), "applies_to": case["applies_to"], "question": case["question"],
            # Real runs store Day 42's OTel trace id; a hash keeps this offline demo reproducible.
            "trace_id": hashlib.sha256(f"{label}:{case['id']}".encode()).hexdigest()[:32],
            "retrieved": [{"doc_id": c.doc_id, "applies_to": c.applies_to, "score": s} for s, c in hits],
            "answer": answer, "citations": cited, "must_cite": case["must_cite"], "passed": passed})
    out = out or Path(f"runs/mini_r2_{label}.jsonl")
    out.parent.mkdir(parents=True, exist_ok=True)
    out.write_text("".join(json.dumps(r) + "\n" for r in records), encoding="utf-8")
    if label in ("baseline", "fix") and Path("evals/datasets/ask_v1.jsonl").exists():
        # Days 45-46 read these names and fields: case_id, version, question, answer, citations, retrieved_ids, context.
        ask = out.parent / f"ask_v1_{label}.jsonl"
        ask.write_text("".join(json.dumps({
            "case_id": r["case_id"], "version": "v1" if label == "baseline" else "v1.1",
            "question": r["question"], "answer": r["answer"], "citations": r["citations"],
            "retrieved_ids": [h["doc_id"] for h in r["retrieved"]], "context": r["context"]}) + "\n"
            for r in records), encoding="utf-8")
    return records


if __name__ == "__main__":
    recs = run_set(load_cases())
    print(f"{sum(r['passed'] for r in recs)}/{len(recs)} passed -> runs/mini_r2_baseline.jsonl")

28/38 passed -> runs/mini_r2_baseline.jsonl

# script_id: day_044_error_analysis/llm_answer
# llm_answer.py: the real generate step for mini_r2. Save as llm_answer.py next to mini_r2.py.
# Install: uv add openai pydantic · Env: OPENAI_API_KEY, ATLAS_MODEL (optional) · Services: none
import os

from openai import OpenAI
from pydantic import BaseModel, Field

from mini_r2 import load_cases, run_set

MODEL = os.environ.get("ATLAS_MODEL", "gpt-6-luna")
client = OpenAI()
INSTRUCTIONS = ("Answer the question about Orbit using only the context. Cite the doc ids you used. "
                "If the context doesn't answer it, say you can't find it and cite nothing.")


class Citation(BaseModel):  # Day 28
    doc_id: str
    quote: str = Field(description="At most 25 words copied exactly from that source.")


class Answer(BaseModel):  # Day 28
    text: str
    citations: list[Citation]

# script_id: day_044_error_analysis/llm_answer


def make_llm_generate(extra_rule: str = "", version_of=None):
    def llm_generate(question: str, hits: list) -> tuple[str, list[str]]:
        if not hits:
            return "I can't find this in the Orbit docs.", []
        rule = extra_rule
        if version_of:  # "all" means the question spans both versions: no single-version rule
            version = version_of(question)
            rule = extra_rule.format(version=version) if version != "all" else ""
        context = "\n\n".join(f"[{c.doc_id} | applies_to={c.applies_to}]\n{c.text}" for _, c in hits)
        resp = client.responses.parse(model=MODEL, instructions=INSTRUCTIONS + rule,
                                      input=f"Context:\n{context}\n\nQuestion: {question}",
                                      text_format=Answer, reasoning={"effort": "low"})
        answer = resp.output_parsed
        if answer is None:  # refusal or truncated output
            return f"(no answer: {resp.status})", []
        return answer.text, [c.doc_id for c in answer.citations]
    return llm_generate


if __name__ == "__main__":
    recs = run_set(load_cases(), generate=make_llm_generate())  # overwrites the offline baseline
    print(f"{sum(r['passed'] for r in recs)}/{len(recs)} passed")

[qa-006] PASS  Does retries=5 on @app.job mean five retries after the first attempt?
   retrieved retries-and-backoff          applies_to=3.x  score=25.2
   retrieved api-reference-flow           applies_to=3.x  score=12.4
   answer: `retries=5` means at most five runs: the first plus four retries.
   cited: ['retries-and-backoff', 'api-reference-flow']   gold: ['retries-and-backoff']
[qa-009] FAIL  Which environment variables does Orbit 2.x read, and do they override orbit.ini?
   retrieved installation                 applies_to=all  score=22.5
   retrieved configuration-reference      applies_to=3.x  score=17.9
   retrieved configuration-reference      applies_to=3.x  score=16.3
   retrieved environment-variables-2x     applies_to=2.x  score=16.0
   answer: 2.x reads configuration from `orbit.ini` and `ORBIT_*` environment variables (...)
   cited: ['installation', 'configuration-reference']   gold: ['environment-variables-2x']
[qa-016] FAIL  How long does Orbit keep dead-lettered jobs?
   retrieved troubleshooting-duplicates   applies_to=all  score=10.1
   retrieved broker-sqs                   applies_to=all  score=8.6
   answer: **Fix:** keep heartbeats well inside the lease, and give genuinely long jobs a larger lease:
   cited: ['troubleshooting-duplicates', 'broker-sqs']   gold: ['dead-letter-queue']
[qa-004] PASS  What does ORB-1202 mean and how should I fix it?
   retrieved dead-letter-queue            applies_to=all  score=7.7
   retrieved error-codes                  applies_to=all  score=7.6
   answer: Fix or wait out the outage, then replay.
   cited: ['dead-letter-queue', 'error-codes']   gold: ['error-codes', 'concepts-jobs', 'performance-tuning']
[qa-024] FAIL  How should I set terminationGracePeriodSeconds for Orbit workers on Kubernetes?
   retrieved concepts-workers             applies_to=3.x  score=11.6
   retrieved troubleshooting-duplicates   applies_to=all  score=10.9
   answer: Under Kubernetes, set `terminationGracePeriodSeconds` to at least your longest expected job duration.
   cited: ['concepts-workers', 'troubleshooting-duplicates']   gold: ['deployment-kubernetes']
[x-na-01] FAIL  Which GPU types does Orbit support for GPU workers?
   retrieved broker-redis                 applies_to=3.x  score=6.6
   retrieved installation                 applies_to=all  score=6.3
   answer: Orbit does not support Redis Cluster (sharded) mode.
   cited: ['broker-redis', 'installation']   gold: []

# script_id: day_044_error_analysis/trace_reviewer
# trace_reviewer.py: one trace at a time; your note goes to annotations.jsonl. Resumable.
# Install: none (stdlib only) · Env: ORBIT_DATA (optional) · Needs: mini_r2.py next to this file
import json
from pathlib import Path

from mini_r2 import load_cases, run_set

RUN, NOTES = Path("runs/mini_r2_baseline.jsonl"), Path("annotations.jsonl")
records = ([json.loads(line) for line in RUN.open(encoding="utf-8")] if RUN.exists()
           else run_set(load_cases()))
done = {json.loads(line)["case_id"] for line in NOTES.open()} if NOTES.exists() else set()
for r in (r for r in records if r["case_id"] not in done):
    print(f"\n[{r['case_id']}] {'PASS' if r['passed'] else 'FAIL'}  {r['question']}")
    for hit in r["retrieved"]:
        print(f"   retrieved {hit['doc_id']:<28} applies_to={hit['applies_to']:<4} score={hit['score']}")
    print(f"   answer: {r['answer'][:200]}\n   cited: {r['citations']}   gold: {r['must_cite']}")
    try:
        note = input("note (first thing that went wrong; Enter = fine; q = quit): ").strip()
    except EOFError:  # no terminal attached (CI, piped input)
        break
    if note == "q":
        break
    with NOTES.open("a", encoding="utf-8") as f:
        f.write(json.dumps({"case_id": r["case_id"], "trace_id": r["trace_id"], "note": note,
                            "mode": None, "passed_check": r["passed"]}) + "\n")

[qa-001] PASS  What are the default values of lease_seconds and heartbeat_seconds in orbit.toml?
   retrieved leases-and-heartbeats        applies_to=all  score=11.3
   ...
note (first thing that went wrong; Enter = fine; q = quit): fine, though the answer is a raw TOML block

# script_id: day_044_error_analysis/analysis
# analysis.py: annotations -> taxonomy -> criteria + code checks (atlas/evals/analysis.py). Save as analysis.py.
# Install: none (stdlib only) · Env: ORBIT_DATA (optional) · Needs: mini_r2.py next to this file
import json
import random
from collections import Counter
from dataclasses import dataclass
from pathlib import Path
from typing import Literal

from mini_r2 import load_cases, load_chunks, run_set

RUN = Path("runs/mini_r2_baseline.jsonl")


@dataclass
class Annotation:
    case_id: str
    trace_id: str
    note: str                 # open code: the first thing that went wrong, in your words
    mode: str | None          # axial code: the pile it landed in (filled in after grouping)
    passed_check: bool        # what the automated check said, so false passes stay visible

# script_id: day_044_error_analysis/analysis


AUTHOR_NOTES = {  # my notes on the offline baseline; yours live in annotations.jsonl
    "qa-002": ("quotes the bug description, not the release that fixed it", "answer_misses_point"),
    "qa-004": ("ORB-1202 defined on error-codes, but a DLQ chunk ranks first", "wrong_page_first"),
    "qa-005": ("answer is a TOML block; never states the three delays", "answer_misses_point"),
    "qa-007": ("changelog 'still open' line outranks the OIDC how-to (rank 4)", "wrong_page_first"),
    "qa-009": ("2.x question; two 3.x config chunks in context, cites a 3.x page", "version_mismatch"),
    "qa-013": ("talks about SQS priority queues; never says if Flows work on SQS", "answer_misses_point"),
    "qa-016": ("dead-letter-queue page not retrieved at all; answers about leases", "search_miss"),
    "qa-019": ("answer is a list of links, no aggregation advice", "answer_misses_point"),
    "qa-021": ("idempotency page only rank 3; quotes a DLQ side fact, no TTL", "wrong_page_first"),
    "qa-024": ("answer is right; check fails because gold expects the k8s page", "eval_bug"),
    "qa-028": ("glossary definition of Step outranks flows-error-handling", "wrong_page_first"),
    "qa-029": ("2.x heartbeat section never retrieved ('heartbeated' != 'heartbeats')", "search_miss"),
    "x-v-03": ("2.x question; all four chunks are 3.x; gives 3.2 prefetch advice", "version_mismatch"),
    "x-v-04": ("2.x question; top chunk is a 3.x Flow API page (ttl_days)", "version_mismatch"),
    "x-v-05": ("3.x question; three 2.x env-var chunks; describes a 2.x override", "version_mismatch"),
    "x-na-01": ("unanswerable; answers with a Redis Cluster fact instead of abstaining", "no_abstain"),
    "x-na-03": ("unanswerable; quotes a tracing table instead of abstaining", "no_abstain"),
}

# script_id: day_044_error_analysis/analysis


def load_records(path: Path = RUN) -> list[dict]:
    if not path.exists():
        return run_set(load_cases())  # no run file yet: make the offline baseline
    return [json.loads(line) for line in path.open(encoding="utf-8")]


def load_annotations(records: list[dict], path: Path = Path("annotations.jsonl")) -> list[Annotation]:
    if path.exists():  # your own notes from trace_reviewer.py win
        return [Annotation(**json.loads(line)) for line in path.open(encoding="utf-8")]
    by_id = {r["case_id"]: r for r in records}
    return [Annotation(cid, by_id[cid]["trace_id"], note, mode, by_id[cid]["passed"])
            for cid, (note, mode) in AUTHOR_NOTES.items() if cid in by_id]


def sample(records: list[dict], n_pass: int = 15, seed: int = 44) -> list[dict]:
    """Every failure plus a seeded random slice of passes: checks can be wrong in both directions."""
    fails = [r for r in records if not r["passed"]]
    passes = [r for r in records if r["passed"]]
    return fails + random.Random(seed).sample(passes, min(n_pass, len(passes)))


if __name__ == "__main__":
    records = load_records()
    notes = {a.case_id: a for a in load_annotations(records)}
    for label, picked in [("failures only", [r for r in records if not r["passed"]]),
                          ("failures + 15 passes", sample(records))]:
        found = [notes[r["case_id"]] for r in picked if r["case_id"] in notes]
        print(f"{label:<21} read {len(picked):>2}  notes {len(found):>2}  from passing cases "
              f"{sum(a.passed_check for a in found)}  modes {len({a.mode for a in found})}")
    seen, order = set(), sample(records, n_pass=len(records))
    for i in range(0, len(order), 10):  # saturation: do new traces still bring new modes?
        batch = {notes[r["case_id"]].mode for r in order[i:i + 10] if r["case_id"] in notes}
        print(f"traces {i + 1:>2}-{min(i + 10, len(order)):<2} new modes: {sorted(batch - seen)}")
        seen |= batch

failures only         read 10  notes 10  from passing cases 0  modes 5
failures + 15 passes  read 25  notes 13  from passing cases 3  modes 6
traces  1-10 new modes: ['eval_bug', 'no_abstain', 'search_miss', 'version_mismatch', 'wrong_page_first']
traces 11-20 new modes: []
traces 21-30 new modes: ['answer_misses_point']
traces 31-38 new modes: []

TypeError: Annotation.__init__() got an unexpected keyword argument 'notes'

# script_id: day_044_error_analysis/analysis


@dataclass
class FailureMode:
    name: str
    definition: str
    count: int
    impact: int      # 1 = annoying, 2 = wastes the user's time, 3 = wrong advice they may act on
    priority: int    # count x impact: a ranking aid, not a law


DEFINITIONS = {
    "version_mismatch": "Answer for one Orbit version is built from pages written for the other.",
    "wrong_page_first": "The right page is retrieved but a weaker page outranks it.",
    "no_abstain": "Docs don't cover the question, yet Atlas answers anyway.",
    "answer_misses_point": "The right page is in context but the answer doesn't address the question.",
    "search_miss": "No page that answers the question is retrieved.",
}
IMPACT = {"version_mismatch": 3, "no_abstain": 3, "wrong_page_first": 2,
          "search_miss": 2, "answer_misses_point": 1}


def build_taxonomy(annotations: list[Annotation], impact: dict[str, int],
                   definitions: dict[str, str] = DEFINITIONS) -> list[FailureMode]:
    """Count product modes; notes about the eval itself (eval_bug) go to a separate fix list."""
    counts = Counter(a.mode or "(uncoded)" for a in annotations if a.note and a.mode != "eval_bug")
    modes = [FailureMode(m, definitions.get(m, "?"), n, impact.get(m, 1), n * impact.get(m, 1))
             for m, n in counts.items()]
    return sorted(modes, key=lambda fm: (-fm.priority, -fm.count, fm.name))


if __name__ == "__main__":
    taxonomy = build_taxonomy(load_annotations(records), IMPACT)
    print(f"\n{'mode':<20} {'count':>5} {'impact':>6} {'priority':>8}")
    for fm in taxonomy:
        print(f"{fm.name:<20} {fm.count:>5} {fm.impact:>6} {fm.priority:>8}")

mode                 count impact priority
version_mismatch         4      3       12
wrong_page_first         4      2        8
no_abstain               2      3        6
answer_misses_point      4      1        4
search_miss              2      2        4

# script_id: day_044_error_analysis/llm_grouping
# llm_grouping.py: a model proposes piles; a human decides. Save next to mini_r2.py and analysis.py.
# Install: uv add openai pydantic · Env: OPENAI_API_KEY, ATLAS_MODEL (optional) · Services: none
import os

from openai import OpenAI
from pydantic import BaseModel, Field

from analysis import load_annotations, load_records

MODEL = os.environ.get("ATLAS_MODEL", "gpt-6-luna")
client = OpenAI()


class ProposedMode(BaseModel):
    name: str = Field(description="snake_case; names the broken step, e.g. search_miss")
    definition: str = Field(description="One sentence a teammate could apply to a new trace.")
    case_ids: list[str]


class Grouping(BaseModel):  # strict schemas need an object at the root, so wrap the list
    modes: list[ProposedMode]


notes = [a for a in load_annotations(load_records()) if a.note]
resp = client.responses.parse(
    model=MODEL, reasoning={"effort": "low"}, text_format=Grouping,
    instructions=("Group these error-analysis notes from a docs Q&A assistant into 4-7 failure modes. "
                  "Each note describes the first thing that went wrong in one trace. "
                  "Put every case id in exactly one mode. Don't invent case ids."),
    input="\n".join(f"{a.case_id}: {a.note}" for a in notes))
grouping = resp.output_parsed
if grouping is None:
    raise SystemExit(f"no grouping returned (status: {resp.status})")
for m in grouping.modes:
    print(f"{m.name:<22} {len(m.case_ids):>2}  {m.definition}")
used = [cid for m in grouping.modes for cid in m.case_ids]
print("missing:", {a.case_id for a in notes} - set(used), "duplicated:", len(used) - len(set(used)))

# script_id: day_044_error_analysis/analysis


@dataclass
class CheckResult:  # Day 20's shape
    name: str
    passed: bool
    detail: str = ""


@dataclass
class Criterion:
    name: str
    definition: str
    pass_example: str
    fail_example: str
    kind: Literal["code", "judge"]


CRITERIA = [
    Criterion("version_matches", "A version-specific question cites no page for the other version.",
              "2.x ORBIT_LEASE question cites environment-variables-2x",
              "2.x ORBIT_PREFETCH question cites configuration-reference (3.x)", "code"),
    Criterion("cites_existing_doc", "Every cited doc id exists in the corpus.",
              "cites ['idempotency']", "cites ['orbit-cloud-sla'] (no such page)", "code"),
    Criterion("grounded", "Every claim in the answer is supported by a cited chunk.",
              "'retries=5 means five runs', quoted", "says 3.1.2 fixed it, citing only contributing", "judge"),
]
APPLIES = {c.doc_id: c.applies_to for c in load_chunks()}


def check_version_matches(case: dict, pred: dict) -> CheckResult:
    clash = {"2.x": "3.x", "3.x": "2.x"}.get(case["applies_to"])
    bad = [d for d in pred["citations"] if APPLIES.get(d) == clash]
    return CheckResult("version_matches", not bad, f"cites {clash} pages {bad}" if bad else "")


def check_cites_existing_doc(case: dict, pred: dict) -> CheckResult:
    unknown = [d for d in pred["citations"] if d not in APPLIES]
    return CheckResult("cites_existing_doc", not unknown, f"unknown ids {unknown}" if unknown else "")

# script_id: day_044_error_analysis/analysis


def slice_of(record: dict) -> str:
    return "unanswerable" if not record["must_cite"] else record["applies_to"]


def compare(before: list[dict], after: list[dict], checks=(check_version_matches,)) -> None:
    """Per-slice pass counts before -> after, so a fix that breaks another slice shows up."""
    def tally(recs: list[dict]) -> Counter:
        t = Counter()
        for r in recs:
            ok = r["passed"] and all(chk(r, r).passed for chk in checks)
            t[slice_of(r)] += ok
            t["overall"] += ok
            t["n:" + slice_of(r)] += 1
        return t
    b, a = tally(before), tally(after)
    print(f"{'slice':<13} {'n':>3} {'before':>7} {'after':>6}")
    for s in sorted({slice_of(r) for r in before}) + ["overall"]:
        n = len(before) if s == "overall" else b["n:" + s]
        print(f"{s:<13} {n:>3} {b[s]:>7} {a[s]:>6}")


if __name__ == "__main__":
    print()
    for chk in (check_version_matches, check_cites_existing_doc):
        fails = [r["case_id"] for r in records if not chk(r, r).passed]
        print(f"{chk.__name__:<26} fails on {len(fails)}: {fails}")
    print("needs a judge (Day 45):", [c.name for c in CRITERIA if c.kind == "judge"])

check_version_matches      fails on 4: ['qa-009', 'x-v-03', 'x-v-04', 'x-v-05']
check_cites_existing_doc   fails on 0: []
needs a judge (Day 45): ['grounded']

# script_id: day_044_error_analysis/fix_version
# fix_version.py: fix the top mode (version_mismatch) and measure it. Save as fix_version.py.
# Install: none offline · Env: ORBIT_DATA (optional); ATLAS_USE_LLM=1 + OPENAI_API_KEY for the LLM path
# Needs: mini_r2.py and analysis.py (and llm_answer.py for the LLM path) next to this file
import re

V2 = re.compile(r"\b2\.(?:x|\d+)\b|orbit\.ini|\bORBIT_(?:[A-Z_]+|\*)")  # 2.9, orbit.ini, ORBIT_LEASE, ORBIT_*
V3 = re.compile(r"\b3\.(?:x|\d+)\b|orbit\.toml")


def detect_version(question: str) -> str:
    """'2.x' or '3.x' for the version a question is about; 'all' when it spans both (migrations)."""
    two, three = bool(V2.search(question)), bool(V3.search(question))
    if two and three:
        return "all"
    return "2.x" if two else "3.x"  # no signal: assume the current release


def naive_detect(question: str) -> str:  # the first version teams write
    return "2.x" if "2" in question else "3.x"


if __name__ == "__main__":
    for q in ["Can I set ORBIT_LEASE=5m on Orbit 2.x?",
              "How do I set worker concurrency with an environment variable in Orbit 3.2?",
              "How does Orbit handle a 2-hour delay on SQS?",
              "Does orbit migrate-config copy my ORBIT_* environment variables into orbit.toml?"]:
        print(f"{naive_detect(q):<4} {detect_version(q):<4} {q}")

2.x  2.x  Can I set ORBIT_LEASE=5m on Orbit 2.x?
2.x  3.x  How do I set worker concurrency with an environment variable in Orbit 3.2?
2.x  3.x  How does Orbit handle a 2-hour delay on SQS?
3.x  all  Does orbit migrate-config copy my ORBIT_* environment variables into orbit.toml?

# script_id: day_044_error_analysis/fix_version
import os

from analysis import compare, load_records
from mini_r2 import extractive_generate, load_cases, run_set

VERSION_RULE = (" The question is about Orbit {version}. Answer only for that version; if the"
                " context is for another version, say so instead of mixing them.")


def version_filter(question: str):
    """Day 27's metadata filter: keep pages for the detected version plus 'all' pages."""
    version = detect_version(question)
    if version == "all":
        return None
    return lambda chunk: chunk.applies_to in (version, "all")


if __name__ == "__main__":
    generate = extractive_generate
    if os.environ.get("ATLAS_USE_LLM"):  # same fix plus one prompt line on the real generator
        from llm_answer import make_llm_generate
        generate = make_llm_generate(VERSION_RULE, version_of=detect_version)
    before = load_records()
    after = run_set(load_cases(), label="fix", where_for=version_filter, generate=generate)
    print()
    compare(before, after)
    changed = [(b["case_id"], b["passed"], a["passed"]) for b, a in zip(before, after)
               if (b["passed"], b["citations"]) != (a["passed"], a["citations"])]
    print("changed cases:", changed)

slice           n  before  after
2.x             7       3      6
3.x            15      12     12
all            13      11     11
unanswerable    3       1      1
overall        38      27     30
changed cases: [('qa-009', False, True), ('x-v-03', False, True), ('x-v-04', True, True), ('x-v-05', False, False)]