#!/usr/bin/env python3
"""Standard-library verifier for the governed-memory model-invariance addendum.

Recomputes every published addendum figure from the row-level artifacts in
../data and fails (exit 1) on any drift. No network, no keys, no third-party
packages.

Usage: python code/verify_model_invariance_metrics.py
"""
from __future__ import annotations

import json
import sys
from math import comb
from pathlib import Path

DATA = Path(__file__).resolve().parent.parent / "data"
ARMS = ("retrieval_only", "recency_aware", "prompt_authority_labeled", "fieldhash_authority")

# ---- pinned published figures -------------------------------------------------
RULE = {  # (model, seed) -> {arm: correct_at_0_5}
    ("qwen", "20260620"): {"retrieval_only": 225, "recency_aware": 236, "prompt_authority_labeled": 263, "fieldhash_authority": 256},
    ("qwen", "20260703"): {"retrieval_only": 222, "recency_aware": 240, "prompt_authority_labeled": 266, "fieldhash_authority": 259},
    ("gemma", "20260620"): {"retrieval_only": 170, "recency_aware": 186, "prompt_authority_labeled": 253, "fieldhash_authority": 253},
    ("gemma", "20260703"): {"retrieval_only": 169, "recency_aware": 183, "prompt_authority_labeled": 253, "fieldhash_authority": 253},
}
BLOCKED = {"20260620": 856, "20260703": 868}
JUDGE = {  # (model, judge) -> {arm: correct}
    ("qwen", "gpt4o"): {"retrieval_only": 147, "recency_aware": 134, "prompt_authority_labeled": 259, "fieldhash_authority": 249},
    ("qwen", "gemini"): {"retrieval_only": 151, "recency_aware": 138, "prompt_authority_labeled": 271, "fieldhash_authority": 268},
    ("gemma", "gpt4o"): {"retrieval_only": 133, "recency_aware": 128, "prompt_authority_labeled": 278, "fieldhash_authority": 279},
    ("gemma", "gemini"): {"retrieval_only": 140, "recency_aware": 132, "prompt_authority_labeled": 289, "fieldhash_authority": 288},
}
FRONTIER = {"recency_aware": 154, "prompt_authority_labeled": 295, "fieldhash_authority": 290}
MCNEMAR = {  # (label) -> (wins, losses)
    "gemma_parity_fh_vs_prompt": (7, 6),
    "qwen_parity_fh_vs_prompt": (20, 30),
    "gemma_fh_vs_frontier_recency": (133, 8),
    "qwen_fh_vs_frontier_recency": (120, 25),
    "gemma_fh_vs_frontier_fh": (4, 15),
    "qwen_fh_vs_frontier_fh": (3, 44),
}

failures: list[str] = []


def check(label: str, actual, expected) -> None:
    ok = actual == expected
    print(f"{'OK ' if ok else 'FAIL'} {label}: {actual}" + ("" if ok else f" (expected {expected})"))
    if not ok:
        failures.append(label)


def rule_counts(path: Path) -> dict:
    d = json.loads(path.read_text())
    out = {a: 0 for a in ARMS}
    for r in d["rows"]:
        if r["arm"] in out and (r.get("primary_score") or 0) >= 0.5:
            out[r["arm"]] += 1
    return out, d


def judge_counts(path: Path) -> dict:
    d = json.loads(path.read_text())
    out = {}
    for r in d["rows"]:
        out.setdefault(r["arm"], 0)
        if (r.get("judge_score") or 0) >= 0.5:
            out[r["arm"]] += 1
    return out


def judge_map(path: Path, arm: str) -> dict:
    d = json.loads(path.read_text())
    return {r["question_key"]: 1 if (r.get("judge_score") or 0) >= 0.5 else 0 for r in d["rows"] if r["arm"] == arm}


def mcnemar(a: dict, b: dict):
    keys = sorted(set(a) & set(b))
    w = sum(1 for k in keys if a[k] and not b[k])
    l = sum(1 for k in keys if b[k] and not a[k])
    n = w + l
    p = 1.0 if n == 0 else min(1.0, sum(comb(n, i) for i in range(0, min(w, l) + 1)) / 2 ** n * 2)
    return w, l, p


def main() -> int:
    # rule-scorer figures, recomputed from row-level primary_score
    for (model, seed), expected in RULE.items():
        counts, run = rule_counts(DATA / f"v512_{model}_{seed}.json")
        for arm in ARMS:
            check(f"rule {model} seed{seed} {arm}", counts[arm], expected[arm])
        check(f"blocked {model} seed{seed}", run["summary"]["fieldhash_authority"]["blocked_records"], BLOCKED[seed])
        check(f"chain {model} seed{seed}", run["summary"]["fieldhash_packet_chain"]["hash_chain_verified"], True)

    # judge figures, recomputed from row-level judge_score
    for (model, judge), expected in JUDGE.items():
        counts = judge_counts(DATA / f"judge512_{model}_{judge}.json")
        for arm, val in expected.items():
            check(f"judge {model} {judge} {arm}", counts.get(arm, 0), val)
    fr = judge_counts(DATA / "judge_frontier_contemporaneous.json")
    for arm, val in FRONTIER.items():
        check(f"judge frontier {arm}", fr.get(arm, 0), val)

    # McNemar discordant pairs, recomputed from paired rows
    fr_rec = judge_map(DATA / "judge_frontier_contemporaneous.json", "recency_aware")
    fr_fh = judge_map(DATA / "judge_frontier_contemporaneous.json", "fieldhash_authority")
    for model in ("gemma", "qwen"):
        fh = judge_map(DATA / f"judge512_{model}_gpt4o.json", "fieldhash_authority")
        pr = judge_map(DATA / f"judge512_{model}_gpt4o.json", "prompt_authority_labeled")
        w, l, _ = mcnemar(fh, pr)
        check(f"mcnemar {model}_parity_fh_vs_prompt", (w, l), MCNEMAR[f"{model}_parity_fh_vs_prompt"])
        w, l, _ = mcnemar(fh, fr_rec)
        check(f"mcnemar {model}_fh_vs_frontier_recency", (w, l), MCNEMAR[f"{model}_fh_vs_frontier_recency"])
        w, l, _ = mcnemar(fh, fr_fh)
        check(f"mcnemar {model}_fh_vs_frontier_fh", (w, l), MCNEMAR[f"{model}_fh_vs_frontier_fh"])

    print()
    if failures:
        print(f"VERIFICATION FAILED: {len(failures)} figure(s) drifted")
        return 1
    print("ALL FIGURES VERIFIED: every published addendum figure recomputes from the row-level artifacts.")
    return 0


if __name__ == "__main__":
    sys.exit(main())
