"""Eval rounds — the in-session model is BOTH author and judge (fully in-house).

  # 1. build + render every design of the round, emit the judging manifest
  BENCH_FILE=eval/rounds/r64.json .cad-venv/bin/python harness/author.py prepare 70

  # 2. the driving session reads each design's reference/full/head images and
  #    scores the rubric in prompts/judge.md, writing verdicts JSON:
  #    {"<name>": {"rubric": {axis: bool...}, "holistic": 0..1, "diffs": [...]}}

  # 3. write the dashboard + commit
  .cad-venv/bin/python harness/author.py finalize 70 /path/verdicts.json

Legacy API judging (OpenRouter, PI_JUDGE_MODEL) remains:
  .cad-venv/bin/python harness/author.py 70
Monitor: http://127.0.0.1:8770/viewer/eval.html
"""

from __future__ import annotations

import json
import os
import shutil
import subprocess
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from harness.judge import judge                       # noqa: E402

BENCH = json.loads((ROOT / os.environ.get("BENCH_FILE", "eval/rounds/r64.json")).read_text())


def to_request(b: dict) -> dict:
    metal = b["metal"].replace("_", " ").replace("950", "").strip()
    for k in ("18k ", "14k ", "10k "):
        metal = metal.replace(k, "")
    return {"name": b["name"], "setting": b["archetype"].replace("_", " "),
            "cut": b["cut"].replace("_", " "), "carat": b["carat"],
            "standard": b["standard"], "size": b["size"], "metal": metal.strip(),
            "reference": b["reference"],
            "camera": b.get("camera", "25:20")}


def write_dashboard(rows: list[dict], iteration: int) -> float:
    from datetime import datetime, timezone
    RESULTS = ROOT / "artifacts" / "eval-dashboard" / "results.json"
    data = json.loads(RESULTS.read_text()) if RESULTS.exists() else {"rounds": []}
    data.setdefault("rounds", [])
    now = datetime.now(timezone.utc).isoformat(timespec="seconds")
    scored = [r for r in rows if r.get("match")]
    mean = sum(r["match"]["score"] for r in scored) / len(rows) if rows else 0.0
    rnd = {"round": f"tune-{iteration}", "started": now, "iteration": iteration,
           "n": len(rows), "passed": len(scored), "mean": round(mean, 3), "designs": rows}
    # append history — one entry PER iteration (re-finalizing the same
    # iteration replaces itself; earlier the single "tune" slot was
    # overwritten every round, so the site never accumulated results)
    data["rounds"] = [r for r in data["rounds"]
                      if r.get("iteration") != iteration
                      and r.get("round") != "tune"] + [rnd]
    data["updated"] = now
    RESULTS.parent.mkdir(parents=True, exist_ok=True)
    RESULTS.write_text(json.dumps(data, indent=2))
    return mean

JUDGE_VOTES = 5   # 3-of-5 majority (r64): with 7 boolean axes x 5 designs, one
#                   stochastic axis flip moves the round mean +-0.03 and swamped
#                   single-knob experiments at 3 votes; 5 votes roughly halves
#                   per-axis flip probability. Worth the extra judge calls.


class BudgetExhausted(RuntimeError):
    """Judge API budget hit (HTTP 403) — abort the round instead of committing
    zero-scored designs (r69 was poisoned exactly this way)."""


def judge_stable(ref, png, req, votes: int = JUDGE_VOTES, head=None) -> dict:
    """Call the judge N times; majority-vote each boolean axis to denoise the score.

    The render is deterministic (same STEP -> same PNG), so all single-call variance is
    the judge's stochasticity. Majority vote over N calls gives a stable per-axis verdict.
    """
    runs = []
    saw_403 = 0
    for _ in range(votes + 2):                 # allow a couple retries for flaky API/parse errors
        if len(runs) >= votes:
            break
        try:
            runs.append(judge(ref, png, req, head_png=head))
        except Exception as e:
            if "403" in str(e):
                saw_403 += 1
                if saw_403 >= 2:               # daily budget hit — stop burning retries
                    raise BudgetExhausted("judge budget exhausted (HTTP 403)")
            print(f"      judge vote failed ({str(e)[:80]}); retrying", flush=True)
    if not runs:
        raise RuntimeError("all judge votes failed")
    axes = runs[0]["rubric"].keys()
    rubric = {a: sum(1 for r in runs if r["rubric"].get(a)) > votes / 2 for a in axes}
    score = round(sum(rubric.values()) / len(rubric), 3)
    holistic = round(sum(r.get("holistic", 0) or 0 for r in runs) / votes, 3)
    # diffs from the run closest to the majority verdict
    best = min(runs, key=lambda r: sum(1 for a in axes if bool(r["rubric"].get(a)) != rubric[a]))
    return {"rubric": rubric, "score": score, "holistic": holistic, "diffs": best["diffs"]}

PYBIN = str(ROOT / ".cad-venv" / "bin" / "python")
STEP = str(ROOT / ".agents/skills/cad/scripts/step")
SNAP = str(ROOT / ".agents/skills/cad/scripts/snapshot")
DESIGNS = ROOT / "designs"
OUTROOT = ROOT / "artifacts" / "authored"


def render_authored(name: str, camera: str, timeout: int = 150) -> dict:
    """Build + snapshot an in-house authored generator (no model call)."""
    gen = DESIGNS / f"{name}.py"
    if not gen.exists():
        return {"png": None, "step": None, "log": f"no design spec: designs/{name}.py"}
    outdir = OUTROOT / name
    outdir.mkdir(parents=True, exist_ok=True)
    local = outdir / f"{name}.py"
    shutil.copy(gen, local)
    step = outdir / f"{name}.step"
    p = subprocess.run([PYBIN, STEP, str(local)], cwd=ROOT,
                       capture_output=True, text=True, timeout=timeout)
    if p.returncode != 0 or not step.exists() or step.stat().st_size == 0:
        return {"png": None, "step": None, "log": (p.stdout + p.stderr).strip()[-700:]}
    png = outdir / f"{name}.png"
    head_png = outdir / f"{name}-head.png"
    # colored judge render (harness/render.py): per-part colors — the STEP
    # pipeline drops Compound child colors, which made every render grey-on-grey
    # judged full view stays CLEAN — sprue stems in frame shrink the ring to a
    # corner and wreck the reference comparison; stems live in -print.png
    r = subprocess.run([PYBIN, str(ROOT / "ringcli" / "render.py"), str(local),
                        str(png), camera],
                       cwd=ROOT, capture_output=True, text=True, timeout=timeout)
    subprocess.run([PYBIN, str(ROOT / "ringcli" / "render.py"), str(local),
                    str(outdir / f"{name}-print.png"), camera, "--supports"],
                   cwd=ROOT, capture_output=True, text=True, timeout=timeout)
    subprocess.run([PYBIN, str(ROOT / "ringcli" / "render.py"), str(local),
                    str(head_png), camera, "--head"],
                   cwd=ROOT, capture_output=True, text=True, timeout=timeout)
    if r.returncode != 0 or not png.exists():
        subprocess.run([PYBIN, SNAP, "--input", str(step), "--output", str(png),
                        "--display", "rendered", "--camera", camera],
                       cwd=ROOT, capture_output=True, text=True, timeout=timeout)
    if not png.exists():
        return {"png": None, "step": str(step), "log": "render failed"}
    return {"png": str(png), "head": str(head_png) if head_png.exists() else None,
            "step": str(step), "log": "built (authored)"}


def main(iteration: int) -> int:
    rows: list[dict] = []
    for b in BENCH:
        req = to_request(b)
        name = req["name"]
        row = {"name": name,
               "request": {"brief": f"{req['carat']}ct {req['cut']} {req['setting']} · author=fable-inhouse",
                           "query": b.get("query", "")},
               "reference": {"img": b["reference"], "source": "fable-inhouse"}}
        print(f"[author {iteration}] {name} ...", flush=True)
        out = render_authored(name, req["camera"])
        if not out["png"]:
            print(f"    BUILD-FAIL: {out['log'][:160]}", flush=True)
            row.update(gate="ERROR", preview=None, turntable=None,
                       gate_failures=[f"authored build failed: {out['log'][:120]}"], match=None)
            rows.append(row)
            write_dashboard(rows, iteration)
            continue
        rel = str(Path(out["png"]).resolve().relative_to(ROOT))
        head = Path(out["head"]) if out.get("head") else None
        try:
            j = judge_stable(ROOT / b["reference"], Path(out["png"]), req, head=head)
        except BudgetExhausted as e:
            print(f"ABORTING ROUND {iteration}: {e} — no commit, rerun after the "
                  "budget resets", flush=True)
            return 2
        except Exception as e:
            print(f"    JUDGE-FAIL: {str(e)[:160]}", flush=True)
            row.update(gate="ERROR", preview=rel, turntable=rel,
                       gate_failures=[f"judge failed: {str(e)[:120]}"], match=None)
            rows.append(row)
            write_dashboard(rows, iteration)
            continue
        fails = [k for k, v in j["rubric"].items() if not v]
        row.update(gate="PASS", preview=rel, turntable=rel,
                   match={"score": j["score"],
                          "verdict": f"fable-inhouse rubric {j['score']} · holistic {j['holistic']} · fails: {', '.join(fails) or 'none'}",
                          "diffs": j["diffs"], "rubric": j["rubric"]})
        print(f"    rubric={j['score']} fails={fails}", flush=True)
        rows.append(row)
        write_dashboard(rows, iteration)

    mean = write_dashboard(rows, iteration)
    built = len([r for r in rows if r.get("match")])
    print(json.dumps({"iteration": iteration, "mean_rubric": round(mean, 3), "built": built}))

    subprocess.run(["git", "add", "-A"], cwd=ROOT, timeout=30)
    subprocess.run(["git", "add", "-f", "artifacts/eval-dashboard/results.json"], cwd=ROOT, timeout=30)
    names = ", ".join(r["name"] for r in rows)
    msg = (f"author {iteration}: mean {mean:.3f} ({built}/{len(rows)} built) fable-inhouse — {names}\n\n"
           "Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>")
    c = subprocess.run(["git", "commit", "-q", "-m", msg], cwd=ROOT, timeout=30)
    if c.returncode == 0:
        subprocess.run(["git", "push", "-q", "origin", "HEAD"], cwd=ROOT, timeout=90)
        print(f"committed + pushed author round {iteration}", flush=True)
    return 0


def prepare(iteration: int) -> int:
    manifest = []
    for b in BENCH:
        req = to_request(b)
        print(f"[prepare {iteration}] {req['name']} ...", flush=True)
        out = render_authored(req["name"], req["camera"])
        manifest.append({"name": req["name"], "request": req,
                         "query": b.get("query", ""),
                         "reference": b["reference"],
                         "png": out.get("png"), "head": out.get("head"),
                         "log": out.get("log")})
    mpath = ROOT / "artifacts" / "eval-dashboard" / f"pending-r{iteration}.json"
    mpath.parent.mkdir(parents=True, exist_ok=True)
    mpath.write_text(json.dumps(manifest, indent=2))
    print(json.dumps({"manifest": str(mpath),
                      "built": sum(1 for m in manifest if m["png"])}))
    return 0


def finalize(iteration: int, verdicts_path: str) -> int:
    manifest = json.loads((ROOT / "artifacts" / "eval-dashboard"
                           / f"pending-r{iteration}.json").read_text())
    verdicts = json.loads(Path(verdicts_path).read_text())
    rows = []
    for m in manifest:
        req = m["request"]
        row = {"name": m["name"],
               "request": {"brief": f"{req['carat']}ct {req['cut']} {req['setting']} · author=fable-inhouse · judge=fable-inhouse",
                           "query": m["query"]},
               "reference": {"img": m["reference"], "source": "fable-inhouse"}}
        v = verdicts.get(m["name"])
        if not m.get("png") or not v:
            row.update(gate="ERROR", preview=None, turntable=None,
                       gate_failures=[m.get("log") or "no verdict"], match=None)
            rows.append(row)
            continue
        rel = str(Path(m["png"]).resolve().relative_to(ROOT))
        rubric = {k: bool(v["rubric"].get(k, False)) for k in
                  ("band_slim", "stone_dominant", "prongs_claws", "accents_fine",
                   "profile_low", "setting_correct", "printable_one_piece")}
        score = round(sum(rubric.values()) / len(rubric), 3)
        fails = [k for k, ok in rubric.items() if not ok]
        row.update(gate="PASS", preview=rel, turntable=rel,
                   match={"score": score,
                          "verdict": f"fable-inhouse rubric {score} · holistic {v.get('holistic', 0)} · fails: {', '.join(fails) or 'none'}",
                          "diffs": v.get("diffs", []), "rubric": rubric})
        rows.append(row)
    mean = write_dashboard(rows, iteration)
    built = len([r for r in rows if r.get("match")])
    print(json.dumps({"iteration": iteration, "mean_rubric": round(mean, 3),
                      "built": built}))
    subprocess.run(["git", "add", "-A"], cwd=ROOT, timeout=30)
    subprocess.run(["git", "add", "-f", "artifacts/eval-dashboard/results.json"], cwd=ROOT, timeout=30)
    names = ", ".join(r["name"] for r in rows)
    msg = (f"author {iteration}: mean {mean:.3f} ({built}/{len(rows)} built) in-house judge — {names}\n\n"
           "Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>")
    c = subprocess.run(["git", "commit", "-q", "-m", msg], cwd=ROOT, timeout=30)
    if c.returncode == 0:
        subprocess.run(["git", "push", "-q", "origin", "HEAD"], cwd=ROOT, timeout=90)
        print(f"committed + pushed round {iteration}", flush=True)
    return 0


if __name__ == "__main__":
    if len(sys.argv) > 1 and sys.argv[1] == "prepare":
        raise SystemExit(prepare(int(sys.argv[2])))
    if len(sys.argv) > 1 and sys.argv[1] == "finalize":
        raise SystemExit(finalize(int(sys.argv[2]), sys.argv[3]))
    raise SystemExit(main(int(sys.argv[1]) if len(sys.argv) > 1 else 0))