#!/usr/bin/env python3
"""Diff a run against the published macOS 26.5.1 transcripts (and the M2 Pro / macOS 27 run).

    python3 compare.py out.json [out-dict.json ...]

The question is NOT "what is the WER" — it is "did Apple's model change". So the headline
is how many transcripts come back BYTE-IDENTICAL to the 26.5.1 baseline. A changed model
cannot produce the same string 90%+ of the time; an unchanged one produces almost nothing
else. The WER column uses a simple normalizer (not the app's TextNormalizer), so treat it
as a sanity check on the identical-counts, not as a publishable number.
"""
import gzip, io, json, os, re, sys, urllib.request

BASE_URL = "https://lyonesse.app/data/speech-benchmark/raw-transcripts-apple.json.gz"
CACHE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "baseline",
                     "raw-transcripts-apple-macos26.json")
M2_27 = os.path.join(os.path.dirname(os.path.abspath(__file__)), "baseline",
                     "m2pro-macos27-speechtranscriber.json")


def published_macos26():
    """The Round 1 transcripts: Apple SpeechAnalyzer, macOS 26.5.1, M2 Pro, 2026-07-13."""
    if not os.path.exists(CACHE):
        os.makedirs(os.path.dirname(CACHE), exist_ok=True)
        print(f"fetching {BASE_URL}")
        raw = gzip.decompress(urllib.request.urlopen(BASE_URL).read())
        with open(CACHE, "wb") as fh:
            fh.write(raw)
    data = json.load(open(CACHE))
    clean = [s for s in data if s["split"] == "test-clean"][0]
    return {t["id"]: t for t in clean["transcripts"]}


def norm(s):
    s = s.lower().replace("’", "'")
    return re.sub(r"[^a-z0-9' ]", " ", s).split()


def edits(a, b):
    row = list(range(len(b) + 1))
    for i in range(1, len(a) + 1):
        prev, row[0] = row[:], i
        for j in range(1, len(b) + 1):
            row[j] = min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + (a[i - 1] != b[j - 1]))
    return row[len(b)]


def wer(ids, hyps, refs):
    errors = words = 0
    for i in ids:
        ref = norm(refs[i])
        words += len(ref)
        errors += edits(ref, norm(hyps.get(i, "")))
    return errors / words * 100 if words else 0.0


def load(path):
    data = json.load(open(path))
    if isinstance(data, dict):
        meta = f"{data['module']} · {data['chip']} {data.get('memoryGB','?')}GB · {data['os']}"
        return meta, {r["id"]: r["hypothesis"] for r in data["rows"]}
    return os.path.basename(path), {r["id"]: r["hypothesis"] for r in data}


def main():
    if len(sys.argv) < 2:
        print(__doc__)
        sys.exit(2)
    old = published_macos26()
    refs = {i: t["reference"] for i, t in old.items()}
    macos26 = {i: t["hypothesis"] for i, t in old.items()}
    others = [("published macOS 26.5.1 (M2 Pro)", macos26)]
    if os.path.exists(M2_27):
        others.append(load(M2_27))

    for path in sys.argv[1:]:
        label, new = load(path)
        others_here = [o for o in others
                       if not (len(sys.argv) > 1 and os.path.abspath(path) == os.path.abspath(M2_27)
                               and o[0] == load(M2_27)[0])]
        ids = [i for i in new if i in refs]
        print(f"\n=== {path}\n    {label}\n    {len(ids)} clips compared, "
              f"{sum(1 for i in ids if not new[i])} empty")
        print(f"    WER (simple normalizer): {wer(ids, new, refs):.2f}%")
        for name, ref_run in others_here:
            common = [i for i in ids if i in ref_run]
            if not common:
                continue
            same = sum(1 for i in common if new[i] == ref_run[i])
            same_norm = sum(1 for i in common if norm(new[i]) == norm(ref_run[i]))
            print(f"    vs {name}: {same}/{len(common)} byte-identical, "
                  f"{same_norm}/{len(common)} identical ignoring punctuation "
                  f"(their WER {wer(common, ref_run, refs):.2f}%)")
            shown = 0
            for i in common:
                if norm(new[i]) != norm(ref_run[i]) and shown < 5:
                    shown += 1
                    print(f"      {i}\n        ref  {refs[i].lower()}\n"
                          f"        them {ref_run[i]}\n        this {new[i]}")
    print("\nReading it: ~90%+ byte-identical to macOS 26.5.1 means the model did not change. "
          "A large WER gap with few identical strings means it did.")


if __name__ == "__main__":
    main()