"""Measure whether Polymarket's price moves BEFORE or AFTER a public score feed.

This settles one question and nothing else: can a retail operator with a public
data source be first? If Polymarket's mid moves before ESPN reports the play,
then no strategy built on that feed can be early -- it can only be late, and
being late is how b00k13's bot lost money on a 7% theoretical edge.

Two phases:

    record   poll ESPN and the Polymarket CLOB in parallel, append to JSONL
    analyse  for each scoring event, find when the price actually moved

Read-only, stdlib only, never authenticates, never orders.

    python research/latency_probe.py record --seconds 600 --out probe.jsonl
    python research/latency_probe.py analyse probe.jsonl
"""

from __future__ import annotations

import argparse
import json
import re
import sys
import threading
import time
import urllib.error
import urllib.request
from collections import defaultdict
from typing import Any

GAMMA = "https://gamma-api.polymarket.com"
CLOB = "https://clob.polymarket.com"
ESPN = "https://site.api.espn.com/apis/site/v2/sports"
UA = {"User-Agent": "nulledge-research/1.0 (read-only latency measurement)"}

# ESPN is the free public feed a retail operator would realistically use.
# It is deliberately the weakest link -- that is the point of the experiment.
ESPN_POLL_S = 2.0
CLOB_POLL_S = 1.0

STOPWORDS = {"the", "fc", "afc", "cf", "sc"}


def fetch(url: str, timeout: int = 15) -> Any:
    req = urllib.request.Request(url, headers=UA)
    with urllib.request.urlopen(req, timeout=timeout) as resp:
        return json.load(resp)


def norm(name: str) -> set[str]:
    """Reduce a team name to comparable tokens.

    ESPN says 'New York Mets', Polymarket says 'New York Mets' in a question
    like 'New York Mets vs. Milwaukee Brewers'. Matching on the last token
    alone breaks on 'New York' collisions, so compare token sets.
    """
    toks = re.findall(r"[a-z]+", name.lower())
    return {t for t in toks if t not in STOPWORDS and len(t) > 2}


def live_espn_games(league: str) -> list[dict]:
    data = fetch(f"{ESPN}/{league}/scoreboard")
    out = []
    for ev in data.get("events", []):
        status = (ev.get("status") or {}).get("type", {})
        if status.get("state") != "in":
            continue
        comp = (ev.get("competitions") or [{}])[0]
        teams = []
        score = {}
        for c in comp.get("competitors", []):
            t = (c.get("team") or {}).get("displayName", "")
            teams.append(t)
            score[t] = c.get("score")
        out.append(
            {
                "id": ev.get("id"),
                "short": ev.get("shortName"),
                "teams": teams,
                "score": score,
                "detail": status.get("detail"),
                # This is the state we watch for changes. Any change means
                # something happened on the field that ESPN has now published.
                "state": json.dumps(
                    {"s": score, "d": status.get("detail")}, sort_keys=True
                ),
            }
        )
    return out


def polymarket_sports_markets(limit: int = 100) -> list[dict]:
    return fetch(
        f"{GAMMA}/markets?limit={limit}&closed=false"
        f"&order=volume24hr&ascending=false"
    )


# A game has several markets: moneyline, totals, spreads. Only the moneyline
# reprices directly on a score change, so the others would blur the signal.
DERIVATIVE_MARKERS = ("o/u", "over/under", "spread", "handicap", "total", "+", "margin")


def match_market(game: dict, markets: list[dict]) -> dict | None:
    """Find the moneyline Polymarket market for an ESPN game by team tokens."""
    want = [norm(t) for t in game["teams"] if t]
    if len(want) != 2:
        return None
    candidates = []
    for m in markets:
        if m.get("feeType") != "sports_fees_v2":
            continue
        question = str(m.get("question", ""))
        q = norm(question)
        # Require BOTH teams present, else a shared city name matches alone.
        if sum(1 for w in want if w and w <= q) != 2:
            continue
        lowered = question.lower()
        if any(marker in lowered for marker in DERIVATIVE_MARKERS):
            continue
        candidates.append((len(question), m))
    if not candidates:
        return None
    # Among survivors the moneyline has the shortest question -- derivatives
    # carry extra qualifiers even when they dodge the marker list.
    candidates.sort(key=lambda pair: pair[0])
    return candidates[0][1]


def token_ids(market: dict) -> list[str]:
    raw = market.get("clobTokenIds")
    if isinstance(raw, str):
        try:
            raw = json.loads(raw)
        except json.JSONDecodeError:
            return []
    return raw if isinstance(raw, list) else []


def mid_price(token_id: str) -> tuple[float | None, float | None]:
    try:
        b = fetch(f"{CLOB}/book?token_id={token_id}", timeout=10)
    except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError):
        return None, None
    bids, asks = b.get("bids") or [], b.get("asks") or []
    if not bids or not asks:
        return None, None
    # CLOB returns bids ascending and asks descending: best is the LAST entry.
    bid, ask = float(bids[-1]["price"]), float(asks[-1]["price"])
    return (bid + ask) / 2.0, ask - bid


class Recorder:
    def __init__(self, path: str, seconds: int, leagues: list[str]):
        self.path = path
        self.seconds = seconds
        self.leagues = leagues
        self.stop = threading.Event()
        self.lock = threading.Lock()
        self.fh = open(path, "a", encoding="utf-8")
        self.pairs: list[dict] = []
        # Wall-clock deadline, not a countdown. If the laptop sleeps mid-run,
        # a countdown timer pauses and the recording bleeds hours into the next
        # day (observed: records at 16:24 the following afternoon). Checking
        # elapsed wall time means a sleep/resume jump ends the run immediately
        # on wake instead of polling for another full duration.
        self.deadline = time.time() + seconds

    def expired(self) -> bool:
        return time.time() >= self.deadline

    def emit(self, rec: dict) -> None:
        rec["t"] = time.time()
        with self.lock:
            self.fh.write(json.dumps(rec) + "\n")
            self.fh.flush()

    def discover(self) -> None:
        markets = polymarket_sports_markets()
        found = []
        for lg in self.leagues:
            try:
                games = live_espn_games(lg)
            except Exception as e:  # noqa: BLE001 - network, keep going
                print(f"  {lg}: ESPN error {e}", file=sys.stderr)
                continue
            for g in games:
                m = match_market(g, markets)
                if not m:
                    continue
                ids = token_ids(m)
                if len(ids) != 2:
                    continue
                found.append(
                    {
                        "game_id": g["id"],
                        "short": g["short"],
                        "league": lg,
                        "question": str(m.get("question"))[:70],
                        "token": ids[0],
                    }
                )
        self.pairs = found

    def poll_espn(self) -> None:
        last: dict[str, str] = {}
        while not self.stop.is_set() and not self.expired():
            for lg in self.leagues:
                try:
                    games = live_espn_games(lg)
                except Exception:  # noqa: BLE001
                    continue
                for g in games:
                    prev = last.get(g["id"])
                    if prev is not None and prev != g["state"]:
                        self.emit(
                            {
                                "src": "espn",
                                "game_id": g["id"],
                                "short": g["short"],
                                "detail": g["detail"],
                                "score": g["score"],
                                "changed": True,
                            }
                        )
                    last[g["id"]] = g["state"]
            self.stop.wait(ESPN_POLL_S)

    def poll_clob(self) -> None:
        last: dict[str, float] = {}
        while not self.stop.is_set() and not self.expired():
            for p in self.pairs:
                mid, spread = mid_price(p["token"])
                if mid is None:
                    continue
                prev = last.get(p["game_id"])
                if prev is None or abs(mid - prev) >= 1e-9:
                    self.emit(
                        {
                            "src": "clob",
                            "game_id": p["game_id"],
                            "short": p["short"],
                            "mid": mid,
                            "spread": spread,
                            "delta": None if prev is None else round(mid - prev, 6),
                        }
                    )
                    last[p["game_id"]] = mid
            self.stop.wait(CLOB_POLL_S)

    def run(self) -> int:
        print("Discovering live games matched to Polymarket markets...")
        self.discover()
        if not self.pairs:
            print("No live games matched a Polymarket sports market. Nothing to do.")
            return 1
        for p in self.pairs:
            print(f"  {p['short']:<14} -> {p['question']}")
        print(f"\nRecording {self.seconds}s to {self.path} "
              f"(ESPN {ESPN_POLL_S}s, CLOB {CLOB_POLL_S}s)...")
        self.emit({"src": "meta", "pairs": self.pairs, "seconds": self.seconds})

        threads = [
            threading.Thread(target=self.poll_espn, daemon=True),
            threading.Thread(target=self.poll_clob, daemon=True),
        ]
        for t in threads:
            t.start()
        try:
            # Poll the deadline in short slices so a wall-clock jump is noticed
            # within a second, rather than sleeping through the whole duration.
            while not self.expired():
                self.stop.wait(1.0)
        except KeyboardInterrupt:
            pass
        self.stop.set()
        for t in threads:
            t.join(timeout=5)
        self.fh.close()
        print("Done.")
        return 0


def _hit_rate(
    events: list[dict],
    clob: dict[str, list[dict]],
    window: float,
    threshold: float,
) -> tuple[int, list[float]]:
    """Count events with a material price move in the preceding window."""
    hits, leads = 0, []
    for e in events:
        moves = [
            m
            for m in clob.get(e["game_id"], [])
            if m.get("delta") is not None and abs(m["delta"]) >= threshold
        ]
        before = [m for m in moves if 0 < e["t"] - m["t"] <= window]
        if before:
            hits += 1
            leads.append(e["t"] - before[-1]["t"])
    return hits, leads


def _mid_at(clob_rows: list[dict], t: float) -> float | None:
    """The market mid last observed at or before time t, else the first after."""
    before = [r for r in clob_rows if r["t"] <= t and r.get("mid") is not None]
    if before:
        return before[-1]["mid"]
    after = [r for r in clob_rows if r.get("mid") is not None]
    return after[0]["mid"] if after else None


def analyse(
    path: str,
    window: float = 30.0,
    threshold: float = 0.005,
    permutations: int = 2000,
    comp_lo: float = 0.10,
    comp_hi: float = 0.90,
) -> int:
    """Test whether price moves PRECEDE score changes more than chance allows.

    The naive version of this test is worthless three times over, and each trap
    is guarded here:

    1. Prices update constantly, so any-move-in-a-lookback finds one almost
       always. Guarded by a permutation null: shuffle event times and see what
       rate random events score against the same price stream.
    2. Inning boundaries ('Middle 5th') carry no price information. Guarded by
       counting only genuine SCORE changes.
    3. A run in a decided game (14-2) cannot move a market already pinned at
       0.99, so it produces a spurious 'no lead'. Guarded by the competitiveness
       filter: an event only counts if the market mid at that moment is between
       comp_lo and comp_hi. This was the flaw that made a blowout read as 0%.

    A sleep/resume gap can also bleed the file into the next day; records past
    the intended window (meta.seconds) are dropped before anything else.
    """
    espn: list[dict] = []
    clob: dict[str, list[dict]] = defaultdict(list)
    first_t: float | None = None
    planned: float | None = None
    raw: list[dict] = []
    with open(path, encoding="utf-8") as fh:
        for line in fh:
            try:
                r = json.loads(line)
            except json.JSONDecodeError:
                continue
            raw.append(r)
            if r.get("t") is not None:
                first_t = r["t"] if first_t is None else min(first_t, r["t"])
            if r.get("src") == "meta":
                planned = r.get("seconds")

    # Contamination guard: clip anything after the intended recording window.
    window_end = (first_t + planned + 30) if (first_t and planned) else None
    dropped = 0
    t_min = t_max = None
    for r in raw:
        t = r.get("t")
        if window_end is not None and t is not None and t > window_end:
            dropped += 1
            continue
        if t is not None:
            t_min = t if t_min is None else min(t_min, t)
            t_max = t if t_max is None else max(t_max, t)
        if r.get("src") == "espn":
            espn.append(r)
        elif r.get("src") == "clob":
            clob[r["game_id"]].append(r)

    if dropped:
        print(f"Dropped {dropped} records after the intended window "
              f"(sleep/resume contamination).")

    if not espn:
        print("No ESPN state changes recorded. Run longer, or during live play.")
        return 1

    # Classify: did the scoreline actually change, or only the inning label?
    prev_score: dict[str, Any] = {}
    scoring, boundary = [], []
    for e in sorted(espn, key=lambda r: r["t"]):
        gid = e["game_id"]
        sc = json.dumps(e.get("score"), sort_keys=True)
        if gid in prev_score and prev_score[gid] != sc:
            scoring.append(e)
        else:
            boundary.append(e)
        prev_score[gid] = sc

    # Competitiveness filter: keep only events where the outcome was still live.
    competitive, decided = [], []
    for e in scoring:
        mid = _mid_at(clob.get(e["game_id"], []), e["t"])
        if mid is not None and comp_lo <= mid <= comp_hi:
            competitive.append(e)
        else:
            decided.append((e, mid))

    n_moves = sum(len(v) for v in clob.values())
    material = sum(
        1
        for v in clob.values()
        for m in v
        if m.get("delta") is not None and abs(m["delta"]) >= threshold
    )
    span = (t_max - t_min) if (t_min and t_max) else 0.0

    print(f"Recording span:        {span / 60:.1f} min")
    print(f"ESPN state changes:    {len(espn)}  "
          f"(score changes: {len(scoring)}, inning boundaries: {len(boundary)})")
    print(f"Scoring events:        {len(scoring)}  "
          f"(competitive: {len(competitive)}, in decided games: {len(decided)})")
    print(f"Price updates:         {n_moves}  "
          f"(>= {threshold:.3f} move: {material})")
    if span > 0 and material:
        print(f"Material move rate:    one per {span / material:.1f}s per book")

    if not competitive:
        print("\nNo scoring events in COMPETITIVE game states. A run in a "
              "decided\ngame cannot move a pinned market, so these carry no "
              "signal.")
        print("Verdict: INCONCLUSIVE. Record during close, live games.")
        return 1

    hits, leads = _hit_rate(competitive, clob, window, threshold)
    rate = hits / len(competitive)

    print(f"\n{'game':<14} {'detail':<18} {'score':<10} {'mid':>6} {'before':>9}")
    print("-" * 64)
    for e in competitive:
        h, ld = _hit_rate([e], clob, window, threshold)
        b = f"-{ld[0]:.1f}s" if ld else "--"
        mid = _mid_at(clob.get(e["game_id"], []), e["t"])
        sc = ",".join(str(v) for v in (e.get("score") or {}).values())
        print(f"{str(e.get('short'))[:14]:<14} {str(e.get('detail'))[:18]:<18} "
              f"{sc[:10]:<10} {mid:>6.2f} {b:>9}")

    # Permutation null: shuffle event times uniformly across the recording.
    import random

    rng = random.Random(20260722)
    null_rates = []
    for _ in range(permutations):
        fake = [
            {"game_id": e["game_id"], "t": rng.uniform(t_min + window, t_max)}
            for e in competitive
        ]
        h, _ = _hit_rate(fake, clob, window, threshold)
        null_rates.append(h / len(competitive))
    null_rates.sort()
    null_mean = sum(null_rates) / len(null_rates)
    p95 = null_rates[int(0.95 * len(null_rates))]
    p_value = sum(1 for r in null_rates if r >= rate) / len(null_rates)

    print(f"\nObserved hit rate:     {rate:.1%}  ({hits}/{len(competitive)})")
    print(f"Permutation null mean: {null_mean:.1%}   95th pct: {p95:.1%}")
    print(f"p-value:               {p_value:.3f}")
    if leads:
        leads.sort()
        print(f"Median lead:           {leads[len(leads) // 2]:.1f}s")

    print()
    if len(competitive) < 20:
        print(f"Verdict: INCONCLUSIVE -- only {len(competitive)} competitive "
              f"events. Need >= 20 before the p-value means anything.")
    elif p_value < 0.05:
        print("Verdict: price moves PRECEDE the public feed beyond chance.")
        print("The feed is downstream of the market; a strategy keyed on it is")
        print("structurally late.")
    else:
        print("Verdict: NO measured lead. The observed rate is within what")
        print("random event times produce against this price-update density.")
    return 0


def main() -> int:
    ap = argparse.ArgumentParser()
    sub = ap.add_subparsers(dest="cmd", required=True)

    rec = sub.add_parser("record")
    rec.add_argument("--seconds", type=int, default=600)
    rec.add_argument("--out", type=str, default="latency_probe.jsonl")
    rec.add_argument(
        "--leagues",
        type=str,
        default="baseball/mlb,basketball/nba,hockey/nhl",
    )

    an = sub.add_parser("analyse")
    an.add_argument("path", type=str)
    an.add_argument("--window", type=float, default=30.0)
    an.add_argument("--threshold", type=float, default=0.005)
    an.add_argument("--permutations", type=int, default=2000)
    an.add_argument("--comp-lo", type=float, default=0.10)
    an.add_argument("--comp-hi", type=float, default=0.90)

    args = ap.parse_args()
    if args.cmd == "record":
        return Recorder(
            args.out, args.seconds, [s.strip() for s in args.leagues.split(",")]
        ).run()
    return analyse(args.path, args.window, args.threshold, args.permutations,
                   args.comp_lo, args.comp_hi)


if __name__ == "__main__":
    sys.exit(main())