#!/usr/bin/env python3
"""2026-10-22 deep pass: bucket counts, fetch windows, dormancy, top-10 verification."""
import json, re, sys
from datetime import datetime, timezone, timedelta
from collections import defaultdict

DATE = "2026-10-22"
path = f"/home/kiro/trend-engine/trends/x/raw/scan_results_{DATE}_FULL.json"
d = json.load(open(path))
q = d["qualifying"]
raw = d["raw"]
handles = d["handles"]

FILM_INDUSTRY = {"DiscussingFilm", "nofilmschool"}
AI_NEWS = {"OpenAI", "AnthropicAI"}
IRRELEVANT = {"AIBuzzNews","AIGuideHQ","AITECHio","AIFrontliner","TheAIColony","AITOPIAai",
              "GoogleLabs","GoogleAI","DonutAI","cladlabs","FaithJaiye2187","KammelaBala",
              "MyStageAI","AlexStudio44","AI_with_jasmin","Ai_Vaidehi","ChrisLaubAI",
              "CINEMA505","AIScottAdams","AISuperDomain","AI_Jasonyu","AIByLeo"}

def pdate(s):
    return datetime.strptime(s, "%a %b %d %H:%M:%S %z %Y")

def qdate(s):
    return datetime.strptime(s.replace(" UTC", ""), "%Y-%m-%d %H:%M").replace(tzinfo=timezone.utc)

core = [x for x in q if x["handle"] not in FILM_INDUSTRY | AI_NEWS | IRRELEVANT]
film = [x for x in q if x["handle"] in FILM_INDUSTRY]
irr  = [x for x in q if x["handle"] in IRRELEVANT]
labs = [x for x in q if x["handle"] in AI_NEWS]
print(f"TOTAL qualifying={len(q)}  core={len(core)} film={len(film)} irr={len(irr)} labs={len(labs)}")
print(f"failures={d.get('failures')!r}  handles={len(handles)}")

core.sort(key=lambda x: -x["score"])
print("\n=== CORE TOP 12 ===")
for i, x in enumerate(core[:12], 1):
    print(f"{i}|@{x['handle']}|{x['likes']}L|{x['rt']}RT|{x['replies']}R|{x['views']}V|{x['date']}|{x['score']:.0f}| {x['text'][:90]}")
print(f"\ncutoff rank10={core[9]['score']:.0f}  rank11={core[10]['score']:.0f}")

# ---- fetch windows / freshness ----
now = datetime.now(timezone.utc)
print("\n=== FETCH WINDOWS (parsed datetimes) ===")
stale = []
for h in handles:
    rs = raw.get(h) or []
    if not rs:
        print(f"@{h}: NO RAW")
        continue
    ds = [pdate(r["createdAt"]) for r in rs]
    mn, mx = min(ds), max(ds)
    age = (now - mx).total_seconds() / 3600
    if age > 24:
        stale.append((h, len(rs), mn, mx, age))
for h, n, mn, mx, age in sorted(stale, key=lambda t: -t[4]):
    print(f"  @{h}: n={n} {mn:%m-%d %H:%M} -> {mx:%m-%d %H:%M} newest {age:.1f}h old")
print(f"  ({len(handles)-len(stale)} handles newer than 24h)")

# ---- dormancy: originals only, in-window ----
WIN = now - timedelta(days=3)
print("\n=== DORMANCY / ZERO-QUALIFYING (originals only) ===")
qh = {x["handle"] for x in q}
for h in handles:
    if h in qh:
        continue
    rs = raw.get(h) or []
    origs = [r for r in rs if r.get("author", {}).get("userName", "").lower() == h.lower()
             and not r.get("retweeted_tweet")]
    if not origs:
        print(f"  @{h}: NO ORIGINALS in {len(rs)} fetched (RT-only)")
        continue
    ds = [pdate(r["createdAt"]) for r in origs]
    newest = max(ds)
    inwin = [r for r in origs if pdate(r["createdAt"]) >= WIN]
    best = max(inwin, key=lambda r: r.get("likeCount", 0)) if inwin else None
    tag = "IN-WINDOW ZERO" if inwin else ("DORMANT" if newest < WIN else "?")
    b = f"best {best.get('likeCount')}L/{best.get('viewCount')}V" if best else ""
    print(f"  @{h}: {len(origs)} orig, newest {newest:%m-%d %H:%M} ({(now-newest).days}d), {len(inwin)} in-window {b} [{tag}]")

# ---- top 12 core verification against raw ----
print("\n=== TOP 12 CORE: author / media / baseline ratio ===")
for i, x in enumerate(core[:12], 1):
    tid = x["url"].rstrip("/").split("/")[-1]
    h = x["handle"]
    match = next((r for r in raw.get(h, []) if str(r.get("id")) == tid), None)
    base = sorted([r.get("likeCount", 0) for r in raw.get(h, []) if str(r.get("id")) != tid], reverse=True)
    bmax = base[0] if base else 0
    ratio = (x["likes"] / bmax) if bmax else float("inf")
    if not match:
        print(f"{i}|@{h}: record NOT FOUND in raw (id={tid})")
        continue
    auth = match.get("author", {}).get("userName")
    med = match.get("media")
    mtype = med[0]["type"] if isinstance(med, list) and med and isinstance(med[0], dict) else "none"
    rt = bool(match.get("retweeted_tweet"))
    print(f"{i}|@{h}|auth={auth}|media={mtype}|RT={rt}|quote={bool(match.get('quoted_tweet'))}|"
          f"{x['likes']}L|ratio={ratio:.1f}x base_max={bmax}|{x['date']}")
    print(f"     {x['text'][:160]}")