#!/usr/bin/env python3
"""Deep pass 2026-10-27: bucket split, corrected core ranking, new-handle profiling,
freshness, originals-only dormancy, top-post verification."""
import json, re, os
from datetime import datetime, timezone, timedelta
from collections import Counter

DATE = "2026-10-27"
RAW = "/home/kiro/trend-engine/trends/x/raw"
d = json.load(open(f"{RAW}/scan_results_{DATE}_FULL.json"))
PREV = f"{RAW}/scan_results_2026-10-26_FULL.json"
q = d["qualifying"]; raws = d["raw"]; handles = d["handles"]

def pd(s):
    try: return datetime.strptime(s, "%a %b %d %H:%M:%S %z %Y")
    except Exception: return None

now = datetime.now(timezone.utc)
cutoff = now.replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=3)
print(f"NOW {now.isoformat()}  CUTOFF {cutoff.isoformat()}  handles {len(handles)}  qualifying {len(q)}")

# --- watchlist diff ---
if os.path.exists(PREV):
    prev = json.load(open(PREV))
    added = [h for h in handles if h not in prev["handles"]]
    removed = [h for h in prev["handles"] if h not in handles]
    print(f"\nWATCHLIST DIFF: +{len(added)} {added}")
    print(f"                -{len(removed)} {removed}")

# --- buckets ---
FILM = {"DiscussingFilm", "nofilmschool"}
LABS = {"OpenAI", "AnthropicAI", "NVIDIAAI"}
IRREL = {"AIBuzzNews","AIGuideHQ","AITECHio","AIFrontliner","TheAIColony","AITOPIAai",
         "GoogleLabs","GoogleAI","DonutAI","cladlabs","FaithJaiye2187","KammelaBala",
         "MyStageAI","AlexStudio44","AI_with_jasmin","Ai_Vaidehi","ChrisLaubAI",
         "CINEMA505","AIScottAdams","AISuperDomain","AI_Jasonyu","AIByLeo"}
# mass-market / off-topic tiers to be excluded from the AI-filmmaking board
OFFTOPIC = {"interesting_aIl", "hubermanlab"}

core = [x for x in q if x["handle"] not in FILM|LABS|IRREL|OFFTOPIC]
offtop = [x for x in q if x["handle"] in OFFTOPIC]
film = [x for x in q if x["handle"] in FILM]
labs = [x for x in q if x["handle"] in LABS]
irr = [x for x in q if x["handle"] in IRREL]
core.sort(key=lambda x: -x["score"])

print(f"\nBUCKETS: core {len(core)} / offtopic(excluded from board) {len(offtop)} / film {len(film)} / labs {len(labs)} / irrelevant {len(irr)}")
print(f"raw-total {len(q)} check {len(core)+len(offtop)+len(film)+len(labs)+len(irr)}")

print("\n===== CORRECTED AI-FILMMAKING TOP 12 =====")
for i, x in enumerate(core[:12], 1):
    print(f"{i}|@{x['handle']}|{x['likes']}L|{x['rt']}RT|{x['replies']}R|{x['views']}V|{x['date']}|{x['score']:.0f}")
    print(f"   {x['text'][:190]}")
    print(f"   {x['url']}")
print(f"\ncutoff rank10={core[9]['score']:.0f}  rank11={core[10]['score']:.0f} (@{core[10]['handle']})")

# --- verify top 12 against raw: author, media, RT/quote/reply ---
print("\n===== TOP-12 RAW VERIFICATION =====")
for x in core[:12]:
    tid = x["url"].rsplit("/", 1)[-1]
    rec = None
    for r in raws.get(x["handle"], []):
        if str(r.get("id")) == tid: rec = r; break
    if not rec:
        print(f"  @{x['handle']} {tid} NOT FOUND IN RAW"); continue
    au = (rec.get("author") or {}).get("userName")
    media = rec.get("media") or []
    mt = media[0].get("type") if media else "none"
    rt = bool(rec.get("retweeted_tweet"))
    quo = bool(rec.get("quoted_tweet"))
    rep = rec.get("isReply")
    lr = x["likes"] / x["views"] * 100 if x["views"] else 0
    print(f"  @{x['handle']} author={au} media={mt} RT={rt} quote={quo} reply={rep} likeRate={lr:.2f}%")

# --- per-handle freshness ---
print("\n===== FRESHNESS (hours since newest fetched, per handle) =====")
fresh = []
for h, tw in raws.items():
    dts = [pd(t.get("createdAt","")) for t in tw]
    dts = [x for x in dts if x]
    if not dts: continue
    mx = max(dts)
    fresh.append(((now - mx).total_seconds()/3600, h, mx, len(tw)))
fresh.sort()
print("FRESHEST:", ", ".join(f"@{h} {hr:.1f}h" for hr,h,_,_ in fresh[:10]))
print("STALEST :", ", ".join(f"@{h} {hr:.1f}h" for hr,h,_,_ in fresh[-14:]))

# --- key handles chronology ---
print("\n===== KEY HANDLE CHRONOLOGY (in-window originals) =====")
for h in ["higgsfield","higgsfield_ai","Flovaai","venturetwins","Kling_ai","runwayml",
          "AIWarper","EHuanglu","CharaspowerAI","c_valenzuelab","ElsaSofia__AI",
          "Soaima_Ai","ItsMaryamAi","openart_ai","AI__TSUBAKI","azed_ai","pika_labs"]:
    tw = raws.get(h, [])
    orig = [t for t in tw if not t.get("retweeted_tweet") and not (t.get("text") or "").startswith("RT @")]
    inw = [t for t in orig if (pd(t.get("createdAt","")) or datetime.min.replace(tzinfo=timezone.utc)) >= cutoff]
    dts = [pd(t.get("createdAt","")) for t in tw]; dts=[x for x in dts if x]
    new = max(dts) if dts else None
    print(f"\n@{h}: fetched {len(tw)}, originals {len(orig)}, in-window originals {len(inw)}, newest {new} ({(now-new).total_seconds()/3600:.1f}h)" if new else f"\n@{h}: no data")
    for t in sorted(inw, key=lambda t: pd(t.get('createdAt')))[-6:]:
        mt = (t.get("media") or [{}])[0].get("type","none")
        print(f"   {pd(t['createdAt']).strftime('%m-%d %H:%M')} {t.get('likeCount',0)}L/{t.get('viewCount',0)}V [{mt}] {(t.get('text') or '')[:120]}")