#!/usr/bin/env python3
"""Deep analysis 2026-11-11: disjoint buckets, raw+organic rankings, like rates,
freshness, originals-only dormancy, watchlist diff, 24h deltas."""
import os, re, json
from datetime import datetime, timezone, timedelta

RAW = "/home/kiro/trend-engine/trends/x/raw"
TODAY = "2026-11-11"
PREV = "2026-11-10"

def load(d):
    with open(f"{RAW}/scan_results_{d}_FULL.json") as f:
        return json.load(f)

cur = load(TODAY)
prev = load(PREV)

CORE = set("""runwayml CuriousRefuge nofilmschool venturetwins Kling_ai paultrillo c_valenzuelab
karenxcheng higgsfield pika_labs EHuanglu azed_ai JeffSynthesized ai_filmmaker Magiermogul
WuxiaRocks JiamingSuen higgsfield_ai Chandan_Perla CharaspowerAI AIWarper ItsMaryamAi
ElsaSofia__AI AI__TSUBAKI Flovaai openart_ai perplexity_ai heyglif""".split())
FILM = {"DiscussingFilm", "nofilmschool"}
LABS = {"OpenAI", "AnthropicAI", "NVIDIAAI"}
OFFTOPIC = {"interesting_aIl"}
HEALTH = {"hubermanlab"}
IRREL = set("""AIBuzzNews AIGuideHQ AITECHio AIFrontliner TheAIColony AITOPIAai GoogleLabs GoogleAI
DonutAI cladlabs FaithJaiye2187 KammelaBala MyStageAI AlexStudio44 AI_with_jasmin Ai_Vaidehi
ChrisLaubAI CINEMA505 AIScottAdams AISuperDomain AI_Jasonyu AIByLeo Soaima_Ai different_ai
via_ai_ Teamily_AI AIsaOneHQ AiWithTariq EinsiaAI AidenChaoyangHe dai2noyuyuta Lab_for_Men
Nevermined_ai numerai uns__nstudio""".split())

q = cur["qualifying"]
print(f"TOTAL QUALIFYING {len(q)}  | scanned {len(cur['handles'])} | failures {len(cur['failures'])}")

# --- disjoint buckets ---
buckets = {"core": [], "film": [], "labs": [], "offtopic": [], "health": [], "irrelevant": []}
for t in q:
    h = t["handle"]
    if h in FILM: buckets["film"].append(t)
    elif h in LABS: buckets["labs"].append(t)
    elif h in OFFTOPIC: buckets["offtopic"].append(t)
    elif h in HEALTH: buckets["health"].append(t)
    elif h in IRREL: buckets["irrelevant"].append(t)
    elif h in CORE: buckets["core"].append(t)
    else: buckets["irrelevant"].append(t); print(f"  UNCLASSIFIED -> irrelevant: @{h}")

print("\n== BUCKETS ==")
tot = 0
for k, v in buckets.items():
    print(f"  {k:12s} {len(v)}")
    tot += len(v)
print(f"  SUM {tot}")

# --- core ranking with like rate ---
def lr(t):
    return (t["likes"] / t["views"] * 100) if t["views"] else 0

core = sorted(buckets["core"], key=lambda x: x["score"], reverse=True)
print("\n== CORE TOP 20 (raw) ==")
for i, t in enumerate(core[:20], 1):
    print(f"{i}|@{t['handle']}|{t['likes']}L|{t['rt']}RT|{t['replies']}R|{t['views']}V|"
          f"{lr(t):.4f}%|{t['date']}|{t['score']:.0f}|{t['url']}")
    print(f"    {t['text'][:230]}")
if len(core) > 10:
    print(f"\n  top10 cutoff {core[9]['score']:.0f} | rank11 {core[10]['score']:.0f} | rank12 {core[11]['score']:.0f}")

print("\n== CORE BY LIKE RATE (top 15, min 2k views) ==")
rr = [t for t in core if t["views"] >= 2000]
rr.sort(key=lr, reverse=True)
for i, t in enumerate(rr[:15], 1):
    print(f"{i}|@{t['handle']}|{t['likes']}L/{t['views']}V={lr(t):.2f}%|{t['date']}|{t['text'][:90]}")

# --- freshness ---
print("\n== FRESHNESS (newest fetched post, hours) ==")
now = datetime.now(timezone.utc)
def pdate(s):
    try: return datetime.strptime(s, "%a %b %d %H:%M:%S %z %Y")
    except Exception: return None
fr = []
for h, tw in cur["raw"].items():
    ds = [pdate(t.get("createdAt","")) for t in tw]
    ds = [d for d in ds if d]
    if ds: fr.append((h, max(ds), len(tw)))
fr.sort(key=lambda x: -(now - x[1]).total_seconds())
print("STALEST (newest fetched post):")
for h, d, n in fr[:14]:
    print(f"  @{h:18s} {d.strftime('%m-%d %H:%M')} {(now-d).total_seconds()/3600:6.1f}h n={n}")
print("FRESHEST:")
for h, d, n in fr[-14:]:
    print(f"  @{h:18s} {d.strftime('%m-%d %H:%M')} {(now-d).total_seconds()/3600:6.1f}h n={n}")

# --- dormancy: newest ORIGINAL only ---
cutoff = datetime(2026, 11, 8, tzinfo=timezone.utc)
print("\n== DORMANCY / ORIGINALS IN WINDOW ==")
rows = []
for h, tw in cur["raw"].items():
    origs = [t for t in tw if not t.get("retweeted_tweet") and t.get("type") != "retweet"
             and not (t.get("text") or "").startswith("RT @") and not t.get("isReply")
             and not (t.get("inReplyToId"))]
    inw = [t for t in origs if (pdate(t.get("createdAt","")) or now) >= cutoff]
    newest = max([pdate(t.get("createdAt","")) for t in origs if pdate(t.get("createdAt",""))] or [None]) if origs else None
    best = max([t.get("likeCount") or 0 for t in inw] or [0]) if inw else 0
    rows.append((h, len(origs), len(inw), newest, best))
rows.sort(key=lambda x: x[2])
for h, no, ni, nw, best in rows:
    if ni == 0:
        hh = (now - nw).total_seconds()/3600 if nw else -1
        print(f"  DORMANT @{h:18s} newest_orig {nw.strftime('%Y-%m-%d %H:%M') if nw else 'NONE':16s} {hh:8.1f}h  origs={no}")
print("\n-- in-window but 0 qualifying (threshold artefacts) --")
zq = {t["handle"] for t in q}
for h, no, ni, nw, best in rows:
    if ni > 0 and h not in zq:
        print(f"  ARTEFACT @{h:18s} inwin_orig={ni:2d} best_likes={best}")

# --- 24h deltas on shared ids ---
print("\n== 24H DELTAS (shared tweet ids, biggest abs view change) ==")
pm = {}
for t in prev["qualifying"]:
    tid = t["url"].rsplit("/",1)[-1]
    pm[tid] = t
deltas = []
for t in q:
    tid = t["url"].rsplit("/",1)[-1]
    if tid in pm:
        p = pm[tid]
        deltas.append((h_ := t["handle"], tid, t["likes"]-p["likes"], t["views"]-p["views"],
                       t["likes"], t["views"], t["date"]))
deltas.sort(key=lambda x: -abs(x[3]))
for h, tid, dl, dv, L, V, d in deltas[:18]:
    print(f"  @{h:16s} {L:>6}L ({dl:+6}) {V:>9}V ({dv:+10}) {d}  https://x.com/{h}/status/{tid}")

# --- watchlist diff ---
print("\n== HANDLE SET DIFF vs prev ==")
cs, ps = set(cur["handles"]), set(prev["handles"])
print(f"  scanned {len(ps)} -> {len(cs)} | added {sorted(cs-ps)} | removed {sorted(ps-cs)}")

# --- coverage ---
print("\n== COVERAGE ==")
print(f"  qualifying handles {len({t['handle'] for t in q})} of {len(cur['handles'])}")
print(f"  zero-qualifying: {sorted(set(cur['handles']) - {t['handle'] for t in q})}")

# --- film bucket AI audit ---
print("\n== FILM BUCKET (%d) AI TERM AUDIT ==" % len(buckets["film"]))
STRONG = r"\b(AI|A\.I\.|artificial intelligence|generative|GenAI|Sora|Kling|Runway|Midjourney|Veo|Seedance|Higgsfield|diffusion|prompt|machine learning|LLM)\b"
hits = 0
for t in sorted(buckets["film"], key=lambda x: -x["score"]):
    txt = t["text"]
    if re.search(STRONG, txt, re.I):
        hits += 1
        print(f"  HIT @{t['handle']} {t['date']} {t['likes']}L/{t['views']}V: {txt[:200]}")
print(f"  STRONG_TERMS hits: {hits} of {len(buckets['film'])}")
print("\n  top film posts:")
for t in sorted(buckets["film"], key=lambda x: -x["score"])[:6]:
    print(f"    @{t['handle']} {t['likes']}L/{t['views']}V {t['date']} {t['text'][:100]}")

# --- labs ---
print("\n== AI LABS (%d) ==" % len(buckets["labs"]))
for t in sorted(buckets["labs"], key=lambda x: -x["score"]):
    print(f"  @{t['handle']:12s} {t['likes']}L/{t['views']}V {t['date']} {t['text'][:150]}")