#!/usr/bin/env python3
"""X monitor deep audit 2026-10-31: corrected buckets, verification, freshness, dormancy."""
import json, re
from datetime import datetime, timezone, timedelta
from collections import Counter

RAW = "/home/kiro/trend-engine/trends/x/raw"
D = json.load(open(f"{RAW}/scan_results_2026-10-31_FULL.json"))
Y = json.load(open(f"{RAW}/scan_results_2026-10-30_FULL.json"))

IRRELEVANT = {
    "AIBuzzNews","AIGuideHQ","AITECHio","AIFrontliner","TheAIColony","AITOPIAai","GoogleLabs",
    "GoogleAI","DonutAI","cladlabs","FaithJaiye2187","KammelaBala","MyStageAI","AlexStudio44",
    "AI_with_jasmin","Ai_Vaidehi","ChrisLaubAI","CINEMA505","OpenAI","AnthropicAI","AIScottAdams",
    "AISuperDomain","AI_Jasonyu","AIByLeo",
}
FILM_INDUSTRY = {"DiscussingFilm","nofilmschool"}
LABS = {"OpenAI","AnthropicAI","NVIDIAAI"}
OFFTOPIC = {"interesting_aIl","hubermanlab"}
NOT_KIRO = {"Teamily_AI"}   # product marketing / agent-SaaS, never filmmaking

cutoff = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=3)
q = D["qualifying"]
raw = D["raw"]

def parse_d(s):
    return datetime.strptime(s, "%a %b %d %H:%M:%S %z %Y")

print(f"CUTOFF {cutoff.isoformat()}   qualifying {len(q)}   handles {len(D['handles'])}   failures {len(D['failures'])}")

def bucket(x):
    h = x["handle"]
    if h in LABS: return "labs"
    if h in OFFTOPIC: return "offtopic"
    if h in NOT_KIRO: return "notkiro"
    if h in FILM_INDUSTRY: return "film"
    if h in IRRELEVANT: return "irrelevant"
    return "core"

bc = Counter(bucket(x) for x in q)
print("BUCKETS(disjoint)", dict(bc), "sum", sum(bc.values()))

core = [x for x in q if bucket(x)=="core"]
off  = [x for x in q if bucket(x)=="offtopic"]
film = [x for x in q if bucket(x)=="film"]
irr  = [x for x in q if bucket(x)=="irrelevant"]
labs = [x for x in q if bucket(x)=="labs"]
nk   = [x for x in q if bucket(x)=="notkiro"]

core.sort(key=lambda x: x["score"], reverse=True)
print("\n=== CORRECTED CORE TOP 22 (raw score) ===")
for i, x in enumerate(core[:22], 1):
    lr = (x["likes"]/x["views"]*100) if x["views"] else 0
    print(f"{i}|@{x['handle']}|{x['likes']}L|{x['rt']}RT|{x['replies']}R|{x['views']}V|{x['date']}|{x['score']:.0f}|lr={lr:.2f}%|{x['text'][:130]}")
print("cutoff rank10", f"{core[9]['score']:.0f}", " rank11", f"{core[10]['score']:.0f}", f"@{core[10]['handle']}")

# organic top 10: drop posts whose like rate is an order of magnitude below organic norm
print("\n=== ORGANIC CORE TOP 12 (like_rate >= 0.15%) ===")
org = [x for x in core if x["views"] and (x["likes"]/x["views"]*100) >= 0.15]
n = 0
for x in org:
    n += 1
    lr = x["likes"]/x["views"]*100
    print(f"{n}|@{x['handle']}|{x['likes']}L|{x['views']}V|{x['date']}|{x['score']:.0f}|lr={lr:.2f}%|{x['text'][:120]}|{x['url']}")
    if n >= 12: break

# ---- watchlist diff ----
added = sorted(set(D["handles"]) - set(Y["handles"]), key=str.lower)
removed = sorted(set(Y["handles"]) - set(D["handles"]), key=str.lower)
print(f"\nWATCHLIST {len(D['handles'])} (yesterday {len(Y['handles'])}) ADDED {added} REMOVED {removed}")

cov = sorted({x["handle"] for x in q}, key=str.lower)
print(f"COVERAGE {len(cov)} handles")
print("PER-HANDLE QUALIFYING:", ", ".join(f"@{h}({n})" for h,n in Counter(x["handle"] for x in q).most_common()))

# ---- verify top 12 against raw ----
print("\n=== TOP-12 RAW VERIFICATION ===")
for i, x in enumerate(core[:12], 1):
    tid = x["url"].rsplit("/",1)[-1]
    h = x["handle"]
    match = None
    for r in raw.get(h, []):
        if str(r.get("id")) == tid:
            match = r; break
    if not match:
        print(f"{i} @{h} NOT FOUND IN RAW {tid}"); continue
    media = match.get("media") or []
    mtype = media[0].get("type") if media else "none"
    lr = (x["likes"]/x["views"]*100) if x["views"] else 0
    print(f"{i} @{h} author={match.get('author',{}).get('userName')} media={mtype} "
          f"rt={bool(match.get('retweeted_tweet'))} quote={bool(match.get('quoted_tweet'))} "
          f"reply={bool(match.get('isReply'))} like_rate={lr:.3f}% | {x['url']}")

# ---- freshness ----
print("\n=== FRESHNESS (hours since newest fetched tweet) ===")
now = datetime.now(timezone.utc)
fr = []
for h, tw in raw.items():
    ds = [parse_d(t["createdAt"]) for t in tw if t.get("createdAt")]
    if not ds: continue
    newest = max(ds)
    fr.append((h, (now-newest).total_seconds()/3600, len(tw), newest,
               sum(1 for d in ds if d >= cutoff)))
fr.sort(key=lambda r: r[1])
print("FRESHEST:", ", ".join(f"@{h} {h2:.1f}h" for h,h2,_,_,_ in fr[:12]))
print("STALEST:", ", ".join(f"@{h} {h2:.1f}h" for h,h2,_,_,_ in fr[-14:]))
print("\nKEY ACCOUNTS:")
for h in ["Flovaai","higgsfield","higgsfield_ai","runwayml","venturetwins","Kling_ai","c_valenzuelab",
          "CharaspowerAI","AIWarper","ItsMaryamAi","Soaima_Ai","ElsaSofia__AI","NVIDIAAI","OpenAI",
          "AnthropicAI","EHuanglu","pika_labs","Kling_ai","openart_ai","Teamily_AI"]:
    for hh,hh2,nn,newest,inw in fr:
        if hh == h:
            print(f"  @{h} n={nn} newest {newest.strftime('%m-%d %H:%M')} ({hh2:.1f}h) in-window {inw}")

# ---- dormancy ----
TIER2 = ["CuriousRefuge","paultrillo","JeffSynthesized","ai_filmmaker","Magiermogul","JiamingSuen","Chandan_Perla"]
print("\n=== DORMANCY (originals only) ===")
for h in TIER2 + ["karenxcheng","WuxiaRocks","AI__TSUBAKI","EinsiaAI","different_ai","via_ai_","numerai","pika_labs"]:
    tw = raw.get(h)
    if not tw:
        print(f"@{h}: NOT FETCHED"); continue
    orig = [t for t in tw if not t.get("retweeted_tweet") and not (t.get("text") or "").startswith("RT @")]
    if not orig:
        print(f"@{h}: 0 originals of {len(tw)} fetched -> RT-ONLY"); continue
    newest = max(parse_d(t["createdAt"]) for t in orig)
    inw = [t for t in orig if parse_d(t["createdAt"]) >= cutoff]
    best = max((t.get("likeCount") or 0) for t in inw) if inw else 0
    verdict = "DORMANT" if not inw else (f"thresh-artefact(best {best}L)" if best < 50 else "qualifying")
    print(f"@{h}: orig {len(orig)}/{len(tw)} newest {newest.strftime('%Y-%m-%d %H:%M')} "
          f"({(now-newest).total_seconds()/3600:.1f}h) in-window-orig {len(inw)} best-likes {best} -> {verdict}")

# ---- film bucket AI audit ----
STRONG = re.compile(r"\b(a\.i\.|seedance|kling|runway|veo|sora|minimax|hailuo|higgsfield|midjourney|"
                    r"nano ?banana|chatgpt|gemini|generative|vfx|ai film|aifilm|deepfake|comfyui|"
                    r"diffusion|(ai|3d|generated) ?animation)\b", re.I)
URL = re.compile(r"https?://\S+")
print("\n=== FILM BUCKET AI AUDIT ===")
print("film qualifying", len(film), " STRONG_TERMS hits:", sum(1 for x in film if STRONG.search(x["text"])))
bare = 0
for x in film:
    stripped = URL.sub(" ", x["text"])
    if re.search(r"\bai\b", stripped, re.I):
        bare += 1
        print("  BARE-AI:", f"@{x['handle']}", x["date"], f"{x['likes']}L/{x['views']}V", "|", x["text"][:200], "|", x["url"])
print("bare-ai-after-URL-strip hits:", bare)
for x in film[:4]:
    print(f"  top: @{x['handle']} {x['likes']}L/{x['views']}V {x['date']} | {x['text'][:120]}")

# ---- offtopic / labs summary ----
print("\n=== OFFTopic ===", Counter(x["handle"] for x in off))
for x in sorted(off, key=lambda z:-z["score"])[:4]:
    print(f"  @{x['handle']} {x['likes']}L/{x['views']}V | {x['text'][:110]}")
print("\n=== LABS ===", Counter(x["handle"] for x in labs))
for x in sorted(labs, key=lambda z:-z["score"]):
    print(f"  @{x['handle']} {x['likes']}L/{x['views']}V {x['date']} | {x['text'][:150]}")
print("\n=== NOT-KIRO ===", Counter(x["handle"] for x in nk))
for x in nk:
    print(f"  @{x['handle']} {x['likes']}L/{x['views']}V | {x['text'][:150]}")