import json, re
from datetime import datetime, timezone, timedelta
RAW="/home/kiro/trend-engine/trends/x/raw"
d=json.load(open(f"{RAW}/scan_results_2026-11-13_FULL.json"))
prev=json.load(open(f"{RAW}/scan_results_2026-11-12_FULL.json"))
CUT=datetime(2026,11,10,tzinfo=timezone.utc)
FMT="%a %b %d %H:%M:%S %z %Y"
def dt(s): return datetime.strptime(s,FMT)
now=datetime.now(timezone.utc)

LABS={"OpenAI","AnthropicAI","NVIDIAAI","GoogleAI"}
OFFTOPIC={"interesting_aIl"}
HEALTH={"hubermanlab"}
FILM={"DiscussingFilm","nofilmschool"}
IRRELEVANT={"AIBuzzNews","AIGuideHQ","AITECHio","AIFrontliner","TheAIColony","AITOPIAai","AISuperDomain",
 "GoogleLabs","DonutAI","cladlabs","FaithJaiye2187","KammelaBala","MyStageAI","AlexStudio44",
 "AI_with_jasmin","Ai_Vaidehi","ChrisLaubAI","CINEMA505","AIScottAdams","AI_Jasonyu","AIByLeo",
 "Teamily_AI","EinsiaAI","numerai","different_ai","via_ai_",
 "AIsaOneHQ","AiWithTariq","Lab_for_Men","dai2noyuyuta","uns__nstudio","Nevermined_ai","AidenChaoyangHe","MysticTrioAi","shikamarutsai"}

qual=d['qualifying']
buckets={"core":[],"film":[],"irrelevant":[],"labs":[],"offtopic":[],"health":[]}
for q in qual:
    h=q['handle']
    if h in LABS: buckets['labs'].append(q)
    elif h in OFFTOPIC: buckets['offtopic'].append(q)
    elif h in HEALTH: buckets['health'].append(q)
    elif h in FILM: buckets['film'].append(q)
    elif h in IRRELEVANT: buckets['irrelevant'].append(q)
    else: buckets['core'].append(q)
print("=== DISJOINT BUCKETS ===")
tot=0
for k,v in buckets.items():
    print(f"  {k:12s} {len(v)}")
    tot+=len(v)
print(f"  SUM {tot} vs qualifying {len(qual)}")

STRONG=re.compile(r"\b(ai|artificial intelligence|generative|vfx|diffusion|sora|runway|midjourney|seedance|stable diffusion|machine learning|neural|deepfake|synthetic media|llm|gpt|comfyui|blender|unreal|render)\b",re.I)
print("\n=== FILM BUCKET AI-TERM MATCHES ===")
film_ai=[q for q in buckets['film'] if STRONG.search(q['text'])]
print(f"  film total {len(buckets['film'])}, ai-term {len(film_ai)}")
for q in film_ai[:6]:
    print(f"  @{q['handle']} {q['likes']}L/{q['views']}V {q['date']} | {q['text'][:160]}")
    print(f"    {q['url']}")
print("  --- film top 8 by likes (excluded from core) ---")
for q in sorted(buckets['film'],key=lambda x:-x['likes'])[:8]:
    lr=q['likes']/q['views']*100 if q['views'] else 0
    print(f"   @{q['handle']} {q['likes']:,}L/{q['views']:,}V {lr:.2f}% | {q['text'][:110]}")

core=sorted(buckets['core'],key=lambda x:-x['score'])
print("\n=== CORE TOP 15 (raw score) with like rate + marginal delta ===")
prevq={}
for q in prev['qualifying']:
    tid=q['url'].rstrip('/').split('/')[-1]; prevq[tid]=q
for i,q in enumerate(core[:15],1):
    tid=q['url'].rstrip('/').split('/')[-1]
    lr=q['likes']/q['views']*100 if q['views'] else 0
    p=prevq.get(tid); marg=""
    if p:
        dl=q['likes']-p['likes']; dv=q['views']-p['views']
        marg=f" | d+{dl}L/+{dv}V = {dv/dl:.0f}v/like" if dl>0 else f" | d+{dl}L/+{dv}V = NO LIKE GROWTH"
    print(f"{i}|@{q['handle']}|{q['likes']}L/{q['views']}V|{lr:.3f}%|score {q['score']:.0f}|{q['date']}{marg}")
    print(f"   {q['text'][:190]}")
    print(f"   {q['url']}")

print("\n=== ORGANIC vs PAID (core) ===")
paid=[q for q in core if q['views'] and q['likes']/q['views']*100 < 0.10]
print(f"organic {len(core)-len(paid)} of {len(core)}; paid-reach (<0.10%) {len(paid)}:")
for q in paid: print(f"  @{q['handle']} {q['likes']}L/{q['views']}V {q['likes']/q['views']*100:.3f}% | {q['text'][:90]}")

print("\n=== ORGANIC-ONLY CORE TOP 10 (like rate >= 0.10%) ===")
org=[q for q in core if q['views'] and q['likes']/q['views']*100>=0.10]
for i,q in enumerate(org[:10],1):
    print(f"{i}|@{q['handle']}|{q['likes']}L/{q['views']}V|{q['likes']/q['views']*100:.3f}%|{q['date']}|{q['text'][:100]}")
print("top10 cutoff", org[9]['score'] if len(org)>9 else "n/a", "| rank11", org[10]['score'] if len(org)>10 else "n/a")

print("\n=== FRESHNESS (max-based) ===")
rows=[]
for h,t in d['raw'].items():
    ts=[dt(x['createdAt']) for x in t if x.get('createdAt')]
    if not ts: continue
    newest=max(ts); n=len(ts); inw=sum(1 for x in ts if x>=CUT)
    rows.append(((now-newest).total_seconds()/3600,h,n,inw,newest))
rows.sort()
for hrs,h,n,inw,newest in rows[:20]:
    print(f"  @{h} {hrs:.1f}h n={n} inwin={inw}")
print("  --- stalest ---")
for hrs,h,n,inw,newest in rows[-12:]:
    print(f"  @{h} {hrs:.1f}h n={n} inwin={inw} newest={newest:%Y-%m-%d %H:%M}")

print("\n=== DORMANCY (originals only, RT+reply filtered) ===")
for h,t in sorted(d['raw'].items()):
    ors=[dt(x['createdAt']) for x in t if x.get('createdAt') and not x.get('retweeted_tweet') and not x.get('isReply')]
    if not ors:
        print(f"  @{h}: NO ORIGINALS in fetched set"); continue
    newest=max(ors); hrs=(now-newest).total_seconds()/3600
    qualh=[q for q in qual if q['handle']==h]
    if hrs>72 and not qualh:
        print(f"  @{h}: DORMANT {hrs:.1f}h newest={newest:%Y-%m-%d %H:%M}")

print("\n=== THRESHOLD ARTEFACTS (in-window originals, 0 qualifying) ===")
for h,t in sorted(d['raw'].items()):
    ors=[x for x in t if x.get('createdAt') and not x.get('retweeted_tweet') and not x.get('isReply') and dt(x['createdAt'])>=CUT]
    if ors and not [q for q in qual if q['handle']==h]:
        print(f"  @{h}: {len(ors)} in-window originals, best {max(x['likeCount'] for x in ors)}L")

print("\n=== NEW HANDLE CHECK: shikamarutsai ===")
t=d['raw'].get('shikamarutsai',[])
ors=[x for x in t if not x.get('retweeted_tweet') and not x.get('isReply')]
print(f"  fetched {len(t)}, originals {len(ors)}, RTs {len(t)-len(ors)}")
for x in sorted(t,key=lambda y:dt(y['createdAt'])):
    print(f"  {x['createdAt'][:22]} | {x.get('likeCount',0)}L/{x.get('viewCount',0)}V | rt={bool(x.get('retweeted_tweet'))} | media={[m.get('type') for m in (x.get('media') or [])]} | {x['text'][:130]}")