#!/usr/bin/env python3
"""Deep analysis 2026-11-07: buckets, freshness, dormancy, offers, deltas."""
import json, re, os
from datetime import datetime, timezone, timedelta

DATE = "2026-11-07"
RAW = "/home/kiro/trend-engine/trends/x/raw"
D = json.load(open(f"{RAW}/scan_results_{DATE}_FULL.json"))
res, failed = D["raw"], D["failures"]

FMT = "%a %b %d %H:%M:%S %z %Y"
now = datetime.now(timezone.utc)
cutoff = datetime.strptime(f"{DATE} 00:00:00 +0000", "%Y-%m-%d %H:%M:%S %z") - timedelta(days=3)

IRREL = set("""AIBuzzNews AIGuideHQ AITECHio AIFrontliner TheAIColony AITOPIAai AISuperDomain
GoogleLabs GoogleAI DonutAI cladlabs FaithJaiye2187 KammelaBala MyStageAI AlexStudio44
AI_with_jasmin Ai_Vaidehi ChrisLaubAI OpenAI AnthropicAI CINEMA505 AIScottAdams AI_Jasonyu AIByLeo
Teamily_AI AidenChaoyangHe openart_ai different_ai via_ai_ AIsaOneHQ numerai ItsMaryamAi Soaima_Ai""".split())
FILM = {"DiscussingFilm", "nofilmschool"}
LABS = {"OpenAI", "AnthropicAI", "NVIDIAAI"}
OFFTOPIC = {"interesting_aIl"}
HEALTH = {"hubermanlab"}

print("=== PER-HANDLE FRESHNESS (newest fetched, hours) ===")
fresh = []
for h, r in res.items():
    rows = r or []
    if not rows:
        continue
    ds = [datetime.strptime(t["createdAt"], FMT) for t in rows if t.get("createdAt")]
    if not ds:
        continue
    mx = max(ds)
    inwin = sum(1 for d in ds if d >= cutoff)
    fresh.append((h, (now - mx).total_seconds()/3600, len(rows), inwin))
fresh.sort(key=lambda x: x[1])
for h, hrs, n, iw in fresh:
    print(f"{h:22s} {hrs:8.1f}h  n={n:3d} inwin={iw:3d}")
print(f"\nSTALE (top 12):")
for h, hrs, n, iw in fresh[-12:][::-1]:
    print(f"{h:22s} {hrs:8.1f}h  n={n:3d} inwin={iw:3d}")

print("\n=== DORMANCY (originals only, in-window) ===")
qual = D.get("qualifying") or []
# rebuild qualifying per handle from raw instead
qcount = {}
for h, r in res.items():
    for t in (r or []):
        if t.get("retweeted_tweet"): continue
        d = datetime.strptime(t["createdAt"], FMT)
        if d < cutoff: continue
        L = t.get("likeCount", 0) or 0; RT = t.get("retweetCount", 0) or 0
        R = t.get("replyCount", 0) or 0; V = t.get("viewCount", 0) or 0
        if L >= 50 or RT >= 10 or V >= 1000:
            qcount.setdefault(h, []).append((d, L, RT, R, V, t.get("id")))
for h, hrs, n, iw in fresh:
    if h in qcount: continue
    if hrs < 72:
        print(f"ARTEFACT? {h:20s} newest {hrs:7.1f}h  0 qualifying")

print("\n=== UNCLASSIFIED HANDLES WITH >0 QUALIFYING ===")
src = open("/home/kiro/.hermes/scripts/x-account-monitor-run.py").read()
def _parse_set(name):
    m = re.search(name + r"\s*=\s*[\[\{](.*?)[\]\}]\s*\n", src, re.S)
    return set(re.findall(r'"([A-Za-z0-9_]+)"', m.group(1))) if m else set()
CORE = _parse_set("CORE")
IRREL = _parse_set("IRRELEVANT") | {"OpenAI", "AnthropicAI"}
print("CORE parsed:", len(CORE), "IRREL parsed:", len(IRREL))
for h in sorted(qcount, key=lambda x: -len(qcount[x])):
    tags = []
    if h in CORE: tags.append("CORE")
    if h in IRREL: tags.append("IRREL")
    if h in FILM: tags.append("FILM")
    if h in LABS: tags.append("LABS")
    if h in OFFTOPIC: tags.append("OFFTOPIC")
    if h in HEALTH: tags.append("HEALTH")
    if not tags: tags.append("*** UNCLASSIFIED ***")
    print(f"{h:22s} {len(qcount[h]):3d}  {','.join(tags)}")

print("\n=== CORE RANKING (raw + organic) ===")
CORESET = CORE - IRREL - FILM - LABS - OFFTOPIC - HEALTH
rows = []
for h, lst in qcount.items():
    if h not in CORESET: continue
    for d, L, RT, R, V, tid in lst:
        score = L + RT*2 + R*3 + V/100
        rate = (L/V*100) if V else 0
        rows.append({"h": h, "d": d, "L": L, "RT": RT, "R": R, "V": V, "score": score,
                     "rate": rate, "id": tid})
rows.sort(key=lambda x: -x["score"])
print("-- RAW TOP 14 --")
for i, x in enumerate(rows[:14], 1):
    print(f"{i:2d}|@{x['h']:18s}|{x['L']:6d}L|{x['RT']:5d}RT|{x['R']:4d}R|{x['V']:9d}V|{x['rate']:5.2f}%|"
          f"{x['d'].strftime('%m-%d %H:%M')}|{x['score']:8.0f}|https://x.com/{x['h']}/status/{x['id']}")
print(f"raw cutoff {rows[9]['score']:.0f} rank11 {rows[10]['score']:.0f}")

paid = [x for x in rows if x["rate"] < 0.15 and x["V"] > 50000]
org = [x for x in rows if x not in paid]
print(f"\n-- ORGANIC ({len(org)} of {len(rows)}); paid-reach excluded: {[(x['h'], x['L'], x['V'], round(x['rate'],3)) for x in paid]}")
for i, x in enumerate(org[:12], 1):
    print(f"{i:2d}|@{x['h']:18s}|{x['L']:6d}L|{x['RT']:5d}RT|{x['R']:4d}R|{x['V']:9d}V|{x['rate']:5.2f}%|"
          f"{x['d'].strftime('%m-%d %H:%M')}")

print("\n=== BUCKET COUNTS ===")
buckets = {}
for h, lst in qcount.items():
    for _ in lst:
        if h in LABS: b = "labs"
        elif h in OFFTOPIC: b = "offtopic"
        elif h in HEALTH: b = "health"
        elif h in FILM: b = "film"
        elif h in IRREL: b = "irrelevant"
        elif h in CORESET: b = "core"
        else: b = "UNCLASSIFIED"
        buckets[b] = buckets.get(b, 0) + 1
tot = sum(len(v) for v in qcount.values())
for k, v in sorted(buckets.items(), key=lambda x: -x[1]):
    print(f"  {k:14s} {v:4d}")
print(f"  TOTAL {tot}")

print("\n=== 24H DELTAS (shared tweet ids vs 11-06) ===")
try:
    P = json.load(open(f"{RAW}/scan_results_2026-11-06_FULL.json"))
    prev = {}
    for h, r in P["raw"].items():
        for t in (r or []):
            prev[t.get("id")] = (t.get("likeCount",0), t.get("viewCount",0), h)
    for x in rows[:16]:
        if x["id"] in prev:
            pL, pV, ph = prev[x["id"]]
            print(f"@{x['h']:18s} {x['d'].strftime('%m-%d %H:%M')} L {pL}->{x['L']} ({x['L']-pL:+d})  V {pV}->{x['V']} ({x['V']-pV:+d})")
except Exception as e:
    print("prev err", e)

print("\n=== OFFER GREP (in-window originals) ===")
PATTERNS = [
 ("flova_launch", r"2 years of Pro|Launch Special"),
 ("seedance_rate", r"\$0\.0\d+/sec|\$0\.\d+/sec|per second"),
 ("pika_free", r"15 free generations|free generations"),
 ("higgsfield_unlimited", r"Unlimited|unlimited"),
 ("openart_discount", r"30% OFF|30% off|discount"),
 ("chroma", r"Chroma Awards|chromaawards"),
 ("capcut_bf", r"capcutblackfriday|Black Friday|black friday"),
 ("davinci", r"[Dd]aVinci"),
 ("topaz", r"Topaz"),
 ("topview", r"Topview"),
 ("nebius", r"Nebius"),
 ("magnific", r"Magnific"),
 ("viggle", r"Viggle"),
 ("glif", r"[Gg]lif\b|glif\.app"),
 ("miracle", r"Miracle"),
 ("gemini_omni", r"Gemini Omni|Omni 1\.2"),
 ("gpt_image_26", r"GPT.?Image.?2\.6|Images 2\.6"),
 ("free_credits", r"free credits|1000 credits|credits for free|giveaway"),
 ("kling_mcp", r"Kling.*MCP|MCP.*Kling"),
 ("runway_dev", r"Runway Dev"),
 ("deepseek", r"[Dd]eep[Ss]eek"),
 ("perplexity", r"[Pp]erplexity"),
 ("einsia", r"Einsia|Frontier3DBench"),
 ("mirofish", r"mirofish|Mirofish"),
]
hits = {k: [] for k, _ in PATTERNS}
for h, r in res.items():
    for t in (r or []):
        if t.get("retweeted_tweet"): continue
        d = datetime.strptime(t["createdAt"], FMT)
        if d < cutoff: continue
        txt = t.get("text") or ""
        for k, pat in PATTERNS:
            if re.search(pat, txt):
                hits[k].append((h, d, t.get("likeCount",0), t.get("viewCount",0), t.get("id"), txt[:200]))
for k, lst in hits.items():
    if not lst:
        print(f"  {k:22s} 0 hits")
        continue
    print(f"  {k:22s} {len(lst)} hits")
    for h, d, L, V, tid, txt in lst[:4]:
        print(f"      @{h:18s} {d.strftime('%m-%d %H:%M')} {L}L/{V}V https://x.com/{h}/status/{tid}")
        print(f"        {txt[:180]}")