#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Wen zitiert die KI? — KI-Sichtbarkeits-Report Deutschland 2026 (SEOlyze)
Repro-Skript: rechnet alle Kernzahlen der Studie aus der offenen Rohdaten-CSV nach.

Aufruf:
    python3 repro_kernzahlen.py [pfad/zu/wen-zitiert-die-ki-2026_rohdaten.csv]

Benötigt nur die Python-Standardbibliothek. Quelle & Daten:
https://www.seolyze.com/studie/wen-zitiert-die-ki
Freie Nutzung mit Quellenangabe (CC BY 4.0).
"""
import csv, math, re, sys
from collections import defaultdict

PATH = sys.argv[1] if len(sys.argv) > 1 else 'wen-zitiert-die-ki-2026_rohdaten.csv'

# ── Publizierte Werte (Stichtag 16.07.2026, Welle 2) ─────────────────────────
EXPECTED = {
    'panel_n':               11180,
    'crawled_n':             11171,
    'aio_quote_pct':         57.6,   # Keywords mit AI Overview / gecrawlte Keywords
    'leitfrage_pct':         66.7,   # #1-Domain zitiert / geladene AIOs
    'aio_quote_sv_pct':      21.7,   # suchvolumengewichtet
    'leitfrage_sv_pct':      78.0,   # suchvolumengewichtet
    'klickwert_gesamt_eur':  32908341,
    'klickwert_aio_pct':     42.2,
    'klickwert_dedup_eur':   28652447,   # nach konservativer Wortstellungs-Deduplizierung
    'klickwert_dedup_aio_pct': 38.6,
    'aio_quote_dedup_pct':   57.5,
    'leitfrage_dedup_pct':   66.2,
    'ymyl_aio_pct':          70.3,
    'nonymyl_aio_pct':       53.0,
}

def wilson(k, n, z=1.96):
    if n == 0: return (0.0, 0.0)
    p = k / n
    d = 1 + z*z/n
    c = (p + z*z/(2*n)) / d
    h = z * math.sqrt(p*(1-p)/n + z*z/(4*n*n)) / d
    return (100*(c-h), 100*(c+h))

def pct(k, n): return 100.0*k/n if n else 0.0

rows = []
with open(PATH, encoding='utf-8-sig') as f:
    lines = [l for l in f if not l.startswith('#')]
    for r in csv.DictReader(lines):
        rows.append(r)

crawled = [r for r in rows if r['crawled'] == '1']
aio     = [r for r in crawled if r['hasAio'] == '1']
loaded  = [r for r in aio if r['aioContentLoaded'] == '1']
cited   = [r for r in loaded if r['pos1Cited'] == '1']

sv        = lambda r: int(r['svAtDraw'])
cpcv      = lambda r: float(r['cpc']) if r['cpc'] else 0.0
klick     = lambda r: sv(r) * cpcv(r)

results = []
def check(key, actual, fmt='{:.1f}'):
    exp = EXPECTED[key]
    ok = abs(actual - exp) < (0.06 if isinstance(exp, float) else 1)
    results.append(ok)
    print(('  [OK]  ' if ok else '  [!!]  ') + f"{key:28s} berechnet {fmt.format(actual):>12s}   publiziert {fmt.format(exp):>12s}")

print(f"Rohdaten: {PATH}")
print(f"\n── Basis ─────────────────────────────────────────────────────")
check('panel_n', len(rows), '{:.0f}')
check('crawled_n', len(crawled), '{:.0f}')

print(f"\n── Kernzahlen (ungewichtet, je Keyword) ──────────────────────")
q = pct(len(aio), len(crawled)); lo, hi = wilson(len(aio), len(crawled))
check('aio_quote_pct', q); print(f"          95%-Wilson-CI: {lo:.1f}–{hi:.1f}")
q = pct(len(cited), len(loaded)); lo, hi = wilson(len(cited), len(loaded))
check('leitfrage_pct', q); print(f"          95%-Wilson-CI: {lo:.1f}–{hi:.1f}  (Basis geladene AIOs: n={len(loaded)})")

print(f"\n── Suchvolumengewichtet ──────────────────────────────────────")
check('aio_quote_sv_pct', pct(sum(map(sv, aio)), sum(map(sv, crawled))))
check('leitfrage_sv_pct', pct(sum(map(sv, cited)), sum(map(sv, loaded))))

print(f"\n── Klickwert (Suchvolumen × CPC, €/Monat) ────────────────────")
tot = sum(map(klick, rows)); taio = sum(klick(r) for r in rows if r['hasAio'] == '1')
check('klickwert_gesamt_eur', round(tot), '{:.0f}')
check('klickwert_aio_pct', pct(taio, tot))

print(f"\n── Sensitivität: konservative Wortstellungs-Deduplizierung ───")
norm = lambda kw: ' '.join(sorted(re.findall(r'\w+', kw.lower())))
groups = defaultdict(list)
for r in rows: groups[norm(r['keyword'])].append(r)
kept = [max(v, key=klick) for v in groups.values()]
dtot = sum(map(klick, kept)); daio = sum(klick(r) for r in kept if r['hasAio'] == '1')
check('klickwert_dedup_eur', round(dtot), '{:.0f}')
check('klickwert_dedup_aio_pct', pct(daio, dtot))
kept_sv = [max(v, key=sv) for v in groups.values()]
dc = [r for r in kept_sv if r['crawled'] == '1']
da = [r for r in dc if r['hasAio'] == '1']
dl = [r for r in da if r['aioContentLoaded'] == '1']
dz = [r for r in dl if r['pos1Cited'] == '1']
check('aio_quote_dedup_pct', pct(len(da), len(dc)))
check('leitfrage_dedup_pct', pct(len(dz), len(dl)))

print(f"\n── YMYL ──────────────────────────────────────────────────────")
y  = [r for r in crawled if r['ymyl'] == '1']
ny = [r for r in crawled if r['ymyl'] != '1']
check('ymyl_aio_pct',    pct(sum(1 for r in y  if r['hasAio'] == '1'), len(y)))
check('nonymyl_aio_pct', pct(sum(1 for r in ny if r['hasAio'] == '1'), len(ny)))

print(f"\n── KI-Antwort-Quote je Branche ───────────────────────────────")
br = defaultdict(lambda: [0, 0])
for r in crawled:
    br[r['branche']][1] += 1
    if r['hasAio'] == '1': br[r['branche']][0] += 1
for b, (k, n) in sorted(br.items(), key=lambda kv: -pct(*kv[1])):
    print(f"          {b:20s} {pct(k, n):5.1f} %   (n={n})")

print(f"\n{'='*62}")
print(f"ERGEBNIS: {sum(results)}/{len(results)} publizierte Kennzahlen reproduziert."
      + ('' if all(results) else '  ABWEICHUNGEN — bitte studie@seolyze.com melden!'))