"""Paired analysis of existing public bucket/scale evaluations; no neural work.

CLI needs only Python's standard library. Input: a local copy of the public Space
JSON pages and the exact EARS training_normalization.json used by evaluation.
"""
import argparse,csv,hashlib,itertools,json,math,statistics,time
from pathlib import Path
LANGS=['en','de','fr','nl','es'];SCALES=[.5,.75,1.]
RESAMPLES=list(itertools.product(range(5),repeat=5))

def mean(values):return statistics.mean(values)
def quantile(values,p):
    values=sorted(values);index=(len(values)-1)*p;lo=math.floor(index);hi=math.ceil(index)
    return values[lo]+(values[hi]-values[lo])*(index-lo)
def interval(language_means,alpha=.05):
    if len(language_means)!=5:raise ValueError('Five language clusters required')
    distribution=[sum(language_means[i] for i in sample)/5 for sample in RESAMPLES]
    return [quantile(distribution,alpha/2),quantile(distribution,1-alpha/2)]
def bhash(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest()
def pair_key(r):return r['language'],r['text'],r['seed']

def validate(records,spec,norm):
    if len(records)!=80:raise ValueError('Exactly80original outputs required: '+spec['bucket'])
    grouped={s:{} for s in [0.,*SCALES]};mu,sigma=norm[spec['head']]
    if sigma<=0:raise ValueError('Invalid normalization')
    for r in records:
        key=pair_key(r);scale=r['scale'];m=r['metrics']
        if scale not in grouped or key in grouped[scale]:raise ValueError('Duplicate/unexpected scale cell')
        for name in ['target_raw','target_normalized','ce','genuineness','wer']:
            if not math.isfinite(m[name]):raise ValueError('Missing/nonfinite metric')
        if abs(m['raw_ears'][spec['head']]-m['target_raw'])>1e-9:raise ValueError('Wrong EARS head')
        cdf=.5*(1+math.erf((m['target_raw']-mu)/sigma/math.sqrt(2)))
        expected=1-cdf if spec['pole']=='low' else cdf
        if abs(expected-m['target_normalized'])>1e-10:raise ValueError('Normalization differs from evaluation')
        grouped[scale][key]=r
    baseline=grouped[0.]
    if len(baseline)!=20 or {lang:sum(k[0]==lang for k in baseline) for lang in LANGS}!={lang:4 for lang in LANGS}:raise ValueError('Missing paired language/text/seed cells')
    for scale in SCALES:
        if set(grouped[scale])!=set(baseline):raise ValueError('Unmatched conditions')
        for key,r in grouped[scale].items():
            base=baseline[key]
            for name in ['bucket','language','text','prompt','frames','seconds','seed','cfg','reference']:
                if r[name]!=base[name]:raise ValueError('Unmatched prompt/reference/frame contract: '+name)
    return grouped

def summarize(pairs,spec,norm,opposite=None,cluster=True):
    sign=-1 if spec['pole']=='low' else 1;sigma=norm[spec['head']][1]
    delta=[sign*(r['metrics']['target_raw']-b['metrics']['target_raw']) for b,r in pairs]
    z=[x/sigma for x in delta]
    language_means=[mean(z[i] for i,(b,r) in enumerate(pairs) if b['language']==lang) for lang in LANGS] if cluster else None
    result=dict(n=len(pairs),baseline_raw=mean(b['metrics']['target_raw'] for b,r in pairs),
        adapter_raw=mean(r['metrics']['target_raw'] for b,r in pairs),
        baseline_normalized=mean(b['metrics']['target_normalized'] for b,r in pairs),
        adapter_normalized=mean(r['metrics']['target_normalized'] for b,r in pairs),
        delta_raw_oriented=mean(delta),delta_z=mean(z),
        delta_normalized=mean(r['metrics']['target_normalized']-b['metrics']['target_normalized'] for b,r in pairs),
        win_rate=sum(d>1e-12 for d in delta)/len(delta),tie_rate=sum(abs(d)<=1e-12 for d in delta)/len(delta),
        delta_ce=mean(r['metrics']['ce']-b['metrics']['ce'] for b,r in pairs),
        delta_genuineness=mean(r['metrics']['genuineness']-b['metrics']['genuineness'] for b,r in pairs),
        delta_wer_pp=100*mean(r['metrics']['wer']-b['metrics']['wer'] for b,r in pairs),
        baseline_cap_rate=mean(not b['ended'] for b,r in pairs),adapter_cap_rate=mean(not r['ended'] for b,r in pairs))
    if cluster:
        result.update(language_means_z=dict(zip(LANGS,language_means)),languages_improved=sum(x>1e-12 for x in language_means),
            ci95_z=interval(language_means),ci9833_z=interval(language_means,.05/3))
    if opposite:
        opposite_delta=[-(r['metrics']['raw_ears'][opposite]-b['metrics']['raw_ears'][opposite])/norm[opposite][1] for b,r in pairs]
        result['opposite_head']=opposite;result['opposite_delta_z']=mean(opposite_delta)
        if cluster:
            languages=[mean(opposite_delta[i] for i,(b,r) in enumerate(pairs) if b['language']==lang) for lang in LANGS]
            result['opposite_ci9833_z']=interval(languages,.05/3);result['opposite_languages_improved']=sum(x>1e-12 for x in languages)
    if cluster:
        result['clearer_gain']=bool(result['delta_z']>0 and result['ci9833_z'][0]>0 and result['languages_improved']>=4
            and (not opposite or result['opposite_ci9833_z'][0]>0 and result['opposite_languages_improved']>=4))
    return result

def analyze(site,normalization,taxonomy):
    n=json.loads(normalization.read_text());norm={name:(float(mu),float(sigma)) for name,mu,sigma in zip(n['score_names'],n['score_mean'],n['score_std'])}
    definitions={b['id']:b for b in json.loads(taxonomy.read_text())};manifest=json.loads((site/'manifest.json').read_text())
    buckets=[];sources={};flat=[]
    for bucket in manifest['completed_buckets']:
        p=site/(bucket+'.json');source=json.loads(p.read_text());spec=source['spec'];records=source['records']
        grouped=validate(records,spec,norm);scales=[];opposite=definitions[bucket].get('opposite')
        for scale in SCALES:
            pairs=[(grouped[0.][k],grouped[scale][k]) for k in sorted(grouped[0.])]
            item=dict(scale=scale,**summarize(pairs,spec,norm,opposite))
            item['languages']={lang:summarize([(b,r) for b,r in pairs if b['language']==lang],spec,norm,opposite,False) for lang in LANGS}
            scales.append(item)
            for scope,values in [('all',item),*item['languages'].items()]:
                row={k:v for k,v in values.items() if not isinstance(v,(dict,list))}
                row.update(bucket=bucket,label=spec['label'],category=spec['category'],head=spec['head'],pole=spec['pole'],scale=scale,scope=scope)
                if scope=='all':row.update(ci9833_low_z=item['ci9833_z'][0],ci9833_high_z=item['ci9833_z'][1])
                flat.append(row)
        best=max(scales,key=lambda x:x['delta_z']);clear=[s for s in scales if s['clearer_gain']]
        pick=max(clear,key=lambda x:x['delta_z']) if clear else best if best['delta_z']>0 else None
        status='clearer_gain' if clear else 'positive_mean_uncertain' if pick else 'no_positive_mean'
        buckets.append(dict(bucket=bucket,label=spec['label'],category=spec['category'],head=spec['head'],pole=spec['pole'],
            trained_languages=spec['languages'],train_clips=spec['train_clips'],normalization_mu=norm[spec['head']][0],normalization_sigma=norm[spec['head']][1],
            opposite_head=opposite,scales=scales,best_mean_scale=best['scale'],best_mean_delta_z=best['delta_z'],
            suggested_scale=pick['scale'] if pick else None,status=status))
        sources[p.name]=bhash(p)
    summary=dict(adapters=len(buckets),comparisons=len(buckets)*3,recordings=len(buckets)*80,
        clearer_gain=sum(b['status']=='clearer_gain' for b in buckets),
        positive_mean_uncertain=sum(b['status']=='positive_mean_uncertain' for b in buckets),
        no_positive_mean=sum(b['status']=='no_positive_mean' for b in buckets),
        normalized_mean_increase=sum(any(s['delta_normalized']>0 for s in b['scales']) for b in buckets),
        raw_and_normalized_increase=sum(any(s['delta_z']>0 and s['delta_normalized']>0 for s in b['scales']) for b in buckets),
        positive_mean_scale_cells=sum(s['delta_z']>0 for b in buckets for s in b['scales']),
        clearer_scale_cells=sum(s['clearer_gain'] for b in buckets for s in b['scales']))
    report=dict(schema='paired-bucket-steering-analysis-v1',created_unix=time.time(),summary=summary,buckets=buckets,
        method=dict(paired_n_per_scale=20,languages=LANGS,outputs_per_language_scale=4,scales=SCALES,cfg=1,
            primary='Desired-direction paired raw target delta divided by its frozen EARS training standard deviation; positive means stronger requested trait',
            normalized='Original Gaussian-CDF target score, inverted for low-pole targets; not a probability; may saturate',
            interval='Exact enumeration of3125bootstrap resamples of5language clusters,retaining both texts and seeds within language',
            clearer_rule='Positive98.33%exploratory cluster-bootstrap lower bound and improvement in at least4/5language means;for contrast requests,both heads must meet these conditions',
            scale_selection='Largest target gain among clearer scales when available;otherwise largest positive mean is an uncertain trial;use base if no scale has a positive mean',
            limitations='Only5fixed speaker-reference/language pairs and2translated texts per language. Selection and intervals use the same grid. No independent held-out scale validation,no global multiplicity claim across72adapters,no human-listener confirmation. Language and reference voice effects are confounded.'),
        input_sha256=sources,normalization_sha256=bhash(normalization),taxonomy_sha256=bhash(taxonomy))
    return report,flat

def main():
    p=argparse.ArgumentParser();p.add_argument('--site',type=Path,required=True);p.add_argument('--normalization',type=Path,required=True);p.add_argument('--taxonomy',type=Path,required=True);p.add_argument('--out',type=Path,required=True);a=p.parse_args()
    report,flat=analyze(a.site,a.normalization,a.taxonomy);a.out.mkdir(parents=True,exist_ok=True)
    (a.out/'steering-comparison.json').write_text(json.dumps(report,ensure_ascii=False,indent=2,allow_nan=False)+'\n')
    with (a.out/'steering-comparison.csv').open('w',newline='') as f:
        fields=sorted({k for row in flat for k in row});writer=csv.DictWriter(f,fieldnames=fields);writer.writeheader();writer.writerows(flat)
    print(json.dumps(report['summary']))
    for b in sorted(report['buckets'],key=lambda b:-b['best_mean_delta_z'])[:12]:
        s=next(s for s in b['scales'] if s['scale']==b['best_mean_scale'])
        print(b['bucket'],b['status'],s['scale'],round(s['delta_z'],3),s['languages_improved'],[round(x,3) for x in s['ci9833_z']],round(s['delta_wer_pp'],2))
if __name__=='__main__':main()