"""Evaluate frozen selection and original baseline on the same fresh test once."""
import hashlib
import json
import math
from pathlib import Path
import subprocess
import sys

root = Path('/home/int/run/cats-dogs')
selection = json.loads((root / 'selection-v2.json').read_text())
best = selection['selected']
run = '/mnt/c/KNP/cats-dogs/run.py'
subprocess.run([sys.executable, run, 'v2_test', '--data-dir', str(root / 'features-v2'),
                '--dataset', 'sparse_hog_train_test', '--train', '4000', '--count', '1000',
                '--columns', str(best['columns']), '--threshold', str(best['threshold']),
                '--quant', 'weights'], check=True)
subprocess.run([sys.executable, run, 'v1_fresh_test', '--data-dir', str(root / 'features-v2'),
                '--dataset', 'baseline_test', '--train', '800', '--count', '1000',
                '--columns', '20', '--threshold', '60'], check=True)
checks = {}
for model, reference, target in [('v2', best['run'], 'v2_test'),
                                 ('v1', 'val_contrast_60', 'v1_fresh_test')]:
    checks[model] = {}
    for name in ('float_weights.csv', 'quant_weights.csv', 'float_neurons.csv', 'quant_neurons.csv'):
        a, b = (root / reference / name).read_bytes(), (root / target / name).read_bytes()
        checks[model][name] = {'equal': a == b, 'sha256': hashlib.sha256(b).hexdigest()}
    assert all(item['equal'] for item in checks[model].values()), f'Weights changed for {model}'
results = {'selection': best, 'reproducibility': checks}
for model, name in [('v1', 'v1_fresh_test'), ('v2', 'v2_test')]:
    metrics = json.loads((root / name / 'metrics.json').read_text())
    n = sum(metrics[k] for k in ('correct', 'wrong', 'unanswered'))
    p, z = metrics['accuracy'], 1.959963984540054
    center = (p + z*z/(2*n)) / (1 + z*z/n)
    half = z * math.sqrt(p*(1-p)/n + z*z/(4*n*n)) / (1 + z*z/n)
    metrics['wilson95'] = [center - half, center + half]
    results[model] = metrics
results['accuracy_gain_percentage_points'] = 100 * (results['v2']['accuracy'] - results['v1']['accuracy'])
(root / 'comparison-v2.json').write_text(json.dumps(results, indent=2) + '\n')
print(json.dumps(results), flush=True)