8f43927575
examples/ is based on KasperskyLab/knp at 55470ed6 with local changes to the mnist-learn altai/blifat models, cuda-backend CMake and model_loader. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
39 lines
2.1 KiB
Python
39 lines
2.1 KiB
Python
"""Evaluate frozen selection and original baseline on the same fresh test once."""
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
|
|
root = Path('/home/int/run/cats-dogs')
|
|
selection = json.loads((root / 'selection-v2.json').read_text())
|
|
best = selection['selected']
|
|
run = '/mnt/c/KNP/cats-dogs/run.py'
|
|
subprocess.run([sys.executable, run, 'v2_test', '--data-dir', str(root / 'features-v2'),
|
|
'--dataset', 'sparse_hog_train_test', '--train', '4000', '--count', '1000',
|
|
'--columns', str(best['columns']), '--threshold', str(best['threshold']),
|
|
'--quant', 'weights'], check=True)
|
|
subprocess.run([sys.executable, run, 'v1_fresh_test', '--data-dir', str(root / 'features-v2'),
|
|
'--dataset', 'baseline_test', '--train', '800', '--count', '1000',
|
|
'--columns', '20', '--threshold', '60'], check=True)
|
|
checks = {}
|
|
for model, reference, target in [('v2', best['run'], 'v2_test'),
|
|
('v1', 'val_contrast_60', 'v1_fresh_test')]:
|
|
checks[model] = {}
|
|
for name in ('float_weights.csv', 'quant_weights.csv', 'float_neurons.csv', 'quant_neurons.csv'):
|
|
a, b = (root / reference / name).read_bytes(), (root / target / name).read_bytes()
|
|
checks[model][name] = {'equal': a == b, 'sha256': hashlib.sha256(b).hexdigest()}
|
|
assert all(item['equal'] for item in checks[model].values()), f'Weights changed for {model}'
|
|
results = {'selection': best, 'reproducibility': checks}
|
|
for model, name in [('v1', 'v1_fresh_test'), ('v2', 'v2_test')]:
|
|
metrics = json.loads((root / name / 'metrics.json').read_text())
|
|
n = sum(metrics[k] for k in ('correct', 'wrong', 'unanswered'))
|
|
p, z = metrics['accuracy'], 1.959963984540054
|
|
center = (p + z*z/(2*n)) / (1 + z*z/n)
|
|
half = z * math.sqrt(p*(1-p)/n + z*z/(4*n*n)) / (1 + z*z/n)
|
|
metrics['wilson95'] = [center - half, center + half]
|
|
results[model] = metrics
|
|
results['accuracy_gain_percentage_points'] = 100 * (results['v2']['accuracy'] - results['v1']['accuracy'])
|
|
(root / 'comparison-v2.json').write_text(json.dumps(results, indent=2) + '\n')
|
|
print(json.dumps(results), flush=True)
|