27 lines
2.1 KiB
Python
27 lines
2.1 KiB
Python
"""Isolated CIFAR cats/dogs splits; no downloads or changes to prior experiments."""
|
|
from pathlib import Path
|
|
import io,json,hashlib,shutil
|
|
import numpy as np
|
|
import pyarrow.parquet as pq
|
|
from PIL import Image
|
|
R=Path('/home/user/run/pets-knp');R.mkdir(exist_ok=True)
|
|
if (R/'data').exists(): raise RuntimeError('Refuse to overwrite prepared data')
|
|
d=R/'data';d.mkdir(); rng=np.random.default_rng(20261015)
|
|
def load(split):
|
|
t=pq.read_table(Path.home()/f'run/cifar/hf_{split}.parquet').to_pydict();xs=[];ys=[];ids=[]
|
|
for i,(im,lab) in enumerate(zip(t['img'],t['label'])):
|
|
if lab in (3,5):
|
|
xs.append(np.asarray(Image.open(io.BytesIO(im['bytes'])).convert('RGB')).transpose(2,0,1));ys.append(lab==5);ids.append(i)
|
|
return np.stack(xs),np.array(ys,np.int32),np.array(ids)
|
|
x,y,ids=load('train');idx=[np.flatnonzero(y==c) for c in range(2)]
|
|
tr=rng.permutation(np.r_[idx[0][:4000],idx[1][:4000]]);va=rng.permutation(np.r_[idx[0][4000:],idx[1][4000:]])
|
|
xt,yt,it=load('test');te=rng.permutation(len(yt))
|
|
for tag,xx,yy,ii in [('train',x[tr],y[tr],ids[tr]),('val',x[va],y[va],ids[va]),('test',xt[te],yt[te],it[te])]:
|
|
np.savez_compressed(d/f'{tag}.npz',rgb=xx,labels=yy,source_ids=ii)
|
|
xx.tofile(d/f'{tag}_images.bin');(d/f'{tag}_labels.txt').write_text('\n'.join(map(str,yy))+'\n')
|
|
if tag=='train':xx[:,:,:,::-1].copy().tofile(d/'train_flip_images.bin')
|
|
m=R/'detectors';m.mkdir()
|
|
for name in ('det_w.bin','det_b.bin','kmeans.npz'):shutil.copyfile(Path.home()/'run/altai-objects/native-cuda/model_readout'/name,m/name)
|
|
(m/'meta.txt').write_text('200 2 135 2 0\n')
|
|
manifest={'source':'uoft-cs/cifar10','classes':['cat','dog'],'train':8000,'validation':2000,'test':2000,'seed':20261015,'split':'first4000 per class train, last1000 validation, official test separate','detectors':'reused unsupervised vehicles training images only; no cats/dogs or official test in detector training','preprocess':'RGB32x32; uploads EXIF transpose and bilinear resize; native CIFAR unchanged','files':{p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(d.iterdir())}}
|
|
(R/'dataset-manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');print(json.dumps(manifest))
|