Files
MeltMind/tools/test_gre24_fit_integration.py

79 lines
5.3 KiB
Python

"""Actual fit/predict/score entry points on tiny synthetic data, one GPU only."""
import json
import os
from pathlib import Path
import resource
import sys
import time
import numpy as np
from gre24_prepare_v2 import sha
from gre24_job_guard import supervise,save
from gre24_gpu_guard import gpu_state,gpu_fit,recorded_peaks
def main():
root=Path(sys.argv[1]);root.mkdir(exist_ok=False)
os.sched_setaffinity(0,[0,1,2,3]);rng=np.random.default_rng(91);good={}
for case in ['learn','validation','test']:
d=root/'parts'/case;d.mkdir(parents=True)
x=rng.uniform(0,1,(128,19));y=.2+.1*x[:,0]+.1*x[:,1]
values={'features':x,'Vf':np.linspace(0,1,128,endpoint=False),'raw_ids':np.arange(128),'seconds':2*y}
if case!='test':values['target']=y # Deliberately absent for test: no fit/predict may need it.
for name,v in values.items():np.save(d/(name+'.npy'),v)
good[case]=dict(output=str(d),nodes=len(y),t_scale_seconds=2.,
output_sha256={p.name:sha(p) for p in d.glob('*.npy')})
splits=[dict(index=i,seed=91+i,train=['learn','validation'],test=['test'],fnn_learning=['learn'],fnn_validation=['validation']) for i in range(5)]
save(root/'splits.json',splits);save(root/'preparation-freeze.json',dict(successful=good,failed=[]))
paths=['tools/gre24_models.py','tools/gre24_fit.py','tools/gre24_score.py','tools/gre24_gpu_guard.py','tools/gre24_job_guard.py']
c=dict(code_sha256={p:sha(p) for p in paths},preparation_freeze_sha256=sha(root/'preparation-freeze.json'),
splits_sha256=sha(root/'splits.json'),dataset_version='synthetic_only',protocol='synthetic smoke, not scientific data',
xgb_threads=1,fnn_cpu_threads=2,gpu_reserved_byte_cap=15*2**30)
save(root/'training-contract.json',c)
receipts={};models={};gpu_seconds=0
for name in ['GBRT','FNN']:
dest=root/'fits'/'split-0'/name;cmd=[sys.executable,'tools/gre24_fit.py',str(root),'0',name]
if name=='GBRT':
r=supervise(cmd,dest,'worker',dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120,
tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1)
else:
begin=time.monotonic();available=sorted((s for s in gpu_state() if s['used_bytes']<128*2**20 and s['utilization']==0),key=lambda s:s['uuid'])
if not available:raise RuntimeError('No idle GPU for bounded synthetic acceptance')
uuid=available[0]['uuid']
r=gpu_fit(cmd,dest,dest,uuid,60.,120.,15*2**30);gpu_seconds=time.monotonic()-begin
receipts[name]=r;save(root/(name+'-guard.json'),r)
if r['status']!='PASS':
save(root/'acceptance.json',dict(status='FAILED',receipts=receipts,GPU_seconds=gpu_seconds));raise RuntimeError(name+' integration failed')
result=json.loads((dest/'result.json').read_text())
access=json.loads((dest/'training-target-access.json').read_text())
assert all(a['case']!='test' for a in access)
models['0/'+name]=dict(status='PASS',checkpoint=result['checkpoint'],checkpoint_sha256=result['checkpoint_sha256'])
for i in range(1,5):
for name in ['GBRT','FNN']:models[f'{i}/{name}']=dict(status='NOT_ATTEMPTED_SYNTHETIC')
save(root/'models-freeze.json',dict(fits=models))
test_seconds=root/'parts/test/seconds.npy';hidden=root/'parts/test/seconds.unavailable'
test_seconds.rename(hidden)
try:
r=supervise([sys.executable,'tools/gre24_score.py',str(root),'predict'],root/'predict-guard','worker',
dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120,tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1)
receipts['predict']=r
if r['status']!='PASS':raise RuntimeError('Prediction failed with test seconds unavailable')
finally:hidden.rename(test_seconds)
r=supervise([sys.executable,'tools/gre24_score.py',str(root),'score'],root/'score-guard','worker',
dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120,tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1)
receipts['score']=r;assert r['status']=='PASS'
summary=json.loads((root/'score-summary.json').read_text())['summary']
assert summary['naive']['rrmse_rms']['full_requested_coverage']
assert summary['FNN']['rrmse_rms']['mean_of_five_split_summaries']['median'] is None
# Failed-result records still contribute their peaks, independently of live memory.
fake=root/'peak-check';fake.mkdir();save(fake/'failure.json',dict(peak_reserved_bytes=16*2**30))
peaks,witness=recorded_peaks(fake);assert peaks['peak_reserved_bytes']==16*2**30
save(root/'acceptance.json',dict(status='PASS',receipts=receipts,GPU_seconds=gpu_seconds,
worker_CPU_seconds=sum(r['cpu_seconds'] for r in receipts.values()),
checks=['real GBRT and FNN entrypoints','test target absent during fits','test seconds unavailable during prediction',
'all predictions frozen before scoring','missing splits not silently averaged','failure VRAM peaks retained'],
scientific_data_read=False,coordinator_CPU_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
print(json.dumps(dict(status='PASS',GPU_seconds=gpu_seconds,worker_CPU_seconds=sum(r['cpu_seconds'] for r in receipts.values()))))
if __name__=='__main__':main()