79 lines
5.3 KiB
Python
79 lines
5.3 KiB
Python
"""Actual fit/predict/score entry points on tiny synthetic data, one GPU only."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import sys
|
|
import time
|
|
import numpy as np
|
|
from gre24_prepare_v2 import sha
|
|
from gre24_job_guard import supervise,save
|
|
from gre24_gpu_guard import gpu_state,gpu_fit,recorded_peaks
|
|
|
|
|
|
def main():
|
|
root=Path(sys.argv[1]);root.mkdir(exist_ok=False)
|
|
os.sched_setaffinity(0,[0,1,2,3]);rng=np.random.default_rng(91);good={}
|
|
for case in ['learn','validation','test']:
|
|
d=root/'parts'/case;d.mkdir(parents=True)
|
|
x=rng.uniform(0,1,(128,19));y=.2+.1*x[:,0]+.1*x[:,1]
|
|
values={'features':x,'Vf':np.linspace(0,1,128,endpoint=False),'raw_ids':np.arange(128),'seconds':2*y}
|
|
if case!='test':values['target']=y # Deliberately absent for test: no fit/predict may need it.
|
|
for name,v in values.items():np.save(d/(name+'.npy'),v)
|
|
good[case]=dict(output=str(d),nodes=len(y),t_scale_seconds=2.,
|
|
output_sha256={p.name:sha(p) for p in d.glob('*.npy')})
|
|
splits=[dict(index=i,seed=91+i,train=['learn','validation'],test=['test'],fnn_learning=['learn'],fnn_validation=['validation']) for i in range(5)]
|
|
save(root/'splits.json',splits);save(root/'preparation-freeze.json',dict(successful=good,failed=[]))
|
|
paths=['tools/gre24_models.py','tools/gre24_fit.py','tools/gre24_score.py','tools/gre24_gpu_guard.py','tools/gre24_job_guard.py']
|
|
c=dict(code_sha256={p:sha(p) for p in paths},preparation_freeze_sha256=sha(root/'preparation-freeze.json'),
|
|
splits_sha256=sha(root/'splits.json'),dataset_version='synthetic_only',protocol='synthetic smoke, not scientific data',
|
|
xgb_threads=1,fnn_cpu_threads=2,gpu_reserved_byte_cap=15*2**30)
|
|
save(root/'training-contract.json',c)
|
|
receipts={};models={};gpu_seconds=0
|
|
for name in ['GBRT','FNN']:
|
|
dest=root/'fits'/'split-0'/name;cmd=[sys.executable,'tools/gre24_fit.py',str(root),'0',name]
|
|
if name=='GBRT':
|
|
r=supervise(cmd,dest,'worker',dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120,
|
|
tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1)
|
|
else:
|
|
begin=time.monotonic();available=sorted((s for s in gpu_state() if s['used_bytes']<128*2**20 and s['utilization']==0),key=lambda s:s['uuid'])
|
|
if not available:raise RuntimeError('No idle GPU for bounded synthetic acceptance')
|
|
uuid=available[0]['uuid']
|
|
r=gpu_fit(cmd,dest,dest,uuid,60.,120.,15*2**30);gpu_seconds=time.monotonic()-begin
|
|
receipts[name]=r;save(root/(name+'-guard.json'),r)
|
|
if r['status']!='PASS':
|
|
save(root/'acceptance.json',dict(status='FAILED',receipts=receipts,GPU_seconds=gpu_seconds));raise RuntimeError(name+' integration failed')
|
|
result=json.loads((dest/'result.json').read_text())
|
|
access=json.loads((dest/'training-target-access.json').read_text())
|
|
assert all(a['case']!='test' for a in access)
|
|
models['0/'+name]=dict(status='PASS',checkpoint=result['checkpoint'],checkpoint_sha256=result['checkpoint_sha256'])
|
|
for i in range(1,5):
|
|
for name in ['GBRT','FNN']:models[f'{i}/{name}']=dict(status='NOT_ATTEMPTED_SYNTHETIC')
|
|
save(root/'models-freeze.json',dict(fits=models))
|
|
test_seconds=root/'parts/test/seconds.npy';hidden=root/'parts/test/seconds.unavailable'
|
|
test_seconds.rename(hidden)
|
|
try:
|
|
r=supervise([sys.executable,'tools/gre24_score.py',str(root),'predict'],root/'predict-guard','worker',
|
|
dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120,tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1)
|
|
receipts['predict']=r
|
|
if r['status']!='PASS':raise RuntimeError('Prediction failed with test seconds unavailable')
|
|
finally:hidden.rename(test_seconds)
|
|
r=supervise([sys.executable,'tools/gre24_score.py',str(root),'score'],root/'score-guard','worker',
|
|
dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120,tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1)
|
|
receipts['score']=r;assert r['status']=='PASS'
|
|
summary=json.loads((root/'score-summary.json').read_text())['summary']
|
|
assert summary['naive']['rrmse_rms']['full_requested_coverage']
|
|
assert summary['FNN']['rrmse_rms']['mean_of_five_split_summaries']['median'] is None
|
|
# Failed-result records still contribute their peaks, independently of live memory.
|
|
fake=root/'peak-check';fake.mkdir();save(fake/'failure.json',dict(peak_reserved_bytes=16*2**30))
|
|
peaks,witness=recorded_peaks(fake);assert peaks['peak_reserved_bytes']==16*2**30
|
|
save(root/'acceptance.json',dict(status='PASS',receipts=receipts,GPU_seconds=gpu_seconds,
|
|
worker_CPU_seconds=sum(r['cpu_seconds'] for r in receipts.values()),
|
|
checks=['real GBRT and FNN entrypoints','test target absent during fits','test seconds unavailable during prediction',
|
|
'all predictions frozen before scoring','missing splits not silently averaged','failure VRAM peaks retained'],
|
|
scientific_data_read=False,coordinator_CPU_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
|
|
print(json.dumps(dict(status='PASS',GPU_seconds=gpu_seconds,worker_CPU_seconds=sum(r['cpu_seconds'] for r in receipts.values()))))
|
|
|
|
|
|
if __name__=='__main__':main()
|