"""Actual fit/predict/score entry points on tiny synthetic data, one GPU only.""" import json import os from pathlib import Path import resource import sys import time import numpy as np from gre24_prepare_v2 import sha from gre24_job_guard import supervise,save from gre24_gpu_guard import gpu_state,gpu_fit,recorded_peaks def main(): root=Path(sys.argv[1]);root.mkdir(exist_ok=False) os.sched_setaffinity(0,[0,1,2,3]);rng=np.random.default_rng(91);good={} for case in ['learn','validation','test']: d=root/'parts'/case;d.mkdir(parents=True) x=rng.uniform(0,1,(128,19));y=.2+.1*x[:,0]+.1*x[:,1] values={'features':x,'Vf':np.linspace(0,1,128,endpoint=False),'raw_ids':np.arange(128),'seconds':2*y} if case!='test':values['target']=y # Deliberately absent for test: no fit/predict may need it. for name,v in values.items():np.save(d/(name+'.npy'),v) good[case]=dict(output=str(d),nodes=len(y),t_scale_seconds=2., output_sha256={p.name:sha(p) for p in d.glob('*.npy')}) splits=[dict(index=i,seed=91+i,train=['learn','validation'],test=['test'],fnn_learning=['learn'],fnn_validation=['validation']) for i in range(5)] save(root/'splits.json',splits);save(root/'preparation-freeze.json',dict(successful=good,failed=[])) paths=['tools/gre24_models.py','tools/gre24_fit.py','tools/gre24_score.py','tools/gre24_gpu_guard.py','tools/gre24_job_guard.py'] c=dict(code_sha256={p:sha(p) for p in paths},preparation_freeze_sha256=sha(root/'preparation-freeze.json'), splits_sha256=sha(root/'splits.json'),dataset_version='synthetic_only',protocol='synthetic smoke, not scientific data', xgb_threads=1,fnn_cpu_threads=2,gpu_reserved_byte_cap=15*2**30) save(root/'training-contract.json',c) receipts={};models={};gpu_seconds=0 for name in ['GBRT','FNN']: dest=root/'fits'/'split-0'/name;cmd=[sys.executable,'tools/gre24_fit.py',str(root),'0',name] if name=='GBRT': r=supervise(cmd,dest,'worker',dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120, tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1) else: begin=time.monotonic();available=sorted((s for s in gpu_state() if s['used_bytes']<128*2**20 and s['utilization']==0),key=lambda s:s['uuid']) if not available:raise RuntimeError('No idle GPU for bounded synthetic acceptance') uuid=available[0]['uuid'] r=gpu_fit(cmd,dest,dest,uuid,60.,120.,15*2**30);gpu_seconds=time.monotonic()-begin receipts[name]=r;save(root/(name+'-guard.json'),r) if r['status']!='PASS': save(root/'acceptance.json',dict(status='FAILED',receipts=receipts,GPU_seconds=gpu_seconds));raise RuntimeError(name+' integration failed') result=json.loads((dest/'result.json').read_text()) access=json.loads((dest/'training-target-access.json').read_text()) assert all(a['case']!='test' for a in access) models['0/'+name]=dict(status='PASS',checkpoint=result['checkpoint'],checkpoint_sha256=result['checkpoint_sha256']) for i in range(1,5): for name in ['GBRT','FNN']:models[f'{i}/{name}']=dict(status='NOT_ATTEMPTED_SYNTHETIC') save(root/'models-freeze.json',dict(fits=models)) test_seconds=root/'parts/test/seconds.npy';hidden=root/'parts/test/seconds.unavailable' test_seconds.rename(hidden) try: r=supervise([sys.executable,'tools/gre24_score.py',str(root),'predict'],root/'predict-guard','worker', dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120,tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1) receipts['predict']=r if r['status']!='PASS':raise RuntimeError('Prediction failed with test seconds unavailable') finally:hidden.rename(test_seconds) r=supervise([sys.executable,'tools/gre24_score.py',str(root),'score'],root/'score-guard','worker', dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120,tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1) receipts['score']=r;assert r['status']=='PASS' summary=json.loads((root/'score-summary.json').read_text())['summary'] assert summary['naive']['rrmse_rms']['full_requested_coverage'] assert summary['FNN']['rrmse_rms']['mean_of_five_split_summaries']['median'] is None # Failed-result records still contribute their peaks, independently of live memory. fake=root/'peak-check';fake.mkdir();save(fake/'failure.json',dict(peak_reserved_bytes=16*2**30)) peaks,witness=recorded_peaks(fake);assert peaks['peak_reserved_bytes']==16*2**30 save(root/'acceptance.json',dict(status='PASS',receipts=receipts,GPU_seconds=gpu_seconds, worker_CPU_seconds=sum(r['cpu_seconds'] for r in receipts.values()), checks=['real GBRT and FNN entrypoints','test target absent during fits','test seconds unavailable during prediction', 'all predictions frozen before scoring','missing splits not silently averaged','failure VRAM peaks retained'], scientific_data_read=False,coordinator_CPU_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]))) print(json.dumps(dict(status='PASS',GPU_seconds=gpu_seconds,worker_CPU_seconds=sum(r['cpu_seconds'] for r in receipts.values())))) if __name__=='__main__':main()