199 lines
17 KiB
Python
199 lines
17 KiB
Python
"""B17: fixed G+channel readouts and tail-weighted direct regression; stage1 only."""
|
|
import argparse,fcntl,json,os,resource,signal,subprocess,sys,time
|
|
from pathlib import Path
|
|
import numpy as np
|
|
import xgboost as xgb
|
|
from phase2_h1_worker_v2 import checked,sha,save
|
|
from phase2_fixed_io import BASE
|
|
from phase2_home_freeze_io import freeze,verify
|
|
from phase2_e2_postfit import validate_roles,extras
|
|
from phase2_b1213_features import cached
|
|
from phase2_b13_e1b import pooled
|
|
from phase2_b17_math import tail_weights,validate_review
|
|
from phase2_b14_gate import field as original_channel_field
|
|
from gre24_models import xgb_model
|
|
from s2_custody_common import prediction_inputs
|
|
from s2_custody_statistics import part_metrics
|
|
from r4bc_guard import supervise
|
|
from s2_recovery_supervisor import tree_usage
|
|
from s2_recovery_monitoring import disk_usage
|
|
|
|
def read(p):return json.loads(Path(p).read_text())
|
|
def config(root):
|
|
c=read(root/'contract.json')
|
|
if c['approval']!='1794915553.067999' or c['total_cpu_seconds']!=10800 or c['new_fits']!=5 or c['new_geometry']!=121:raise PermissionError('Fixed B17 scope')
|
|
if np.__version__!='1.26.4' or xgb.__version__!='2.1.4':raise ValueError('Pinned libraries')
|
|
for p,h in {**c['source_sha256'],**c['code_sha256']}.items():checked(BASE/p,h)
|
|
validate_roles(c)
|
|
if set(c['new_indices']) & set(c['reused_indices']) or len(c['new_indices'])!=121 or len(c['reused_indices'])!=431 or set(c['new_indices'])|set(c['reused_indices'])!=set(range(552)):raise PermissionError('Exact missing121 and reused431 only')
|
|
return c
|
|
|
|
def job(index):
|
|
if index not in range(5):raise ValueError('Only five fits')
|
|
return index,'B17'
|
|
def jobroot(root,index):return root/'cells'/f'{index:02d}'
|
|
def channel_extra(root,row):
|
|
e=read(root/'B14-manifest.json')['by_case'][row['case']]
|
|
a=np.load(checked(e['path'],e['sha256']),allow_pickle=False)
|
|
if a.shape!=(row['nodes'],2) or np.isinf(a).any():raise ValueError('B14 readout identity/shape')
|
|
return a
|
|
|
|
def implementation_review(root):
|
|
verify(root,'implementation-review');launch=verify(root,'launch')
|
|
binding=dict(reviewed_commit=launch['commit'],contract_sha256=sha(root/'contract.json'),launch_proof_sha256=sha(root/'launch-git-freeze.json'),code_sha256=read(root/'contract.json')['code_sha256'])
|
|
return validate_review(root/'review.json',binding,BASE)
|
|
|
|
|
|
def decision(metrics):
|
|
m=metrics['last_location'];d=m['new_minus_A']
|
|
if m['components']!=dict(new=204,A=204) or d['high'] is None:return 'PARK_UNDEFINED'
|
|
return 'SCREEN_PASS_NOT_CONFIRMED' if d['value']<=-.010 and d['high']<0 else 'PRUNE_B17_FIXED_RECIPE'
|
|
|
|
|
|
def train(root,c,index):
|
|
verify(root,'features');verify(root,'channels');fold,arm=job(index);rows=c['folds'][fold]['rows']['learning'];dest=jobroot(root,index)/'train';dest.mkdir(parents=True,exist_ok=False)
|
|
entries={e['case']:e for e in read(root/'features-manifest.json')['entries']};n=sum(r['nodes'] for r in rows);X=np.empty((n,23),np.float32);Y=np.empty(n,np.float32);W=np.empty(n,np.float64);offset=0;access=[]
|
|
for row in rows:
|
|
f=cached(row,'features');e=extras(row,'G',entries);raw=cached(row,'raw_ids')
|
|
if f.shape!=(row['nodes'],19) or not np.isfinite(f).all() or len(raw)!=row['nodes']:raise ValueError('Frozen learning feature identity')
|
|
access.append(dict(case=row['case'],fold=fold,arm=arm,role='learning',learning_pool=100,target_sha256=row['v5_cache']['output_sha256']['target.npy'],raw_ids_sha256=row['v5_cache']['output_sha256']['raw_ids.npy'],nodes=row['nodes']));save(dest/'target-access.json',access)
|
|
y=cached(row,'target');end=offset+len(y)
|
|
if y.shape!=(row['nodes'],) or not np.isfinite(y).all():raise ValueError('Learning target shape/nonfinite')
|
|
X[offset:end,:19]=f;X[offset:end,19:21]=e;X[offset:end,21:]=channel_extra(root,row);Y[offset:end]=y;W[offset:end]=tail_weights(cached(row,'seconds'),cached(row,'mass'));offset=end
|
|
save(dest/'progress.json',dict(stage='LOAD_LEARNING',parts=len(access),nodes=offset))
|
|
if offset!=n:raise ValueError('Learning count')
|
|
model=xgb_model(0,1);save(dest/'progress.json',dict(stage='FIT',nodes=n,parts=len(access),columns=23))
|
|
model.fit(X,Y,sample_weight=W);model.save_model(dest/'model.ubj');save(dest/'xgboost-config.json',json.loads(model.get_booster().save_config()))
|
|
save(dest/'result.json',dict(status='FIT_COMPLETE',fold=fold,arm=arm,checkpoint_sha256=sha(dest/'model.ubj'),nodes=n,parts=len(rows),columns=23,weights_per_part_node_mean_one=True,tail_mass_fraction=.10,tail_weight_ratio=4,test_labels_read=False))
|
|
|
|
def query_identity(q,z):
|
|
for key in ['raw_ids','counts','coordinates_over_Rg','near_wall']:
|
|
if not np.array_equal(q[key],z[key]):raise ValueError('Frozen G query mismatch: '+key)
|
|
|
|
def predict(root,c,index):
|
|
verify(root,'models');fold,arm=job(index);d=jobroot(root,index);model=xgb.XGBRegressor();model.load_model(d/'train/model.ubj');dest=d/'predict';dest.mkdir(exist_ok=False)
|
|
entries={e['case']:e for e in read(root/'features-manifest.json')['entries']};G={e['case']:e for e in read(c['G_prediction_manifest'])['entries']};out=[]
|
|
for row in c['folds'][fold]['rows']['test']:
|
|
q=prediction_inputs(row);raw=cached(row,'raw_ids');ix=np.searchsorted(raw,q['raw_ids'])
|
|
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],q['raw_ids']):raise ValueError('Test query IDs')
|
|
g=G[row['case']]
|
|
if g['fold']!=fold or g['component_id']!=row['component_id']:raise ValueError('G fold/component mismatch')
|
|
with np.load(checked(g['path'],g['sha256']),allow_pickle=False) as z:query_identity(q,z)
|
|
f=cached(row,'features');e=extras(row,'G',entries);X=np.column_stack([f[ix],e[ix],channel_extra(root,row)[ix]]).astype(np.float32);pred=np.asarray(model.predict(X),np.float64)*row['preparation']['t_set']
|
|
if not np.isfinite(pred).all():raise ValueError('Nonfinite prediction')
|
|
p=dest/(row['case']+'.npz');np.savez_compressed(p,pred=pred,**{k:q[k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']})
|
|
out.append(dict(case=row['case'],component_id=row['component_id'],fold=fold,arm=arm,path=str(p),sha256=sha(p),model_sha256=sha(d/'train/model.ubj'),G_prediction_sha256=g['sha256']))
|
|
save(dest/'progress.json',dict(parts=len(out),test_labels_read=False))
|
|
save(dest/'manifest.json',dict(entries=out,test_labels_read=False,units='seconds'))
|
|
|
|
def score(root,c):
|
|
verify(root,'predictions');implementation_review(root);dest=root/'score';dest.mkdir(exist_ok=False);m=read(root/'predictions-manifest.json')['entries'];by={r['case']:r for r in c['parts']};G={e['case']:e for e in read(c['G_prediction_manifest'])['entries']};old={r['case']:r for r in read(c['G_score'])['rows']}
|
|
if len(m)!=552 or {e['case'] for e in m}!=set(by):raise ValueError('Complete552 prediction roster required')
|
|
rows=[];access=[]
|
|
for e in m:
|
|
case=e['case'];row=by[case];g=G[case];prior=old[case]
|
|
if e['fold']!=g['fold'] or prior['fold']!=g['fold'] or prior['component_id']!=row['component_id']:raise ValueError('Paired role identity')
|
|
access.append(dict(case=case,role='test',fold=e['fold'],target_sha256=row['v5_cache']['output_sha256']['seconds.npy']));save(dest/'target-access.json',access)
|
|
truth=cached(row,'seconds');raw=cached(row,'raw_ids')
|
|
with np.load(checked(g['path'],g['sha256']),allow_pickle=False) as z,np.load(checked(e['path'],e['sha256']),allow_pickle=False) as v:
|
|
query_identity(z,v);ix=np.searchsorted(raw,v['raw_ids'])
|
|
if np.any(ix>=len(raw)) or not np.array_equal(raw[ix],v['raw_ids']):raise ValueError('Score query IDs')
|
|
metric=part_metrics(v['pred'],truth[ix],v['counts'],v['near_wall'],v['raw_ids'],v['coordinates_over_Rg'],int(case.split('_')[0]))
|
|
rows.append(dict(case=case,component_id=row['component_id'],fold=e['fold'],new=metric,G=prior['new'],A=prior['A'],G_valid=prior['G_valid'],negative_predictions=int(np.sum(v['pred']<0)),true_max_ties=int(np.sum(truth[ix]==truth[ix].max())),candidate_max_ties=int(np.sum(v['pred']==v['pred'].max())),G_max_ties=int(np.sum(z['pred']==z['pred'].max()))))
|
|
save(dest/'progress.json',dict(parts=len(rows)))
|
|
comparisons={a:pooled([dict(r,A=r[a]) for r in rows]) for a in ['G']};result=decision(comparisons['G'])
|
|
differences=[r['new']['last_location']-r['G']['last_location'] for r in rows if r['new']['last_location'] is not None and r['G']['last_location'] is not None]
|
|
save(dest/'result.json',dict(status='COMPLETE',decision=result,parts=552,components=204,new_minus=comparisons,primary='new minus frozen E1b G last-location mean<=-.010 and CI high<0; screening only',stage2_authorized=False,part_last_difference_median=float(np.median(differences)) if differences else None,rows=rows,original_G_metrics_reused=True,invalid_G_parts_retained=sum(not r['G_valid'] for r in rows),negative_predictions=sum(r['negative_predictions'] for r in rows),validation_used=False,new_decodes=0))
|
|
|
|
def worker(root,mode,index):
|
|
c=config(root)
|
|
if mode=='score':score(root,c)
|
|
elif mode=='field':original_channel_field(root,c,index)
|
|
else:{'train':train,'predict':predict}[mode](root,c,index)
|
|
|
|
def guard(root,mode,index,slot):
|
|
c=config(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]]);start=time.process_time();cap=c['job_caps'][mode];d=root/'guards'/f'{mode}-{index:02d}'
|
|
g=supervise([sys.executable,'tools/phase2_b17.py','worker',str(root),mode,str(index)],d,dict(cpu_seconds=cap,wall_seconds=cap if mode=='field' else cap+180,rss_bytes=8*2**30,disk_bytes=c['disk_bytes']),global_stop=root/'global-stop.json')
|
|
save(d/'guard-result.json',dict(mode=mode,index=index,guard=g,guard_cpu_seconds=time.process_time()-start));return 0 if g['status']=='PASS' else 1
|
|
|
|
def admission(total,active_count,job_cap,limit=10800):return total+(active_count+1)*job_cap+120<=limit
|
|
|
|
def run(root):
|
|
c=config(root);os.sched_setaffinity(0,[30,31]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'started.json').exists():raise FileExistsError('Single attempt B17')
|
|
verify(root,'launch');verify(root,'features');start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);settled=0.;records=[];active={};requested=[]
|
|
def total():
|
|
u=resource.getrusage(resource.RUSAGE_CHILDREN);return c['setup_seconds']+max(settled,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start+sum(tree_usage(p.pid)[0] for p,_,_ in active.values())
|
|
def limits():
|
|
sample=dict(cpu_seconds=total(),rss_bytes=sum(tree_usage(p.pid)[1] for p,_,_ in active.values()),disk_bytes=disk_usage(root),signals=list(requested));reasons=[]
|
|
if sample['cpu_seconds']>10800-120:reasons.append('TOTAL_CPU_RESERVE')
|
|
if sample['rss_bytes']>32*2**30:reasons.append('RSS_LIMIT')
|
|
if sample['disk_bytes']>c['disk_bytes']:reasons.append('COMBINED_GEOMETRY_POSTFIT_DISK_LIMIT')
|
|
if requested:reasons.append('COORDINATOR_SIGNAL')
|
|
if active and any(m=='field' for _,_,m in active.values()) and sample['cpu_seconds']>c['geometry_cpu_cap']-120:reasons.append('GEOMETRY_RESERVE_FOR_POSTFIT')
|
|
if reasons:save(root/'stop-trigger.json',dict(reasons=reasons,sample=sample));raise RuntimeError('RESOURCE_OR_SIGNAL_STOP')
|
|
def batch(mode,indices):
|
|
nonlocal settled
|
|
pending=list(indices)
|
|
while pending or active:
|
|
limits()
|
|
for slot,(p,i,m) in list(active.items()):
|
|
if p.poll() is not None:
|
|
r=read(root/'guards'/f'{m}-{i:02d}'/'guard-result.json');settled+=r['guard']['cpu_seconds']+r['guard_cpu_seconds'];records.append(r);del active[slot]
|
|
if p.returncode or r['guard']['status']!='PASS':
|
|
if m=='field' and r['guard'].get('stop_reason') in ['cpu_limit','wall_limit','cpu_limit_after_exit','wall_limit_after_exit'] and not r['guard'].get('monitor_errors'):
|
|
d=root/'parts'/f'{i:04d}';row=c['parts'][i];np.save(d/'extra.npy',np.full((row['nodes'],2),np.nan));save(d/'result.json',dict(status='RESOURCE_FAIL',case=row['case'],component_id=row['component_id'],index=i,extra_sha256=sha(d/'extra.npy'),reason=r['guard']['stop_reason'],labels_read=False,partial_families_retained_not_merged=True))
|
|
else:raise RuntimeError('WORKER_NOT_PASS')
|
|
for slot in range(4):
|
|
if pending and slot not in active:
|
|
if not admission(total(),len(active),c['job_caps'][mode],c['geometry_cpu_cap'] if mode=='field' else 10800):
|
|
if not active:raise RuntimeError('TOTAL_ADMISSION_RESERVE')
|
|
break
|
|
i=pending.pop(0)
|
|
if mode=='field':(root/'parts'/f'{i:04d}').mkdir(parents=True,exist_ok=False)
|
|
p=subprocess.Popen([sys.executable,'tools/phase2_b17.py','guard',str(root),mode,str(i),str(slot)],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True);active[slot]=(p,i,mode)
|
|
save(root/'live-progress.json',dict(stage=mode,pending=len(pending),active=[dict(slot=s,pid=p.pid,index=i,mode=m) for s,(p,i,m) in active.items()],terminal=len(records),cumulative_cpu_seconds=total(),unix=time.time()));time.sleep(.5)
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),new_geometry=121,reused_geometry=431))
|
|
old={s:signal.signal(s,lambda signum,frame:requested.append(signum)) for s in [signal.SIGTERM,signal.SIGINT]}
|
|
try:
|
|
batch('field',c['new_indices'][:10])
|
|
used=sum(r['guard']['cpu_seconds']+r['guard_cpu_seconds'] for r in records if r['mode']=='field')
|
|
projected=1.25*used*sum(c['parts'][i]['nodes'] for i in c['new_indices'])/sum(c['parts'][i]['nodes'] for i in c['new_indices'][:10])+c['setup_seconds']
|
|
save(root/'first-ten-cost-check.json',dict(projected_geometry_cpu_seconds=projected,geometry_cap=c['geometry_cpu_cap'],postfit_reserve=c['postfit_reserve'],passed=projected<=c['geometry_cpu_cap']))
|
|
if projected>c['geometry_cpu_cap']:raise RuntimeError('FIRST_TEN_COST_PROJECTION_STOP')
|
|
batch('field',c['new_indices'][10:])
|
|
by_case=dict(c['reused_channels'])
|
|
for i in c['new_indices']:
|
|
d=root/'parts'/f'{i:04d}';r=read(d/'result.json');p=checked(d/'extra.npy',r['extra_sha256']);by_case[c['parts'][i]['case']]=dict(path=str(p),sha256=r['extra_sha256'],result_path=str(d/'result.json'),result_sha256=sha(d/'result.json'),status=r['status'])
|
|
if set(by_case)!={r['case'] for r in c['parts']}:raise ValueError('All552 channel manifest')
|
|
save(root/'B14-manifest.json',dict(by_case=by_case,new=121,reused=431,labels_read=False))
|
|
freeze(root,'channels',[root/'contract.json',root/'B14-manifest.json',*[BASE/p for p in c['code_sha256']]])
|
|
batch('train',range(5));freeze(root,'models',[root/'contract.json',*[jobroot(root,i)/'train'/n for i in range(5) for n in ['model.ubj','result.json','target-access.json']],*[BASE/p for p in c['code_sha256']]])
|
|
batch('predict',range(5));entries=[]
|
|
for i in range(5):entries.extend(read(jobroot(root,i)/'predict/manifest.json')['entries'])
|
|
save(root/'predictions-manifest.json',dict(entries=entries,all552_terminal=True,reused_G_manifest=c['G_prediction_manifest'],reused_G_manifest_sha256=c['source_sha256'][c['G_prediction_manifest']],labels_read=False))
|
|
freeze(root,'predictions',[root/'contract.json',root/'predictions-manifest.json',root/'features-manifest.json',root/'B14-manifest.json',Path(c['G_prediction_manifest']),Path(c['G_score']),*[BASE/p for p in c['code_sha256']]])
|
|
wait_start=time.monotonic()
|
|
while not (root/'implementation-review-authorization.json').exists():
|
|
limits();save(root/'live-progress.json',dict(stage='WAIT_IMPLEMENTATION_REVIEW_BEFORE_SCORE',cumulative_cpu_seconds=total(),wait_wall_seconds=time.monotonic()-wait_start,unix=time.time()));time.sleep(2)
|
|
implementation_review(root);batch('score',[0]);limits();save(root/'complete.json',dict(status='COMPLETE',decision=read(root/'score/result.json')['decision'],cumulative_cpu_seconds=total(),records=records,new_fits=5,new_predictions=552,reused_G_predictions=552,accounting_complete=True,retry=False))
|
|
freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',root/'score/target-access.json'])
|
|
save(root/'post-terminal-freeze-cost.json',dict(cumulative_cpu_seconds=total(),includes_terminal_freeze=True,within_cap=total()<=10800))
|
|
except BaseException as exc:
|
|
save(root/'global-stop.json',dict(reason=type(exc).__name__+': '+str(exc)))
|
|
for p,_,_ in active.values():
|
|
try:os.kill(p.pid,signal.SIGTERM)
|
|
except ProcessLookupError:pass
|
|
for p,i,m in active.values():
|
|
p.wait();f=root/'guards'/f'{m}-{i:02d}'/'guard-result.json'
|
|
if f.exists():records.append(read(f))
|
|
active.clear();save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),cumulative_cpu_seconds=total(),records=records,accounting_complete=True,retry=False));raise
|
|
finally:
|
|
for s,h in old.items():signal.signal(s,h)
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','worker','guard']);p.add_argument('root',type=Path);p.add_argument('mode',nargs='?');p.add_argument('index',nargs='?',type=int);p.add_argument('slot',nargs='?',type=int);a=p.parse_args()
|
|
if a.action=='run':run(a.root)
|
|
elif a.action=='worker':worker(a.root,a.mode,a.index)
|
|
else:raise SystemExit(guard(a.root,a.mode,a.index,a.slot))
|