96 lines
7.8 KiB
Python
96 lines
7.8 KiB
Python
"""7848 B15 prediction continuation: same per-part science, model/first prediction reused."""
|
|
import argparse,fcntl,json,os,resource,signal,subprocess,sys,time
|
|
from pathlib import Path
|
|
import numpy as np
|
|
import xgboost as xgb
|
|
import joblib
|
|
from phase2_b15_gate import load,entries,positive_speed,flow_mesh,solve_eikonal,strict_volume_cdf,prediction_inputs,cached,score
|
|
from phase2_b1213_gate import config as original_config
|
|
from phase2_h1_worker_v2 import checked,sha,save
|
|
from phase2_fixed_io import BASE,verify
|
|
from phase2_local_freeze_io import freeze
|
|
from r4bc_guard import supervise
|
|
from s2_recovery_supervisor import tree_usage
|
|
from s2_recovery_monitoring import disk_usage
|
|
|
|
def read(p):return json.loads(Path(p).read_text())
|
|
def config(root):
|
|
c=original_config(root)
|
|
if c['continuation_approval']!='1794907848.106509' or c['total_cpu_cap']!=8000:raise PermissionError('One prediction-only extension')
|
|
if c['first_prediction']['case']!=c['rows']['validation'][0]['case'] or len(c['rows']['validation'])!=111:raise ValueError('Frozen first/remaining validation identity')
|
|
checked(root/'phases/train/model.ubj',c['reused_model_sha256'])
|
|
return c
|
|
|
|
def projection(charged,completed_cpu,completed_nodes,total_nodes):
|
|
if completed_nodes<=0 or total_nodes<completed_nodes or min(charged,completed_cpu)<0:raise ValueError('Invalid completed-cost projection')
|
|
return charged+1.25*completed_cpu*(total_nodes-completed_nodes)/completed_nodes
|
|
|
|
def predict(root,c):
|
|
verify(root,'model');dest=root/'phases/predict';by=entries(root);model=xgb.XGBRegressor();model.load_model(root/'phases/train/model.ubj');baseline=joblib.load(checked(c['baseline']['model_path'],c['baseline']['model_sha256']));manifest=[dict(c['first_prediction'])]
|
|
for row in c['rows']['validation'][1:]:
|
|
begin=time.process_time()
|
|
with load(root,by[row['case']])[0] as z:
|
|
if not np.all(z['geometry_valid']):raise ValueError('Validation contains invalid geometry; no tet deletion')
|
|
a=positive_speed(model.predict(z['X']));g=flow_mesh((z['points']-z['center'])/float(z['Rg']),z['tets'])
|
|
begin_solver=time.process_time();phi=solve_eikonal(g,z['sources'],a);solver_cpu=time.process_time()-begin_solver;U=strict_volume_cdf(phi,z['mass']);pred_all=c['clock_c']*U
|
|
np.savez_compressed(dest/(row['case']+'-field.npz'),raw_ids=z['raw_ids'],raw_cells=z['raw_cells'],speed=a,phi=phi,U=U,pred_normalized=pred_all)
|
|
q=prediction_inputs(row);ix=np.searchsorted(z['raw_ids'],q['raw_ids'])
|
|
if np.any(ix>=len(z['raw_ids'])) or not np.array_equal(z['raw_ids'][ix],q['raw_ids']):raise ValueError('Query identity')
|
|
f=cached(row,'features');A=np.asarray(baseline.predict(f[ix].astype(np.float32)),np.float64);p=dest/(row['case']+'.npz')
|
|
np.savez_compressed(p,pred=pred_all[ix],A=A,raw_ids=q['raw_ids'],counts=q['counts'],coordinates_over_Rg=q['coordinates_over_Rg'])
|
|
manifest.append(dict(case=row['case'],component_id=row['component_id'],path=str(p),sha256=sha(p),full_field_sha256=sha(dest/(row['case']+'-field.npz')),geometry_valid=True,solver_cpu_seconds=solver_cpu,cpu_seconds=time.process_time()-begin,clock_learning_only=c['clock_c']))
|
|
save(dest/'progress.json',dict(parts=len(manifest),last=manifest[-1],completed_prediction_cpu_seconds=sum(e['cpu_seconds'] for e in manifest),validation_targets_read=False))
|
|
save(dest/'manifest.json',dict(entries=manifest,validation_labels_read=False,all_validation_complete=True))
|
|
|
|
def guard(root,mode,cap):
|
|
c=config(root);os.sched_setaffinity(0,[4]);start=time.process_time();d=root/'guards'/mode
|
|
g=supervise([sys.executable,'tools/phase2_b15_prediction_continue.py','worker',str(root),mode],d,dict(cpu_seconds=cap,wall_seconds=cap+120,rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes']-c['referenced_storage_bytes']),global_stop=root/'global-stop.json')
|
|
save(d/'guard-result.json',dict(mode=mode,guard=g,guard_cpu_seconds=time.process_time()-start));return 0 if g['status']=='PASS' else 1
|
|
|
|
def run(root):
|
|
c=config(root);os.sched_setaffinity(0,[29]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'started.json').exists():raise FileExistsError('Single prediction continuation')
|
|
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);records=[];settled=0.;active=None;requested=[]
|
|
def total():
|
|
u=resource.getrusage(resource.RUSAGE_CHILDREN);return c['prior_cpu_seconds']+60+max(settled,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start+(tree_usage(active.pid)[0] if active else 0)
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),prior_cpu_seconds=c['prior_cpu_seconds'],reused_models=1,reused_predictions=1,remaining_predictions=110,refit=False))
|
|
old={s:signal.signal(s,lambda signum,frame:requested.append(signum)) for s in [signal.SIGTERM,signal.SIGINT]}
|
|
try:
|
|
for mode in ['predict','score']:
|
|
if mode=='score':
|
|
freeze(root,'predictions',[root/'contract.json',root/'phases/predict/manifest.json',*[BASE/p for p in c['code_sha256']]]);(root/'phases/score').mkdir()
|
|
cap=8000-total()-120
|
|
if cap<=0:raise RuntimeError('TOTAL_RESERVE_EXHAUSTED')
|
|
active=subprocess.Popen([sys.executable,'tools/phase2_b15_prediction_continue.py','guard',str(root),mode,str(cap)],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True)
|
|
while active.poll() is None:
|
|
charged=total();usage=tree_usage(active.pid);sample=dict(cpu_seconds=charged,disk_bytes=c['referenced_storage_bytes']+disk_usage(root),rss_bytes=usage[1],signals=list(requested));reasons=[];estimate=charged
|
|
if mode=='predict':
|
|
progress=read(root/'phases/predict/progress.json');n=progress['parts'];weights=[r['nodes'] for r in c['rows']['validation']];estimate=projection(charged,progress['completed_prediction_cpu_seconds'],sum(weights[:n]),sum(weights))
|
|
if estimate>8000:reasons.append('PROJECTION_EXCEEDS_8000')
|
|
if charged>8000-120:reasons.append('TOTAL_CPU_RESERVE')
|
|
if sample['disk_bytes']>c['disk_bytes']:reasons.append('CUMULATIVE_DIRECTORY_DISK_LIMIT')
|
|
if sample['rss_bytes']>c['rss_bytes']:reasons.append('RSS_LIMIT')
|
|
if requested:reasons.append('COORDINATOR_SIGNAL')
|
|
if reasons:save(root/'stop-trigger.json',dict(reasons=reasons,sample=sample,phase=mode,projected_total=estimate));raise RuntimeError('RESOURCE_OR_SIGNAL_STOP')
|
|
save(root/'live-progress.json',dict(stage=mode,guard_pid=active.pid,cumulative_cpu_seconds=charged,projected_total=estimate,unix=time.time()));time.sleep(.2)
|
|
result=read(root/'guards'/mode/'guard-result.json');records.append(result);settled+=result['guard']['cpu_seconds']+result['guard_cpu_seconds'];returncode=active.returncode;active=None
|
|
if returncode or result['guard']['status']!='PASS':raise RuntimeError('WORKER_NOT_PASS')
|
|
save(root/'complete.json',dict(status='COMPLETE',decision=read(root/'phases/score/result.json')['decision'],cumulative_cpu_seconds=total(),records=records,refit=False,reused_first_prediction=True,accounting_complete=True))
|
|
except BaseException as exc:
|
|
save(root/'global-stop.json',dict(reason=type(exc).__name__+': '+str(exc)))
|
|
if active:
|
|
try:os.kill(active.pid,signal.SIGTERM)
|
|
except ProcessLookupError:pass
|
|
active.wait();p=root/'guards'/mode/'guard-result.json'
|
|
if p.exists():records.append(read(p))
|
|
active=None
|
|
save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),cumulative_cpu_seconds=total(),records=records,refit=False,retry=False,accounting_complete=True));raise
|
|
finally:
|
|
for s,h in old.items():signal.signal(s,h)
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','guard','worker']);p.add_argument('root',type=Path);p.add_argument('mode',nargs='?');p.add_argument('cap',nargs='?',type=float);a=p.parse_args()
|
|
if a.action=='run':run(a.root)
|
|
elif a.action=='guard':raise SystemExit(guard(a.root,a.mode,a.cap))
|
|
else:
|
|
c=config(a.root);(predict if a.mode=='predict' else score)(a.root,c)
|