Files
MeltMind/tools/phase2_b15_prediction_continue.py
T

96 lines
7.8 KiB
Python

"""7848 B15 prediction continuation: same per-part science, model/first prediction reused."""
import argparse,fcntl,json,os,resource,signal,subprocess,sys,time
from pathlib import Path
import numpy as np
import xgboost as xgb
import joblib
from phase2_b15_gate import load,entries,positive_speed,flow_mesh,solve_eikonal,strict_volume_cdf,prediction_inputs,cached,score
from phase2_b1213_gate import config as original_config
from phase2_h1_worker_v2 import checked,sha,save
from phase2_fixed_io import BASE,verify
from phase2_local_freeze_io import freeze
from r4bc_guard import supervise
from s2_recovery_supervisor import tree_usage
from s2_recovery_monitoring import disk_usage
def read(p):return json.loads(Path(p).read_text())
def config(root):
c=original_config(root)
if c['continuation_approval']!='1794907848.106509' or c['total_cpu_cap']!=8000:raise PermissionError('One prediction-only extension')
if c['first_prediction']['case']!=c['rows']['validation'][0]['case'] or len(c['rows']['validation'])!=111:raise ValueError('Frozen first/remaining validation identity')
checked(root/'phases/train/model.ubj',c['reused_model_sha256'])
return c
def projection(charged,completed_cpu,completed_nodes,total_nodes):
if completed_nodes<=0 or total_nodes<completed_nodes or min(charged,completed_cpu)<0:raise ValueError('Invalid completed-cost projection')
return charged+1.25*completed_cpu*(total_nodes-completed_nodes)/completed_nodes
def predict(root,c):
verify(root,'model');dest=root/'phases/predict';by=entries(root);model=xgb.XGBRegressor();model.load_model(root/'phases/train/model.ubj');baseline=joblib.load(checked(c['baseline']['model_path'],c['baseline']['model_sha256']));manifest=[dict(c['first_prediction'])]
for row in c['rows']['validation'][1:]:
begin=time.process_time()
with load(root,by[row['case']])[0] as z:
if not np.all(z['geometry_valid']):raise ValueError('Validation contains invalid geometry; no tet deletion')
a=positive_speed(model.predict(z['X']));g=flow_mesh((z['points']-z['center'])/float(z['Rg']),z['tets'])
begin_solver=time.process_time();phi=solve_eikonal(g,z['sources'],a);solver_cpu=time.process_time()-begin_solver;U=strict_volume_cdf(phi,z['mass']);pred_all=c['clock_c']*U
np.savez_compressed(dest/(row['case']+'-field.npz'),raw_ids=z['raw_ids'],raw_cells=z['raw_cells'],speed=a,phi=phi,U=U,pred_normalized=pred_all)
q=prediction_inputs(row);ix=np.searchsorted(z['raw_ids'],q['raw_ids'])
if np.any(ix>=len(z['raw_ids'])) or not np.array_equal(z['raw_ids'][ix],q['raw_ids']):raise ValueError('Query identity')
f=cached(row,'features');A=np.asarray(baseline.predict(f[ix].astype(np.float32)),np.float64);p=dest/(row['case']+'.npz')
np.savez_compressed(p,pred=pred_all[ix],A=A,raw_ids=q['raw_ids'],counts=q['counts'],coordinates_over_Rg=q['coordinates_over_Rg'])
manifest.append(dict(case=row['case'],component_id=row['component_id'],path=str(p),sha256=sha(p),full_field_sha256=sha(dest/(row['case']+'-field.npz')),geometry_valid=True,solver_cpu_seconds=solver_cpu,cpu_seconds=time.process_time()-begin,clock_learning_only=c['clock_c']))
save(dest/'progress.json',dict(parts=len(manifest),last=manifest[-1],completed_prediction_cpu_seconds=sum(e['cpu_seconds'] for e in manifest),validation_targets_read=False))
save(dest/'manifest.json',dict(entries=manifest,validation_labels_read=False,all_validation_complete=True))
def guard(root,mode,cap):
c=config(root);os.sched_setaffinity(0,[4]);start=time.process_time();d=root/'guards'/mode
g=supervise([sys.executable,'tools/phase2_b15_prediction_continue.py','worker',str(root),mode],d,dict(cpu_seconds=cap,wall_seconds=cap+120,rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes']-c['referenced_storage_bytes']),global_stop=root/'global-stop.json')
save(d/'guard-result.json',dict(mode=mode,guard=g,guard_cpu_seconds=time.process_time()-start));return 0 if g['status']=='PASS' else 1
def run(root):
c=config(root);os.sched_setaffinity(0,[29]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Single prediction continuation')
start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);records=[];settled=0.;active=None;requested=[]
def total():
u=resource.getrusage(resource.RUSAGE_CHILDREN);return c['prior_cpu_seconds']+60+max(settled,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start+(tree_usage(active.pid)[0] if active else 0)
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),prior_cpu_seconds=c['prior_cpu_seconds'],reused_models=1,reused_predictions=1,remaining_predictions=110,refit=False))
old={s:signal.signal(s,lambda signum,frame:requested.append(signum)) for s in [signal.SIGTERM,signal.SIGINT]}
try:
for mode in ['predict','score']:
if mode=='score':
freeze(root,'predictions',[root/'contract.json',root/'phases/predict/manifest.json',*[BASE/p for p in c['code_sha256']]]);(root/'phases/score').mkdir()
cap=8000-total()-120
if cap<=0:raise RuntimeError('TOTAL_RESERVE_EXHAUSTED')
active=subprocess.Popen([sys.executable,'tools/phase2_b15_prediction_continue.py','guard',str(root),mode,str(cap)],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,start_new_session=True)
while active.poll() is None:
charged=total();usage=tree_usage(active.pid);sample=dict(cpu_seconds=charged,disk_bytes=c['referenced_storage_bytes']+disk_usage(root),rss_bytes=usage[1],signals=list(requested));reasons=[];estimate=charged
if mode=='predict':
progress=read(root/'phases/predict/progress.json');n=progress['parts'];weights=[r['nodes'] for r in c['rows']['validation']];estimate=projection(charged,progress['completed_prediction_cpu_seconds'],sum(weights[:n]),sum(weights))
if estimate>8000:reasons.append('PROJECTION_EXCEEDS_8000')
if charged>8000-120:reasons.append('TOTAL_CPU_RESERVE')
if sample['disk_bytes']>c['disk_bytes']:reasons.append('CUMULATIVE_DIRECTORY_DISK_LIMIT')
if sample['rss_bytes']>c['rss_bytes']:reasons.append('RSS_LIMIT')
if requested:reasons.append('COORDINATOR_SIGNAL')
if reasons:save(root/'stop-trigger.json',dict(reasons=reasons,sample=sample,phase=mode,projected_total=estimate));raise RuntimeError('RESOURCE_OR_SIGNAL_STOP')
save(root/'live-progress.json',dict(stage=mode,guard_pid=active.pid,cumulative_cpu_seconds=charged,projected_total=estimate,unix=time.time()));time.sleep(.2)
result=read(root/'guards'/mode/'guard-result.json');records.append(result);settled+=result['guard']['cpu_seconds']+result['guard_cpu_seconds'];returncode=active.returncode;active=None
if returncode or result['guard']['status']!='PASS':raise RuntimeError('WORKER_NOT_PASS')
save(root/'complete.json',dict(status='COMPLETE',decision=read(root/'phases/score/result.json')['decision'],cumulative_cpu_seconds=total(),records=records,refit=False,reused_first_prediction=True,accounting_complete=True))
except BaseException as exc:
save(root/'global-stop.json',dict(reason=type(exc).__name__+': '+str(exc)))
if active:
try:os.kill(active.pid,signal.SIGTERM)
except ProcessLookupError:pass
active.wait();p=root/'guards'/mode/'guard-result.json'
if p.exists():records.append(read(p))
active=None
save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),cumulative_cpu_seconds=total(),records=records,refit=False,retry=False,accounting_complete=True));raise
finally:
for s,h in old.items():signal.signal(s,h)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','guard','worker']);p.add_argument('root',type=Path);p.add_argument('mode',nargs='?');p.add_argument('cap',nargs='?',type=float);a=p.parse_args()
if a.action=='run':run(a.root)
elif a.action=='guard':raise SystemExit(guard(a.root,a.mode,a.cap))
else:
c=config(a.root);(predict if a.mode=='predict' else score)(a.root,c)