Files
MeltMind/tools/phase2_gate_outcome_continuation.py
T

114 lines
12 KiB
Python

"""One labelled v2 continuation: original333 retained, only160 never attempted."""
import argparse,fcntl,json,math,os,resource,signal,subprocess,sys,time
from pathlib import Path
import psutil
from phase2_fixed_io import BASE
from phase2_h1_worker_v2 import checked,sha,save
from phase2_home_freeze_io import verify,freeze
from phase2_gate_outcome_queue import config as original_config
from phase2_gate_outcome_features import prepare
from r4bc_guard import supervise
from s2_recovery_supervisor import tree_usage
from s2_recovery_monitoring import disk_usage
def read(p):return json.loads(Path(p).read_text())
def project(prior,old_setup,terminal_nodes,remaining_nodes,setup=600,tail=120,postfit=3600):
if not all(math.isfinite(x) and x>=0 for x in [prior,old_setup,terminal_nodes,remaining_nodes,setup,tail,postfit]) or prior<=old_setup or terminal_nodes<=0 or remaining_nodes<=0:raise ValueError('Invalid original cost/node accounting')
rate=(prior-old_setup)/terminal_nodes;remaining=1.15*rate*remaining_nodes;total=prior+remaining+setup+tail+postfit
return dict(prior_cpu_once=prior,original_setup_already_in_prior=old_setup,terminal_nodes=terminal_nodes,remaining_nodes=remaining_nodes,variable_cpu_per_node=rate,safety_factor=1.15,remaining_variable_cpu_seconds=remaining,new_setup_seconds=setup,terminal_reserve_seconds=tail,postfit_reserve=postfit,projected_total_cpu_seconds=total,total_cpu_seconds=min(total,110000.),within_final_ceiling=total<=110000.,projection_rule='Old settled CPU once + 1.15*(old settled minus fixed setup)/terminal nodes*remaining nodes + new setup + terminal reserve + postfit reserve')
def scope(c,old,done):
records=done['records'];ids=[r['index'] for r in records]
if done['status']!='STOPPED' or done['reason']!='RuntimeError: Full per-part reservation exceeds remaining budget' or done['accounting_complete'] is not True or done['new_v2_labels_read']!=0:raise PermissionError('Only original accounted budget stop')
if len(ids)!=333 or len(set(ids))!=333 or set(ids)!=set(range(333)) or any(r['guard']['status']!='PASS' or r['guard']['returncode']!=0 for r in records):raise ValueError('Actual333 PASS terminal set required')
if c['approval']!='1794927353.190299' or c['new_indices']!=list(range(333,493)) or c['reused_indices']!=list(range(333)):raise PermissionError('Only160 never attempted; no retries')
if len(old['parts'])!=493 or c['prior_cpu_seconds']!=done['cumulative_cpu_seconds'] or c['prior_cpu_seconds']!=66936.020178167:raise ValueError('Actual old ledger once')
p=project(done['cumulative_cpu_seconds'],old['setup_seconds'],sum(old['parts'][i]['row']['nodes'] for i in ids),sum(old['parts'][i]['row']['nodes'] for i in c['new_indices']))
if c['projection']!=p or not p['within_final_ceiling'] or c['total_cpu_seconds']!=p['total_cpu_seconds'] or c['geometry_cpu_cap']!=p['total_cpu_seconds']-3600:raise PermissionError('Measured final total cap')
if c['worker_cpus']!=old['worker_cpus'] or c['worker_cpus']!=list(range(16,28)) or c['postfit_reserve']!=3600 or c['setup_seconds']!=600 or c['terminal_reserve']!=120:raise PermissionError('Original CPU allocation and reserves')
return p
def config(root):
c=read(root/'contract.json');verify(root,'launch');oldroot=Path(c['original_root']);old=original_config(oldroot);done=read(checked(oldroot/'complete.json',c['original_complete_sha256']))
checked(oldroot/'contract.json',c['original_contract_sha256']);scope(c,old,done)
for p,h in c['code_sha256'].items():checked(p,h)
checked(root/'reused-manifest.json',c['reused_manifest_sha256']);return c,old,done
def original_exited(oldroot):
for p in psutil.process_iter(['pid','cmdline']):
args=p.info['cmdline'] or []
if str(oldroot) in args and any(Path(x).name=='phase2_gate_outcome_queue.py' for x in args):raise PermissionError('Original queue/guard/worker still alive')
def reused_identity(entry,item,record):
d=Path(entry['path']);result=read(checked(d/'result.json',entry['result_sha256']));receipt=read(checked(d/'guard-result.json',entry['guard_result_sha256']))
if receipt!=record or entry['index']!=record['index'] or entry['case']!=item['row']['case'] or result['case']!=entry['case'] or result['index']!=entry['index'] or result['output_sha256']!=entry['output_sha256'] or result['new_labels_read'] is not False or result['new_fits']!=0 or result['gate_raw_ids']!=item['gate_raw_ids'] or result['status']!='FEATURES_COMPLETE_NO_PREDICTIONS':raise ValueError('Original terminal/result/gate identity')
for n,h in entry['output_sha256'].items():checked(d/n,h)
def guard(root,index,slot):
c,old,done=config(root)
if index not in c['new_indices'] or not 0<=slot<12:raise PermissionError('No completed-part retry')
os.sched_setaffinity(0,[c['worker_cpus'][slot]]);start=time.process_time();d=root/'parts'/f'{index:04d}'
g=supervise([sys.executable,'tools/phase2_gate_outcome_continuation.py','worker',str(root),str(index)],d,dict(cpu_seconds=1800,wall_seconds=1800,rss_bytes=4*2**30,disk_bytes=2*2**30),global_stop=root/'global-stop.json')
save(d/'guard-result.json',dict(index=index,guard=g,guard_cpu_seconds=time.process_time()-start));return 0 if g['status']=='PASS' else 1
def worker(root,index):
c,old,done=config(root)
if index not in c['new_indices']:raise PermissionError('No old completed-part execution')
prepare(root,old,index) # Unchanged scientific function and original indexed part contract.
def reservation(used,active,cap):return used+(active+1)*1800+120<=cap
def run(root):
c,old,done=config(root);oldroot=Path(c['original_root']);original_exited(oldroot);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Single labelled continuation')
for p in psutil.process_iter(['pid','cmdline']):
if p.pid!=os.getpid() and any(Path(x).name=='phase2_gate_outcome_continuation.py' for x in p.info['cmdline'] or []):raise RuntimeError('Existing continuation process')
if any((oldroot/'parts'/f'{i:04d}').exists() for i in c['new_indices']):raise PermissionError('Never-attempted part has original output directory')
reuse=read(root/'reused-manifest.json')['entries'];prior_by={r['index']:r for r in done['records']}
if len(reuse)!=333 or {e['index'] for e in reuse}!=set(c['reused_indices']):raise ValueError('All333 immutable reuse required')
for e in reuse:reused_identity(e,old['parts'][e['index']],prior_by[e['index']])
os.sched_setaffinity(0,[30,31]);start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);settled=0.;records=[];active={};requested=[];pending=list(c['new_indices'])
def total():
u=resource.getrusage(resource.RUSAGE_CHILDREN);return c['prior_cpu_seconds']+c['setup_seconds']+max(settled,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start+sum(tree_usage(p.pid)[0] for p,_ in active.values())
def collect(slot,p,index):
nonlocal settled
rr=read(root/'parts'/f'{index:04d}'/'guard-result.json');records.append(rr);settled+=rr['guard']['cpu_seconds']+rr['guard_cpu_seconds'];del active[slot]
if p.returncode or rr['index']!=index or rr['guard']['status']!='PASS':raise RuntimeError('NEW_FEATURE_WORKER_STOP_NO_RETRY')
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),original_root=str(oldroot),prior_cpu_once=c['prior_cpu_seconds'],reused_terminal=333,never_attempted=160,final_total_cpu_cap=c['total_cpu_seconds'],projection=c['projection'],postfit_started=False,new_labels_read=False))
handlers={s:signal.signal(s,lambda n,f:requested.append(n)) for s in [signal.SIGINT,signal.SIGTERM]}
try:
while pending or active:
if requested or total()>c['geometry_cpu_cap']-120 or disk_usage(root)+disk_usage(oldroot)>old['disk_bytes'] or sum(tree_usage(p.pid)[1] for p,_ in active.values())>48*2**30:raise RuntimeError('RESOURCE_OR_SIGNAL_STOP')
for slot,(p,i) in list(active.items()):
if p.poll() is not None:collect(slot,p,i)
for slot in range(12):
if pending and slot not in active:
if not reservation(total(),len(active),c['geometry_cpu_cap']):
if not active:raise RuntimeError('Full per-part reservation exceeds final budget; no extension')
break
i=pending.pop(0);p=subprocess.Popen([sys.executable,'tools/phase2_gate_outcome_continuation.py','guard',str(root),str(i),str(slot)],start_new_session=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL);active[slot]=(p,i)
save(root/'live-progress.json',dict(stage='LABELLED_GATE_V2_CONTINUATION_FEATURES_ONLY',reused_terminal=333,new_terminal=len(records),canonical_terminal=333+len(records),pending=len(pending),active=[dict(slot=s,pid=p.pid,index=i) for s,(p,i) in active.items()],prior_cpu_once=c['prior_cpu_seconds'],cumulative_cpu_seconds=total(),geometry_cpu_cap=c['geometry_cpu_cap'],total_cpu_cap=c['total_cpu_seconds'],unix=time.time()));time.sleep(.5)
if len(records)!=160 or {r['index'] for r in records}!=set(c['new_indices']):raise ValueError('Exactly160 new terminal attempts required')
entries=[dict(e,origin='ORIGINAL_TERMINAL_HASH_REUSE') for e in reuse]
for rr in records:
i=rr['index'];d=root/'parts'/f'{i:04d}';result=read(d/'result.json');entry=dict(index=i,case=old['parts'][i]['row']['case'],path=str(d),result_sha256=sha(d/'result.json'),guard_result_sha256=sha(d/'guard-result.json'),output_sha256=result['output_sha256'],origin='NEVER_ATTEMPTED_CONTINUATION')
reused_identity(entry,old['parts'][i],rr);entries.append(entry)
for e in reuse:reused_identity(e,old['parts'][e['index']],prior_by[e['index']])
if total()>c['geometry_cpu_cap']-120:raise RuntimeError('No terminal reserve')
save(root/'features-manifest.json',dict(recomputed=sorted(entries,key=lambda e:e['index']),exact_reuse=old['exact_reuse'],invalid_cases=old['invalid_cases'],parts_total=552,original333_reused=True,new160_completed=True,original_stop_preserved=True,new_labels_read=False,new_fits=0,new_predictions=0))
save(root/'complete.json',dict(status='CANONICAL_FEATURES_COMPLETE_POSTFIT_REVIEW_PENDING',original_complete_sha256=c['original_complete_sha256'],original_reused_terminal=333,new_records=records,canonical_terminal=493,prior_cpu_once=c['prior_cpu_seconds'],cumulative_cpu_seconds=total(),total_cpu_seconds=c['total_cpu_seconds'],accounting_complete=True,postfit_started=False,new_fits=0,new_predictions=0,new_v2_labels_read=0,retry=False))
freeze(root,'features',[root/'contract.json',root/'complete.json',root/'features-manifest.json',root/'reused-manifest.json',oldroot/'complete.json',*[Path(p) for p in c['code_sha256']]])
cost=total();save(root/'post-freeze-cost.json',dict(cumulative_cpu_seconds=cost,geometry_cpu_cap=c['geometry_cpu_cap'],total_cpu_seconds=c['total_cpu_seconds'],postfit_reserve=3600,within_cap=cost<=c['geometry_cpu_cap']))
if cost>c['geometry_cpu_cap']:raise RuntimeError('Final freeze exceeded geometry cap')
except BaseException as exc:
save(root/'global-stop.json',dict(reason=type(exc).__name__+': '+str(exc)))
for p,_ in active.values():
try:os.kill(p.pid,signal.SIGTERM)
except ProcessLookupError:pass
errors=[]
for slot,(p,i) in list(active.items()):
p.wait();f=root/'parts'/f'{i:04d}'/'guard-result.json'
if f.exists():
rr=read(f);records.append(rr);settled+=rr['guard']['cpu_seconds']+rr['guard_cpu_seconds']
else:errors.append(dict(index=i,reason='MISSING_GUARD_RECEIPT'))
active.clear();save(root/'complete.json',dict(status='STOPPED',reason=str(exc),original_reused_terminal=333,new_records=records,never_dispatched=pending,prior_cpu_once=c['prior_cpu_seconds'],cumulative_cpu_seconds=total(),accounting_complete=not errors,missing_receipts=errors,original_stop_preserved=True,postfit_started=False,new_v2_labels_read=0,retry=False));raise
finally:
for s,h in handlers.items():signal.signal(s,h)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('action',choices=['run','guard','worker']);p.add_argument('root',type=Path);p.add_argument('index',type=int,nargs='?');p.add_argument('slot',type=int,nargs='?');a=p.parse_args()
if a.action=='run':run(a.root)
elif a.action=='guard':raise SystemExit(guard(a.root,a.index,a.slot))
else:worker(a.root,a.index)