"""One labelled v2 continuation: original333 retained, only160 never attempted.""" import argparse,fcntl,json,math,os,resource,signal,subprocess,sys,time from pathlib import Path import psutil from phase2_fixed_io import BASE from phase2_h1_worker_v2 import checked,sha,save from phase2_home_freeze_io import verify,freeze from phase2_gate_outcome_queue import config as original_config from phase2_gate_outcome_features import prepare from r4bc_guard import supervise from s2_recovery_supervisor import tree_usage from s2_recovery_monitoring import disk_usage def read(p):return json.loads(Path(p).read_text()) def project(prior,old_setup,terminal_nodes,remaining_nodes,setup=600,tail=120,postfit=3600): if not all(math.isfinite(x) and x>=0 for x in [prior,old_setup,terminal_nodes,remaining_nodes,setup,tail,postfit]) or prior<=old_setup or terminal_nodes<=0 or remaining_nodes<=0:raise ValueError('Invalid original cost/node accounting') rate=(prior-old_setup)/terminal_nodes;remaining=1.15*rate*remaining_nodes;total=prior+remaining+setup+tail+postfit return dict(prior_cpu_once=prior,original_setup_already_in_prior=old_setup,terminal_nodes=terminal_nodes,remaining_nodes=remaining_nodes,variable_cpu_per_node=rate,safety_factor=1.15,remaining_variable_cpu_seconds=remaining,new_setup_seconds=setup,terminal_reserve_seconds=tail,postfit_reserve=postfit,projected_total_cpu_seconds=total,total_cpu_seconds=min(total,110000.),within_final_ceiling=total<=110000.,projection_rule='Old settled CPU once + 1.15*(old settled minus fixed setup)/terminal nodes*remaining nodes + new setup + terminal reserve + postfit reserve') def scope(c,old,done): records=done['records'];ids=[r['index'] for r in records] if done['status']!='STOPPED' or done['reason']!='RuntimeError: Full per-part reservation exceeds remaining budget' or done['accounting_complete'] is not True or done['new_v2_labels_read']!=0:raise PermissionError('Only original accounted budget stop') if len(ids)!=333 or len(set(ids))!=333 or set(ids)!=set(range(333)) or any(r['guard']['status']!='PASS' or r['guard']['returncode']!=0 for r in records):raise ValueError('Actual333 PASS terminal set required') if c['approval']!='1794927353.190299' or c['new_indices']!=list(range(333,493)) or c['reused_indices']!=list(range(333)):raise PermissionError('Only160 never attempted; no retries') if len(old['parts'])!=493 or c['prior_cpu_seconds']!=done['cumulative_cpu_seconds'] or c['prior_cpu_seconds']!=66936.020178167:raise ValueError('Actual old ledger once') p=project(done['cumulative_cpu_seconds'],old['setup_seconds'],sum(old['parts'][i]['row']['nodes'] for i in ids),sum(old['parts'][i]['row']['nodes'] for i in c['new_indices'])) if c['projection']!=p or not p['within_final_ceiling'] or c['total_cpu_seconds']!=p['total_cpu_seconds'] or c['geometry_cpu_cap']!=p['total_cpu_seconds']-3600:raise PermissionError('Measured final total cap') if c['worker_cpus']!=old['worker_cpus'] or c['worker_cpus']!=list(range(16,28)) or c['postfit_reserve']!=3600 or c['setup_seconds']!=600 or c['terminal_reserve']!=120:raise PermissionError('Original CPU allocation and reserves') return p def config(root): c=read(root/'contract.json');verify(root,'launch');oldroot=Path(c['original_root']);old=original_config(oldroot);done=read(checked(oldroot/'complete.json',c['original_complete_sha256'])) checked(oldroot/'contract.json',c['original_contract_sha256']);scope(c,old,done) for p,h in c['code_sha256'].items():checked(p,h) checked(root/'reused-manifest.json',c['reused_manifest_sha256']);return c,old,done def original_exited(oldroot): for p in psutil.process_iter(['pid','cmdline']): args=p.info['cmdline'] or [] if str(oldroot) in args and any(Path(x).name=='phase2_gate_outcome_queue.py' for x in args):raise PermissionError('Original queue/guard/worker still alive') def reused_identity(entry,item,record): d=Path(entry['path']);result=read(checked(d/'result.json',entry['result_sha256']));receipt=read(checked(d/'guard-result.json',entry['guard_result_sha256'])) if receipt!=record or entry['index']!=record['index'] or entry['case']!=item['row']['case'] or result['case']!=entry['case'] or result['index']!=entry['index'] or result['output_sha256']!=entry['output_sha256'] or result['new_labels_read'] is not False or result['new_fits']!=0 or result['gate_raw_ids']!=item['gate_raw_ids'] or result['status']!='FEATURES_COMPLETE_NO_PREDICTIONS':raise ValueError('Original terminal/result/gate identity') for n,h in entry['output_sha256'].items():checked(d/n,h) def guard(root,index,slot): c,old,done=config(root) if index not in c['new_indices'] or not 0<=slot<12:raise PermissionError('No completed-part retry') os.sched_setaffinity(0,[c['worker_cpus'][slot]]);start=time.process_time();d=root/'parts'/f'{index:04d}' g=supervise([sys.executable,'tools/phase2_gate_outcome_continuation.py','worker',str(root),str(index)],d,dict(cpu_seconds=1800,wall_seconds=1800,rss_bytes=4*2**30,disk_bytes=2*2**30),global_stop=root/'global-stop.json') save(d/'guard-result.json',dict(index=index,guard=g,guard_cpu_seconds=time.process_time()-start));return 0 if g['status']=='PASS' else 1 def worker(root,index): c,old,done=config(root) if index not in c['new_indices']:raise PermissionError('No old completed-part execution') prepare(root,old,index) # Unchanged scientific function and original indexed part contract. def reservation(used,active,cap):return used+(active+1)*1800+120<=cap def run(root): c,old,done=config(root);oldroot=Path(c['original_root']);original_exited(oldroot);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (root/'started.json').exists():raise FileExistsError('Single labelled continuation') for p in psutil.process_iter(['pid','cmdline']): if p.pid!=os.getpid() and any(Path(x).name=='phase2_gate_outcome_continuation.py' for x in p.info['cmdline'] or []):raise RuntimeError('Existing continuation process') if any((oldroot/'parts'/f'{i:04d}').exists() for i in c['new_indices']):raise PermissionError('Never-attempted part has original output directory') reuse=read(root/'reused-manifest.json')['entries'];prior_by={r['index']:r for r in done['records']} if len(reuse)!=333 or {e['index'] for e in reuse}!=set(c['reused_indices']):raise ValueError('All333 immutable reuse required') for e in reuse:reused_identity(e,old['parts'][e['index']],prior_by[e['index']]) os.sched_setaffinity(0,[30,31]);start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);settled=0.;records=[];active={};requested=[];pending=list(c['new_indices']) def total(): u=resource.getrusage(resource.RUSAGE_CHILDREN);return c['prior_cpu_seconds']+c['setup_seconds']+max(settled,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start+sum(tree_usage(p.pid)[0] for p,_ in active.values()) def collect(slot,p,index): nonlocal settled rr=read(root/'parts'/f'{index:04d}'/'guard-result.json');records.append(rr);settled+=rr['guard']['cpu_seconds']+rr['guard_cpu_seconds'];del active[slot] if p.returncode or rr['index']!=index or rr['guard']['status']!='PASS':raise RuntimeError('NEW_FEATURE_WORKER_STOP_NO_RETRY') save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),original_root=str(oldroot),prior_cpu_once=c['prior_cpu_seconds'],reused_terminal=333,never_attempted=160,final_total_cpu_cap=c['total_cpu_seconds'],projection=c['projection'],postfit_started=False,new_labels_read=False)) handlers={s:signal.signal(s,lambda n,f:requested.append(n)) for s in [signal.SIGINT,signal.SIGTERM]} try: while pending or active: if requested or total()>c['geometry_cpu_cap']-120 or disk_usage(root)+disk_usage(oldroot)>old['disk_bytes'] or sum(tree_usage(p.pid)[1] for p,_ in active.values())>48*2**30:raise RuntimeError('RESOURCE_OR_SIGNAL_STOP') for slot,(p,i) in list(active.items()): if p.poll() is not None:collect(slot,p,i) for slot in range(12): if pending and slot not in active: if not reservation(total(),len(active),c['geometry_cpu_cap']): if not active:raise RuntimeError('Full per-part reservation exceeds final budget; no extension') break i=pending.pop(0);p=subprocess.Popen([sys.executable,'tools/phase2_gate_outcome_continuation.py','guard',str(root),str(i),str(slot)],start_new_session=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL);active[slot]=(p,i) save(root/'live-progress.json',dict(stage='LABELLED_GATE_V2_CONTINUATION_FEATURES_ONLY',reused_terminal=333,new_terminal=len(records),canonical_terminal=333+len(records),pending=len(pending),active=[dict(slot=s,pid=p.pid,index=i) for s,(p,i) in active.items()],prior_cpu_once=c['prior_cpu_seconds'],cumulative_cpu_seconds=total(),geometry_cpu_cap=c['geometry_cpu_cap'],total_cpu_cap=c['total_cpu_seconds'],unix=time.time()));time.sleep(.5) if len(records)!=160 or {r['index'] for r in records}!=set(c['new_indices']):raise ValueError('Exactly160 new terminal attempts required') entries=[dict(e,origin='ORIGINAL_TERMINAL_HASH_REUSE') for e in reuse] for rr in records: i=rr['index'];d=root/'parts'/f'{i:04d}';result=read(d/'result.json');entry=dict(index=i,case=old['parts'][i]['row']['case'],path=str(d),result_sha256=sha(d/'result.json'),guard_result_sha256=sha(d/'guard-result.json'),output_sha256=result['output_sha256'],origin='NEVER_ATTEMPTED_CONTINUATION') reused_identity(entry,old['parts'][i],rr);entries.append(entry) for e in reuse:reused_identity(e,old['parts'][e['index']],prior_by[e['index']]) if total()>c['geometry_cpu_cap']-120:raise RuntimeError('No terminal reserve') save(root/'features-manifest.json',dict(recomputed=sorted(entries,key=lambda e:e['index']),exact_reuse=old['exact_reuse'],invalid_cases=old['invalid_cases'],parts_total=552,original333_reused=True,new160_completed=True,original_stop_preserved=True,new_labels_read=False,new_fits=0,new_predictions=0)) save(root/'complete.json',dict(status='CANONICAL_FEATURES_COMPLETE_POSTFIT_REVIEW_PENDING',original_complete_sha256=c['original_complete_sha256'],original_reused_terminal=333,new_records=records,canonical_terminal=493,prior_cpu_once=c['prior_cpu_seconds'],cumulative_cpu_seconds=total(),total_cpu_seconds=c['total_cpu_seconds'],accounting_complete=True,postfit_started=False,new_fits=0,new_predictions=0,new_v2_labels_read=0,retry=False)) freeze(root,'features',[root/'contract.json',root/'complete.json',root/'features-manifest.json',root/'reused-manifest.json',oldroot/'complete.json',*[Path(p) for p in c['code_sha256']]]) cost=total();save(root/'post-freeze-cost.json',dict(cumulative_cpu_seconds=cost,geometry_cpu_cap=c['geometry_cpu_cap'],total_cpu_seconds=c['total_cpu_seconds'],postfit_reserve=3600,within_cap=cost<=c['geometry_cpu_cap'])) if cost>c['geometry_cpu_cap']:raise RuntimeError('Final freeze exceeded geometry cap') except BaseException as exc: save(root/'global-stop.json',dict(reason=type(exc).__name__+': '+str(exc))) for p,_ in active.values(): try:os.kill(p.pid,signal.SIGTERM) except ProcessLookupError:pass errors=[] for slot,(p,i) in list(active.items()): p.wait();f=root/'parts'/f'{i:04d}'/'guard-result.json' if f.exists(): rr=read(f);records.append(rr);settled+=rr['guard']['cpu_seconds']+rr['guard_cpu_seconds'] else:errors.append(dict(index=i,reason='MISSING_GUARD_RECEIPT')) active.clear();save(root/'complete.json',dict(status='STOPPED',reason=str(exc),original_reused_terminal=333,new_records=records,never_dispatched=pending,prior_cpu_once=c['prior_cpu_seconds'],cumulative_cpu_seconds=total(),accounting_complete=not errors,missing_receipts=errors,original_stop_preserved=True,postfit_started=False,new_v2_labels_read=0,retry=False));raise finally: for s,h in handlers.items():signal.signal(s,h) if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('action',choices=['run','guard','worker']);p.add_argument('root',type=Path);p.add_argument('index',type=int,nargs='?');p.add_argument('slot',type=int,nargs='?');a=p.parse_args() if a.action=='run':run(a.root) elif a.action=='guard':raise SystemExit(guard(a.root,a.index,a.slot)) else:worker(a.root,a.index)