"""Single O1 geometry queue with first-ten gate and bounded, non-retry attempts.""" import argparse import fcntl import json import math import os from pathlib import Path import signal import subprocess import sys import time import psutil from o1_partition import checked, sha, common_roster, can_reserve, InputViolation from o1_prepare import contract, read from r4bc_guard import supervise from s2_recovery_supervisor import save, tree_usage from s2_recovery_monitoring import disk_usage def part_dir(root, c, index): return Path(root)/'parts'/c['parts'][index]['case'] def guard(root, index, slot): c = contract(root); os.sched_setaffinity(0, [c['worker_cpus'][slot]]) dest = part_dir(root, c, index) receipt = supervise([sys.executable, 'tools/o1_prepare.py', str(root), str(index)], dest, c['part_limits'], global_stop=root/'global-stop.json') output = dest/'output/result.json'; failure = dest/'output/failure.json' failed = read(failure) if failure.exists() else None unexplained = receipt['status'] != 'PASS' and not receipt['stop_reason'] and failed is None global_failure = bool(receipt['monitor_errors']) or bool(failed and failed['global_stop']) or unexplained if receipt['status'] == 'PASS' and not output.exists(): receipt.update(status='FAILED', stop_reason='MISSING_RESULT'); global_failure = True record = dict(index=index,case=c['parts'][index]['case'],worker=receipt, result_path=str(output) if output.exists() else None, result_sha256=sha(output) if output.exists() else None, global_failure=global_failure,guard_cpu_seconds=time.process_time()) save(dest/'guard-result.json', record) return 0 if receipt['status'] == 'PASS' else 1 def cost_check(c, records): by = {r['index']:r for r in records} samples = []; rows = [] for i in c['first_ten']: r = by.get(i) if r is None: return dict(passed=False, reason='FIRST_TEN_INCOMPLETE') result = read(checked(r['result_path'],r['result_sha256'])) if r['result_path'] else {} rows.append(dict(case=r['case'],worker_status=r['worker']['status'], part_status=result.get('status'),reason=result.get('reason'), cpu_seconds=r['worker']['cpu_seconds'],tets=c['parts'][i]['tets'], candidate_calls=result.get('candidate_calls',0))) if r['worker']['status']=='PASS' and result.get('candidate_calls')==32: samples.append(r['worker']['cpu_seconds']/c['parts'][i]['tets']) spent = c['initial_preparation_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in records) if not samples: return dict(passed=False,reason='NO_COMPLETE_32_SEED_COST_SAMPLE',samples=rows,spent_cpu_seconds=spent) rate = max(samples) remaining = sum(c['parts'][i]['tets'] for i in c['dispatch'] if i not in by)*rate return dict(passed=bool(spent+remaining <= c['limits']['preparation_cpu_seconds']), reason='PROJECTED_WITHIN_CAP' if spent+remaining <= c['limits']['preparation_cpu_seconds'] else 'COST_PROJECTION_STOP', samples=rows,measured_max_cpu_per_tet=rate,remaining_projected_cpu_seconds=remaining, spent_cpu_seconds=spent,total_projected_cpu_seconds=spent+remaining, method='Maximum measured complete-32-candidate CPU/tet times remaining tet counts; heuristic only; hard guards remain authoritative.') def freeze_results(root, c, records, complete): results = {p['case']:dict(status='INELIGIBLE',reason=p['metadata_failure']) for p in c['parts'] if p.get('metadata_failure')} for r in records: if r['worker']['status']=='PASS': result = read(checked(r['result_path'],r['result_sha256'])) else: result = dict(status='WORKER_FAILED',reason=r['worker']['stop_reason']) results[r['case']] = result for p in c['parts']: results.setdefault(p['case'],dict(status='NEVER_ATTEMPTED')) save(root/'all-part-results.json',results) if not complete: return None kept, bad = common_roster(c['parts'],results) groups = sorted({p['component_id'] for p in kept}) strengths = {} for group in groups: rows = [results[p['case']] for p in kept if p['component_id']==group] strengths[group] = {key:sum(r['diagnostics'][key]['interface']['value'] for r in rows)/len(rows) for key in ['T/G','T/C']} strong = sum(v['T/G']>=.1 and v['T/C']>=.1 for v in strengths.values()) output = dict(status='GEOMETRY_COMMON_ROSTER_FROZEN_NO_FITS',original_parts=617,original_components=209, parts=kept,paired_components=groups,excluded_components=bad, component_interface_strength=strengths,strong_components=strong, coverage_gate=len(groups)>=168,intervention_gate=strong>=math.ceil(.7*len(groups)) if groups else False, retained_parts=len(kept),retained_nodes=sum(p['nodes'] for p in kept), all_part_results_sha256=sha(root/'all-part-results.json'), results={r['case']:dict(path=r['result_path'],sha256=r['result_sha256']) for r in records}, fit_started=False,labels_read=False) save(root/'common-roster.json',output) return sha(root/'common-roster.json') def run(root): root = Path(root) with (root/'queue.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) return locked(root) def locked(root): c=contract(root); launch=read(root/'launch-freeze.json') checked(root/'contract.json',launch['contract_sha256']) if (root/'started.json').exists(): raise FileExistsError('O1 queue already attempted; no restart') os.sched_setaffinity(0,[1]); start=time.monotonic() save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'), implementation_commit=launch['implementation_commit'],cpu_affinity=sorted(os.sched_getaffinity(0)), mode='GEOMETRY_PREPARATION_ONLY',fit_started=False)) active={};records=[];cursor=0;gate=False;cut=None;reason=None;errors=[];accounting=True;requested=[None] def interrupt(sig,frame): requested[0]='COORDINATOR_SIGNAL_'+str(sig) for s in (signal.SIGTERM,signal.SIGINT): signal.signal(s,interrupt) env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') try: while True: for slot,item in list(active.items()): if item['process'].poll() is None: continue path=part_dir(root,c,item['index'])/'guard-result.json' if not path.exists(): accounting=False; reason='MISSING_GUARD_RECEIPT' else: r=read(path) if not math.isfinite(r['worker']['cpu_seconds']) or r['worker']['cpu_seconds']<0: accounting=False;reason='INVALID_ACCOUNTING' records.append(r) if r['global_failure']: reason='WORKER_GLOBAL_FAILURE' item['log'].close();del active[slot] spent=c['initial_preparation_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in records) reason=reason or requested[0] try: usage=[tree_usage(item['process'].pid) for item in active.values()] live=sum(u[0] for u in usage);rss=sum(u[1] for u in usage)+psutil.Process().memory_info().rss disk=disk_usage(root);available=psutil.virtual_memory().available if live+spent>c['limits']['preparation_cpu_seconds']: reason='GLOBAL_PREPARATION_CPU_LIMIT' if rss>c['limits']['total_rss_bytes']: reason='GLOBAL_RSS_LIMIT' if disk>=c['limits']['disk_bytes']: reason='GLOBAL_DISK_LIMIT' except Exception as exc: errors.append(dict(type=type(exc).__name__,message=str(exc)));reason='GLOBAL_MONITOR_ERROR' live=rss=disk=available=0 if not gate and not active and len(records)==len(c['first_ten']) and not reason and not cut: check=cost_check(c,records);save(root/'first-ten-cost-check.json',check) gate=check['passed'] if not gate: cut=check['reason'] waiting=None if not reason and not cut: while cursor=len(c['first_ten']):break if not can_reserve(spent,len(active),c['limits']['preparation_cpu_seconds']): if not active: cut='NOT_RUN_BUDGET' else: waiting='WAIT_ACTIVE_BUDGET_SETTLEMENT' break if available<10*2**30: waiting='WAIT_AVAILABLE_MEMORY';break contract(root) index=c['dispatch'][cursor];slot=next(i for i in range(4) if i not in active) dest=part_dir(root,c,index);dest.mkdir(parents=True,exist_ok=True) if (dest/'worker.started.json').exists(): raise InputViolation('Attempt already exists') log=(dest/'guard.log').open('x') proc=subprocess.Popen([sys.executable,'tools/run_o1_prepare.py','guard',str(root),str(index),str(slot)], stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True) active[slot]=dict(process=proc,index=index,log=log);cursor+=1;available-=8*2**30 if reason: if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time())) for item in active.values(): if not item.get('stop_sent'): try:item['process'].send_signal(signal.SIGTERM) except ProcessLookupError:pass item['stop_sent']=True save(root/'progress.json',dict(terminal_attempts=len(records),metadata_excluded=sum(bool(p.get('metadata_failure')) for p in c['parts']), active={str(s):dict(index=v['index'],case=c['parts'][v['index']]['case'],guard_pid=v['process'].pid) for s,v in active.items()}, preparation_cpu_seconds=spent,live_guard_and_worker_cpu_seconds=live,reserved_active_cpu_seconds=len(active)*1800, total_rss_bytes=rss,disk_bytes=disk,first_ten_gate_passed=gate,cut=cut,global_stop=reason,waiting=waiting, monitor_errors=errors,wall_seconds=time.monotonic()-start,fit_started=False)) if not active and (reason or cut or cursor==len(c['dispatch'])):break time.sleep(.2) except Exception as exc: reason='COORDINATOR_EXCEPTION' save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc))) save(root/'global-stop.json',dict(reason=reason)) finally: for item in active.values(): try:item['process'].send_signal(signal.SIGTERM) except ProcessLookupError:pass for item in active.values(): item['process'].wait();item['log'].close();p=part_dir(root,c,item['index'])/'guard-result.json' if p.exists():records.append(read(p)) else:accounting=False complete=not reason and not cut and accounting and len(records)==len(c['dispatch']) roster_hash=freeze_results(root,c,records,complete) save(root/'complete.json',dict(status='PREPARATION_COMPLETE' if complete else 'PREPARATION_STOPPED', global_stop=reason,cut=cut,accounting_complete=accounting,records=records, never_attempted=[c['parts'][i]['case'] for i in c['dispatch'][cursor:]], preparation_cpu_seconds=c['initial_preparation_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in records), guard_cpu_seconds=sum(r.get('guard_cpu_seconds',0) for r in records),coordinator_cpu_seconds=time.process_time(), common_roster_sha256=roster_hash,monitor_errors=errors,fit_started=False,new_label_decodes=0, next='No auto fit launch; R4bc terminal dependency and model implementation still required')) return 0 if complete else 1 if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('mode',choices=['run','guard']);p.add_argument('root',type=Path) p.add_argument('index',type=int,nargs='?');p.add_argument('slot',type=int,nargs='?');a=p.parse_args() raise SystemExit(run(a.root) if a.mode=='run' else guard(a.root,a.index,a.slot))