Files
MeltMind/tools/run_o1_prepare_v2.py
T

237 lines
14 KiB
Python

"""Independent O1 engineering continuation: corrected contract schema, unchanged science."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
import psutil
from o1_partition import checked, sha, common_roster, can_reserve, InputViolation
from o1_prepare import contract, read
from r4bc_guard import supervise
from s2_recovery_supervisor import save, tree_usage
from s2_recovery_monitoring import disk_usage
def validate_runtime_contract(c):
expected = dict(worker_cpu_seconds_total=144000, preparation_cpu_seconds=43200,
fit_cpu_seconds=86400, post_cpu_seconds=14400,
total_tree_rss_bytes=36*2**30, disk_bytes=100*2**30,
workers_max=4, retries=0, gpu=False, new_target_decodes=0)
for key, value in expected.items():
if c['limits'].get(key) != value:
raise InputViolation('Runtime contract field mismatch: '+key)
if c['worker_cpus'] != [0,2,4,6] or c['coordinator_cpu'] != 1:
raise InputViolation('CPU assignment mismatch')
if c['part_limits'] != dict(cpu_seconds=1800,wall_seconds=1800,rss_bytes=8*2**30,disk_bytes=100*2**30):
raise InputViolation('Per-part limits mismatch')
if not 0 <= c['initial_preparation_cpu_seconds'] < 43200:
raise InputViolation('Invalid initial preparation accounting')
if len(c['first_ten']) != 10 or c['first_ten'] != c['dispatch'][:10]:
raise InputViolation('First-ten identity mismatch')
if len(c['dispatch']) != len(set(c['dispatch'])) or any(i not in range(len(c['parts'])) for i in c['dispatch']):
raise InputViolation('Duplicate or invalid dispatch')
return True
def part_dir(root, c, index):
return Path(root)/'parts'/c['parts'][index]['case']
def resource_reason(c, spent, live, rss, disk):
if spent+live > c['limits']['preparation_cpu_seconds']: return 'GLOBAL_PREPARATION_CPU_LIMIT'
if rss > c['limits']['total_tree_rss_bytes']: return 'GLOBAL_RSS_LIMIT'
if disk >= c['limits']['disk_bytes']: return 'GLOBAL_DISK_LIMIT'
return None
def guard(root, index, slot):
c = contract(root); validate_runtime_contract(c); os.sched_setaffinity(0, [c['worker_cpus'][slot]])
dest = part_dir(root, c, index)
receipt = supervise([sys.executable, 'tools/o1_prepare.py', str(root), str(index)], dest,
c['part_limits'], global_stop=root/'global-stop.json')
output = dest/'output/result.json'; failure = dest/'output/failure.json'
failed = read(failure) if failure.exists() else None
unexplained = receipt['status'] != 'PASS' and not receipt['stop_reason'] and failed is None
global_failure = bool(receipt['monitor_errors']) or bool(failed and failed['global_stop']) or unexplained
if receipt['status'] == 'PASS' and not output.exists():
receipt.update(status='FAILED', stop_reason='MISSING_RESULT'); global_failure = True
record = dict(index=index,case=c['parts'][index]['case'],worker=receipt,
result_path=str(output) if output.exists() else None,
result_sha256=sha(output) if output.exists() else None,
global_failure=global_failure,guard_cpu_seconds=time.process_time())
save(dest/'guard-result.json', record)
return 0 if receipt['status'] == 'PASS' else 1
def cost_check(c, records):
by = {r['index']:r for r in records}
samples = []; rows = []
for i in c['first_ten']:
r = by.get(i)
if r is None:
return dict(passed=False, reason='FIRST_TEN_INCOMPLETE')
result = read(checked(r['result_path'],r['result_sha256'])) if r['result_path'] else {}
rows.append(dict(case=r['case'],worker_status=r['worker']['status'],
part_status=result.get('status'),reason=result.get('reason'),
cpu_seconds=r['worker']['cpu_seconds'],tets=c['parts'][i]['tets'],
candidate_calls=result.get('candidate_calls',0)))
if r['worker']['status']=='PASS' and result.get('candidate_calls')==32:
samples.append(r['worker']['cpu_seconds']/c['parts'][i]['tets'])
spent = c['initial_preparation_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in records)
if not samples:
return dict(passed=False,reason='NO_COMPLETE_32_SEED_COST_SAMPLE',samples=rows,spent_cpu_seconds=spent)
rate = max(samples)
remaining = sum(c['parts'][i]['tets'] for i in c['dispatch'] if i not in by)*rate
return dict(passed=bool(spent+remaining <= c['limits']['preparation_cpu_seconds']),
reason='PROJECTED_WITHIN_CAP' if spent+remaining <= c['limits']['preparation_cpu_seconds'] else 'COST_PROJECTION_STOP',
samples=rows,measured_max_cpu_per_tet=rate,remaining_projected_cpu_seconds=remaining,
spent_cpu_seconds=spent,total_projected_cpu_seconds=spent+remaining,
method='Maximum measured complete-32-candidate CPU/tet times remaining tet counts; heuristic only; hard guards remain authoritative.')
def freeze_results(root, c, records, complete):
results = {p['case']:dict(status='INELIGIBLE',reason=p['metadata_failure']) for p in c['parts'] if p.get('metadata_failure')}
for r in records:
if r['worker']['status']=='PASS':
result = read(checked(r['result_path'],r['result_sha256']))
else:
result = dict(status='WORKER_FAILED',reason=r['worker']['stop_reason'])
results[r['case']] = result
for p in c['parts']:
results.setdefault(p['case'],dict(status='NEVER_ATTEMPTED'))
save(root/'all-part-results.json',results)
if not complete:
return None
kept, bad = common_roster(c['parts'],results)
groups = sorted({p['component_id'] for p in kept})
strengths = {}
for group in groups:
rows = [results[p['case']] for p in kept if p['component_id']==group]
strengths[group] = {key:sum(r['diagnostics'][key]['interface']['value'] for r in rows)/len(rows) for key in ['T/G','T/C']}
strong = sum(v['T/G']>=.1 and v['T/C']>=.1 for v in strengths.values())
output = dict(status='GEOMETRY_COMMON_ROSTER_FROZEN_NO_FITS',original_parts=617,original_components=209,
parts=kept,paired_components=groups,excluded_components=bad,
component_interface_strength=strengths,strong_components=strong,
coverage_gate=len(groups)>=168,intervention_gate=strong>=math.ceil(.7*len(groups)) if groups else False,
retained_parts=len(kept),retained_nodes=sum(p['nodes'] for p in kept),
all_part_results_sha256=sha(root/'all-part-results.json'),
results={r['case']:dict(path=r['result_path'],sha256=r['result_sha256']) for r in records},
fit_started=False,labels_read=False)
save(root/'common-roster.json',output)
return sha(root/'common-roster.json')
def run(root):
root = Path(root)
with (root/'queue.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return locked(root)
def locked(root):
c=contract(root); validate_runtime_contract(c); launch=read(root/'launch-freeze.json')
checked(root/'contract.json',launch['contract_sha256'])
if (root/'started.json').exists(): raise FileExistsError('O1 queue already attempted; no restart')
os.sched_setaffinity(0,[1]); start=time.monotonic()
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),
implementation_commit=launch['implementation_commit'],cpu_affinity=sorted(os.sched_getaffinity(0)),
mode='GEOMETRY_PREPARATION_ONLY',fit_started=False))
active={};records=[];cursor=0;gate=False;cut=None;reason=None;errors=[];accounting=True;requested=[None]
def interrupt(sig,frame): requested[0]='COORDINATOR_SIGNAL_'+str(sig)
for s in (signal.SIGTERM,signal.SIGINT): signal.signal(s,interrupt)
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
try:
while True:
for slot,item in list(active.items()):
if item['process'].poll() is None: continue
path=part_dir(root,c,item['index'])/'guard-result.json'
if not path.exists():
accounting=False; reason='MISSING_GUARD_RECEIPT'
else:
r=read(path)
if not math.isfinite(r['worker']['cpu_seconds']) or r['worker']['cpu_seconds']<0:
accounting=False;reason='INVALID_ACCOUNTING'
records.append(r)
if r['global_failure']: reason='WORKER_GLOBAL_FAILURE'
item['log'].close();del active[slot]
spent=c['initial_preparation_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in records)
reason=reason or requested[0]
try:
usage=[tree_usage(item['process'].pid) for item in active.values()]
live=sum(u[0] for u in usage);rss=sum(u[1] for u in usage)+psutil.Process().memory_info().rss
disk=disk_usage(root);available=psutil.virtual_memory().available
reason=reason or resource_reason(c,spent,live,rss,disk)
except Exception as exc:
errors.append(dict(type=type(exc).__name__,message=str(exc)));reason='GLOBAL_MONITOR_ERROR'
live=rss=disk=available=0
if not gate and not active and len(records)==len(c['first_ten']) and not reason and not cut:
check=cost_check(c,records);save(root/'first-ten-cost-check.json',check)
gate=check['passed']
if not gate: cut=check['reason']
waiting=None
if not reason and not cut:
while cursor<len(c['dispatch']) and len(active)<(4 if gate else 1):
if not gate and cursor>=len(c['first_ten']):break
if not can_reserve(spent,len(active),c['limits']['preparation_cpu_seconds']):
if not active: cut='NOT_RUN_BUDGET'
else: waiting='WAIT_ACTIVE_BUDGET_SETTLEMENT'
break
if available<10*2**30: waiting='WAIT_AVAILABLE_MEMORY';break
contract(root)
index=c['dispatch'][cursor];slot=next(i for i in range(4) if i not in active)
dest=part_dir(root,c,index);dest.mkdir(parents=True,exist_ok=True)
if (dest/'worker.started.json').exists(): raise InputViolation('Attempt already exists')
log=(dest/'guard.log').open('x')
proc=subprocess.Popen([sys.executable,'tools/run_o1_prepare_v2.py','guard',str(root),str(index),str(slot)],
stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True)
active[slot]=dict(process=proc,index=index,log=log);cursor+=1;available-=8*2**30
if reason:
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
for item in active.values():
if not item.get('stop_sent'):
try:item['process'].send_signal(signal.SIGTERM)
except ProcessLookupError:pass
item['stop_sent']=True
save(root/'progress.json',dict(terminal_attempts=len(records),metadata_excluded=sum(bool(p.get('metadata_failure')) for p in c['parts']),
active={str(s):dict(index=v['index'],case=c['parts'][v['index']]['case'],guard_pid=v['process'].pid) for s,v in active.items()},
preparation_cpu_seconds=spent,live_guard_and_worker_cpu_seconds=live,reserved_active_cpu_seconds=len(active)*1800,
total_rss_bytes=rss,disk_bytes=disk,first_ten_gate_passed=gate,cut=cut,global_stop=reason,waiting=waiting,
monitor_errors=errors,wall_seconds=time.monotonic()-start,fit_started=False))
if not active and (reason or cut or cursor==len(c['dispatch'])):break
time.sleep(.2)
except Exception as exc:
reason='COORDINATOR_EXCEPTION'
save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc)))
save(root/'global-stop.json',dict(reason=reason))
finally:
for item in active.values():
try:item['process'].send_signal(signal.SIGTERM)
except ProcessLookupError:pass
for item in active.values():
item['process'].wait();item['log'].close();p=part_dir(root,c,item['index'])/'guard-result.json'
if p.exists():records.append(read(p))
else:accounting=False
complete=not reason and not cut and accounting and len(records)==len(c['dispatch'])
roster_hash=freeze_results(root,c,records,complete)
save(root/'complete.json',dict(status='PREPARATION_COMPLETE' if complete else 'PREPARATION_STOPPED',
global_stop=reason,cut=cut,accounting_complete=accounting,records=records,
never_attempted=[c['parts'][i]['case'] for i in c['dispatch'][cursor:]],
preparation_cpu_seconds=c['initial_preparation_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in records),
guard_cpu_seconds=sum(r.get('guard_cpu_seconds',0) for r in records),coordinator_cpu_seconds=time.process_time(),
common_roster_sha256=roster_hash,monitor_errors=errors,fit_started=False,new_label_decodes=0,
next='No auto fit launch; R4bc terminal dependency and model implementation still required'))
return 0 if complete else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('mode',choices=['run','guard']);p.add_argument('root',type=Path)
p.add_argument('index',type=int,nargs='?');p.add_argument('slot',type=int,nargs='?');a=p.parse_args()
raise SystemExit(run(a.root) if a.mode=='run' else guard(a.root,a.index,a.slot))