Files
MeltMind/tools/run_o1_v2.py
T

237 lines
14 KiB
Python

"""Scientific O1 v2 preparation: fixed 60-part feasibility barrier before full roster."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
import psutil
from o1_partition import checked, sha, common_roster, can_reserve, InputViolation
from o1_prepare import contract, read
from o1_v2 import feasibility
from r4bc_guard import supervise
from s2_recovery_supervisor import save, tree_usage
from s2_recovery_monitoring import disk_usage
def validate_runtime_contract(c):
expected = dict(worker_cpu_seconds_total=144000, preparation_cpu_seconds=43200,
fit_cpu_seconds=86400, post_cpu_seconds=14400,
total_tree_rss_bytes=36*2**30, disk_bytes=100*2**30,
workers_max=4, retries=0, gpu=False, new_target_decodes=0)
for key, value in expected.items():
if c['limits'].get(key) != value:
raise InputViolation('Runtime contract field mismatch: '+key)
if c['worker_cpus'] != [0,2,4,6] or c['coordinator_cpu'] != 1:
raise InputViolation('CPU assignment mismatch')
if c['part_limits'] != dict(cpu_seconds=1800,wall_seconds=1800,rss_bytes=8*2**30,disk_bytes=100*2**30):
raise InputViolation('Per-part limits mismatch')
if not 0 <= c['initial_preparation_cpu_seconds'] < 43200:
raise InputViolation('Invalid initial preparation accounting')
if len(c['panel_indices']) != 60 or c['panel_indices'] != c['dispatch'][:60]:
raise InputViolation('Original sixty-panel identity mismatch')
if c.get('scientific_spec_sha256') != '632688217e59dd134922245e524ff1c0f63933be64600be21379f7aa74947a5f':
raise InputViolation('Scientific v2 specification mismatch')
if len(c['dispatch']) != len(set(c['dispatch'])) or any(i not in range(len(c['parts'])) for i in c['dispatch']):
raise InputViolation('Duplicate or invalid dispatch')
return True
def part_dir(root, c, index):
return Path(root)/'parts'/c['parts'][index]['case']
def resource_reason(c, spent, live, rss, disk):
if spent+live > c['limits']['preparation_cpu_seconds']: return 'GLOBAL_PREPARATION_CPU_LIMIT'
if rss > c['limits']['total_tree_rss_bytes']: return 'GLOBAL_RSS_LIMIT'
if disk >= c['limits']['disk_bytes']: return 'GLOBAL_DISK_LIMIT'
return None
def guard(root, index, slot):
c = contract(root); validate_runtime_contract(c); os.sched_setaffinity(0, [c['worker_cpus'][slot]])
dest = part_dir(root, c, index)
receipt = supervise([sys.executable, 'tools/o1_v2_prepare.py', str(root), str(index)], dest,
c['part_limits'], global_stop=root/'global-stop.json')
output = dest/'output/result.json'; failure = dest/'output/failure.json'
failed = read(failure) if failure.exists() else None
unexplained = receipt['status'] != 'PASS' and not receipt['stop_reason'] and failed is None
global_failure = bool(receipt['monitor_errors']) or bool(failed and failed['global_stop']) or unexplained
if receipt['status'] == 'PASS' and not output.exists():
receipt.update(status='FAILED', stop_reason='MISSING_RESULT'); global_failure = True
record = dict(index=index,case=c['parts'][index]['case'],worker=receipt,
result_path=str(output) if output.exists() else None,
result_sha256=sha(output) if output.exists() else None,
global_failure=global_failure,guard_cpu_seconds=time.process_time())
save(dest/'guard-result.json', record)
return 0 if receipt['status'] == 'PASS' else 1
def panel_checkpoint(c, records):
by={r['index']:r for r in records};results={}
if set(by)!=set(c['panel_indices']):
raise InputViolation('Panel receipts differ from frozen sixty')
for i in c['panel_indices']:
r=by[i]
if r['worker']['status']=='PASS':
results[r['case']]=read(checked(r['result_path'],r['result_sha256']))
else:
results[r['case']]=dict(status='WORKER_FAILED',reason=r['worker']['stop_reason'])
panel_ids=[c['parts'][i]['case'] for i in c['panel_indices']]
check=feasibility(c['parts'],results,panel_ids)
check['reason']='V2_PANEL_PASSED' if check['passed'] else 'V2_PANEL_IDENTIFIABILITY_STOP'
check['parts']=[dict(case=p,component_id=next(r['component_id'] for r in c['parts'] if r['case']==p),
status=results[p]['status'],reason=results[p].get('reason'),
diagnostics=results[p].get('diagnostics')) for p in panel_ids]
check['part_receipts']={r['case']:r['result_sha256'] for r in records}
return check
def freeze_results(root, c, records, complete):
results = {p['case']:dict(status='INELIGIBLE',reason=p['metadata_failure']) for p in c['parts'] if p.get('metadata_failure')}
for r in records:
if r['worker']['status']=='PASS':
result = read(checked(r['result_path'],r['result_sha256']))
else:
result = dict(status='WORKER_FAILED',reason=r['worker']['stop_reason'])
results[r['case']] = result
for p in c['parts']:
results.setdefault(p['case'],dict(status='NEVER_ATTEMPTED'))
save(root/'all-part-results.json',results)
if not complete:
return None
kept, bad = common_roster(c['parts'],results)
groups = sorted({p['component_id'] for p in kept})
strengths = {}
for group in groups:
rows = [results[p['case']] for p in kept if p['component_id']==group]
strengths[group] = {key:sum(r['diagnostics'][key]['interface']['value'] for r in rows)/len(rows) for key in ['T/G','T/C']}
strong = sum(v['T/G']>=.1 and v['T/C']>=.1 for v in strengths.values())
output = dict(status='GEOMETRY_COMMON_ROSTER_FROZEN_NO_FITS',original_parts=617,original_components=209,
parts=kept,paired_components=groups,excluded_components=bad,
component_interface_strength=strengths,strong_components=strong,
coverage_gate=len(groups)>=168,intervention_gate=strong>=math.ceil(.7*len(groups)) if groups else False,
retained_parts=len(kept),retained_nodes=sum(p['nodes'] for p in kept),
all_part_results_sha256=sha(root/'all-part-results.json'),
results={r['case']:dict(path=r['result_path'],sha256=r['result_sha256']) for r in records},
fit_started=False,labels_read=False)
full_gate=feasibility(c['parts'],results,[p['case'] for p in c['parts']],full=True)
save(root/'full-component-gate.json',full_gate)
output['original_component_gate_passed']=full_gate['passed']
save(root/'common-roster.json',output)
return sha(root/'common-roster.json')
def run(root):
root = Path(root)
with (root/'queue.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return locked(root)
def locked(root):
c=contract(root); validate_runtime_contract(c); launch=read(root/'launch-freeze.json')
checked(root/'contract.json',launch['contract_sha256'])
if (root/'started.json').exists(): raise FileExistsError('O1 queue already attempted; no restart')
os.sched_setaffinity(0,[1]); start=time.monotonic()
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),
implementation_commit=launch['implementation_commit'],cpu_affinity=sorted(os.sched_getaffinity(0)),
mode='SCIENTIFIC_V2_GEOMETRY_PREPARATION_ONLY',fit_started=False))
active={};records=[];cursor=0;gate=False;cut=None;reason=None;errors=[];accounting=True;requested=[None]
def interrupt(sig,frame): requested[0]='COORDINATOR_SIGNAL_'+str(sig)
for s in (signal.SIGTERM,signal.SIGINT): signal.signal(s,interrupt)
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
try:
while True:
for slot,item in list(active.items()):
if item['process'].poll() is None: continue
path=part_dir(root,c,item['index'])/'guard-result.json'
if not path.exists():
accounting=False; reason='MISSING_GUARD_RECEIPT'
else:
r=read(path)
if not math.isfinite(r['worker']['cpu_seconds']) or r['worker']['cpu_seconds']<0:
accounting=False;reason='INVALID_ACCOUNTING'
records.append(r)
if r['global_failure']: reason='WORKER_GLOBAL_FAILURE'
item['log'].close();del active[slot]
spent=c['initial_preparation_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in records)
reason=reason or requested[0]
try:
usage=[tree_usage(item['process'].pid) for item in active.values()]
live=sum(u[0] for u in usage);rss=sum(u[1] for u in usage)+psutil.Process().memory_info().rss
disk=disk_usage(root);available=psutil.virtual_memory().available
reason=reason or resource_reason(c,spent,live,rss,disk)
except Exception as exc:
errors.append(dict(type=type(exc).__name__,message=str(exc)));reason='GLOBAL_MONITOR_ERROR'
live=rss=disk=available=0
if not gate and not active and len(records)==len(c['panel_indices']) and not reason and not cut:
check=panel_checkpoint(c,records);save(root/'panel-feasibility.json',check)
gate=check['passed']
if not gate: cut=check['reason']
waiting=None
if not reason and not cut:
while cursor<len(c['dispatch']) and len(active)<4:
if not gate and cursor>=len(c['panel_indices']):break
if not can_reserve(spent,len(active),c['limits']['preparation_cpu_seconds']):
if not active: cut='NOT_RUN_BUDGET'
else: waiting='WAIT_ACTIVE_BUDGET_SETTLEMENT'
break
if available<10*2**30: waiting='WAIT_AVAILABLE_MEMORY';break
contract(root)
index=c['dispatch'][cursor];slot=next(i for i in range(4) if i not in active)
dest=part_dir(root,c,index);dest.mkdir(parents=True,exist_ok=True)
if (dest/'worker.started.json').exists(): raise InputViolation('Attempt already exists')
log=(dest/'guard.log').open('x')
proc=subprocess.Popen([sys.executable,'tools/run_o1_v2.py','guard',str(root),str(index),str(slot)],
stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True)
active[slot]=dict(process=proc,index=index,log=log);cursor+=1;available-=8*2**30
if reason:
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
for item in active.values():
if not item.get('stop_sent'):
try:item['process'].send_signal(signal.SIGTERM)
except ProcessLookupError:pass
item['stop_sent']=True
save(root/'progress.json',dict(terminal_attempts=len(records),metadata_excluded=sum(bool(p.get('metadata_failure')) for p in c['parts']),
active={str(s):dict(index=v['index'],case=c['parts'][v['index']]['case'],guard_pid=v['process'].pid) for s,v in active.items()},
preparation_cpu_seconds=spent,live_guard_and_worker_cpu_seconds=live,reserved_active_cpu_seconds=len(active)*1800,
total_rss_bytes=rss,disk_bytes=disk,panel_gate_passed=gate,cut=cut,global_stop=reason,waiting=waiting,
monitor_errors=errors,wall_seconds=time.monotonic()-start,fit_started=False))
if not active and (reason or cut or cursor==len(c['dispatch'])):break
time.sleep(.2)
except Exception as exc:
reason='COORDINATOR_EXCEPTION'
save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc)))
save(root/'global-stop.json',dict(reason=reason))
finally:
for item in active.values():
try:item['process'].send_signal(signal.SIGTERM)
except ProcessLookupError:pass
for item in active.values():
item['process'].wait();item['log'].close();p=part_dir(root,c,item['index'])/'guard-result.json'
if p.exists():records.append(read(p))
else:accounting=False
complete=not reason and not cut and accounting and len(records)==len(c['dispatch'])
roster_hash=freeze_results(root,c,records,complete)
save(root/'complete.json',dict(status='V2_PREPARATION_COMPLETE' if complete else ('V2_PANEL_IDENTIFIABILITY_STOP' if cut=='V2_PANEL_IDENTIFIABILITY_STOP' else 'V2_ENGINEERING_OR_BUDGET_STOP'),
global_stop=reason,cut=cut,accounting_complete=accounting,records=records,
never_attempted=[c['parts'][i]['case'] for i in c['dispatch'][cursor:]],
preparation_cpu_seconds=c['initial_preparation_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in records),
guard_cpu_seconds=sum(r.get('guard_cpu_seconds',0) for r in records),coordinator_cpu_seconds=time.process_time(),
common_roster_sha256=roster_hash,monitor_errors=errors,fit_started=False,new_label_decodes=0,
next='No further redesign if panel/full component gate fails; no auto fit launch; R4bc terminal dependency and model implementation still required'))
return 0 if complete else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('mode',choices=['run','guard']);p.add_argument('root',type=Path)
p.add_argument('index',type=int,nargs='?');p.add_argument('slot',type=int,nargs='?');a=p.parse_args()
raise SystemExit(run(a.root) if a.mode=='run' else guard(a.root,a.index,a.slot))