Preserve O1 zero-dispatch stop and freeze approved schema continuation
This commit is contained in:
1 parent
afec2dce6a
commit
ff8d9cfd42
20 files changed
+64734
No files matched your search
@@ -0,0 +1,236 @@
|
||||
"""Independent O1 engineering continuation: corrected contract schema, unchanged science."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import psutil
|
||||
|
||||
from o1_partition import checked, sha, common_roster, can_reserve, InputViolation
|
||||
from o1_prepare import contract, read
|
||||
from r4bc_guard import supervise
|
||||
from s2_recovery_supervisor import save, tree_usage
|
||||
from s2_recovery_monitoring import disk_usage
|
||||
|
||||
|
||||
def validate_runtime_contract(c):
|
||||
expected = dict(worker_cpu_seconds_total=144000, preparation_cpu_seconds=43200,
|
||||
fit_cpu_seconds=86400, post_cpu_seconds=14400,
|
||||
total_tree_rss_bytes=36*2**30, disk_bytes=100*2**30,
|
||||
workers_max=4, retries=0, gpu=False, new_target_decodes=0)
|
||||
for key, value in expected.items():
|
||||
if c['limits'].get(key) != value:
|
||||
raise InputViolation('Runtime contract field mismatch: '+key)
|
||||
if c['worker_cpus'] != [0,2,4,6] or c['coordinator_cpu'] != 1:
|
||||
raise InputViolation('CPU assignment mismatch')
|
||||
if c['part_limits'] != dict(cpu_seconds=1800,wall_seconds=1800,rss_bytes=8*2**30,disk_bytes=100*2**30):
|
||||
raise InputViolation('Per-part limits mismatch')
|
||||
if not 0 <= c['initial_preparation_cpu_seconds'] < 43200:
|
||||
raise InputViolation('Invalid initial preparation accounting')
|
||||
if len(c['first_ten']) != 10 or c['first_ten'] != c['dispatch'][:10]:
|
||||
raise InputViolation('First-ten identity mismatch')
|
||||
if len(c['dispatch']) != len(set(c['dispatch'])) or any(i not in range(len(c['parts'])) for i in c['dispatch']):
|
||||
raise InputViolation('Duplicate or invalid dispatch')
|
||||
return True
|
||||
|
||||
|
||||
def part_dir(root, c, index):
|
||||
return Path(root)/'parts'/c['parts'][index]['case']
|
||||
|
||||
|
||||
def resource_reason(c, spent, live, rss, disk):
|
||||
if spent+live > c['limits']['preparation_cpu_seconds']: return 'GLOBAL_PREPARATION_CPU_LIMIT'
|
||||
if rss > c['limits']['total_tree_rss_bytes']: return 'GLOBAL_RSS_LIMIT'
|
||||
if disk >= c['limits']['disk_bytes']: return 'GLOBAL_DISK_LIMIT'
|
||||
return None
|
||||
|
||||
|
||||
def guard(root, index, slot):
|
||||
c = contract(root); validate_runtime_contract(c); os.sched_setaffinity(0, [c['worker_cpus'][slot]])
|
||||
dest = part_dir(root, c, index)
|
||||
receipt = supervise([sys.executable, 'tools/o1_prepare.py', str(root), str(index)], dest,
|
||||
c['part_limits'], global_stop=root/'global-stop.json')
|
||||
output = dest/'output/result.json'; failure = dest/'output/failure.json'
|
||||
failed = read(failure) if failure.exists() else None
|
||||
unexplained = receipt['status'] != 'PASS' and not receipt['stop_reason'] and failed is None
|
||||
global_failure = bool(receipt['monitor_errors']) or bool(failed and failed['global_stop']) or unexplained
|
||||
if receipt['status'] == 'PASS' and not output.exists():
|
||||
receipt.update(status='FAILED', stop_reason='MISSING_RESULT'); global_failure = True
|
||||
record = dict(index=index,case=c['parts'][index]['case'],worker=receipt,
|
||||
result_path=str(output) if output.exists() else None,
|
||||
result_sha256=sha(output) if output.exists() else None,
|
||||
global_failure=global_failure,guard_cpu_seconds=time.process_time())
|
||||
save(dest/'guard-result.json', record)
|
||||
return 0 if receipt['status'] == 'PASS' else 1
|
||||
|
||||
|
||||
def cost_check(c, records):
|
||||
by = {r['index']:r for r in records}
|
||||
samples = []; rows = []
|
||||
for i in c['first_ten']:
|
||||
r = by.get(i)
|
||||
if r is None:
|
||||
return dict(passed=False, reason='FIRST_TEN_INCOMPLETE')
|
||||
result = read(checked(r['result_path'],r['result_sha256'])) if r['result_path'] else {}
|
||||
rows.append(dict(case=r['case'],worker_status=r['worker']['status'],
|
||||
part_status=result.get('status'),reason=result.get('reason'),
|
||||
cpu_seconds=r['worker']['cpu_seconds'],tets=c['parts'][i]['tets'],
|
||||
candidate_calls=result.get('candidate_calls',0)))
|
||||
if r['worker']['status']=='PASS' and result.get('candidate_calls')==32:
|
||||
samples.append(r['worker']['cpu_seconds']/c['parts'][i]['tets'])
|
||||
spent = c['initial_preparation_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in records)
|
||||
if not samples:
|
||||
return dict(passed=False,reason='NO_COMPLETE_32_SEED_COST_SAMPLE',samples=rows,spent_cpu_seconds=spent)
|
||||
rate = max(samples)
|
||||
remaining = sum(c['parts'][i]['tets'] for i in c['dispatch'] if i not in by)*rate
|
||||
return dict(passed=bool(spent+remaining <= c['limits']['preparation_cpu_seconds']),
|
||||
reason='PROJECTED_WITHIN_CAP' if spent+remaining <= c['limits']['preparation_cpu_seconds'] else 'COST_PROJECTION_STOP',
|
||||
samples=rows,measured_max_cpu_per_tet=rate,remaining_projected_cpu_seconds=remaining,
|
||||
spent_cpu_seconds=spent,total_projected_cpu_seconds=spent+remaining,
|
||||
method='Maximum measured complete-32-candidate CPU/tet times remaining tet counts; heuristic only; hard guards remain authoritative.')
|
||||
|
||||
|
||||
def freeze_results(root, c, records, complete):
|
||||
results = {p['case']:dict(status='INELIGIBLE',reason=p['metadata_failure']) for p in c['parts'] if p.get('metadata_failure')}
|
||||
for r in records:
|
||||
if r['worker']['status']=='PASS':
|
||||
result = read(checked(r['result_path'],r['result_sha256']))
|
||||
else:
|
||||
result = dict(status='WORKER_FAILED',reason=r['worker']['stop_reason'])
|
||||
results[r['case']] = result
|
||||
for p in c['parts']:
|
||||
results.setdefault(p['case'],dict(status='NEVER_ATTEMPTED'))
|
||||
save(root/'all-part-results.json',results)
|
||||
if not complete:
|
||||
return None
|
||||
kept, bad = common_roster(c['parts'],results)
|
||||
groups = sorted({p['component_id'] for p in kept})
|
||||
strengths = {}
|
||||
for group in groups:
|
||||
rows = [results[p['case']] for p in kept if p['component_id']==group]
|
||||
strengths[group] = {key:sum(r['diagnostics'][key]['interface']['value'] for r in rows)/len(rows) for key in ['T/G','T/C']}
|
||||
strong = sum(v['T/G']>=.1 and v['T/C']>=.1 for v in strengths.values())
|
||||
output = dict(status='GEOMETRY_COMMON_ROSTER_FROZEN_NO_FITS',original_parts=617,original_components=209,
|
||||
parts=kept,paired_components=groups,excluded_components=bad,
|
||||
component_interface_strength=strengths,strong_components=strong,
|
||||
coverage_gate=len(groups)>=168,intervention_gate=strong>=math.ceil(.7*len(groups)) if groups else False,
|
||||
retained_parts=len(kept),retained_nodes=sum(p['nodes'] for p in kept),
|
||||
all_part_results_sha256=sha(root/'all-part-results.json'),
|
||||
results={r['case']:dict(path=r['result_path'],sha256=r['result_sha256']) for r in records},
|
||||
fit_started=False,labels_read=False)
|
||||
save(root/'common-roster.json',output)
|
||||
return sha(root/'common-roster.json')
|
||||
|
||||
|
||||
def run(root):
|
||||
root = Path(root)
|
||||
with (root/'queue.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
return locked(root)
|
||||
|
||||
|
||||
def locked(root):
|
||||
c=contract(root); validate_runtime_contract(c); launch=read(root/'launch-freeze.json')
|
||||
checked(root/'contract.json',launch['contract_sha256'])
|
||||
if (root/'started.json').exists(): raise FileExistsError('O1 queue already attempted; no restart')
|
||||
os.sched_setaffinity(0,[1]); start=time.monotonic()
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),
|
||||
implementation_commit=launch['implementation_commit'],cpu_affinity=sorted(os.sched_getaffinity(0)),
|
||||
mode='GEOMETRY_PREPARATION_ONLY',fit_started=False))
|
||||
active={};records=[];cursor=0;gate=False;cut=None;reason=None;errors=[];accounting=True;requested=[None]
|
||||
def interrupt(sig,frame): requested[0]='COORDINATOR_SIGNAL_'+str(sig)
|
||||
for s in (signal.SIGTERM,signal.SIGINT): signal.signal(s,interrupt)
|
||||
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
||||
try:
|
||||
while True:
|
||||
for slot,item in list(active.items()):
|
||||
if item['process'].poll() is None: continue
|
||||
path=part_dir(root,c,item['index'])/'guard-result.json'
|
||||
if not path.exists():
|
||||
accounting=False; reason='MISSING_GUARD_RECEIPT'
|
||||
else:
|
||||
r=read(path)
|
||||
if not math.isfinite(r['worker']['cpu_seconds']) or r['worker']['cpu_seconds']<0:
|
||||
accounting=False;reason='INVALID_ACCOUNTING'
|
||||
records.append(r)
|
||||
if r['global_failure']: reason='WORKER_GLOBAL_FAILURE'
|
||||
item['log'].close();del active[slot]
|
||||
spent=c['initial_preparation_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in records)
|
||||
reason=reason or requested[0]
|
||||
try:
|
||||
usage=[tree_usage(item['process'].pid) for item in active.values()]
|
||||
live=sum(u[0] for u in usage);rss=sum(u[1] for u in usage)+psutil.Process().memory_info().rss
|
||||
disk=disk_usage(root);available=psutil.virtual_memory().available
|
||||
reason=reason or resource_reason(c,spent,live,rss,disk)
|
||||
except Exception as exc:
|
||||
errors.append(dict(type=type(exc).__name__,message=str(exc)));reason='GLOBAL_MONITOR_ERROR'
|
||||
live=rss=disk=available=0
|
||||
if not gate and not active and len(records)==len(c['first_ten']) and not reason and not cut:
|
||||
check=cost_check(c,records);save(root/'first-ten-cost-check.json',check)
|
||||
gate=check['passed']
|
||||
if not gate: cut=check['reason']
|
||||
waiting=None
|
||||
if not reason and not cut:
|
||||
while cursor<len(c['dispatch']) and len(active)<(4 if gate else 1):
|
||||
if not gate and cursor>=len(c['first_ten']):break
|
||||
if not can_reserve(spent,len(active),c['limits']['preparation_cpu_seconds']):
|
||||
if not active: cut='NOT_RUN_BUDGET'
|
||||
else: waiting='WAIT_ACTIVE_BUDGET_SETTLEMENT'
|
||||
break
|
||||
if available<10*2**30: waiting='WAIT_AVAILABLE_MEMORY';break
|
||||
contract(root)
|
||||
index=c['dispatch'][cursor];slot=next(i for i in range(4) if i not in active)
|
||||
dest=part_dir(root,c,index);dest.mkdir(parents=True,exist_ok=True)
|
||||
if (dest/'worker.started.json').exists(): raise InputViolation('Attempt already exists')
|
||||
log=(dest/'guard.log').open('x')
|
||||
proc=subprocess.Popen([sys.executable,'tools/run_o1_prepare_v2.py','guard',str(root),str(index),str(slot)],
|
||||
stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True)
|
||||
active[slot]=dict(process=proc,index=index,log=log);cursor+=1;available-=8*2**30
|
||||
if reason:
|
||||
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
|
||||
for item in active.values():
|
||||
if not item.get('stop_sent'):
|
||||
try:item['process'].send_signal(signal.SIGTERM)
|
||||
except ProcessLookupError:pass
|
||||
item['stop_sent']=True
|
||||
save(root/'progress.json',dict(terminal_attempts=len(records),metadata_excluded=sum(bool(p.get('metadata_failure')) for p in c['parts']),
|
||||
active={str(s):dict(index=v['index'],case=c['parts'][v['index']]['case'],guard_pid=v['process'].pid) for s,v in active.items()},
|
||||
preparation_cpu_seconds=spent,live_guard_and_worker_cpu_seconds=live,reserved_active_cpu_seconds=len(active)*1800,
|
||||
total_rss_bytes=rss,disk_bytes=disk,first_ten_gate_passed=gate,cut=cut,global_stop=reason,waiting=waiting,
|
||||
monitor_errors=errors,wall_seconds=time.monotonic()-start,fit_started=False))
|
||||
if not active and (reason or cut or cursor==len(c['dispatch'])):break
|
||||
time.sleep(.2)
|
||||
except Exception as exc:
|
||||
reason='COORDINATOR_EXCEPTION'
|
||||
save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc)))
|
||||
save(root/'global-stop.json',dict(reason=reason))
|
||||
finally:
|
||||
for item in active.values():
|
||||
try:item['process'].send_signal(signal.SIGTERM)
|
||||
except ProcessLookupError:pass
|
||||
for item in active.values():
|
||||
item['process'].wait();item['log'].close();p=part_dir(root,c,item['index'])/'guard-result.json'
|
||||
if p.exists():records.append(read(p))
|
||||
else:accounting=False
|
||||
complete=not reason and not cut and accounting and len(records)==len(c['dispatch'])
|
||||
roster_hash=freeze_results(root,c,records,complete)
|
||||
save(root/'complete.json',dict(status='PREPARATION_COMPLETE' if complete else 'PREPARATION_STOPPED',
|
||||
global_stop=reason,cut=cut,accounting_complete=accounting,records=records,
|
||||
never_attempted=[c['parts'][i]['case'] for i in c['dispatch'][cursor:]],
|
||||
preparation_cpu_seconds=c['initial_preparation_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in records),
|
||||
guard_cpu_seconds=sum(r.get('guard_cpu_seconds',0) for r in records),coordinator_cpu_seconds=time.process_time(),
|
||||
common_roster_sha256=roster_hash,monitor_errors=errors,fit_started=False,new_label_decodes=0,
|
||||
next='No auto fit launch; R4bc terminal dependency and model implementation still required'))
|
||||
return 0 if complete else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('mode',choices=['run','guard']);p.add_argument('root',type=Path)
|
||||
p.add_argument('index',type=int,nargs='?');p.add_argument('slot',type=int,nargs='?');a=p.parse_args()
|
||||
raise SystemExit(run(a.root) if a.mode=='run' else guard(a.root,a.index,a.slot))
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Regression of the zero-dispatch contract/schema integration failure."""
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
from o1_partition import InputViolation
|
||||
from run_o1_prepare_v2 import validate_runtime_contract,resource_reason
|
||||
|
||||
|
||||
class EngineeringAcceptance(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# Actual immutable metadata contract, not a differently named test fixture.
|
||||
self.c=json.loads(Path('artifacts/o1-preparation-2026-11-17/contract.json').read_text())
|
||||
|
||||
def test_actual_frozen_contract_passes_startup_and_first_monitor(self):
|
||||
self.assertTrue(validate_runtime_contract(self.c))
|
||||
self.assertIsNone(resource_reason(self.c,self.c['initial_preparation_cpu_seconds'],0,100000000,0))
|
||||
self.assertIn('total_tree_rss_bytes',self.c['limits'])
|
||||
self.assertNotIn('total_rss_bytes',self.c['limits'])
|
||||
|
||||
def test_missing_or_legacy_memory_key_refused_before_dispatch(self):
|
||||
self.c['limits']['total_rss_bytes']=self.c['limits'].pop('total_tree_rss_bytes')
|
||||
with self.assertRaisesRegex(InputViolation,'total_tree_rss_bytes'):
|
||||
validate_runtime_contract(self.c)
|
||||
|
||||
def test_actual_contract_all_resource_boundaries(self):
|
||||
self.assertEqual(resource_reason(self.c,43200,1,0,0),'GLOBAL_PREPARATION_CPU_LIMIT')
|
||||
self.assertEqual(resource_reason(self.c,0,0,36*2**30+1,0),'GLOBAL_RSS_LIMIT')
|
||||
self.assertEqual(resource_reason(self.c,0,0,0,100*2**30),'GLOBAL_DISK_LIMIT')
|
||||
self.assertIsNone(resource_reason(self.c,43200,0,36*2**30,100*2**30-1))
|
||||
|
||||
def test_budget_widening_rejected(self):
|
||||
self.c['limits']['preparation_cpu_seconds']+=1
|
||||
with self.assertRaises(InputViolation):validate_runtime_contract(self.c)
|
||||
|
||||
def test_duplicate_or_changed_first_ten_rejected(self):
|
||||
self.c['dispatch'][11]=self.c['dispatch'][10]
|
||||
with self.assertRaises(InputViolation):validate_runtime_contract(self.c)
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user