Preserve O1 zero-dispatch stop and freeze approved schema continuation

This commit is contained in:
Frank14f committed 2026-11-17 04:57:18 +08:00
1 parent afec2dce6a
commit ff8d9cfd42
20 files changed
+64734

No files matched your search

+236
View File
@@ -0,0 +1,236 @@
"""Independent O1 engineering continuation: corrected contract schema, unchanged science."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
import psutil
from o1_partition import checked, sha, common_roster, can_reserve, InputViolation
from o1_prepare import contract, read
from r4bc_guard import supervise
from s2_recovery_supervisor import save, tree_usage
from s2_recovery_monitoring import disk_usage
def validate_runtime_contract(c):
expected = dict(worker_cpu_seconds_total=144000, preparation_cpu_seconds=43200,
fit_cpu_seconds=86400, post_cpu_seconds=14400,
total_tree_rss_bytes=36*2**30, disk_bytes=100*2**30,
workers_max=4, retries=0, gpu=False, new_target_decodes=0)
for key, value in expected.items():
if c['limits'].get(key) != value:
raise InputViolation('Runtime contract field mismatch: '+key)
if c['worker_cpus'] != [0,2,4,6] or c['coordinator_cpu'] != 1:
raise InputViolation('CPU assignment mismatch')
if c['part_limits'] != dict(cpu_seconds=1800,wall_seconds=1800,rss_bytes=8*2**30,disk_bytes=100*2**30):
raise InputViolation('Per-part limits mismatch')
if not 0 <= c['initial_preparation_cpu_seconds'] < 43200:
raise InputViolation('Invalid initial preparation accounting')
if len(c['first_ten']) != 10 or c['first_ten'] != c['dispatch'][:10]:
raise InputViolation('First-ten identity mismatch')
if len(c['dispatch']) != len(set(c['dispatch'])) or any(i not in range(len(c['parts'])) for i in c['dispatch']):
raise InputViolation('Duplicate or invalid dispatch')
return True
def part_dir(root, c, index):
return Path(root)/'parts'/c['parts'][index]['case']
def resource_reason(c, spent, live, rss, disk):
if spent+live > c['limits']['preparation_cpu_seconds']: return 'GLOBAL_PREPARATION_CPU_LIMIT'
if rss > c['limits']['total_tree_rss_bytes']: return 'GLOBAL_RSS_LIMIT'
if disk >= c['limits']['disk_bytes']: return 'GLOBAL_DISK_LIMIT'
return None
def guard(root, index, slot):
c = contract(root); validate_runtime_contract(c); os.sched_setaffinity(0, [c['worker_cpus'][slot]])
dest = part_dir(root, c, index)
receipt = supervise([sys.executable, 'tools/o1_prepare.py', str(root), str(index)], dest,
c['part_limits'], global_stop=root/'global-stop.json')
output = dest/'output/result.json'; failure = dest/'output/failure.json'
failed = read(failure) if failure.exists() else None
unexplained = receipt['status'] != 'PASS' and not receipt['stop_reason'] and failed is None
global_failure = bool(receipt['monitor_errors']) or bool(failed and failed['global_stop']) or unexplained
if receipt['status'] == 'PASS' and not output.exists():
receipt.update(status='FAILED', stop_reason='MISSING_RESULT'); global_failure = True
record = dict(index=index,case=c['parts'][index]['case'],worker=receipt,
result_path=str(output) if output.exists() else None,
result_sha256=sha(output) if output.exists() else None,
global_failure=global_failure,guard_cpu_seconds=time.process_time())
save(dest/'guard-result.json', record)
return 0 if receipt['status'] == 'PASS' else 1
def cost_check(c, records):
by = {r['index']:r for r in records}
samples = []; rows = []
for i in c['first_ten']:
r = by.get(i)
if r is None:
return dict(passed=False, reason='FIRST_TEN_INCOMPLETE')
result = read(checked(r['result_path'],r['result_sha256'])) if r['result_path'] else {}
rows.append(dict(case=r['case'],worker_status=r['worker']['status'],
part_status=result.get('status'),reason=result.get('reason'),
cpu_seconds=r['worker']['cpu_seconds'],tets=c['parts'][i]['tets'],
candidate_calls=result.get('candidate_calls',0)))
if r['worker']['status']=='PASS' and result.get('candidate_calls')==32:
samples.append(r['worker']['cpu_seconds']/c['parts'][i]['tets'])
spent = c['initial_preparation_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in records)
if not samples:
return dict(passed=False,reason='NO_COMPLETE_32_SEED_COST_SAMPLE',samples=rows,spent_cpu_seconds=spent)
rate = max(samples)
remaining = sum(c['parts'][i]['tets'] for i in c['dispatch'] if i not in by)*rate
return dict(passed=bool(spent+remaining <= c['limits']['preparation_cpu_seconds']),
reason='PROJECTED_WITHIN_CAP' if spent+remaining <= c['limits']['preparation_cpu_seconds'] else 'COST_PROJECTION_STOP',
samples=rows,measured_max_cpu_per_tet=rate,remaining_projected_cpu_seconds=remaining,
spent_cpu_seconds=spent,total_projected_cpu_seconds=spent+remaining,
method='Maximum measured complete-32-candidate CPU/tet times remaining tet counts; heuristic only; hard guards remain authoritative.')
def freeze_results(root, c, records, complete):
results = {p['case']:dict(status='INELIGIBLE',reason=p['metadata_failure']) for p in c['parts'] if p.get('metadata_failure')}
for r in records:
if r['worker']['status']=='PASS':
result = read(checked(r['result_path'],r['result_sha256']))
else:
result = dict(status='WORKER_FAILED',reason=r['worker']['stop_reason'])
results[r['case']] = result
for p in c['parts']:
results.setdefault(p['case'],dict(status='NEVER_ATTEMPTED'))
save(root/'all-part-results.json',results)
if not complete:
return None
kept, bad = common_roster(c['parts'],results)
groups = sorted({p['component_id'] for p in kept})
strengths = {}
for group in groups:
rows = [results[p['case']] for p in kept if p['component_id']==group]
strengths[group] = {key:sum(r['diagnostics'][key]['interface']['value'] for r in rows)/len(rows) for key in ['T/G','T/C']}
strong = sum(v['T/G']>=.1 and v['T/C']>=.1 for v in strengths.values())
output = dict(status='GEOMETRY_COMMON_ROSTER_FROZEN_NO_FITS',original_parts=617,original_components=209,
parts=kept,paired_components=groups,excluded_components=bad,
component_interface_strength=strengths,strong_components=strong,
coverage_gate=len(groups)>=168,intervention_gate=strong>=math.ceil(.7*len(groups)) if groups else False,
retained_parts=len(kept),retained_nodes=sum(p['nodes'] for p in kept),
all_part_results_sha256=sha(root/'all-part-results.json'),
results={r['case']:dict(path=r['result_path'],sha256=r['result_sha256']) for r in records},
fit_started=False,labels_read=False)
save(root/'common-roster.json',output)
return sha(root/'common-roster.json')
def run(root):
root = Path(root)
with (root/'queue.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return locked(root)
def locked(root):
c=contract(root); validate_runtime_contract(c); launch=read(root/'launch-freeze.json')
checked(root/'contract.json',launch['contract_sha256'])
if (root/'started.json').exists(): raise FileExistsError('O1 queue already attempted; no restart')
os.sched_setaffinity(0,[1]); start=time.monotonic()
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),
implementation_commit=launch['implementation_commit'],cpu_affinity=sorted(os.sched_getaffinity(0)),
mode='GEOMETRY_PREPARATION_ONLY',fit_started=False))
active={};records=[];cursor=0;gate=False;cut=None;reason=None;errors=[];accounting=True;requested=[None]
def interrupt(sig,frame): requested[0]='COORDINATOR_SIGNAL_'+str(sig)
for s in (signal.SIGTERM,signal.SIGINT): signal.signal(s,interrupt)
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
try:
while True:
for slot,item in list(active.items()):
if item['process'].poll() is None: continue
path=part_dir(root,c,item['index'])/'guard-result.json'
if not path.exists():
accounting=False; reason='MISSING_GUARD_RECEIPT'
else:
r=read(path)
if not math.isfinite(r['worker']['cpu_seconds']) or r['worker']['cpu_seconds']<0:
accounting=False;reason='INVALID_ACCOUNTING'
records.append(r)
if r['global_failure']: reason='WORKER_GLOBAL_FAILURE'
item['log'].close();del active[slot]
spent=c['initial_preparation_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in records)
reason=reason or requested[0]
try:
usage=[tree_usage(item['process'].pid) for item in active.values()]
live=sum(u[0] for u in usage);rss=sum(u[1] for u in usage)+psutil.Process().memory_info().rss
disk=disk_usage(root);available=psutil.virtual_memory().available
reason=reason or resource_reason(c,spent,live,rss,disk)
except Exception as exc:
errors.append(dict(type=type(exc).__name__,message=str(exc)));reason='GLOBAL_MONITOR_ERROR'
live=rss=disk=available=0
if not gate and not active and len(records)==len(c['first_ten']) and not reason and not cut:
check=cost_check(c,records);save(root/'first-ten-cost-check.json',check)
gate=check['passed']
if not gate: cut=check['reason']
waiting=None
if not reason and not cut:
while cursor<len(c['dispatch']) and len(active)<(4 if gate else 1):
if not gate and cursor>=len(c['first_ten']):break
if not can_reserve(spent,len(active),c['limits']['preparation_cpu_seconds']):
if not active: cut='NOT_RUN_BUDGET'
else: waiting='WAIT_ACTIVE_BUDGET_SETTLEMENT'
break
if available<10*2**30: waiting='WAIT_AVAILABLE_MEMORY';break
contract(root)
index=c['dispatch'][cursor];slot=next(i for i in range(4) if i not in active)
dest=part_dir(root,c,index);dest.mkdir(parents=True,exist_ok=True)
if (dest/'worker.started.json').exists(): raise InputViolation('Attempt already exists')
log=(dest/'guard.log').open('x')
proc=subprocess.Popen([sys.executable,'tools/run_o1_prepare_v2.py','guard',str(root),str(index),str(slot)],
stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True)
active[slot]=dict(process=proc,index=index,log=log);cursor+=1;available-=8*2**30
if reason:
if not (root/'global-stop.json').exists():save(root/'global-stop.json',dict(reason=reason,unix=time.time()))
for item in active.values():
if not item.get('stop_sent'):
try:item['process'].send_signal(signal.SIGTERM)
except ProcessLookupError:pass
item['stop_sent']=True
save(root/'progress.json',dict(terminal_attempts=len(records),metadata_excluded=sum(bool(p.get('metadata_failure')) for p in c['parts']),
active={str(s):dict(index=v['index'],case=c['parts'][v['index']]['case'],guard_pid=v['process'].pid) for s,v in active.items()},
preparation_cpu_seconds=spent,live_guard_and_worker_cpu_seconds=live,reserved_active_cpu_seconds=len(active)*1800,
total_rss_bytes=rss,disk_bytes=disk,first_ten_gate_passed=gate,cut=cut,global_stop=reason,waiting=waiting,
monitor_errors=errors,wall_seconds=time.monotonic()-start,fit_started=False))
if not active and (reason or cut or cursor==len(c['dispatch'])):break
time.sleep(.2)
except Exception as exc:
reason='COORDINATOR_EXCEPTION'
save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc)))
save(root/'global-stop.json',dict(reason=reason))
finally:
for item in active.values():
try:item['process'].send_signal(signal.SIGTERM)
except ProcessLookupError:pass
for item in active.values():
item['process'].wait();item['log'].close();p=part_dir(root,c,item['index'])/'guard-result.json'
if p.exists():records.append(read(p))
else:accounting=False
complete=not reason and not cut and accounting and len(records)==len(c['dispatch'])
roster_hash=freeze_results(root,c,records,complete)
save(root/'complete.json',dict(status='PREPARATION_COMPLETE' if complete else 'PREPARATION_STOPPED',
global_stop=reason,cut=cut,accounting_complete=accounting,records=records,
never_attempted=[c['parts'][i]['case'] for i in c['dispatch'][cursor:]],
preparation_cpu_seconds=c['initial_preparation_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in records),
guard_cpu_seconds=sum(r.get('guard_cpu_seconds',0) for r in records),coordinator_cpu_seconds=time.process_time(),
common_roster_sha256=roster_hash,monitor_errors=errors,fit_started=False,new_label_decodes=0,
next='No auto fit launch; R4bc terminal dependency and model implementation still required'))
return 0 if complete else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('mode',choices=['run','guard']);p.add_argument('root',type=Path)
p.add_argument('index',type=int,nargs='?');p.add_argument('slot',type=int,nargs='?');a=p.parse_args()
raise SystemExit(run(a.root) if a.mode=='run' else guard(a.root,a.index,a.slot))
+42
View File
@@ -0,0 +1,42 @@
"""Regression of the zero-dispatch contract/schema integration failure."""
import copy
import json
from pathlib import Path
import unittest
from o1_partition import InputViolation
from run_o1_prepare_v2 import validate_runtime_contract,resource_reason
class EngineeringAcceptance(unittest.TestCase):
def setUp(self):
# Actual immutable metadata contract, not a differently named test fixture.
self.c=json.loads(Path('artifacts/o1-preparation-2026-11-17/contract.json').read_text())
def test_actual_frozen_contract_passes_startup_and_first_monitor(self):
self.assertTrue(validate_runtime_contract(self.c))
self.assertIsNone(resource_reason(self.c,self.c['initial_preparation_cpu_seconds'],0,100000000,0))
self.assertIn('total_tree_rss_bytes',self.c['limits'])
self.assertNotIn('total_rss_bytes',self.c['limits'])
def test_missing_or_legacy_memory_key_refused_before_dispatch(self):
self.c['limits']['total_rss_bytes']=self.c['limits'].pop('total_tree_rss_bytes')
with self.assertRaisesRegex(InputViolation,'total_tree_rss_bytes'):
validate_runtime_contract(self.c)
def test_actual_contract_all_resource_boundaries(self):
self.assertEqual(resource_reason(self.c,43200,1,0,0),'GLOBAL_PREPARATION_CPU_LIMIT')
self.assertEqual(resource_reason(self.c,0,0,36*2**30+1,0),'GLOBAL_RSS_LIMIT')
self.assertEqual(resource_reason(self.c,0,0,0,100*2**30),'GLOBAL_DISK_LIMIT')
self.assertIsNone(resource_reason(self.c,43200,0,36*2**30,100*2**30-1))
def test_budget_widening_rejected(self):
self.c['limits']['preparation_cpu_seconds']+=1
with self.assertRaises(InputViolation):validate_runtime_contract(self.c)
def test_duplicate_or_changed_first_ten_rejected(self):
self.c['dispatch'][11]=self.c['dispatch'][10]
with self.assertRaises(InputViolation):validate_runtime_contract(self.c)
if __name__=='__main__':unittest.main(verbosity=2)