254 lines
17 KiB
Python
254 lines
17 KiB
Python
"""Single H1 execution, bounded guards, actual Git barriers, no retries."""
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import psutil
|
|
from r4bc_guard import supervise
|
|
from s2_recovery_supervisor import tree_usage
|
|
from s2_recovery_monitoring import disk_usage
|
|
from phase2_h1_worker_v2 import contract,sha,checked,save
|
|
|
|
BASE=Path('/home/frank14f/optane/MeltMind')
|
|
WRITER=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
|
|
ROOT=BASE/'artifacts/phase2-h1-execution-2026-11-17'
|
|
|
|
|
|
def read(p):return json.loads(Path(p).read_text())
|
|
|
|
|
|
def prepare(acceptance):
|
|
start=time.process_time();packet=read(BASE/'artifacts/phase2-h1-planning-2026-11-17/b1-review-packet-v1.json')
|
|
for p,h in packet['files'].items():checked(BASE/p,h)
|
|
for p,h in packet['legacy_source_sha256'].items():checked(BASE/p,h)
|
|
tests=read(acceptance)
|
|
if tests['status']!='PASS' or tests['cpu_seconds']>=50:raise ValueError('Synthetic acceptance/bounded setup missing')
|
|
frozen=read(BASE/'artifacts/phase2-h1-planning-2026-11-17/clock-freeze-v1/membership-and-sources.json')
|
|
for p,h in frozen['source_sha256'].items():checked(BASE/p,h)
|
|
grouped=read(BASE/'artifacts/r4bc-execution-2026-11-16/contract.json')
|
|
prior_path='artifacts/r4bc-final-postfit-2026-11-17/predictions-freeze.json';prior=read(BASE/prior_path)
|
|
c=dict(version='phase2-h1-res-v1',main_review='1794888554.440209',reviewed_contract_commit='e751e5b261820b6c2df1dc1f218cde7275b2ebec',
|
|
parts=frozen['parts'],jobs=frozen['cells'],grouped_rows={r['case']:r for r in grouped['parts']},
|
|
source_sha256={**frozen['source_sha256'],**packet['legacy_source_sha256'],prior_path:sha(BASE/prior_path)},
|
|
baseline_predictions={},archives={},initial_setup_charge_seconds=60.,setup_charge_kind='conservative metadata/synthetic reserve, not measured worker use',
|
|
synthetic_acceptance_path=str(acceptance),synthetic_acceptance_sha256=sha(acceptance),
|
|
budget=dict(total=72000.,geometry=3540.,fit=43200.,postfit=25200.,disk_bytes=20*1024**3,total_rss_bytes=36*1024**3),
|
|
tree_rule=dict(sign='H1-A',field_noninferiority_margin=.001,field_margin_percentage_points=.10,scope='grouped100'),new_decodes=0,gpu=False,retry=False)
|
|
for job in c['jobs']:
|
|
checked(job['baseline_model']['path'],job['baseline_model']['sha256'])
|
|
if job['protocol']=='grouped':
|
|
entries=[e for e in prior['entries'] if e['arm']=='A' and e['fold']==job['fold'] and e['pool']==job['pool']]
|
|
if {e['case'] for e in entries}!=set(job['test']):raise ValueError('Original A prediction roster mismatch')
|
|
c['baseline_predictions'][str(job['index'])]={e['case']:e for e in entries}
|
|
for protocol,version in [('grouped','v5'),('v4_random','v4')]:
|
|
archive=packet['archive_metadata'][version];c['archives'][protocol]=dict(path=archive['path'],stat=archive['original_archive_stat'])
|
|
new=['phase2_h1_execution','phase2_h1_worker_v2','phase2_h1_geometry','phase2_h1_topology','phase2_h1_postfit','phase2_h1_decision','phase2_residual_math']
|
|
old=['gre24_models','gre24_prepare_v2','gre24_features_v2','audit_samples','r4bc_guard','r4bc_common','s2_custody_common','s2_custody_statistics','s2_recovery_supervisor','s2_recovery_monitoring']
|
|
c['code_sha256']={f'tools/{name}.py':sha(BASE/f'tools/{name}.py') for name in new+old}
|
|
for p,h in tests['code_sha256'].items():checked(BASE/p,h)
|
|
if time.process_time()-start+tests['cpu_seconds']+.570199085>=60:raise ValueError('Setup reserve exceeded')
|
|
ROOT.mkdir(exist_ok=False);save(ROOT/'contract.json',c)
|
|
save(ROOT/'preparation-receipt.json',dict(status='PASS_METADATA_ONLY',actual_cpu_seconds=time.process_time()-start,
|
|
setup_charge_seconds=60.,synthetic_cpu_seconds=tests['cpu_seconds'],project_targets_read=False,
|
|
contract_sha256=sha(ROOT/'contract.json')))
|
|
print(ROOT,flush=True)
|
|
|
|
|
|
def freeze(root,name,relative_names,extra_paths=()):
|
|
with Path('/tmp/meltmind-single-writer.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX)
|
|
def git(*args):return subprocess.check_output(['git','-C',str(WRITER),*args],text=True).strip()
|
|
if git('branch','--show-current')!='codex/s2-execution-diff-20261114' or git('status','--porcelain'):raise RuntimeError('Single writer not clean on approved branch')
|
|
files=[root/n for n in relative_names]+[Path(p) for p in extra_paths]
|
|
for p in files:
|
|
rel=p.resolve().relative_to(BASE);dest=WRITER/rel;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,dest)
|
|
paths=[str(p.resolve().relative_to(BASE)) for p in files]
|
|
subprocess.run(['git','-C',str(WRITER),'add','--',*paths],check=True)
|
|
if git('diff','--cached','--name-only'):
|
|
subprocess.run(['git','-C',str(WRITER),'commit','-m','H1 '+name+' immutable evidence'],check=True)
|
|
commit=git('rev-parse','HEAD')
|
|
for p,rel in zip(files,paths):
|
|
data=subprocess.check_output(['git','-C',str(WRITER),'show',commit+':'+rel])
|
|
if hashlib.sha256(data).hexdigest()!=sha(p):raise RuntimeError('Git byte freeze mismatch')
|
|
proof=dict(commit=commit,checkout=str(WRITER),sha256={n:sha(root/n) for n in relative_names},
|
|
extra_sha256={str(p):sha(p) for p in extra_paths},remote_published=False)
|
|
save(root/(name+'-git-freeze.json'),proof)
|
|
return proof
|
|
|
|
|
|
def guard(root,task_path):
|
|
c=contract(root);task=read(task_path);os.sched_setaffinity(0,task['cpus']);start=time.process_time()
|
|
dest=root/'guards'/task['name']
|
|
result=supervise(task['command'],dest,task['limits'],global_stop=root/'global-stop.json')
|
|
result['guard_cpu_seconds']=time.process_time()-start
|
|
save(dest/'guard-result.json',result)
|
|
return 0 if result['status']=='PASS' else 1
|
|
|
|
|
|
def projected_fits(c,records):
|
|
by={r['job_index']:r for r in records};remaining=0.
|
|
for first in (0,15):
|
|
if first not in by or by[first]['guard']['status']!='PASS':return dict(pass_gate=False,reason='FIRST_CELL_FAILED_OR_MISSING')
|
|
reference=c['jobs'][first]
|
|
for job in c['jobs']:
|
|
if job['protocol']==reference['protocol'] and job['index'] not in by:
|
|
remaining+=1.25*by[first]['guard']['cpu_seconds']*job['learning_nodes']/reference['learning_nodes']
|
|
spent=sum(r['guard']['cpu_seconds']+r['guard']['guard_cpu_seconds'] for r in records)
|
|
return dict(pass_gate=spent+remaining<=c['budget']['fit'],spent=spent,remaining_projection=remaining,
|
|
full_fit_projection=spent+remaining,postfit_reserved=c['budget']['postfit'],factor=1.25)
|
|
|
|
|
|
class Runner:
|
|
def __init__(self,root,c):
|
|
self.root=root;self.c=c;self.charges={'geometry':0.,'fit':0.,'postfit':0.};self.records=[];self.stop=None
|
|
self.active={};self.accounting_complete=True
|
|
self.stage_seconds={};self.start_cpu=time.process_time()
|
|
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,self.interrupt)
|
|
def interrupt(self,sig,frame):self.stop='COORDINATOR_SIGNAL_'+str(sig)
|
|
def total(self):return 60+sum(self.charges.values())+time.process_time()-self.start_cpu
|
|
def jobs(self,stage,items,parallel=1):
|
|
try:return self._jobs(stage,items,parallel)
|
|
except BaseException:
|
|
self.stop=self.stop or 'COORDINATOR_EXCEPTION'
|
|
save(self.root/'global-stop.json',dict(reason=self.stop))
|
|
for proc,_,_ in self.active.values():
|
|
if proc.poll() is None:proc.send_signal(signal.SIGTERM)
|
|
for proc,task,log in self.active.values():
|
|
try:proc.wait(timeout=30)
|
|
except subprocess.TimeoutExpired:
|
|
self.accounting_complete=False
|
|
for child in psutil.Process(proc.pid).children(recursive=True):child.kill()
|
|
proc.kill();proc.wait()
|
|
p=self.root/'guards'/task['name']/'guard-result.json'
|
|
if p.exists():
|
|
g=read(p);self.charges[stage]+=g['cpu_seconds']+g['guard_cpu_seconds']
|
|
self.records.append(dict(name=task['name'],stage=stage,job_index=task.get('job_index'),guard=g))
|
|
else:self.accounting_complete=False
|
|
log.close()
|
|
self.active={}
|
|
raise
|
|
def _jobs(self,stage,items,parallel=1):
|
|
active={};pending=list(items);finished=[];cap=self.c['budget'][stage]
|
|
self.active=active
|
|
if stage=='postfit' and not (len(items)==1 and items[0]['name']=='summary'):cap-=120.
|
|
while pending or active:
|
|
for key,(proc,task,log) in list(active.items()):
|
|
code=proc.poll()
|
|
if code is None:continue
|
|
receipt=self.root/'guards'/task['name']/'guard-result.json'
|
|
if not receipt.exists():self.stop='MISSING_GUARD_ACCOUNTING';self.accounting_complete=False;raise RuntimeError(self.stop)
|
|
g=read(receipt);charged=g['cpu_seconds']+g['guard_cpu_seconds'];self.charges[stage]+=charged
|
|
record=dict(name=task['name'],stage=stage,job_index=task.get('job_index'),protocol=task.get('protocol'),case=task.get('case'),guard=g)
|
|
finished.append(record);self.records.append(record);log.close();del active[key]
|
|
if g['monitor_errors']:self.stop='MONITOR_ERROR'
|
|
if g['status']!='PASS' and g['stop_reason'] is None and stage!='postfit':self.stop='WORKER_EXCEPTION_REQUIRES_REVIEW'
|
|
print('TERMINAL',task['name'],g['status'],flush=True)
|
|
try:
|
|
usage=[tree_usage(proc.pid) for proc,_,_ in active.values()]
|
|
used_disk=disk_usage(self.root)
|
|
except Exception:
|
|
self.stop='GLOBAL_MONITOR_ERROR';usage=[];used_disk=0
|
|
live=sum(x[0] for x in usage)
|
|
if self.charges[stage]+live>=cap or self.total()+live>=72000:self.stop='CUMULATIVE_CPU_LIMIT'
|
|
if sum(x[1] for x in usage)+psutil.Process().memory_info().rss>self.c['budget']['total_rss_bytes']:self.stop='GLOBAL_RSS_LIMIT'
|
|
if used_disk>=self.c['budget']['disk_bytes']:self.stop='GLOBAL_DISK_LIMIT'
|
|
if self.stop:
|
|
if not (self.root/'global-stop.json').exists():save(self.root/'global-stop.json',dict(reason=self.stop))
|
|
pending=[]
|
|
for proc,_,_ in active.values():
|
|
if proc.poll() is None:proc.send_signal(signal.SIGTERM)
|
|
while pending and len(active)<parallel and not self.stop:
|
|
task=pending[0];reserved=sum(t['limits']['cpu_seconds'] for _,t,_ in active.values())
|
|
if self.charges[stage]+reserved+task['limits']['cpu_seconds']>cap:
|
|
if active:break
|
|
self.stop='STAGE_RESERVATION_LIMIT';break
|
|
pending.pop(0);p=self.root/'tasks'/(task['name']+'.json');p.parent.mkdir(exist_ok=True);save(p,task)
|
|
log=(self.root/'tasks'/(task['name']+'.log')).open('x')
|
|
env=dict(os.environ,CUDA_VISIBLE_DEVICES='',OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1')
|
|
proc=subprocess.Popen([sys.executable,'tools/phase2_h1_execution.py','guard',str(self.root),str(p)],stdout=log,stderr=subprocess.STDOUT,env=env)
|
|
active[task['name']]=(proc,task,log)
|
|
save(self.root/'live-progress.json',dict(unix=time.time(),stage=stage,charges=self.charges,total_cpu_charge=self.total(),
|
|
active=[dict(name=t['name'],guard_pid=p.pid) for p,t,_ in active.values()],terminal=len(self.records),stop=self.stop))
|
|
if not active and not pending:break
|
|
time.sleep(.5)
|
|
return finished
|
|
|
|
|
|
def task(name,command,cpus,cap=1800.,**kw):
|
|
return dict(name=name,command=[sys.executable,*command],cpus=cpus,
|
|
limits=dict(cpu_seconds=cap,wall_seconds=cap,rss_bytes=8*1024**3,disk_bytes=20*1024**3),**kw)
|
|
|
|
|
|
def run(root):
|
|
with (root/'run.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'started.json').exists():raise FileExistsError('Never restart an H1 run')
|
|
c=contract(root);launch=read(root/'launch-freeze.json');checked(root/'contract.json',launch['contract_sha256'])
|
|
os.sched_setaffinity(0,[30,31]);runner=Runner(root,c)
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),implementation_commit=launch['commit']))
|
|
try:
|
|
rec=runner.jobs('geometry',[task('geometry',['tools/phase2_h1_geometry.py',str(root)],[8],3540.)])
|
|
if runner.stop or not rec or rec[0]['guard']['status']!='PASS':raise RuntimeError('Geometry preparation not complete')
|
|
freeze(root,'geometry',['contract.json','geometry/manifest.json','geometry/complete.json'])
|
|
def fit_task(i):
|
|
j=c['jobs'][i];cpus=[8+2*(i%4)] if j['protocol']=='grouped' else list(range(8,16))
|
|
return task(f'fit-{i:02d}',['tools/phase2_h1_worker_v2.py',str(root),'fit',str(i)],cpus,job_index=i)
|
|
fits=runner.jobs('fit',[fit_task(0)])
|
|
if runner.stop or fits[0]['guard']['status']!='PASS':raise RuntimeError('First grouped cell stopped')
|
|
fits+=runner.jobs('fit',[fit_task(15)])
|
|
cost=projected_fits(c,fits);save(root/'first-cell-cost-check.json',cost)
|
|
if not cost['pass_gate'] or runner.stop:raise RuntimeError('First-cell gate stopped')
|
|
fits+=runner.jobs('fit',[fit_task(i) for i in range(15) if i!=0],parallel=4)
|
|
if not runner.stop:fits+=runner.jobs('fit',[fit_task(i) for i in range(16,20)],parallel=1)
|
|
models=[];by={r['job_index']:r for r in fits};extras=[]
|
|
for j in c['jobs']:
|
|
i=j['index'];d=root/'fits'/f'job-{i:02d}';r=by.get(i);result=read(d/'result.json') if (d/'result.json').exists() else None
|
|
valid=bool(r and r['guard']['status']=='PASS' and result)
|
|
item=dict(index=i,status='PASS' if valid else 'NA',reason=None if valid else 'FAILED_OR_UNATTEMPTED')
|
|
if valid:
|
|
checked(result['checkpoint'],result['checkpoint_sha256']);item.update(checkpoint=result['checkpoint'],checkpoint_sha256=result['checkpoint_sha256'],clock_path=str(d/'clock.json'),clock_sha256=result['clock_sha256'])
|
|
extras += [d/'model.ubj',d/'clock.json',d/'result.json']
|
|
models.append(item)
|
|
save(root/'models-terminal.json',dict(models=models));save(root/'batch-complete.json',dict(attempted=len(fits),global_stop=runner.stop,accounting_complete=True,records=fits))
|
|
if runner.stop or len(fits)!=20:raise RuntimeError('Full fit barrier not met')
|
|
freeze(root,'models',['contract.json','batch-complete.json','models-terminal.json'],extras)
|
|
predtasks=[task(f"predict-{m['index']:02d}",['tools/phase2_h1_worker_v2.py',str(root),'predict',str(m['index'])],[8+2*(m['index']%4)],job_index=m['index']) for m in models if m['status']=='PASS']
|
|
predictions=runner.jobs('postfit',predtasks,parallel=4);entries=[]
|
|
for r in predictions:
|
|
if r['guard']['status']!='PASS':continue
|
|
j=c['jobs'][r['job_index']];d=root/'predictions'/f"job-{j['index']:02d}"
|
|
result=read(d/'result.json');checked(d/'index.json',result['index_sha256'])
|
|
for entry in read(d/'index.json'):
|
|
checked(entry['path'],entry['sha256']);entries.append(dict(**entry,index=j['index'],protocol=j['protocol'],fold=j['fold'],pool=j['pool']))
|
|
if runner.stop:raise RuntimeError('Prediction stop')
|
|
save(root/'predictions-freeze.json',dict(entries=entries,all_prediction_attempts_terminal=len(predictions)==len(predtasks),accounting_complete=True,
|
|
attempts=predictions,models=models,contract_sha256=sha(root/'contract.json')))
|
|
freeze(root,'scoring',['contract.json','predictions-freeze.json','geometry/manifest.json'])
|
|
pairs=sorted({(e['protocol'],e['case']) for e in entries})
|
|
scoretasks=[task(f'score-{k:04d}',['tools/phase2_h1_postfit.py',str(root),'part',p,case],[8+2*(k%4)],300.,protocol=p,case=case) for k,(p,case) in enumerate(pairs)]
|
|
runner.jobs('postfit',scoretasks,parallel=4)
|
|
# A budget stop preserves every completed field; summary is launched
|
|
# only if a separately reserved 120 CPU seconds remains.
|
|
if runner.stop:raise RuntimeError('Scoring stopped; partial evidence preserved')
|
|
last=runner.jobs('postfit',[task('summary',['tools/phase2_h1_postfit.py',str(root),'summary'],[8],120.)])
|
|
if runner.stop or last[0]['guard']['status']!='PASS':raise RuntimeError('Summary failed')
|
|
save(root/'complete.json',dict(status='COMPLETE',accounting_complete=runner.accounting_complete,charges=runner.charges,total_cpu_charge=runner.total(),records=runner.records,new_decodes=0))
|
|
except BaseException as exc:
|
|
runner.stop=runner.stop or type(exc).__name__+': '+str(exc)
|
|
save(root/'complete.json',dict(status='STOPPED',reason=runner.stop,accounting_complete=runner.accounting_complete,charges=runner.charges,total_cpu_charge=runner.total(),records=runner.records,new_decodes=0))
|
|
raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('mode',choices=['prepare','run','guard']);p.add_argument('path',type=Path);p.add_argument('task',type=Path,nargs='?');a=p.parse_args()
|
|
if a.mode=='prepare':prepare(a.path)
|
|
elif a.mode=='guard':raise SystemExit(guard(a.path,a.task))
|
|
else:run(a.path)
|