"""Single H1 execution, bounded guards, actual Git barriers, no retries.""" import argparse import fcntl import hashlib import json import os from pathlib import Path import shutil import signal import subprocess import sys import time import psutil from r4bc_guard import supervise from s2_recovery_supervisor import tree_usage from s2_recovery_monitoring import disk_usage from phase2_h1_worker_v2 import contract,sha,checked,save BASE=Path('/home/frank14f/optane/MeltMind') WRITER=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8') ROOT=BASE/'artifacts/phase2-h1-execution-2026-11-17' def read(p):return json.loads(Path(p).read_text()) def prepare(acceptance): start=time.process_time();packet=read(BASE/'artifacts/phase2-h1-planning-2026-11-17/b1-review-packet-v1.json') for p,h in packet['files'].items():checked(BASE/p,h) for p,h in packet['legacy_source_sha256'].items():checked(BASE/p,h) tests=read(acceptance) if tests['status']!='PASS' or tests['cpu_seconds']>=50:raise ValueError('Synthetic acceptance/bounded setup missing') frozen=read(BASE/'artifacts/phase2-h1-planning-2026-11-17/clock-freeze-v1/membership-and-sources.json') for p,h in frozen['source_sha256'].items():checked(BASE/p,h) grouped=read(BASE/'artifacts/r4bc-execution-2026-11-16/contract.json') prior_path='artifacts/r4bc-final-postfit-2026-11-17/predictions-freeze.json';prior=read(BASE/prior_path) c=dict(version='phase2-h1-res-v1',main_review='1794888554.440209',reviewed_contract_commit='e751e5b261820b6c2df1dc1f218cde7275b2ebec', parts=frozen['parts'],jobs=frozen['cells'],grouped_rows={r['case']:r for r in grouped['parts']}, source_sha256={**frozen['source_sha256'],**packet['legacy_source_sha256'],prior_path:sha(BASE/prior_path)}, baseline_predictions={},archives={},initial_setup_charge_seconds=60.,setup_charge_kind='conservative metadata/synthetic reserve, not measured worker use', synthetic_acceptance_path=str(acceptance),synthetic_acceptance_sha256=sha(acceptance), budget=dict(total=72000.,geometry=3540.,fit=43200.,postfit=25200.,disk_bytes=20*1024**3,total_rss_bytes=36*1024**3), tree_rule=dict(sign='H1-A',field_noninferiority_margin=.001,field_margin_percentage_points=.10,scope='grouped100'),new_decodes=0,gpu=False,retry=False) for job in c['jobs']: checked(job['baseline_model']['path'],job['baseline_model']['sha256']) if job['protocol']=='grouped': entries=[e for e in prior['entries'] if e['arm']=='A' and e['fold']==job['fold'] and e['pool']==job['pool']] if {e['case'] for e in entries}!=set(job['test']):raise ValueError('Original A prediction roster mismatch') c['baseline_predictions'][str(job['index'])]={e['case']:e for e in entries} for protocol,version in [('grouped','v5'),('v4_random','v4')]: archive=packet['archive_metadata'][version];c['archives'][protocol]=dict(path=archive['path'],stat=archive['original_archive_stat']) new=['phase2_h1_execution','phase2_h1_worker_v2','phase2_h1_geometry','phase2_h1_topology','phase2_h1_postfit','phase2_h1_decision','phase2_residual_math'] old=['gre24_models','gre24_prepare_v2','gre24_features_v2','audit_samples','r4bc_guard','r4bc_common','s2_custody_common','s2_custody_statistics','s2_recovery_supervisor','s2_recovery_monitoring'] c['code_sha256']={f'tools/{name}.py':sha(BASE/f'tools/{name}.py') for name in new+old} for p,h in tests['code_sha256'].items():checked(BASE/p,h) if time.process_time()-start+tests['cpu_seconds']+.570199085>=60:raise ValueError('Setup reserve exceeded') ROOT.mkdir(exist_ok=False);save(ROOT/'contract.json',c) save(ROOT/'preparation-receipt.json',dict(status='PASS_METADATA_ONLY',actual_cpu_seconds=time.process_time()-start, setup_charge_seconds=60.,synthetic_cpu_seconds=tests['cpu_seconds'],project_targets_read=False, contract_sha256=sha(ROOT/'contract.json'))) print(ROOT,flush=True) def freeze(root,name,relative_names,extra_paths=()): with Path('/tmp/meltmind-single-writer.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX) def git(*args):return subprocess.check_output(['git','-C',str(WRITER),*args],text=True).strip() if git('branch','--show-current')!='codex/s2-execution-diff-20261114' or git('status','--porcelain'):raise RuntimeError('Single writer not clean on approved branch') files=[root/n for n in relative_names]+[Path(p) for p in extra_paths] for p in files: rel=p.resolve().relative_to(BASE);dest=WRITER/rel;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,dest) paths=[str(p.resolve().relative_to(BASE)) for p in files] subprocess.run(['git','-C',str(WRITER),'add','--',*paths],check=True) if git('diff','--cached','--name-only'): subprocess.run(['git','-C',str(WRITER),'commit','-m','H1 '+name+' immutable evidence'],check=True) commit=git('rev-parse','HEAD') for p,rel in zip(files,paths): data=subprocess.check_output(['git','-C',str(WRITER),'show',commit+':'+rel]) if hashlib.sha256(data).hexdigest()!=sha(p):raise RuntimeError('Git byte freeze mismatch') proof=dict(commit=commit,checkout=str(WRITER),sha256={n:sha(root/n) for n in relative_names}, extra_sha256={str(p):sha(p) for p in extra_paths},remote_published=False) save(root/(name+'-git-freeze.json'),proof) return proof def guard(root,task_path): c=contract(root);task=read(task_path);os.sched_setaffinity(0,task['cpus']);start=time.process_time() dest=root/'guards'/task['name'] result=supervise(task['command'],dest,task['limits'],global_stop=root/'global-stop.json') result['guard_cpu_seconds']=time.process_time()-start save(dest/'guard-result.json',result) return 0 if result['status']=='PASS' else 1 def projected_fits(c,records): by={r['job_index']:r for r in records};remaining=0. for first in (0,15): if first not in by or by[first]['guard']['status']!='PASS':return dict(pass_gate=False,reason='FIRST_CELL_FAILED_OR_MISSING') reference=c['jobs'][first] for job in c['jobs']: if job['protocol']==reference['protocol'] and job['index'] not in by: remaining+=1.25*by[first]['guard']['cpu_seconds']*job['learning_nodes']/reference['learning_nodes'] spent=sum(r['guard']['cpu_seconds']+r['guard']['guard_cpu_seconds'] for r in records) return dict(pass_gate=spent+remaining<=c['budget']['fit'],spent=spent,remaining_projection=remaining, full_fit_projection=spent+remaining,postfit_reserved=c['budget']['postfit'],factor=1.25) class Runner: def __init__(self,root,c): self.root=root;self.c=c;self.charges={'geometry':0.,'fit':0.,'postfit':0.};self.records=[];self.stop=None self.active={};self.accounting_complete=True self.stage_seconds={};self.start_cpu=time.process_time() for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,self.interrupt) def interrupt(self,sig,frame):self.stop='COORDINATOR_SIGNAL_'+str(sig) def total(self):return 60+sum(self.charges.values())+time.process_time()-self.start_cpu def jobs(self,stage,items,parallel=1): try:return self._jobs(stage,items,parallel) except BaseException: self.stop=self.stop or 'COORDINATOR_EXCEPTION' save(self.root/'global-stop.json',dict(reason=self.stop)) for proc,_,_ in self.active.values(): if proc.poll() is None:proc.send_signal(signal.SIGTERM) for proc,task,log in self.active.values(): try:proc.wait(timeout=30) except subprocess.TimeoutExpired: self.accounting_complete=False for child in psutil.Process(proc.pid).children(recursive=True):child.kill() proc.kill();proc.wait() p=self.root/'guards'/task['name']/'guard-result.json' if p.exists(): g=read(p);self.charges[stage]+=g['cpu_seconds']+g['guard_cpu_seconds'] self.records.append(dict(name=task['name'],stage=stage,job_index=task.get('job_index'),guard=g)) else:self.accounting_complete=False log.close() self.active={} raise def _jobs(self,stage,items,parallel=1): active={};pending=list(items);finished=[];cap=self.c['budget'][stage] self.active=active if stage=='postfit' and not (len(items)==1 and items[0]['name']=='summary'):cap-=120. while pending or active: for key,(proc,task,log) in list(active.items()): code=proc.poll() if code is None:continue receipt=self.root/'guards'/task['name']/'guard-result.json' if not receipt.exists():self.stop='MISSING_GUARD_ACCOUNTING';self.accounting_complete=False;raise RuntimeError(self.stop) g=read(receipt);charged=g['cpu_seconds']+g['guard_cpu_seconds'];self.charges[stage]+=charged record=dict(name=task['name'],stage=stage,job_index=task.get('job_index'),protocol=task.get('protocol'),case=task.get('case'),guard=g) finished.append(record);self.records.append(record);log.close();del active[key] if g['monitor_errors']:self.stop='MONITOR_ERROR' if g['status']!='PASS' and g['stop_reason'] is None and stage!='postfit':self.stop='WORKER_EXCEPTION_REQUIRES_REVIEW' print('TERMINAL',task['name'],g['status'],flush=True) try: usage=[tree_usage(proc.pid) for proc,_,_ in active.values()] used_disk=disk_usage(self.root) except Exception: self.stop='GLOBAL_MONITOR_ERROR';usage=[];used_disk=0 live=sum(x[0] for x in usage) if self.charges[stage]+live>=cap or self.total()+live>=72000:self.stop='CUMULATIVE_CPU_LIMIT' if sum(x[1] for x in usage)+psutil.Process().memory_info().rss>self.c['budget']['total_rss_bytes']:self.stop='GLOBAL_RSS_LIMIT' if used_disk>=self.c['budget']['disk_bytes']:self.stop='GLOBAL_DISK_LIMIT' if self.stop: if not (self.root/'global-stop.json').exists():save(self.root/'global-stop.json',dict(reason=self.stop)) pending=[] for proc,_,_ in active.values(): if proc.poll() is None:proc.send_signal(signal.SIGTERM) while pending and len(active)cap: if active:break self.stop='STAGE_RESERVATION_LIMIT';break pending.pop(0);p=self.root/'tasks'/(task['name']+'.json');p.parent.mkdir(exist_ok=True);save(p,task) log=(self.root/'tasks'/(task['name']+'.log')).open('x') env=dict(os.environ,CUDA_VISIBLE_DEVICES='',OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1') proc=subprocess.Popen([sys.executable,'tools/phase2_h1_execution.py','guard',str(self.root),str(p)],stdout=log,stderr=subprocess.STDOUT,env=env) active[task['name']]=(proc,task,log) save(self.root/'live-progress.json',dict(unix=time.time(),stage=stage,charges=self.charges,total_cpu_charge=self.total(), active=[dict(name=t['name'],guard_pid=p.pid) for p,t,_ in active.values()],terminal=len(self.records),stop=self.stop)) if not active and not pending:break time.sleep(.5) return finished def task(name,command,cpus,cap=1800.,**kw): return dict(name=name,command=[sys.executable,*command],cpus=cpus, limits=dict(cpu_seconds=cap,wall_seconds=cap,rss_bytes=8*1024**3,disk_bytes=20*1024**3),**kw) def run(root): with (root/'run.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (root/'started.json').exists():raise FileExistsError('Never restart an H1 run') c=contract(root);launch=read(root/'launch-freeze.json');checked(root/'contract.json',launch['contract_sha256']) os.sched_setaffinity(0,[30,31]);runner=Runner(root,c) save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),implementation_commit=launch['commit'])) try: rec=runner.jobs('geometry',[task('geometry',['tools/phase2_h1_geometry.py',str(root)],[8],3540.)]) if runner.stop or not rec or rec[0]['guard']['status']!='PASS':raise RuntimeError('Geometry preparation not complete') freeze(root,'geometry',['contract.json','geometry/manifest.json','geometry/complete.json']) def fit_task(i): j=c['jobs'][i];cpus=[8+2*(i%4)] if j['protocol']=='grouped' else list(range(8,16)) return task(f'fit-{i:02d}',['tools/phase2_h1_worker_v2.py',str(root),'fit',str(i)],cpus,job_index=i) fits=runner.jobs('fit',[fit_task(0)]) if runner.stop or fits[0]['guard']['status']!='PASS':raise RuntimeError('First grouped cell stopped') fits+=runner.jobs('fit',[fit_task(15)]) cost=projected_fits(c,fits);save(root/'first-cell-cost-check.json',cost) if not cost['pass_gate'] or runner.stop:raise RuntimeError('First-cell gate stopped') fits+=runner.jobs('fit',[fit_task(i) for i in range(15) if i!=0],parallel=4) if not runner.stop:fits+=runner.jobs('fit',[fit_task(i) for i in range(16,20)],parallel=1) models=[];by={r['job_index']:r for r in fits};extras=[] for j in c['jobs']: i=j['index'];d=root/'fits'/f'job-{i:02d}';r=by.get(i);result=read(d/'result.json') if (d/'result.json').exists() else None valid=bool(r and r['guard']['status']=='PASS' and result) item=dict(index=i,status='PASS' if valid else 'NA',reason=None if valid else 'FAILED_OR_UNATTEMPTED') if valid: checked(result['checkpoint'],result['checkpoint_sha256']);item.update(checkpoint=result['checkpoint'],checkpoint_sha256=result['checkpoint_sha256'],clock_path=str(d/'clock.json'),clock_sha256=result['clock_sha256']) extras += [d/'model.ubj',d/'clock.json',d/'result.json'] models.append(item) save(root/'models-terminal.json',dict(models=models));save(root/'batch-complete.json',dict(attempted=len(fits),global_stop=runner.stop,accounting_complete=True,records=fits)) if runner.stop or len(fits)!=20:raise RuntimeError('Full fit barrier not met') freeze(root,'models',['contract.json','batch-complete.json','models-terminal.json'],extras) predtasks=[task(f"predict-{m['index']:02d}",['tools/phase2_h1_worker_v2.py',str(root),'predict',str(m['index'])],[8+2*(m['index']%4)],job_index=m['index']) for m in models if m['status']=='PASS'] predictions=runner.jobs('postfit',predtasks,parallel=4);entries=[] for r in predictions: if r['guard']['status']!='PASS':continue j=c['jobs'][r['job_index']];d=root/'predictions'/f"job-{j['index']:02d}" result=read(d/'result.json');checked(d/'index.json',result['index_sha256']) for entry in read(d/'index.json'): checked(entry['path'],entry['sha256']);entries.append(dict(**entry,index=j['index'],protocol=j['protocol'],fold=j['fold'],pool=j['pool'])) if runner.stop:raise RuntimeError('Prediction stop') save(root/'predictions-freeze.json',dict(entries=entries,all_prediction_attempts_terminal=len(predictions)==len(predtasks),accounting_complete=True, attempts=predictions,models=models,contract_sha256=sha(root/'contract.json'))) freeze(root,'scoring',['contract.json','predictions-freeze.json','geometry/manifest.json']) pairs=sorted({(e['protocol'],e['case']) for e in entries}) scoretasks=[task(f'score-{k:04d}',['tools/phase2_h1_postfit.py',str(root),'part',p,case],[8+2*(k%4)],300.,protocol=p,case=case) for k,(p,case) in enumerate(pairs)] runner.jobs('postfit',scoretasks,parallel=4) # A budget stop preserves every completed field; summary is launched # only if a separately reserved 120 CPU seconds remains. if runner.stop:raise RuntimeError('Scoring stopped; partial evidence preserved') last=runner.jobs('postfit',[task('summary',['tools/phase2_h1_postfit.py',str(root),'summary'],[8],120.)]) if runner.stop or last[0]['guard']['status']!='PASS':raise RuntimeError('Summary failed') save(root/'complete.json',dict(status='COMPLETE',accounting_complete=runner.accounting_complete,charges=runner.charges,total_cpu_charge=runner.total(),records=runner.records,new_decodes=0)) except BaseException as exc: runner.stop=runner.stop or type(exc).__name__+': '+str(exc) save(root/'complete.json',dict(status='STOPPED',reason=runner.stop,accounting_complete=runner.accounting_complete,charges=runner.charges,total_cpu_charge=runner.total(),records=runner.records,new_decodes=0)) raise if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('mode',choices=['prepare','run','guard']);p.add_argument('path',type=Path);p.add_argument('task',type=Path,nargs='?');a=p.parse_args() if a.mode=='prepare':prepare(a.path) elif a.mode=='guard':raise SystemExit(guard(a.path,a.task)) else:run(a.path)