Files
MeltMind/tools/phase2_h1_execution.py
T

254 lines
17 KiB
Python

"""Single H1 execution, bounded guards, actual Git barriers, no retries."""
import argparse
import fcntl
import hashlib
import json
import os
from pathlib import Path
import shutil
import signal
import subprocess
import sys
import time
import psutil
from r4bc_guard import supervise
from s2_recovery_supervisor import tree_usage
from s2_recovery_monitoring import disk_usage
from phase2_h1_worker_v2 import contract,sha,checked,save
BASE=Path('/home/frank14f/optane/MeltMind')
WRITER=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
ROOT=BASE/'artifacts/phase2-h1-execution-2026-11-17'
def read(p):return json.loads(Path(p).read_text())
def prepare(acceptance):
start=time.process_time();packet=read(BASE/'artifacts/phase2-h1-planning-2026-11-17/b1-review-packet-v1.json')
for p,h in packet['files'].items():checked(BASE/p,h)
for p,h in packet['legacy_source_sha256'].items():checked(BASE/p,h)
tests=read(acceptance)
if tests['status']!='PASS' or tests['cpu_seconds']>=50:raise ValueError('Synthetic acceptance/bounded setup missing')
frozen=read(BASE/'artifacts/phase2-h1-planning-2026-11-17/clock-freeze-v1/membership-and-sources.json')
for p,h in frozen['source_sha256'].items():checked(BASE/p,h)
grouped=read(BASE/'artifacts/r4bc-execution-2026-11-16/contract.json')
prior_path='artifacts/r4bc-final-postfit-2026-11-17/predictions-freeze.json';prior=read(BASE/prior_path)
c=dict(version='phase2-h1-res-v1',main_review='1794888554.440209',reviewed_contract_commit='e751e5b261820b6c2df1dc1f218cde7275b2ebec',
parts=frozen['parts'],jobs=frozen['cells'],grouped_rows={r['case']:r for r in grouped['parts']},
source_sha256={**frozen['source_sha256'],**packet['legacy_source_sha256'],prior_path:sha(BASE/prior_path)},
baseline_predictions={},archives={},initial_setup_charge_seconds=60.,setup_charge_kind='conservative metadata/synthetic reserve, not measured worker use',
synthetic_acceptance_path=str(acceptance),synthetic_acceptance_sha256=sha(acceptance),
budget=dict(total=72000.,geometry=3540.,fit=43200.,postfit=25200.,disk_bytes=20*1024**3,total_rss_bytes=36*1024**3),
tree_rule=dict(sign='H1-A',field_noninferiority_margin=.001,field_margin_percentage_points=.10,scope='grouped100'),new_decodes=0,gpu=False,retry=False)
for job in c['jobs']:
checked(job['baseline_model']['path'],job['baseline_model']['sha256'])
if job['protocol']=='grouped':
entries=[e for e in prior['entries'] if e['arm']=='A' and e['fold']==job['fold'] and e['pool']==job['pool']]
if {e['case'] for e in entries}!=set(job['test']):raise ValueError('Original A prediction roster mismatch')
c['baseline_predictions'][str(job['index'])]={e['case']:e for e in entries}
for protocol,version in [('grouped','v5'),('v4_random','v4')]:
archive=packet['archive_metadata'][version];c['archives'][protocol]=dict(path=archive['path'],stat=archive['original_archive_stat'])
new=['phase2_h1_execution','phase2_h1_worker_v2','phase2_h1_geometry','phase2_h1_topology','phase2_h1_postfit','phase2_h1_decision','phase2_residual_math']
old=['gre24_models','gre24_prepare_v2','gre24_features_v2','audit_samples','r4bc_guard','r4bc_common','s2_custody_common','s2_custody_statistics','s2_recovery_supervisor','s2_recovery_monitoring']
c['code_sha256']={f'tools/{name}.py':sha(BASE/f'tools/{name}.py') for name in new+old}
for p,h in tests['code_sha256'].items():checked(BASE/p,h)
if time.process_time()-start+tests['cpu_seconds']+.570199085>=60:raise ValueError('Setup reserve exceeded')
ROOT.mkdir(exist_ok=False);save(ROOT/'contract.json',c)
save(ROOT/'preparation-receipt.json',dict(status='PASS_METADATA_ONLY',actual_cpu_seconds=time.process_time()-start,
setup_charge_seconds=60.,synthetic_cpu_seconds=tests['cpu_seconds'],project_targets_read=False,
contract_sha256=sha(ROOT/'contract.json')))
print(ROOT,flush=True)
def freeze(root,name,relative_names,extra_paths=()):
with Path('/tmp/meltmind-single-writer.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX)
def git(*args):return subprocess.check_output(['git','-C',str(WRITER),*args],text=True).strip()
if git('branch','--show-current')!='codex/s2-execution-diff-20261114' or git('status','--porcelain'):raise RuntimeError('Single writer not clean on approved branch')
files=[root/n for n in relative_names]+[Path(p) for p in extra_paths]
for p in files:
rel=p.resolve().relative_to(BASE);dest=WRITER/rel;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(p,dest)
paths=[str(p.resolve().relative_to(BASE)) for p in files]
subprocess.run(['git','-C',str(WRITER),'add','--',*paths],check=True)
if git('diff','--cached','--name-only'):
subprocess.run(['git','-C',str(WRITER),'commit','-m','H1 '+name+' immutable evidence'],check=True)
commit=git('rev-parse','HEAD')
for p,rel in zip(files,paths):
data=subprocess.check_output(['git','-C',str(WRITER),'show',commit+':'+rel])
if hashlib.sha256(data).hexdigest()!=sha(p):raise RuntimeError('Git byte freeze mismatch')
proof=dict(commit=commit,checkout=str(WRITER),sha256={n:sha(root/n) for n in relative_names},
extra_sha256={str(p):sha(p) for p in extra_paths},remote_published=False)
save(root/(name+'-git-freeze.json'),proof)
return proof
def guard(root,task_path):
c=contract(root);task=read(task_path);os.sched_setaffinity(0,task['cpus']);start=time.process_time()
dest=root/'guards'/task['name']
result=supervise(task['command'],dest,task['limits'],global_stop=root/'global-stop.json')
result['guard_cpu_seconds']=time.process_time()-start
save(dest/'guard-result.json',result)
return 0 if result['status']=='PASS' else 1
def projected_fits(c,records):
by={r['job_index']:r for r in records};remaining=0.
for first in (0,15):
if first not in by or by[first]['guard']['status']!='PASS':return dict(pass_gate=False,reason='FIRST_CELL_FAILED_OR_MISSING')
reference=c['jobs'][first]
for job in c['jobs']:
if job['protocol']==reference['protocol'] and job['index'] not in by:
remaining+=1.25*by[first]['guard']['cpu_seconds']*job['learning_nodes']/reference['learning_nodes']
spent=sum(r['guard']['cpu_seconds']+r['guard']['guard_cpu_seconds'] for r in records)
return dict(pass_gate=spent+remaining<=c['budget']['fit'],spent=spent,remaining_projection=remaining,
full_fit_projection=spent+remaining,postfit_reserved=c['budget']['postfit'],factor=1.25)
class Runner:
def __init__(self,root,c):
self.root=root;self.c=c;self.charges={'geometry':0.,'fit':0.,'postfit':0.};self.records=[];self.stop=None
self.active={};self.accounting_complete=True
self.stage_seconds={};self.start_cpu=time.process_time()
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,self.interrupt)
def interrupt(self,sig,frame):self.stop='COORDINATOR_SIGNAL_'+str(sig)
def total(self):return 60+sum(self.charges.values())+time.process_time()-self.start_cpu
def jobs(self,stage,items,parallel=1):
try:return self._jobs(stage,items,parallel)
except BaseException:
self.stop=self.stop or 'COORDINATOR_EXCEPTION'
save(self.root/'global-stop.json',dict(reason=self.stop))
for proc,_,_ in self.active.values():
if proc.poll() is None:proc.send_signal(signal.SIGTERM)
for proc,task,log in self.active.values():
try:proc.wait(timeout=30)
except subprocess.TimeoutExpired:
self.accounting_complete=False
for child in psutil.Process(proc.pid).children(recursive=True):child.kill()
proc.kill();proc.wait()
p=self.root/'guards'/task['name']/'guard-result.json'
if p.exists():
g=read(p);self.charges[stage]+=g['cpu_seconds']+g['guard_cpu_seconds']
self.records.append(dict(name=task['name'],stage=stage,job_index=task.get('job_index'),guard=g))
else:self.accounting_complete=False
log.close()
self.active={}
raise
def _jobs(self,stage,items,parallel=1):
active={};pending=list(items);finished=[];cap=self.c['budget'][stage]
self.active=active
if stage=='postfit' and not (len(items)==1 and items[0]['name']=='summary'):cap-=120.
while pending or active:
for key,(proc,task,log) in list(active.items()):
code=proc.poll()
if code is None:continue
receipt=self.root/'guards'/task['name']/'guard-result.json'
if not receipt.exists():self.stop='MISSING_GUARD_ACCOUNTING';self.accounting_complete=False;raise RuntimeError(self.stop)
g=read(receipt);charged=g['cpu_seconds']+g['guard_cpu_seconds'];self.charges[stage]+=charged
record=dict(name=task['name'],stage=stage,job_index=task.get('job_index'),protocol=task.get('protocol'),case=task.get('case'),guard=g)
finished.append(record);self.records.append(record);log.close();del active[key]
if g['monitor_errors']:self.stop='MONITOR_ERROR'
if g['status']!='PASS' and g['stop_reason'] is None and stage!='postfit':self.stop='WORKER_EXCEPTION_REQUIRES_REVIEW'
print('TERMINAL',task['name'],g['status'],flush=True)
try:
usage=[tree_usage(proc.pid) for proc,_,_ in active.values()]
used_disk=disk_usage(self.root)
except Exception:
self.stop='GLOBAL_MONITOR_ERROR';usage=[];used_disk=0
live=sum(x[0] for x in usage)
if self.charges[stage]+live>=cap or self.total()+live>=72000:self.stop='CUMULATIVE_CPU_LIMIT'
if sum(x[1] for x in usage)+psutil.Process().memory_info().rss>self.c['budget']['total_rss_bytes']:self.stop='GLOBAL_RSS_LIMIT'
if used_disk>=self.c['budget']['disk_bytes']:self.stop='GLOBAL_DISK_LIMIT'
if self.stop:
if not (self.root/'global-stop.json').exists():save(self.root/'global-stop.json',dict(reason=self.stop))
pending=[]
for proc,_,_ in active.values():
if proc.poll() is None:proc.send_signal(signal.SIGTERM)
while pending and len(active)<parallel and not self.stop:
task=pending[0];reserved=sum(t['limits']['cpu_seconds'] for _,t,_ in active.values())
if self.charges[stage]+reserved+task['limits']['cpu_seconds']>cap:
if active:break
self.stop='STAGE_RESERVATION_LIMIT';break
pending.pop(0);p=self.root/'tasks'/(task['name']+'.json');p.parent.mkdir(exist_ok=True);save(p,task)
log=(self.root/'tasks'/(task['name']+'.log')).open('x')
env=dict(os.environ,CUDA_VISIBLE_DEVICES='',OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1')
proc=subprocess.Popen([sys.executable,'tools/phase2_h1_execution.py','guard',str(self.root),str(p)],stdout=log,stderr=subprocess.STDOUT,env=env)
active[task['name']]=(proc,task,log)
save(self.root/'live-progress.json',dict(unix=time.time(),stage=stage,charges=self.charges,total_cpu_charge=self.total(),
active=[dict(name=t['name'],guard_pid=p.pid) for p,t,_ in active.values()],terminal=len(self.records),stop=self.stop))
if not active and not pending:break
time.sleep(.5)
return finished
def task(name,command,cpus,cap=1800.,**kw):
return dict(name=name,command=[sys.executable,*command],cpus=cpus,
limits=dict(cpu_seconds=cap,wall_seconds=cap,rss_bytes=8*1024**3,disk_bytes=20*1024**3),**kw)
def run(root):
with (root/'run.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Never restart an H1 run')
c=contract(root);launch=read(root/'launch-freeze.json');checked(root/'contract.json',launch['contract_sha256'])
os.sched_setaffinity(0,[30,31]);runner=Runner(root,c)
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),implementation_commit=launch['commit']))
try:
rec=runner.jobs('geometry',[task('geometry',['tools/phase2_h1_geometry.py',str(root)],[8],3540.)])
if runner.stop or not rec or rec[0]['guard']['status']!='PASS':raise RuntimeError('Geometry preparation not complete')
freeze(root,'geometry',['contract.json','geometry/manifest.json','geometry/complete.json'])
def fit_task(i):
j=c['jobs'][i];cpus=[8+2*(i%4)] if j['protocol']=='grouped' else list(range(8,16))
return task(f'fit-{i:02d}',['tools/phase2_h1_worker_v2.py',str(root),'fit',str(i)],cpus,job_index=i)
fits=runner.jobs('fit',[fit_task(0)])
if runner.stop or fits[0]['guard']['status']!='PASS':raise RuntimeError('First grouped cell stopped')
fits+=runner.jobs('fit',[fit_task(15)])
cost=projected_fits(c,fits);save(root/'first-cell-cost-check.json',cost)
if not cost['pass_gate'] or runner.stop:raise RuntimeError('First-cell gate stopped')
fits+=runner.jobs('fit',[fit_task(i) for i in range(15) if i!=0],parallel=4)
if not runner.stop:fits+=runner.jobs('fit',[fit_task(i) for i in range(16,20)],parallel=1)
models=[];by={r['job_index']:r for r in fits};extras=[]
for j in c['jobs']:
i=j['index'];d=root/'fits'/f'job-{i:02d}';r=by.get(i);result=read(d/'result.json') if (d/'result.json').exists() else None
valid=bool(r and r['guard']['status']=='PASS' and result)
item=dict(index=i,status='PASS' if valid else 'NA',reason=None if valid else 'FAILED_OR_UNATTEMPTED')
if valid:
checked(result['checkpoint'],result['checkpoint_sha256']);item.update(checkpoint=result['checkpoint'],checkpoint_sha256=result['checkpoint_sha256'],clock_path=str(d/'clock.json'),clock_sha256=result['clock_sha256'])
extras += [d/'model.ubj',d/'clock.json',d/'result.json']
models.append(item)
save(root/'models-terminal.json',dict(models=models));save(root/'batch-complete.json',dict(attempted=len(fits),global_stop=runner.stop,accounting_complete=True,records=fits))
if runner.stop or len(fits)!=20:raise RuntimeError('Full fit barrier not met')
freeze(root,'models',['contract.json','batch-complete.json','models-terminal.json'],extras)
predtasks=[task(f"predict-{m['index']:02d}",['tools/phase2_h1_worker_v2.py',str(root),'predict',str(m['index'])],[8+2*(m['index']%4)],job_index=m['index']) for m in models if m['status']=='PASS']
predictions=runner.jobs('postfit',predtasks,parallel=4);entries=[]
for r in predictions:
if r['guard']['status']!='PASS':continue
j=c['jobs'][r['job_index']];d=root/'predictions'/f"job-{j['index']:02d}"
result=read(d/'result.json');checked(d/'index.json',result['index_sha256'])
for entry in read(d/'index.json'):
checked(entry['path'],entry['sha256']);entries.append(dict(**entry,index=j['index'],protocol=j['protocol'],fold=j['fold'],pool=j['pool']))
if runner.stop:raise RuntimeError('Prediction stop')
save(root/'predictions-freeze.json',dict(entries=entries,all_prediction_attempts_terminal=len(predictions)==len(predtasks),accounting_complete=True,
attempts=predictions,models=models,contract_sha256=sha(root/'contract.json')))
freeze(root,'scoring',['contract.json','predictions-freeze.json','geometry/manifest.json'])
pairs=sorted({(e['protocol'],e['case']) for e in entries})
scoretasks=[task(f'score-{k:04d}',['tools/phase2_h1_postfit.py',str(root),'part',p,case],[8+2*(k%4)],300.,protocol=p,case=case) for k,(p,case) in enumerate(pairs)]
runner.jobs('postfit',scoretasks,parallel=4)
# A budget stop preserves every completed field; summary is launched
# only if a separately reserved 120 CPU seconds remains.
if runner.stop:raise RuntimeError('Scoring stopped; partial evidence preserved')
last=runner.jobs('postfit',[task('summary',['tools/phase2_h1_postfit.py',str(root),'summary'],[8],120.)])
if runner.stop or last[0]['guard']['status']!='PASS':raise RuntimeError('Summary failed')
save(root/'complete.json',dict(status='COMPLETE',accounting_complete=runner.accounting_complete,charges=runner.charges,total_cpu_charge=runner.total(),records=runner.records,new_decodes=0))
except BaseException as exc:
runner.stop=runner.stop or type(exc).__name__+': '+str(exc)
save(root/'complete.json',dict(status='STOPPED',reason=runner.stop,accounting_complete=runner.accounting_complete,charges=runner.charges,total_cpu_charge=runner.total(),records=runner.records,new_decodes=0))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('mode',choices=['prepare','run','guard']);p.add_argument('path',type=Path);p.add_argument('task',type=Path,nargs='?');a=p.parse_args()
if a.mode=='prepare':prepare(a.path)
elif a.mode=='guard':raise SystemExit(guard(a.path,a.task))
else:run(a.path)