Files
MeltMind/tools/phase2_h1_recovery.py
T

138 lines
10 KiB
Python

"""Claude1794892722: one labelled recovery, no fits or original-source edits."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import resource
import signal
import shutil
import subprocess
import sys
import time
import psutil
from phase2_h1_execution import Runner,task,freeze,read,BASE
from phase2_h1_worker_v2 import contract,checked,sha,save
from phase2_h1_recovery_gate import engineering
OLD=BASE/'artifacts/phase2-h1-execution-2026-11-17'
ROOT=BASE/'artifacts/phase2-h1-labelled-recovery-2026-11-17'
FAILED=[0,1,3,4,10]
def authorize(root,phase):
proof=read(root/(phase+'-git-freeze.json'))
for name,h in proof['sha256'].items():
p=root/name;checked(p,h);rel=str(p.relative_to(BASE))
data=subprocess.check_output(['git','show',proof['commit']+':'+rel],cwd=proof['checkout'])
if hashlib.sha256(data).hexdigest()!=h:raise ValueError('Coordinator Git check failed')
for p,h in proof.get('extra_sha256',{}).items():checked(p,h)
save(root/(phase+'-authorization.json'),dict(engineering_sha256=sha(root/'engineering-contract.json'),proof_sha256=sha(root/(phase+'-git-freeze.json')),verified_git_commit=proof['commit']))
def failure_scope(frozen):
bad=[r['job_index'] for r in frozen['attempts'] if r['guard']['status']!='PASS']
if sorted(bad)!=FAILED or len(frozen['attempts'])!=20:raise ValueError('Recovery scope differs from approved five')
if any(r['guard']['stop_reason']!='process_count_limit' for r in frozen['attempts'] if r['job_index'] in FAILED):raise ValueError('Recovery cause differs')
return bad
def prepare(acceptance):
a=read(acceptance)
if a['status']!='PASS':raise ValueError('Recovery synthetic acceptance missing')
for p,h in a['code_sha256'].items():checked(BASE/p,h)
original=read(OLD/'predictions-freeze.json');failure_scope(original);contract(OLD)
names=['phase2_h1_recovery','phase2_h1_recovery_gate','phase2_h1_recovery_worker','phase2_h1_recovery_postfit']
ROOT.mkdir(exist_ok=False)
for name in ['contract.json','batch-complete.json','models-terminal.json','geometry/manifest.json','geometry/complete.json']:
p=ROOT/name;p.parent.mkdir(parents=True,exist_ok=True);shutil.copyfile(OLD/name,p)
save(ROOT/'engineering-contract.json',dict(approval='1794892722.876299',version='B1-labelled-infrastructure-recovery-v1',old_root=str(OLD),failed_prediction_jobs=FAILED,
original_model_commit='265fa06518262d8173b229d2cfbe104a36ce19ec',old_predictions_sha256=sha(OLD/'predictions-freeze.json'),
original_contract_sha256=sha(OLD/'contract.json'),code_sha256={f'tools/{n}.py':sha(BASE/f'tools/{n}.py') for n in names},
acceptance_sha256=sha(acceptance),acceptance_path=str(acceptance),setup_charge_seconds=60,total_cpu_cap=72000,postfit_cpu_cap=25200,
original_chain_must_exit_before_workers=True,refit=False,retry=False,new_decodes=0,scope='five failed predictions once, canonical20 freeze, unchanged scoring arithmetic, full decision'))
class RecoveryRunner(Runner):
def __init__(self,root,c,baseline,waiting_cpu):
super().__init__(root,c);self.baseline=baseline;self.charges['postfit']=baseline['charges']['postfit'];self.waiting_cpu=waiting_cpu
u=resource.getrusage(resource.RUSAGE_CHILDREN);self.child_start=u.ru_utime+u.ru_stime
def total(self):
u=resource.getrusage(resource.RUSAGE_CHILDREN)
settled=sum(self.charges.values())-self.baseline['charges']['postfit']
new=60+self.waiting_cpu+max(settled,u.ru_utime+u.ru_stime-self.child_start)+time.process_time()-self.start_cpu
if self.baseline['charges']['postfit']+new>=25200:self.stop=self.stop or 'CUMULATIVE_POSTFIT_CPU_LIMIT'
return self.baseline['total_cpu_charge']+new
def run(root):
eng=engineering(root);start=time.process_time();os.sched_setaffinity(0,[30,31]);runner=None
def interrupted(sig,frame):raise InterruptedError('Recovery waiting coordinator signal '+str(sig))
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,interrupted)
import fcntl
with (root/'run.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('One labelled recovery only')
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),engineering_sha256=sha(root/'engineering-contract.json')))
while not (OLD/'complete.json').exists():
save(root/'live-progress.json',dict(stage='WAIT_ORIGINAL_B1_TERMINAL',unix=time.time(),waiting_cpu_seconds=time.process_time()-start))
time.sleep(10)
for p in psutil.process_iter(['pid','cmdline']):
args=p.info['cmdline'] or []
if str(OLD) in args and any(str(x).endswith(('phase2_h1_execution.py','phase2_h1_worker_v2.py','phase2_h1_postfit.py')) for x in args):raise RuntimeError('Original process still live after terminal receipt')
baseline=read(OLD/'complete.json')
if not baseline['accounting_complete']:raise RuntimeError('Original accounting incomplete')
original=read(checked(OLD/'predictions-freeze.json',eng['old_predictions_sha256']));failure_scope(original)
c=contract(root);save(root/'baseline.json',dict(original_complete=baseline,original_complete_sha256=sha(OLD/'complete.json'),original_predictions_sha256=eng['old_predictions_sha256'],classification='B1-v1 incomplete; no grouped100 main tree decision'))
# Keep a reserve for this coordinator, Git checks, and setup inside the
# original postfit cap; the 20 CPU-hour overall cap is not reopened.
c['budget']['postfit']=25200-600
runner=RecoveryRunner(root,c,baseline,time.process_time()-start)
try:
oldproof=read(OLD/'models-git-freeze.json')
if oldproof['commit']!=eng['original_model_commit']:raise ValueError('Frozen model commit changed')
for name,h in oldproof['sha256'].items():
checked(OLD/name,h);rel=str((OLD/name).relative_to(BASE));data=subprocess.check_output(['git','show',oldproof['commit']+':'+rel],cwd=oldproof['checkout'])
if hashlib.sha256(data).hexdigest()!=h:raise ValueError('Original model Git identity differs')
for p,h in oldproof['extra_sha256'].items():checked(p,h)
freeze(root,'models',['contract.json','batch-complete.json','models-terminal.json'])
authorize(root,'models')
predtasks=[task(f'predict-{i:02d}',['tools/phase2_h1_recovery_worker.py',str(root),str(i)],[8+2*(i%4)],job_index=i) for i in FAILED]
recovered=runner.jobs('postfit',predtasks,parallel=4)
if runner.stop or len(recovered)!=5 or any(r['guard']['status']!='PASS' for r in recovered):raise RuntimeError('Labelled prediction recovery incomplete')
entries=list(original['entries'])
for rec in recovered:
i=rec['job_index'];j=c['jobs'][i];d=root/'predictions'/f'job-{i:02d}';result=read(d/'result.json');checked(d/'index.json',result['index_sha256'])
for e in read(d/'index.json'):entries.append(dict(**e,index=i,protocol=j['protocol'],fold=j['fold'],pool=j['pool']))
expected={(j['index'],case) for j in c['jobs'] for case in j['test']}
if {(e['index'],e['case']) for e in entries}!=expected or len(entries)!=len(expected):raise ValueError('Canonical prediction roster incomplete/duplicate')
for e in entries:checked(e['path'],e['sha256'])
canonical=[r for r in original['attempts'] if r['job_index'] not in FAILED]+recovered
save(root/'predictions-freeze.json',dict(entries=entries,all_prediction_attempts_terminal=True,accounting_complete=True,attempts=canonical,
original_failed_attempts=[r for r in original['attempts'] if r['job_index'] in FAILED],labelled_recovery_attempts=recovered,
models=original['models'],contract_sha256=sha(root/'contract.json'),version='B1-v2',original_predictions_sha256=eng['old_predictions_sha256']))
freeze(root,'scoring',['contract.json','predictions-freeze.json','geometry/manifest.json'])
authorize(root,'scoring')
pairs=sorted({(e['protocol'],e['case']) for e in entries})
scoretasks=[task(f'score-{i:04d}',['tools/phase2_h1_recovery_postfit.py',str(root),'part',p,case],[8+2*(i%4)],300.,protocol=p,case=case) for i,(p,case) in enumerate(pairs)]
runner.jobs('postfit',scoretasks,parallel=4)
if runner.stop:raise RuntimeError('v2 score budget/monitor stop')
summary=runner.jobs('postfit',[task('summary',['tools/phase2_h1_recovery_postfit.py',str(root),'summary'],[8],120.)])
if runner.stop or summary[0]['guard']['status']!='PASS':raise RuntimeError('v2 summary failure')
total=runner.total();new=total-baseline['total_cpu_charge']
if runner.stop or total>72000 or baseline['charges']['postfit']+new>25200:raise RuntimeError('Cumulative recovery cap')
save(root/'complete.json',dict(status='COMPLETE',accounting_complete=runner.accounting_complete,old_total_cpu_seconds=baseline['total_cpu_charge'],
new_cpu_seconds=new,cumulative_total_cpu_seconds=total,cumulative_postfit_cpu_seconds=baseline['charges']['postfit']+new,records=runner.records,refit=False,retry=False))
except BaseException as exc:
save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),accounting_complete=runner.accounting_complete,
cumulative_total_cpu_seconds=runner.total(),records=runner.records,original_receipts_preserved=True,refit=False,retry=False));raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('mode',choices=['prepare','run']);p.add_argument('path',type=Path);a=p.parse_args()
try:(prepare if a.mode=='prepare' else run)(a.path)
except BaseException as exc:
if a.mode=='run' and (a.path/'started.json').exists() and not (a.path/'complete.json').exists():
save(a.path/'complete.json',dict(status='STOPPED_BEFORE_RECOVERY_WORKERS',reason=type(exc).__name__+': '+str(exc),new_worker_cpu_seconds=0,coordinator_cpu_seconds=time.process_time(),original_receipts_preserved=True))
raise