Files
MeltMind/tools/run_r4a_recovery.py
T

103 lines
6.6 KiB
Python

"""Single labelled1470 recovery, then immutable PASS freeze and approved scoring."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
from gre24_prepare_v2 import sha
from r4a_part import contract
from run_r4a import kill_guard_worker
from s2_custody_common import checked
from s2_recovery_supervisor import save,supervise
from s2_recovery_monitoring import disk_usage
CASE='00001470_a45f6c99e2154f3b94955175_step_054'
def verify_scope(c,e,prior):
if e['recovery_indices']!=[568] or c['parts'][568]['case']!=CASE:raise ValueError('Only one1470 recovery permitted')
if prior['never_attempted'] or prior['missing_receipt_indices'] or not prior['accounting_complete']:raise ValueError('Original terminal differs')
records={r['index']:r for r in prior['records']};r=records[568]
if r['worker']['returncode']!=-9 or r['worker']['stop_reason'] is not None:raise ValueError('1470 original evidence differs')
if e['prior_charged_worker_cpu_seconds']!=prior['charged_worker_cpu_seconds']:raise ValueError('Prior ledger mismatch')
if e['prior_charged_worker_cpu_seconds']+e['new_initial_cpu_seconds']+1802>28800:raise ValueError('Insufficient shared compute budget')
def run(root):
root=Path(root)
with (root/'recovery.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return locked(root)
def locked(root):
c=contract(root);e=json.loads((root/'continuation-contract.json').read_text());original=Path(e['original_root'])
for p,h in {**e['code_sha256'],**e['source_sha256']}.items():checked(p,h)
if sha(root/'contract.json')!=sha(original/'contract.json'):raise ValueError('Scientific contract must be byte identical')
prior=json.loads((original/'preparation-complete.json').read_text());verify_scope(c,e,prior)
if (root/'recovery-started.json').exists():raise FileExistsError('Never restart labelled recovery')
os.sched_setaffinity(0,[25]);started=time.monotonic();stop=[None]
def interrupted(*_):stop[0]=stop[0] or 'COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
save(root/'recovery-started.json',dict(pid=os.getpid(),unix=time.time(),recovery_indices=[568],
contract_sha256=sha(root/'contract.json'),continuation_contract_sha256=sha(root/'continuation-contract.json')))
dest=root/'parts'/CASE;dest.mkdir(parents=True,exist_ok=False)
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
with (dest/'guard.log').open('x') as log:
proc=subprocess.Popen([sys.executable,'tools/run_r4a.py',str(root),'--guard','568','--slot','0'],stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True)
while proc.poll() is None:
if disk_usage(original)+disk_usage(root)>=20*2**30:stop[0]=stop[0] or 'GLOBAL_DISK_LIMIT'
if stop[0]:kill_guard_worker(proc.pid)
save(root/'progress.json',dict(stage='LABELLED_1470_DISTANCE_ONLY_RECOVERY',guard_pid=proc.pid,
prior_charged_worker_cpu_seconds=e['prior_charged_worker_cpu_seconds'],new_initial_cpu_seconds=e['new_initial_cpu_seconds'],
stop_reason=stop[0],wall_seconds=time.monotonic()-started))
time.sleep(.5)
guard_path=dest/'guard-result.json'
if not guard_path.exists():raise RuntimeError('Missing recovery guard receipt; no scoring')
recovery=json.loads(guard_path.read_text())
spent=e['prior_charged_worker_cpu_seconds']+e['new_initial_cpu_seconds']+recovery['worker']['cpu_seconds']
if recovery['worker']['monitor_errors']:stop[0]=stop[0] or 'MONITOR_ERROR'
if spent>28800:stop[0]=stop[0] or 'COMPUTE_CPU_LIMIT'
contract(root) # A global source mismatch never becomes a local failure.
for p,h in e['code_sha256'].items():checked(p,h)
parts={};failures=[]
for record in prior['records']:
chosen=recovery if record['index']==568 else record
if chosen['worker']['status']!='PASS':
failures.append(dict(case=chosen['case'],source='labelled_recovery' if chosen is recovery else 'original',worker=chosen['worker']));continue
result=checked(chosen['result_path'],chosen['result_sha256']);data=json.loads(result.read_text())
pred=result.parent/'query-predictions.npz';digest=data['output_sha256'][pred.name];checked(pred,digest)
parts[chosen['case']]=dict(result=str(result),result_sha256=sha(result),predictions=str(pred),predictions_sha256=digest,
source='labelled_recovery' if chosen is recovery else 'original')
post=None
if stop[0] is None:
save(root/'pass-predictions-freeze.json',dict(contract_sha256=sha(root/'contract.json'),
continuation_contract_sha256=sha(root/'continuation-contract.json'),parts=parts,canonical_failures=failures,
original_preparation_complete_sha256=sha(original/'preparation-complete.json'),recovery_guard_sha256=sha(guard_path),
original_1470_receipt_preserved=True,classification_ruling='1794830693.515879',new_labels_read=False))
remaining=20*2**30-disk_usage(original)-disk_usage(root)
if remaining<=0:stop[0]='GLOBAL_DISK_LIMIT'
else:
os.sched_setaffinity(0,[24])
post=supervise([sys.executable,'tools/r4a_poststop_score.py',str(root)],root/'postfit','worker',
dict(total_cpu_seconds=7200,job_cpu_seconds=7200,job_wall_seconds=7200,
tree_rss_bytes=4*2**30,disk_bytes=remaining,max_processes=1),poll=.2)
os.sched_setaffinity(0,[25]);spent+=post['cpu_seconds']
if post['status']!='PASS' or post['monitor_errors']:stop[0]='POSTPROCESS_FAILED'
save(root/'continuation-complete.json',dict(status='COMPLETE' if post and post['status']=='PASS' and stop[0] is None else 'STOPPED',
stop_reason=stop[0],recovery=recovery,canonical_pass=len(parts),canonical_failures=failures,postfit=post,
charged_worker_cpu_seconds=spent,total_worker_cpu_cap=36000,postprocess_cpu_cap=7200,
prior_charged_worker_cpu_seconds=e['prior_charged_worker_cpu_seconds'],new_initial_cpu_seconds=e['new_initial_cpu_seconds'],
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-started,
labelled_recoveries=1,other_retries=0,original_receipts_unchanged=True))
return 0 if post and post['status']=='PASS' and stop[0] is None else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))