"""Single labelled1470 recovery, then immutable PASS freeze and approved scoring.""" import argparse import fcntl import json import os from pathlib import Path import resource import signal import subprocess import sys import time from gre24_prepare_v2 import sha from r4a_part import contract from run_r4a import kill_guard_worker from s2_custody_common import checked from s2_recovery_supervisor import save,supervise from s2_recovery_monitoring import disk_usage CASE='00001470_a45f6c99e2154f3b94955175_step_054' def verify_scope(c,e,prior): if e['recovery_indices']!=[568] or c['parts'][568]['case']!=CASE:raise ValueError('Only one1470 recovery permitted') if prior['never_attempted'] or prior['missing_receipt_indices'] or not prior['accounting_complete']:raise ValueError('Original terminal differs') records={r['index']:r for r in prior['records']};r=records[568] if r['worker']['returncode']!=-9 or r['worker']['stop_reason'] is not None:raise ValueError('1470 original evidence differs') if e['prior_charged_worker_cpu_seconds']!=prior['charged_worker_cpu_seconds']:raise ValueError('Prior ledger mismatch') if e['prior_charged_worker_cpu_seconds']+e['new_initial_cpu_seconds']+1802>28800:raise ValueError('Insufficient shared compute budget') def run(root): root=Path(root) with (root/'recovery.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) return locked(root) def locked(root): c=contract(root);e=json.loads((root/'continuation-contract.json').read_text());original=Path(e['original_root']) for p,h in {**e['code_sha256'],**e['source_sha256']}.items():checked(p,h) if sha(root/'contract.json')!=sha(original/'contract.json'):raise ValueError('Scientific contract must be byte identical') prior=json.loads((original/'preparation-complete.json').read_text());verify_scope(c,e,prior) if (root/'recovery-started.json').exists():raise FileExistsError('Never restart labelled recovery') os.sched_setaffinity(0,[25]);started=time.monotonic();stop=[None] def interrupted(*_):stop[0]=stop[0] or 'COORDINATOR_INTERRUPTED' signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted) save(root/'recovery-started.json',dict(pid=os.getpid(),unix=time.time(),recovery_indices=[568], contract_sha256=sha(root/'contract.json'),continuation_contract_sha256=sha(root/'continuation-contract.json'))) dest=root/'parts'/CASE;dest.mkdir(parents=True,exist_ok=False) env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') with (dest/'guard.log').open('x') as log: proc=subprocess.Popen([sys.executable,'tools/run_r4a.py',str(root),'--guard','568','--slot','0'],stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True) while proc.poll() is None: if disk_usage(original)+disk_usage(root)>=20*2**30:stop[0]=stop[0] or 'GLOBAL_DISK_LIMIT' if stop[0]:kill_guard_worker(proc.pid) save(root/'progress.json',dict(stage='LABELLED_1470_DISTANCE_ONLY_RECOVERY',guard_pid=proc.pid, prior_charged_worker_cpu_seconds=e['prior_charged_worker_cpu_seconds'],new_initial_cpu_seconds=e['new_initial_cpu_seconds'], stop_reason=stop[0],wall_seconds=time.monotonic()-started)) time.sleep(.5) guard_path=dest/'guard-result.json' if not guard_path.exists():raise RuntimeError('Missing recovery guard receipt; no scoring') recovery=json.loads(guard_path.read_text()) spent=e['prior_charged_worker_cpu_seconds']+e['new_initial_cpu_seconds']+recovery['worker']['cpu_seconds'] if recovery['worker']['monitor_errors']:stop[0]=stop[0] or 'MONITOR_ERROR' if spent>28800:stop[0]=stop[0] or 'COMPUTE_CPU_LIMIT' contract(root) # A global source mismatch never becomes a local failure. for p,h in e['code_sha256'].items():checked(p,h) parts={};failures=[] for record in prior['records']: chosen=recovery if record['index']==568 else record if chosen['worker']['status']!='PASS': failures.append(dict(case=chosen['case'],source='labelled_recovery' if chosen is recovery else 'original',worker=chosen['worker']));continue result=checked(chosen['result_path'],chosen['result_sha256']);data=json.loads(result.read_text()) pred=result.parent/'query-predictions.npz';digest=data['output_sha256'][pred.name];checked(pred,digest) parts[chosen['case']]=dict(result=str(result),result_sha256=sha(result),predictions=str(pred),predictions_sha256=digest, source='labelled_recovery' if chosen is recovery else 'original') post=None if stop[0] is None: save(root/'pass-predictions-freeze.json',dict(contract_sha256=sha(root/'contract.json'), continuation_contract_sha256=sha(root/'continuation-contract.json'),parts=parts,canonical_failures=failures, original_preparation_complete_sha256=sha(original/'preparation-complete.json'),recovery_guard_sha256=sha(guard_path), original_1470_receipt_preserved=True,classification_ruling='1794830693.515879',new_labels_read=False)) remaining=20*2**30-disk_usage(original)-disk_usage(root) if remaining<=0:stop[0]='GLOBAL_DISK_LIMIT' else: os.sched_setaffinity(0,[24]) post=supervise([sys.executable,'tools/r4a_poststop_score.py',str(root)],root/'postfit','worker', dict(total_cpu_seconds=7200,job_cpu_seconds=7200,job_wall_seconds=7200, tree_rss_bytes=4*2**30,disk_bytes=remaining,max_processes=1),poll=.2) os.sched_setaffinity(0,[25]);spent+=post['cpu_seconds'] if post['status']!='PASS' or post['monitor_errors']:stop[0]='POSTPROCESS_FAILED' save(root/'continuation-complete.json',dict(status='COMPLETE' if post and post['status']=='PASS' and stop[0] is None else 'STOPPED', stop_reason=stop[0],recovery=recovery,canonical_pass=len(parts),canonical_failures=failures,postfit=post, charged_worker_cpu_seconds=spent,total_worker_cpu_cap=36000,postprocess_cpu_cap=7200, prior_charged_worker_cpu_seconds=e['prior_charged_worker_cpu_seconds'],new_initial_cpu_seconds=e['new_initial_cpu_seconds'], coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-started, labelled_recoveries=1,other_retries=0,original_receipts_unchanged=True)) return 0 if post and post['status']=='PASS' and stop[0] is None else 1 if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))