103 lines
6.6 KiB
Python
103 lines
6.6 KiB
Python
"""Single labelled1470 recovery, then immutable PASS freeze and approved scoring."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from gre24_prepare_v2 import sha
|
|
from r4a_part import contract
|
|
from run_r4a import kill_guard_worker
|
|
from s2_custody_common import checked
|
|
from s2_recovery_supervisor import save,supervise
|
|
from s2_recovery_monitoring import disk_usage
|
|
|
|
CASE='00001470_a45f6c99e2154f3b94955175_step_054'
|
|
|
|
|
|
def verify_scope(c,e,prior):
|
|
if e['recovery_indices']!=[568] or c['parts'][568]['case']!=CASE:raise ValueError('Only one1470 recovery permitted')
|
|
if prior['never_attempted'] or prior['missing_receipt_indices'] or not prior['accounting_complete']:raise ValueError('Original terminal differs')
|
|
records={r['index']:r for r in prior['records']};r=records[568]
|
|
if r['worker']['returncode']!=-9 or r['worker']['stop_reason'] is not None:raise ValueError('1470 original evidence differs')
|
|
if e['prior_charged_worker_cpu_seconds']!=prior['charged_worker_cpu_seconds']:raise ValueError('Prior ledger mismatch')
|
|
if e['prior_charged_worker_cpu_seconds']+e['new_initial_cpu_seconds']+1802>28800:raise ValueError('Insufficient shared compute budget')
|
|
|
|
|
|
def run(root):
|
|
root=Path(root)
|
|
with (root/'recovery.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
return locked(root)
|
|
|
|
|
|
def locked(root):
|
|
c=contract(root);e=json.loads((root/'continuation-contract.json').read_text());original=Path(e['original_root'])
|
|
for p,h in {**e['code_sha256'],**e['source_sha256']}.items():checked(p,h)
|
|
if sha(root/'contract.json')!=sha(original/'contract.json'):raise ValueError('Scientific contract must be byte identical')
|
|
prior=json.loads((original/'preparation-complete.json').read_text());verify_scope(c,e,prior)
|
|
if (root/'recovery-started.json').exists():raise FileExistsError('Never restart labelled recovery')
|
|
os.sched_setaffinity(0,[25]);started=time.monotonic();stop=[None]
|
|
def interrupted(*_):stop[0]=stop[0] or 'COORDINATOR_INTERRUPTED'
|
|
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
|
save(root/'recovery-started.json',dict(pid=os.getpid(),unix=time.time(),recovery_indices=[568],
|
|
contract_sha256=sha(root/'contract.json'),continuation_contract_sha256=sha(root/'continuation-contract.json')))
|
|
dest=root/'parts'/CASE;dest.mkdir(parents=True,exist_ok=False)
|
|
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
|
with (dest/'guard.log').open('x') as log:
|
|
proc=subprocess.Popen([sys.executable,'tools/run_r4a.py',str(root),'--guard','568','--slot','0'],stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True)
|
|
while proc.poll() is None:
|
|
if disk_usage(original)+disk_usage(root)>=20*2**30:stop[0]=stop[0] or 'GLOBAL_DISK_LIMIT'
|
|
if stop[0]:kill_guard_worker(proc.pid)
|
|
save(root/'progress.json',dict(stage='LABELLED_1470_DISTANCE_ONLY_RECOVERY',guard_pid=proc.pid,
|
|
prior_charged_worker_cpu_seconds=e['prior_charged_worker_cpu_seconds'],new_initial_cpu_seconds=e['new_initial_cpu_seconds'],
|
|
stop_reason=stop[0],wall_seconds=time.monotonic()-started))
|
|
time.sleep(.5)
|
|
guard_path=dest/'guard-result.json'
|
|
if not guard_path.exists():raise RuntimeError('Missing recovery guard receipt; no scoring')
|
|
recovery=json.loads(guard_path.read_text())
|
|
spent=e['prior_charged_worker_cpu_seconds']+e['new_initial_cpu_seconds']+recovery['worker']['cpu_seconds']
|
|
if recovery['worker']['monitor_errors']:stop[0]=stop[0] or 'MONITOR_ERROR'
|
|
if spent>28800:stop[0]=stop[0] or 'COMPUTE_CPU_LIMIT'
|
|
contract(root) # A global source mismatch never becomes a local failure.
|
|
for p,h in e['code_sha256'].items():checked(p,h)
|
|
parts={};failures=[]
|
|
for record in prior['records']:
|
|
chosen=recovery if record['index']==568 else record
|
|
if chosen['worker']['status']!='PASS':
|
|
failures.append(dict(case=chosen['case'],source='labelled_recovery' if chosen is recovery else 'original',worker=chosen['worker']));continue
|
|
result=checked(chosen['result_path'],chosen['result_sha256']);data=json.loads(result.read_text())
|
|
pred=result.parent/'query-predictions.npz';digest=data['output_sha256'][pred.name];checked(pred,digest)
|
|
parts[chosen['case']]=dict(result=str(result),result_sha256=sha(result),predictions=str(pred),predictions_sha256=digest,
|
|
source='labelled_recovery' if chosen is recovery else 'original')
|
|
post=None
|
|
if stop[0] is None:
|
|
save(root/'pass-predictions-freeze.json',dict(contract_sha256=sha(root/'contract.json'),
|
|
continuation_contract_sha256=sha(root/'continuation-contract.json'),parts=parts,canonical_failures=failures,
|
|
original_preparation_complete_sha256=sha(original/'preparation-complete.json'),recovery_guard_sha256=sha(guard_path),
|
|
original_1470_receipt_preserved=True,classification_ruling='1794830693.515879',new_labels_read=False))
|
|
remaining=20*2**30-disk_usage(original)-disk_usage(root)
|
|
if remaining<=0:stop[0]='GLOBAL_DISK_LIMIT'
|
|
else:
|
|
os.sched_setaffinity(0,[24])
|
|
post=supervise([sys.executable,'tools/r4a_poststop_score.py',str(root)],root/'postfit','worker',
|
|
dict(total_cpu_seconds=7200,job_cpu_seconds=7200,job_wall_seconds=7200,
|
|
tree_rss_bytes=4*2**30,disk_bytes=remaining,max_processes=1),poll=.2)
|
|
os.sched_setaffinity(0,[25]);spent+=post['cpu_seconds']
|
|
if post['status']!='PASS' or post['monitor_errors']:stop[0]='POSTPROCESS_FAILED'
|
|
save(root/'continuation-complete.json',dict(status='COMPLETE' if post and post['status']=='PASS' and stop[0] is None else 'STOPPED',
|
|
stop_reason=stop[0],recovery=recovery,canonical_pass=len(parts),canonical_failures=failures,postfit=post,
|
|
charged_worker_cpu_seconds=spent,total_worker_cpu_cap=36000,postprocess_cpu_cap=7200,
|
|
prior_charged_worker_cpu_seconds=e['prior_charged_worker_cpu_seconds'],new_initial_cpu_seconds=e['new_initial_cpu_seconds'],
|
|
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-started,
|
|
labelled_recoveries=1,other_retries=0,original_receipts_unchanged=True))
|
|
return 0 if post and post['status']=='PASS' and stop[0] is None else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();raise SystemExit(run(a.root))
|