Implement approved R4a paired propagation with guarded budgets and synthetic acceptance
This commit is contained in:
1 parent
f9633bd219
commit
7c85d1a29a
12 files changed
+59816
No files matched your search
@@ -0,0 +1,141 @@
|
||||
"""One detached R4(a) queue: fail-stop, no retry, no scoring after a failure."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import psutil
|
||||
from gre24_prepare_v2 import sha
|
||||
from r4a_part import contract
|
||||
from r4a_core import may_dispatch, scoring_allowed
|
||||
from s2_recovery_supervisor import save, supervise, stop_group
|
||||
from s2_recovery_monitoring import disk_usage
|
||||
|
||||
|
||||
def guard(root,index,slot):
|
||||
c=contract(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]])
|
||||
dest=root/'parts'/c['parts'][index]['case']
|
||||
receipt=supervise([sys.executable,'tools/r4a_part.py',str(root),str(index)],dest,'worker',
|
||||
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,
|
||||
tree_rss_bytes=4*2**30,disk_bytes=c['disk_bytes'],max_processes=1),poll=.2)
|
||||
result=dest/'output/result.json'
|
||||
if receipt['status']=='PASS' and (not result.exists() or json.loads(result.read_text())['status']!='PASS'):
|
||||
receipt.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_PASS_RESULT')
|
||||
save(dest/'guard-result.json',dict(index=index,case=c['parts'][index]['case'],worker=receipt,
|
||||
result_path=str(result) if result.exists() else None,result_sha256=sha(result) if result.exists() else None,
|
||||
guard_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
|
||||
return 0 if receipt['status']=='PASS' else 1
|
||||
|
||||
|
||||
def kill_guard_worker(pid):
|
||||
# Keep the supervisor alive to reap its worker and write the true failure receipt.
|
||||
try:
|
||||
for child in psutil.Process(pid).children():stop_group(child.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
|
||||
|
||||
def fresh(root):
|
||||
if (root/'started.json').exists() or (root/'complete.json').exists():
|
||||
raise FileExistsError('Existing attempt: never restart R4 queue')
|
||||
|
||||
|
||||
def run(root):
|
||||
with (root/'queue.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
return run_locked(root)
|
||||
|
||||
|
||||
def run_locked(root):
|
||||
c=contract(root);fresh(root);os.sched_setaffinity(0,[25])
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json')))
|
||||
active={};done=[];next_i=0;reason=None;start=time.monotonic();error=None
|
||||
def interrupted(*_):
|
||||
nonlocal reason
|
||||
reason=reason or 'COORDINATOR_INTERRUPTED'
|
||||
signal.signal(signal.SIGINT,interrupted);signal.signal(signal.SIGTERM,interrupted)
|
||||
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
||||
try:
|
||||
while active or (next_i<len(c['parts']) and reason is None):
|
||||
for slot,item in list(active.items()):
|
||||
if item['process'].poll() is None:continue
|
||||
receipt=root/'parts'/c['parts'][item['index']]['case']/'guard-result.json'
|
||||
if not receipt.exists():raise RuntimeError('MISSING_GUARD_RECEIPT')
|
||||
r=json.loads(receipt.read_text());done.append(r);item['log'].close();del active[slot]
|
||||
if r['worker']['status']!='PASS':reason=reason or 'PART_FAILED_NO_AUTOMATIC_SCORING'
|
||||
spent=c['initial_worker_cpu_seconds']+sum(x['worker']['cpu_seconds'] for x in done)
|
||||
used=disk_usage(root)
|
||||
if used>=c['disk_bytes']:reason=reason or 'GLOBAL_DISK_LIMIT'
|
||||
if spent>=28800:reason=reason or 'COMPUTE_CPU_LIMIT'
|
||||
while reason is None and len(active)<4 and next_i<len(c['parts']):
|
||||
if not may_dispatch(spent,len(active)):
|
||||
if not active:reason='INSUFFICIENT_COMPUTE_RESERVATION'
|
||||
break
|
||||
slot=next(s for s in range(4) if s not in active);index=next_i;next_i+=1
|
||||
dest=root/'parts'/c['parts'][index]['case'];dest.mkdir(parents=True,exist_ok=False)
|
||||
log=(dest/'guard.log').open('x')
|
||||
proc=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(index),'--slot',str(slot)],
|
||||
stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True)
|
||||
active[slot]=dict(index=index,process=proc,log=log)
|
||||
if reason:
|
||||
for item in active.values():kill_guard_worker(item['process'].pid)
|
||||
save(root/'progress.json',dict(completed=len(done),attempted=next_i,total=len(c['parts']),
|
||||
passed=sum(x['worker']['status']=='PASS' for x in done),active={str(s):v['index'] for s,v in active.items()},
|
||||
settled_worker_cpu_seconds=spent,reserved_active_worker_cpu_seconds=len(active)*1802,
|
||||
stop_reason=reason,disk_bytes=used,wall_seconds=time.monotonic()-start))
|
||||
if active:time.sleep(.5)
|
||||
except Exception as exc:
|
||||
reason=reason or 'INFRASTRUCTURE_EXCEPTION';error=dict(message=str(exc),traceback=traceback.format_exc())
|
||||
finally:
|
||||
for item in active.values():kill_guard_worker(item['process'].pid)
|
||||
for item in active.values():
|
||||
item['process'].wait();item['log'].close()
|
||||
p=root/'parts'/c['parts'][item['index']]['case']/'guard-result.json'
|
||||
if p.exists():done.append(json.loads(p.read_text()))
|
||||
active={}
|
||||
done.sort(key=lambda r:r['index']);spent=c['initial_worker_cpu_seconds']+sum(x['worker']['cpu_seconds'] for x in done)
|
||||
allowed=scoring_allowed(done,len(c['parts']),reason)
|
||||
terminal=dict(status='ALL_PASS' if allowed else 'STOPPED',stop_reason=reason,records=done,
|
||||
never_attempted=list(range(next_i,len(c['parts']))),attempted=next_i,
|
||||
missing_receipt_indices=sorted(set(range(next_i))-{r['index'] for r in done}),
|
||||
charged_worker_cpu_seconds=spent,initial_worker_cpu_seconds=c['initial_worker_cpu_seconds'],
|
||||
error=error,accounting_complete=len(done)==next_i)
|
||||
save(root/'preparation-complete.json',terminal)
|
||||
post=None
|
||||
if allowed:
|
||||
parts={}
|
||||
for r in done:
|
||||
result=Path(r['result_path']);checked_result=json.loads(result.read_text())
|
||||
if sha(result)!=r['result_sha256']:raise ValueError('Result changed before freeze')
|
||||
p=result.parent/'query-predictions.npz';digest=checked_result['output_sha256'][p.name]
|
||||
if sha(p)!=digest:raise ValueError('Prediction changed before freeze')
|
||||
parts[r['case']]=dict(result=str(result),result_sha256=sha(result),predictions=str(p),predictions_sha256=digest)
|
||||
save(root/'predictions-freeze.json',dict(contract_sha256=sha(root/'contract.json'),
|
||||
preparation_complete_sha256=sha(root/'preparation-complete.json'),parts=parts,labels_read=False))
|
||||
remaining_disk=c['disk_bytes']-disk_usage(root)
|
||||
if remaining_disk<=0:reason='POSTPROCESS_DISK_EXHAUSTED'
|
||||
else:
|
||||
# Inherit a non-v4 CPU; one guarded process for all cache-only scoring.
|
||||
os.sched_setaffinity(0,[24])
|
||||
post=supervise([sys.executable,'tools/r4a_score.py',str(root)],root/'postfit','worker',
|
||||
dict(total_cpu_seconds=7200,job_cpu_seconds=7200,job_wall_seconds=7200,
|
||||
tree_rss_bytes=4*2**30,disk_bytes=remaining_disk,max_processes=1),poll=.2)
|
||||
os.sched_setaffinity(0,[25]);spent+=post['cpu_seconds']
|
||||
if post['status']!='PASS':reason='POSTPROCESS_FAILED'
|
||||
save(root/'complete.json',dict(status='COMPLETE' if allowed and post and post['status']=='PASS' else 'STOPPED',
|
||||
stop_reason=reason,charged_worker_cpu_seconds=spent,worker_cpu_cap=36000,
|
||||
preparation_complete_sha256=sha(root/'preparation-complete.json'),postfit=post,
|
||||
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
guard_cpu_seconds=sum(x.get('guard_cpu_seconds',0.) for x in done),
|
||||
wall_seconds=time.monotonic()-start,retries=0,accounting_complete=len(done)==next_i))
|
||||
return 0 if allowed and post and post['status']=='PASS' else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int,default=0)
|
||||
a=p.parse_args();raise SystemExit(run(a.root) if a.guard is None else guard(a.root,a.guard,a.slot))
|
||||
Reference in new issue
Block a user