142 lines
8.0 KiB
Python
142 lines
8.0 KiB
Python
"""One detached R4(a) queue: fail-stop, no retry, no scoring after a failure."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import psutil
|
|
from gre24_prepare_v2 import sha
|
|
from r4a_part import contract
|
|
from r4a_core import may_dispatch, scoring_allowed
|
|
from s2_recovery_supervisor import save, supervise, stop_group
|
|
from s2_recovery_monitoring import disk_usage
|
|
|
|
|
|
def guard(root,index,slot):
|
|
c=contract(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]])
|
|
dest=root/'parts'/c['parts'][index]['case']
|
|
receipt=supervise([sys.executable,'tools/r4a_part.py',str(root),str(index)],dest,'worker',
|
|
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,
|
|
tree_rss_bytes=4*2**30,disk_bytes=c['disk_bytes'],max_processes=1),poll=.2)
|
|
result=dest/'output/result.json'
|
|
if receipt['status']=='PASS' and (not result.exists() or json.loads(result.read_text())['status']!='PASS'):
|
|
receipt.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_PASS_RESULT')
|
|
save(dest/'guard-result.json',dict(index=index,case=c['parts'][index]['case'],worker=receipt,
|
|
result_path=str(result) if result.exists() else None,result_sha256=sha(result) if result.exists() else None,
|
|
guard_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
|
|
return 0 if receipt['status']=='PASS' else 1
|
|
|
|
|
|
def kill_guard_worker(pid):
|
|
# Keep the supervisor alive to reap its worker and write the true failure receipt.
|
|
try:
|
|
for child in psutil.Process(pid).children():stop_group(child.pid)
|
|
except psutil.NoSuchProcess:pass
|
|
|
|
|
|
def fresh(root):
|
|
if (root/'started.json').exists() or (root/'complete.json').exists():
|
|
raise FileExistsError('Existing attempt: never restart R4 queue')
|
|
|
|
|
|
def run(root):
|
|
with (root/'queue.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
return run_locked(root)
|
|
|
|
|
|
def run_locked(root):
|
|
c=contract(root);fresh(root);os.sched_setaffinity(0,[25])
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json')))
|
|
active={};done=[];next_i=0;reason=None;start=time.monotonic();error=None
|
|
def interrupted(*_):
|
|
nonlocal reason
|
|
reason=reason or 'COORDINATOR_INTERRUPTED'
|
|
signal.signal(signal.SIGINT,interrupted);signal.signal(signal.SIGTERM,interrupted)
|
|
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
|
try:
|
|
while active or (next_i<len(c['parts']) and reason is None):
|
|
for slot,item in list(active.items()):
|
|
if item['process'].poll() is None:continue
|
|
receipt=root/'parts'/c['parts'][item['index']]['case']/'guard-result.json'
|
|
if not receipt.exists():raise RuntimeError('MISSING_GUARD_RECEIPT')
|
|
r=json.loads(receipt.read_text());done.append(r);item['log'].close();del active[slot]
|
|
if r['worker']['status']!='PASS':reason=reason or 'PART_FAILED_NO_AUTOMATIC_SCORING'
|
|
spent=c['initial_worker_cpu_seconds']+sum(x['worker']['cpu_seconds'] for x in done)
|
|
used=disk_usage(root)
|
|
if used>=c['disk_bytes']:reason=reason or 'GLOBAL_DISK_LIMIT'
|
|
if spent>=28800:reason=reason or 'COMPUTE_CPU_LIMIT'
|
|
while reason is None and len(active)<4 and next_i<len(c['parts']):
|
|
if not may_dispatch(spent,len(active)):
|
|
if not active:reason='INSUFFICIENT_COMPUTE_RESERVATION'
|
|
break
|
|
slot=next(s for s in range(4) if s not in active);index=next_i;next_i+=1
|
|
dest=root/'parts'/c['parts'][index]['case'];dest.mkdir(parents=True,exist_ok=False)
|
|
log=(dest/'guard.log').open('x')
|
|
proc=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(index),'--slot',str(slot)],
|
|
stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True)
|
|
active[slot]=dict(index=index,process=proc,log=log)
|
|
if reason:
|
|
for item in active.values():kill_guard_worker(item['process'].pid)
|
|
save(root/'progress.json',dict(completed=len(done),attempted=next_i,total=len(c['parts']),
|
|
passed=sum(x['worker']['status']=='PASS' for x in done),active={str(s):v['index'] for s,v in active.items()},
|
|
settled_worker_cpu_seconds=spent,reserved_active_worker_cpu_seconds=len(active)*1802,
|
|
stop_reason=reason,disk_bytes=used,wall_seconds=time.monotonic()-start))
|
|
if active:time.sleep(.5)
|
|
except Exception as exc:
|
|
reason=reason or 'INFRASTRUCTURE_EXCEPTION';error=dict(message=str(exc),traceback=traceback.format_exc())
|
|
finally:
|
|
for item in active.values():kill_guard_worker(item['process'].pid)
|
|
for item in active.values():
|
|
item['process'].wait();item['log'].close()
|
|
p=root/'parts'/c['parts'][item['index']]['case']/'guard-result.json'
|
|
if p.exists():done.append(json.loads(p.read_text()))
|
|
active={}
|
|
done.sort(key=lambda r:r['index']);spent=c['initial_worker_cpu_seconds']+sum(x['worker']['cpu_seconds'] for x in done)
|
|
allowed=scoring_allowed(done,len(c['parts']),reason)
|
|
terminal=dict(status='ALL_PASS' if allowed else 'STOPPED',stop_reason=reason,records=done,
|
|
never_attempted=list(range(next_i,len(c['parts']))),attempted=next_i,
|
|
missing_receipt_indices=sorted(set(range(next_i))-{r['index'] for r in done}),
|
|
charged_worker_cpu_seconds=spent,initial_worker_cpu_seconds=c['initial_worker_cpu_seconds'],
|
|
error=error,accounting_complete=len(done)==next_i)
|
|
save(root/'preparation-complete.json',terminal)
|
|
post=None
|
|
if allowed:
|
|
parts={}
|
|
for r in done:
|
|
result=Path(r['result_path']);checked_result=json.loads(result.read_text())
|
|
if sha(result)!=r['result_sha256']:raise ValueError('Result changed before freeze')
|
|
p=result.parent/'query-predictions.npz';digest=checked_result['output_sha256'][p.name]
|
|
if sha(p)!=digest:raise ValueError('Prediction changed before freeze')
|
|
parts[r['case']]=dict(result=str(result),result_sha256=sha(result),predictions=str(p),predictions_sha256=digest)
|
|
save(root/'predictions-freeze.json',dict(contract_sha256=sha(root/'contract.json'),
|
|
preparation_complete_sha256=sha(root/'preparation-complete.json'),parts=parts,labels_read=False))
|
|
remaining_disk=c['disk_bytes']-disk_usage(root)
|
|
if remaining_disk<=0:reason='POSTPROCESS_DISK_EXHAUSTED'
|
|
else:
|
|
# Inherit a non-v4 CPU; one guarded process for all cache-only scoring.
|
|
os.sched_setaffinity(0,[24])
|
|
post=supervise([sys.executable,'tools/r4a_score.py',str(root)],root/'postfit','worker',
|
|
dict(total_cpu_seconds=7200,job_cpu_seconds=7200,job_wall_seconds=7200,
|
|
tree_rss_bytes=4*2**30,disk_bytes=remaining_disk,max_processes=1),poll=.2)
|
|
os.sched_setaffinity(0,[25]);spent+=post['cpu_seconds']
|
|
if post['status']!='PASS':reason='POSTPROCESS_FAILED'
|
|
save(root/'complete.json',dict(status='COMPLETE' if allowed and post and post['status']=='PASS' else 'STOPPED',
|
|
stop_reason=reason,charged_worker_cpu_seconds=spent,worker_cpu_cap=36000,
|
|
preparation_complete_sha256=sha(root/'preparation-complete.json'),postfit=post,
|
|
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
|
guard_cpu_seconds=sum(x.get('guard_cpu_seconds',0.) for x in done),
|
|
wall_seconds=time.monotonic()-start,retries=0,accounting_complete=len(done)==next_i))
|
|
return 0 if allowed and post and post['status']=='PASS' else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int,default=0)
|
|
a=p.parse_args();raise SystemExit(run(a.root) if a.guard is None else guard(a.root,a.guard,a.slot))
|