100 lines
6.1 KiB
Python
100 lines
6.1 KiB
Python
"""One guarded Gre24 preparation queue, preserving failed attempts and total CPU."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from gre24_prepare_v2 import checked,sha
|
|
from s2_recovery_supervisor import save,supervise,stop_group
|
|
|
|
|
|
def guard(root,index,slot):
|
|
c=checked(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]])
|
|
part=c['parts'][index];dest=root/'parts'/part['case'];dest.mkdir(parents=True,exist_ok=True)
|
|
cap=c['limits']['per_part_cpu_seconds']
|
|
r=supervise([sys.executable,'tools/gre24_prepare_v2.py',str(root),str(index)],dest,'worker',
|
|
dict(total_cpu_seconds=cap+1,job_cpu_seconds=cap,job_wall_seconds=c['limits']['per_part_wall_seconds'],
|
|
tree_rss_bytes=c['limits']['per_part_rss_bytes'],disk_bytes=2*2**30,max_processes=1),poll=.2)
|
|
result=dest/'output/result.json'
|
|
if r['status']=='PASS' and (not result.exists() or json.loads(result.read_text())['status']!='PASS'):
|
|
r.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT')
|
|
save(dest/'guard-result.json',dict(worker=r,index=index,case=part['case'],result_path=str(result) if result.exists() else None,
|
|
result_sha256=sha(result) if result.exists() else None,guard_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
|
|
return 0 if r['status']=='PASS' else 1
|
|
|
|
|
|
def run(root):
|
|
with (root/'queue.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
return run_locked(root)
|
|
|
|
|
|
def run_locked(root):
|
|
c=checked(root)
|
|
if (root/'queue-started.json').exists():raise FileExistsError('Never restart this queue')
|
|
os.sched_setaffinity(0,c['coordinator_cpus'])
|
|
save(root/'queue-started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json')))
|
|
active={};done=[];next_i=0;reason=None;first=False;started=time.monotonic()
|
|
cap=c['limits']['total_worker_cpu_seconds'];per=c['limits']['per_part_cpu_seconds']
|
|
def interrupted(*_):
|
|
nonlocal reason
|
|
reason='COORDINATOR_INTERRUPTED'
|
|
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
|
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
|
try:
|
|
while active or (next_i<len(c['parts']) and reason is None):
|
|
for slot,item in list(active.items()):
|
|
if item['process'].poll() is None:continue
|
|
receipt=root/'parts'/c['parts'][item['index']]['case']/'guard-result.json'
|
|
if not receipt.exists():
|
|
reason='MISSING_GUARD_RECEIPT';raise RuntimeError(reason)
|
|
r=json.loads(receipt.read_text());done.append(r);item['log'].close();del active[slot]
|
|
print(json.dumps(dict(event='PREPARATION_TERMINAL',index=r['index'],case=r['case'],status=r['worker']['status'],completed=len(done))),flush=True)
|
|
spent=c['limits']['preparation_initial_cpu_reserve_seconds']+sum(r['worker']['cpu_seconds'] for r in done)
|
|
if not first and done and not active:
|
|
first=True;r=done[0];spec=c['parts'][r['index']]
|
|
projection=r['worker']['cpu_seconds']*sum(p['raw_nodes'] for p in c['parts'])/spec['raw_nodes']*1.25
|
|
save(root/'first-part-checkpoint.json',dict(first=r['case'],status=r['worker']['status'],
|
|
projected_preparation_cpu_seconds=projection,projection_is_heuristic=True,
|
|
total_worker_cap_seconds=cap,first_result_sha256=r.get('result_sha256')))
|
|
if r['worker']['status']!='PASS':reason='FIRST_PART_FAILED'
|
|
elif projection>cap:reason='FIRST_PART_PROJECTED_PREPARATION_OVER_TOTAL_CAP'
|
|
ceiling=len(c['parts']) if first else 1
|
|
while reason is None and len(active)<c['limits']['workers'] and next_i<ceiling:
|
|
# Reserve each active job's full cap plus one sampling margin.
|
|
if spent+(len(active)+1)*(per+2)>cap:
|
|
if not active:reason='REMAINING_CPU_INSUFFICIENT_FOR_ANOTHER_CAPPED_ATTEMPT'
|
|
break
|
|
slot=next(s for s in range(c['limits']['workers']) if s not in active)
|
|
index=c['dispatch_order'][next_i];dest=root/'parts'/c['parts'][index]['case'];dest.mkdir(parents=True,exist_ok=True)
|
|
log=(dest/'guard.log').open('x')
|
|
process=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(index),'--slot',str(slot)],
|
|
stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True)
|
|
active[slot]=dict(process=process,log=log,index=index);next_i+=1
|
|
save(root/'progress.json',dict(completed=len(done),attempted=next_i,total=len(c['parts']),
|
|
active={str(k):v['index'] for k,v in active.items()},settled_worker_cpu_seconds=spent,
|
|
reserved_active_worker_cpu_seconds=len(active)*(per+2),stop_reason=reason,
|
|
first_part_checked=first,elapsed_seconds=time.monotonic()-started))
|
|
if active:time.sleep(.5)
|
|
finally:
|
|
for item in active.values():
|
|
stop_group(item['process'].pid);item['process'].wait();item['log'].close()
|
|
attempted={r['index'] for r in done}
|
|
save(root/'complete.json',dict(status='COMPLETE' if len(done)==len(c['parts']) else 'STOPPED',stop_reason=reason,
|
|
records=done,never_attempted=[i for i in c['dispatch_order'] if i not in attempted],
|
|
worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
|
|
initial_cpu_reserve_seconds=c['limits']['preparation_initial_cpu_reserve_seconds'],
|
|
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
|
accounting_complete=not active,wall_seconds=time.monotonic()-started))
|
|
return 0 if len(done)==len(c['parts']) else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int,default=0)
|
|
a=p.parse_args();raise SystemExit(run(a.root) if a.guard is None else guard(a.root,a.guard,a.slot))
|