Files
MeltMind/tools/run_gre24_preparation.py

100 lines
6.1 KiB
Python

"""One guarded Gre24 preparation queue, preserving failed attempts and total CPU."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
from gre24_prepare import checked,sha
from s2_recovery_supervisor import save,supervise,stop_group
def guard(root,index,slot):
c=checked(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]])
part=c['parts'][index];dest=root/'parts'/part['case'];dest.mkdir(parents=True,exist_ok=True)
cap=c['limits']['per_part_cpu_seconds']
r=supervise([sys.executable,'tools/gre24_prepare.py',str(root),str(index)],dest,'worker',
dict(total_cpu_seconds=cap+1,job_cpu_seconds=cap,job_wall_seconds=c['limits']['per_part_wall_seconds'],
tree_rss_bytes=c['limits']['per_part_rss_bytes'],disk_bytes=2*2**30,max_processes=1),poll=.2)
result=dest/'output/result.json'
if r['status']=='PASS' and (not result.exists() or json.loads(result.read_text())['status']!='PASS'):
r.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT')
save(dest/'guard-result.json',dict(worker=r,index=index,case=part['case'],result_path=str(result) if result.exists() else None,
result_sha256=sha(result) if result.exists() else None,guard_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
return 0 if r['status']=='PASS' else 1
def run(root):
with (root/'queue.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return run_locked(root)
def run_locked(root):
c=checked(root)
if (root/'queue-started.json').exists():raise FileExistsError('Never restart this queue')
os.sched_setaffinity(0,c['coordinator_cpus'])
save(root/'queue-started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json')))
active={};done=[];next_i=0;reason=None;first=False;started=time.monotonic()
cap=c['limits']['total_worker_cpu_seconds'];per=c['limits']['per_part_cpu_seconds']
def interrupted(*_):
nonlocal reason
reason='COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
try:
while active or (next_i<len(c['parts']) and reason is None):
for slot,item in list(active.items()):
if item['process'].poll() is None:continue
receipt=root/'parts'/c['parts'][item['index']]['case']/'guard-result.json'
if not receipt.exists():
reason='MISSING_GUARD_RECEIPT';raise RuntimeError(reason)
r=json.loads(receipt.read_text());done.append(r);item['log'].close();del active[slot]
print(json.dumps(dict(event='PREPARATION_TERMINAL',index=r['index'],case=r['case'],status=r['worker']['status'],completed=len(done))),flush=True)
spent=c['limits']['preparation_initial_cpu_reserve_seconds']+sum(r['worker']['cpu_seconds'] for r in done)
if not first and done and not active:
first=True;r=done[0];spec=c['parts'][r['index']]
projection=r['worker']['cpu_seconds']*sum(p['raw_nodes'] for p in c['parts'])/spec['raw_nodes']*1.25
save(root/'first-part-checkpoint.json',dict(first=r['case'],status=r['worker']['status'],
projected_preparation_cpu_seconds=projection,projection_is_heuristic=True,
total_worker_cap_seconds=cap,first_result_sha256=r.get('result_sha256')))
if r['worker']['status']!='PASS':reason='FIRST_PART_FAILED'
elif projection>cap:reason='FIRST_PART_PROJECTED_PREPARATION_OVER_TOTAL_CAP'
ceiling=len(c['parts']) if first else 1
while reason is None and len(active)<c['limits']['workers'] and next_i<ceiling:
# Reserve each active job's full cap plus one sampling margin.
if spent+(len(active)+1)*(per+2)>cap:
if not active:reason='REMAINING_CPU_INSUFFICIENT_FOR_ANOTHER_CAPPED_ATTEMPT'
break
slot=next(s for s in range(c['limits']['workers']) if s not in active)
index=c['dispatch_order'][next_i];dest=root/'parts'/c['parts'][index]['case'];dest.mkdir(parents=True,exist_ok=True)
log=(dest/'guard.log').open('x')
process=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(index),'--slot',str(slot)],
stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True)
active[slot]=dict(process=process,log=log,index=index);next_i+=1
save(root/'progress.json',dict(completed=len(done),attempted=next_i,total=len(c['parts']),
active={str(k):v['index'] for k,v in active.items()},settled_worker_cpu_seconds=spent,
reserved_active_worker_cpu_seconds=len(active)*(per+2),stop_reason=reason,
first_part_checked=first,elapsed_seconds=time.monotonic()-started))
if active:time.sleep(.5)
finally:
for item in active.values():
stop_group(item['process'].pid);item['process'].wait();item['log'].close()
attempted={r['index'] for r in done}
save(root/'complete.json',dict(status='COMPLETE' if len(done)==len(c['parts']) else 'STOPPED',stop_reason=reason,
records=done,never_attempted=[i for i in c['dispatch_order'] if i not in attempted],
worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
initial_cpu_reserve_seconds=c['limits']['preparation_initial_cpu_reserve_seconds'],
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
accounting_complete=not active,wall_seconds=time.monotonic()-started))
return 0 if len(done)==len(c['parts']) else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int,default=0)
a=p.parse_args();raise SystemExit(run(a.root) if a.guard is None else guard(a.root,a.guard,a.slot))