"""One guarded Gre24 preparation queue, preserving failed attempts and total CPU.""" import argparse import fcntl import json import os from pathlib import Path import resource import signal import subprocess import sys import time from gre24_prepare_v2 import checked,sha from s2_recovery_supervisor import save,supervise,stop_group def guard(root,index,slot): c=checked(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]]) part=c['parts'][index];dest=root/'parts'/part['case'];dest.mkdir(parents=True,exist_ok=True) cap=c['limits']['per_part_cpu_seconds'] r=supervise([sys.executable,'tools/gre24_prepare_v2.py',str(root),str(index)],dest,'worker', dict(total_cpu_seconds=cap+1,job_cpu_seconds=cap,job_wall_seconds=c['limits']['per_part_wall_seconds'], tree_rss_bytes=c['limits']['per_part_rss_bytes'],disk_bytes=2*2**30,max_processes=1),poll=.2) result=dest/'output/result.json' if r['status']=='PASS' and (not result.exists() or json.loads(result.read_text())['status']!='PASS'): r.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT') save(dest/'guard-result.json',dict(worker=r,index=index,case=part['case'],result_path=str(result) if result.exists() else None, result_sha256=sha(result) if result.exists() else None,guard_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]))) return 0 if r['status']=='PASS' else 1 def run(root): with (root/'queue.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) return run_locked(root) def run_locked(root): c=checked(root) if (root/'queue-started.json').exists():raise FileExistsError('Never restart this queue') os.sched_setaffinity(0,c['coordinator_cpus']) save(root/'queue-started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'))) active={};done=[];next_i=0;reason=None;first=False;started=time.monotonic() cap=c['limits']['total_worker_cpu_seconds'];per=c['limits']['per_part_cpu_seconds'] def interrupted(*_): nonlocal reason reason='COORDINATOR_INTERRUPTED' signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted) env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') try: while active or (next_icap:reason='FIRST_PART_PROJECTED_PREPARATION_OVER_TOTAL_CAP' ceiling=len(c['parts']) if first else 1 while reason is None and len(active)cap: if not active:reason='REMAINING_CPU_INSUFFICIENT_FOR_ANOTHER_CAPPED_ATTEMPT' break slot=next(s for s in range(c['limits']['workers']) if s not in active) index=c['dispatch_order'][next_i];dest=root/'parts'/c['parts'][index]['case'];dest.mkdir(parents=True,exist_ok=True) log=(dest/'guard.log').open('x') process=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(index),'--slot',str(slot)], stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True) active[slot]=dict(process=process,log=log,index=index);next_i+=1 save(root/'progress.json',dict(completed=len(done),attempted=next_i,total=len(c['parts']), active={str(k):v['index'] for k,v in active.items()},settled_worker_cpu_seconds=spent, reserved_active_worker_cpu_seconds=len(active)*(per+2),stop_reason=reason, first_part_checked=first,elapsed_seconds=time.monotonic()-started)) if active:time.sleep(.5) finally: for item in active.values(): stop_group(item['process'].pid);item['process'].wait();item['log'].close() attempted={r['index'] for r in done} save(root/'complete.json',dict(status='COMPLETE' if len(done)==len(c['parts']) else 'STOPPED',stop_reason=reason, records=done,never_attempted=[i for i in c['dispatch_order'] if i not in attempted], worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done), initial_cpu_reserve_seconds=c['limits']['preparation_initial_cpu_reserve_seconds'], coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), accounting_complete=not active,wall_seconds=time.monotonic()-started)) return 0 if len(done)==len(c['parts']) else 1 if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int,default=0) a=p.parse_args();raise SystemExit(run(a.root) if a.guard is None else guard(a.root,a.guard,a.slot))