"""One detached R4(a) queue: fail-stop, no retry, no scoring after a failure.""" import argparse import fcntl import json import os from pathlib import Path import resource import signal import subprocess import sys import time import traceback import psutil from gre24_prepare_v2 import sha from r4a_part import contract from r4a_core import may_dispatch, scoring_allowed from s2_recovery_supervisor import save, supervise, stop_group from s2_recovery_monitoring import disk_usage def guard(root,index,slot): c=contract(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]]) dest=root/'parts'/c['parts'][index]['case'] receipt=supervise([sys.executable,'tools/r4a_part.py',str(root),str(index)],dest,'worker', dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800, tree_rss_bytes=4*2**30,disk_bytes=c['disk_bytes'],max_processes=1),poll=.2) result=dest/'output/result.json' if receipt['status']=='PASS' and (not result.exists() or json.loads(result.read_text())['status']!='PASS'): receipt.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_PASS_RESULT') save(dest/'guard-result.json',dict(index=index,case=c['parts'][index]['case'],worker=receipt, result_path=str(result) if result.exists() else None,result_sha256=sha(result) if result.exists() else None, guard_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]))) return 0 if receipt['status']=='PASS' else 1 def kill_guard_worker(pid): # Keep the supervisor alive to reap its worker and write the true failure receipt. try: for child in psutil.Process(pid).children():stop_group(child.pid) except psutil.NoSuchProcess:pass def fresh(root): if (root/'started.json').exists() or (root/'complete.json').exists(): raise FileExistsError('Existing attempt: never restart R4 queue') def run(root): with (root/'queue.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) return run_locked(root) def run_locked(root): c=contract(root);fresh(root);os.sched_setaffinity(0,[25]) save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'))) active={};done=[];next_i=0;reason=None;start=time.monotonic();error=None def interrupted(*_): nonlocal reason reason=reason or 'COORDINATOR_INTERRUPTED' signal.signal(signal.SIGINT,interrupted);signal.signal(signal.SIGTERM,interrupted) env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') try: while active or (next_i=c['disk_bytes']:reason=reason or 'GLOBAL_DISK_LIMIT' if spent>=28800:reason=reason or 'COMPUTE_CPU_LIMIT' while reason is None and len(active)<4 and next_i