"""Separate CPU-only full-node custody queue, bounded independently of S3-A.""" import argparse import fcntl import json import math import os from pathlib import Path import resource import signal import subprocess import sys import time import traceback from freeze_g1_components import sha from s2_custody_common import read_json, checked from s2_recovery_supervisor import save, supervise, tree_usage from s2_recovery_monitoring import disk_usage, Monitor, budget_reason from run_s2_pilot_preparation import terminate from naive_ordering_common import contract def guard(root,index,slot): c=contract(root);lim=c['limits'];os.sched_setaffinity(0,[c['worker_cpus'][slot]]) dest=root/'parts'/f'part-{index:03d}';dest.mkdir(parents=True,exist_ok=True) w=supervise([sys.executable,'tools/run_naive_ordering_part.py',str(root),str(index)],dest,'worker', dict(total_cpu_seconds=lim['part_cpu_seconds']+1,job_cpu_seconds=lim['part_cpu_seconds'], job_wall_seconds=lim['part_wall_seconds'],tree_rss_bytes=lim['part_rss_bytes'],disk_bytes=lim['part_disk_bytes'],max_processes=1),poll=.2) result=dest/'output/result.json' if w['status']=='PASS' and (not result.exists() or read_json(result)['status']!='COMPLETE'): w.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT') save(dest/'guard-result.json',dict(index=index,case=c['cases'][index],worker=w, guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]), result_path=str(result) if result.exists() else None,result_sha256=sha(result) if result.exists() else None)) return 0 if w['status']=='PASS' else 1 def run(root): with (root/'batch.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) return run_locked(root) def run_locked(root): c=contract(root);lim=c['limits'];total=len(c['cases']) if (root/'batch-started.json').exists():raise FileExistsError('Never repeat a custody batch/part') os.sched_setaffinity(0,c['coordinator_cpus']) start=time.monotonic();save(root/'batch-started.json',dict(pid=os.getpid(),unix=time.time(), contract_sha256=sha(root/'contract.json'),code_freeze=read_json(root/'code-freeze.json'))) monitor=Monitor(root/'monitor-errors.jsonl');active={};done=[];cursor=0;reason=None;checked_first=False;accounting=True live=mem=disk=0 env=dict(os.environ,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') def interrupted(*_): nonlocal reason reason='NAIVE_COORDINATOR_INTERRUPTED' signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted) def stop():terminate({k:(v['proc'],v['log']) for k,v in active.items()}) def collect(): nonlocal reason,accounting for slot,item in list(active.items()): code=item['proc'].poll() if code is None:continue p=root/'parts'/f"part-{item['index']:03d}"/'guard-result.json' if not p.exists(): reason='MISSING_GUARD_RECEIPT';accounting=False save(root/'infrastructure-stop.json',dict(index=item['index'],returncode=code,reason=reason,accounting_complete=False)) else: receipt=read_json(p) if receipt['index']!=item['index'] or not math.isfinite(receipt['worker']['cpu_seconds']) or receipt['worker']['cpu_seconds']<0:raise ValueError('Receipt mismatch') if receipt.get('result_path'):checked(receipt['result_path'],receipt['result_sha256']) done.append(receipt) print(json.dumps(dict(event='NAIVE_PART_TERMINAL',index=item['index'],status=receipt['worker']['status'],completed=len(done))),flush=True) item['log'].close();del active[slot] try: while active or (cursorlim['preparation_cpu_seconds']: if not active:reason='INSUFFICIENT_REMAINING_PART_CAP' break slot=next(k for k in range(lim['workers']) if k not in active) dest=root/'parts'/f'part-{cursor:03d}';dest.mkdir(parents=True,exist_ok=True);log=(dest/'guard.log').open('x') p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(cursor),'--slot',str(slot)],env=env, stdout=log,stderr=subprocess.STDOUT,start_new_session=True) active[slot]=dict(proc=p,log=log,index=cursor,cpu_peak=0.);cursor+=1 save(root/'progress.json',dict(completed=len(done),active={str(k):v['index'] for k,v in active.items()},next_index=cursor, worker_CPU_s=spent,live_guard_and_worker_CPU_s=live,RSS=mem,disk_bytes=disk,first_checked=checked_first, stop_reason=reason,monitor_errors=monitor.errors,fresh_accounting=fresh,elapsed_seconds=time.monotonic()-start)) if active or not fresh:time.sleep(.5) except Exception as exc: reason='COORDINATOR_EXCEPTION' save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc())) finally: stop() for item in active.values():item['proc'].wait() collect() complete=reason is None and len(done)==total and accounting passed=sum(r['worker']['status']=='PASS' for r in done) save(root/'batch-complete.json',dict(status='ALL_PARTS_TERMINAL' if complete else 'NAIVE_PREPARATION_INCOMPLETE', stop_reason=reason,accounting_complete=accounting,jobs=done,passed=passed,failed=len(done)-passed, not_run_indices=[i for i in range(total) if i not in {r['index'] for r in done}], worker_CPU_s=sum(v['worker']['cpu_seconds'] for v in done),guard_CPU_s=sum(v['guard_CPU_s'] for v in done), coordinator_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start, monitor_errors=monitor.errors,contract_sha256=sha(root/'contract.json'),retry=False)) return 0 if complete and passed==total else 1 if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int);a=p.parse_args() raise SystemExit(guard(a.root,a.guard,a.slot) if a.guard is not None else run(a.root))