129 lines
7.8 KiB
Python
129 lines
7.8 KiB
Python
"""Separate CPU-only full-node custody queue, bounded independently of S3-A."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from freeze_g1_components import sha
|
|
from s2_custody_common import read_json, checked
|
|
from s2_recovery_supervisor import save, supervise, tree_usage
|
|
from s2_recovery_monitoring import disk_usage, Monitor, budget_reason
|
|
from run_s2_pilot_preparation import terminate
|
|
from naive_ordering_common import contract
|
|
|
|
|
|
def guard(root,index,slot):
|
|
c=contract(root);lim=c['limits'];os.sched_setaffinity(0,[c['worker_cpus'][slot]])
|
|
dest=root/'parts'/f'part-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
|
|
w=supervise([sys.executable,'tools/run_naive_ordering_part.py',str(root),str(index)],dest,'worker',
|
|
dict(total_cpu_seconds=lim['part_cpu_seconds']+1,job_cpu_seconds=lim['part_cpu_seconds'],
|
|
job_wall_seconds=lim['part_wall_seconds'],tree_rss_bytes=lim['part_rss_bytes'],disk_bytes=lim['part_disk_bytes'],max_processes=1),poll=.2)
|
|
result=dest/'output/result.json'
|
|
if w['status']=='PASS' and (not result.exists() or read_json(result)['status']!='COMPLETE'):
|
|
w.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT')
|
|
save(dest/'guard-result.json',dict(index=index,case=c['cases'][index],worker=w,
|
|
guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
|
result_path=str(result) if result.exists() else None,result_sha256=sha(result) if result.exists() else None))
|
|
return 0 if w['status']=='PASS' else 1
|
|
|
|
|
|
def run(root):
|
|
with (root/'batch.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
return run_locked(root)
|
|
|
|
|
|
def run_locked(root):
|
|
c=contract(root);lim=c['limits'];total=len(c['cases'])
|
|
if (root/'batch-started.json').exists():raise FileExistsError('Never repeat a custody batch/part')
|
|
os.sched_setaffinity(0,c['coordinator_cpus'])
|
|
start=time.monotonic();save(root/'batch-started.json',dict(pid=os.getpid(),unix=time.time(),
|
|
contract_sha256=sha(root/'contract.json'),code_freeze=read_json(root/'code-freeze.json')))
|
|
monitor=Monitor(root/'monitor-errors.jsonl');active={};done=[];cursor=0;reason=None;checked_first=False;accounting=True
|
|
live=mem=disk=0
|
|
env=dict(os.environ,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
|
def interrupted(*_):
|
|
nonlocal reason
|
|
reason='NAIVE_COORDINATOR_INTERRUPTED'
|
|
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
|
def stop():terminate({k:(v['proc'],v['log']) for k,v in active.items()})
|
|
def collect():
|
|
nonlocal reason,accounting
|
|
for slot,item in list(active.items()):
|
|
code=item['proc'].poll()
|
|
if code is None:continue
|
|
p=root/'parts'/f"part-{item['index']:03d}"/'guard-result.json'
|
|
if not p.exists():
|
|
reason='MISSING_GUARD_RECEIPT';accounting=False
|
|
save(root/'infrastructure-stop.json',dict(index=item['index'],returncode=code,reason=reason,accounting_complete=False))
|
|
else:
|
|
receipt=read_json(p)
|
|
if receipt['index']!=item['index'] or not math.isfinite(receipt['worker']['cpu_seconds']) or receipt['worker']['cpu_seconds']<0:raise ValueError('Receipt mismatch')
|
|
if receipt.get('result_path'):checked(receipt['result_path'],receipt['result_sha256'])
|
|
done.append(receipt)
|
|
print(json.dumps(dict(event='NAIVE_PART_TERMINAL',index=item['index'],status=receipt['worker']['status'],completed=len(done))),flush=True)
|
|
item['log'].close();del active[slot]
|
|
try:
|
|
while active or (cursor<total and reason is None):
|
|
collect()
|
|
def usage():
|
|
samples=[]
|
|
for item in active.values():
|
|
x=tree_usage(item['proc'].pid);item['cpu_peak']=max(item['cpu_peak'],x[0]);samples.append((item['cpu_peak'],x[1]))
|
|
return sum(v[0] for v in samples),sum(v[1] for v in samples)
|
|
(live,mem),fresh=monitor.sample('live',usage,(live,mem));disk,ok=monitor.sample('disk',lambda:disk_usage(root),disk);fresh &= ok
|
|
if not fresh:reason='GLOBAL_MONITOR_ACCOUNTING_ERROR'
|
|
spent=sum(v['worker']['cpu_seconds'] for v in done)
|
|
over=budget_reason(0,spent,live,disk,cpu_limit=lim['preparation_cpu_seconds'],disk_limit=lim['disk_bytes'])
|
|
if over:reason=over
|
|
if not checked_first and len(done)==1 and not active and reason is None:
|
|
first=done[0]['worker'];forecast=first['cpu_seconds']*(1+1.25*(total-1))
|
|
approved=first['status']=='PASS' and forecast<=lim['preparation_cpu_seconds']
|
|
save(root/'first-part-cost-check.json',dict(continue_within_cap=approved,first_worker_CPU_s=first['cpu_seconds'],
|
|
forecast_CPU_s=forecast,method='First-part total worker CPU scaled to617 with25percent reserve; heuristic'))
|
|
checked_first=True
|
|
if not approved:reason='FIRST_PART_FAILURE_OR_COST_OVER_CAP'
|
|
if reason:stop()
|
|
ceiling=total if checked_first else 1;parallel=lim['workers'] if checked_first else 1
|
|
while fresh and reason is None and len(active)<parallel and cursor<ceiling:
|
|
if spent+live+(len(active)+1)*lim['part_cpu_seconds']>lim['preparation_cpu_seconds']:
|
|
if not active:reason='INSUFFICIENT_REMAINING_PART_CAP'
|
|
break
|
|
slot=next(k for k in range(lim['workers']) if k not in active)
|
|
dest=root/'parts'/f'part-{cursor:03d}';dest.mkdir(parents=True,exist_ok=True);log=(dest/'guard.log').open('x')
|
|
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(cursor),'--slot',str(slot)],env=env,
|
|
stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
|
active[slot]=dict(proc=p,log=log,index=cursor,cpu_peak=0.);cursor+=1
|
|
save(root/'progress.json',dict(completed=len(done),active={str(k):v['index'] for k,v in active.items()},next_index=cursor,
|
|
worker_CPU_s=spent,live_guard_and_worker_CPU_s=live,RSS=mem,disk_bytes=disk,first_checked=checked_first,
|
|
stop_reason=reason,monitor_errors=monitor.errors,fresh_accounting=fresh,elapsed_seconds=time.monotonic()-start))
|
|
if active or not fresh:time.sleep(.5)
|
|
except Exception as exc:
|
|
reason='COORDINATOR_EXCEPTION'
|
|
save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc()))
|
|
finally:
|
|
stop()
|
|
for item in active.values():item['proc'].wait()
|
|
collect()
|
|
complete=reason is None and len(done)==total and accounting
|
|
passed=sum(r['worker']['status']=='PASS' for r in done)
|
|
save(root/'batch-complete.json',dict(status='ALL_PARTS_TERMINAL' if complete else 'NAIVE_PREPARATION_INCOMPLETE',
|
|
stop_reason=reason,accounting_complete=accounting,jobs=done,passed=passed,failed=len(done)-passed,
|
|
not_run_indices=[i for i in range(total) if i not in {r['index'] for r in done}],
|
|
worker_CPU_s=sum(v['worker']['cpu_seconds'] for v in done),guard_CPU_s=sum(v['guard_CPU_s'] for v in done),
|
|
coordinator_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
|
|
monitor_errors=monitor.errors,contract_sha256=sha(root/'contract.json'),retry=False))
|
|
return 0 if complete and passed==total else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int);a=p.parse_args()
|
|
raise SystemExit(guard(a.root,a.guard,a.slot) if a.guard is not None else run(a.root))
|