Freeze corrected full-volume ordering diagnostic and one-time custody contract
This commit is contained in:
1 parent
6d907acc7c
commit
d2adb6bcce
14 files changed
+2599
No files matched your search
@@ -0,0 +1,128 @@
|
||||
"""Separate CPU-only full-node custody queue, bounded independently of S3-A."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from freeze_g1_components import sha
|
||||
from s2_custody_common import read_json, checked
|
||||
from s2_recovery_supervisor import save, supervise, tree_usage
|
||||
from s2_recovery_monitoring import disk_usage, Monitor, budget_reason
|
||||
from run_s2_pilot_preparation import terminate
|
||||
from naive_ordering_common import contract
|
||||
|
||||
|
||||
def guard(root,index,slot):
|
||||
c=contract(root);lim=c['limits'];os.sched_setaffinity(0,[c['worker_cpus'][slot]])
|
||||
dest=root/'parts'/f'part-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
|
||||
w=supervise([sys.executable,'tools/run_naive_ordering_part.py',str(root),str(index)],dest,'worker',
|
||||
dict(total_cpu_seconds=lim['part_cpu_seconds']+1,job_cpu_seconds=lim['part_cpu_seconds'],
|
||||
job_wall_seconds=lim['part_wall_seconds'],tree_rss_bytes=lim['part_rss_bytes'],disk_bytes=lim['part_disk_bytes'],max_processes=1),poll=.2)
|
||||
result=dest/'output/result.json'
|
||||
if w['status']=='PASS' and (not result.exists() or read_json(result)['status']!='COMPLETE'):
|
||||
w.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT')
|
||||
save(dest/'guard-result.json',dict(index=index,case=c['cases'][index],worker=w,
|
||||
guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
result_path=str(result) if result.exists() else None,result_sha256=sha(result) if result.exists() else None))
|
||||
return 0 if w['status']=='PASS' else 1
|
||||
|
||||
|
||||
def run(root):
|
||||
with (root/'batch.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
return run_locked(root)
|
||||
|
||||
|
||||
def run_locked(root):
|
||||
c=contract(root);lim=c['limits'];total=len(c['cases'])
|
||||
if (root/'batch-started.json').exists():raise FileExistsError('Never repeat a custody batch/part')
|
||||
os.sched_setaffinity(0,c['coordinator_cpus'])
|
||||
start=time.monotonic();save(root/'batch-started.json',dict(pid=os.getpid(),unix=time.time(),
|
||||
contract_sha256=sha(root/'contract.json'),code_freeze=read_json(root/'code-freeze.json')))
|
||||
monitor=Monitor(root/'monitor-errors.jsonl');active={};done=[];cursor=0;reason=None;checked_first=False;accounting=True
|
||||
live=mem=disk=0
|
||||
env=dict(os.environ,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
||||
def interrupted(*_):
|
||||
nonlocal reason
|
||||
reason='NAIVE_COORDINATOR_INTERRUPTED'
|
||||
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
||||
def stop():terminate({k:(v['proc'],v['log']) for k,v in active.items()})
|
||||
def collect():
|
||||
nonlocal reason,accounting
|
||||
for slot,item in list(active.items()):
|
||||
code=item['proc'].poll()
|
||||
if code is None:continue
|
||||
p=root/'parts'/f"part-{item['index']:03d}"/'guard-result.json'
|
||||
if not p.exists():
|
||||
reason='MISSING_GUARD_RECEIPT';accounting=False
|
||||
save(root/'infrastructure-stop.json',dict(index=item['index'],returncode=code,reason=reason,accounting_complete=False))
|
||||
else:
|
||||
receipt=read_json(p)
|
||||
if receipt['index']!=item['index'] or not math.isfinite(receipt['worker']['cpu_seconds']) or receipt['worker']['cpu_seconds']<0:raise ValueError('Receipt mismatch')
|
||||
if receipt.get('result_path'):checked(receipt['result_path'],receipt['result_sha256'])
|
||||
done.append(receipt)
|
||||
print(json.dumps(dict(event='NAIVE_PART_TERMINAL',index=item['index'],status=receipt['worker']['status'],completed=len(done))),flush=True)
|
||||
item['log'].close();del active[slot]
|
||||
try:
|
||||
while active or (cursor<total and reason is None):
|
||||
collect()
|
||||
def usage():
|
||||
samples=[]
|
||||
for item in active.values():
|
||||
x=tree_usage(item['proc'].pid);item['cpu_peak']=max(item['cpu_peak'],x[0]);samples.append((item['cpu_peak'],x[1]))
|
||||
return sum(v[0] for v in samples),sum(v[1] for v in samples)
|
||||
(live,mem),fresh=monitor.sample('live',usage,(live,mem));disk,ok=monitor.sample('disk',lambda:disk_usage(root),disk);fresh &= ok
|
||||
if not fresh:reason='GLOBAL_MONITOR_ACCOUNTING_ERROR'
|
||||
spent=sum(v['worker']['cpu_seconds'] for v in done)
|
||||
over=budget_reason(0,spent,live,disk,cpu_limit=lim['preparation_cpu_seconds'],disk_limit=lim['disk_bytes'])
|
||||
if over:reason=over
|
||||
if not checked_first and len(done)==1 and not active and reason is None:
|
||||
first=done[0]['worker'];forecast=first['cpu_seconds']*(1+1.25*(total-1))
|
||||
approved=first['status']=='PASS' and forecast<=lim['preparation_cpu_seconds']
|
||||
save(root/'first-part-cost-check.json',dict(continue_within_cap=approved,first_worker_CPU_s=first['cpu_seconds'],
|
||||
forecast_CPU_s=forecast,method='First-part total worker CPU scaled to617 with25percent reserve; heuristic'))
|
||||
checked_first=True
|
||||
if not approved:reason='FIRST_PART_FAILURE_OR_COST_OVER_CAP'
|
||||
if reason:stop()
|
||||
ceiling=total if checked_first else 1;parallel=lim['workers'] if checked_first else 1
|
||||
while fresh and reason is None and len(active)<parallel and cursor<ceiling:
|
||||
if spent+live+(len(active)+1)*lim['part_cpu_seconds']>lim['preparation_cpu_seconds']:
|
||||
if not active:reason='INSUFFICIENT_REMAINING_PART_CAP'
|
||||
break
|
||||
slot=next(k for k in range(lim['workers']) if k not in active)
|
||||
dest=root/'parts'/f'part-{cursor:03d}';dest.mkdir(parents=True,exist_ok=True);log=(dest/'guard.log').open('x')
|
||||
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(cursor),'--slot',str(slot)],env=env,
|
||||
stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
active[slot]=dict(proc=p,log=log,index=cursor,cpu_peak=0.);cursor+=1
|
||||
save(root/'progress.json',dict(completed=len(done),active={str(k):v['index'] for k,v in active.items()},next_index=cursor,
|
||||
worker_CPU_s=spent,live_guard_and_worker_CPU_s=live,RSS=mem,disk_bytes=disk,first_checked=checked_first,
|
||||
stop_reason=reason,monitor_errors=monitor.errors,fresh_accounting=fresh,elapsed_seconds=time.monotonic()-start))
|
||||
if active or not fresh:time.sleep(.5)
|
||||
except Exception as exc:
|
||||
reason='COORDINATOR_EXCEPTION'
|
||||
save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc()))
|
||||
finally:
|
||||
stop()
|
||||
for item in active.values():item['proc'].wait()
|
||||
collect()
|
||||
complete=reason is None and len(done)==total and accounting
|
||||
passed=sum(r['worker']['status']=='PASS' for r in done)
|
||||
save(root/'batch-complete.json',dict(status='ALL_PARTS_TERMINAL' if complete else 'NAIVE_PREPARATION_INCOMPLETE',
|
||||
stop_reason=reason,accounting_complete=accounting,jobs=done,passed=passed,failed=len(done)-passed,
|
||||
not_run_indices=[i for i in range(total) if i not in {r['index'] for r in done}],
|
||||
worker_CPU_s=sum(v['worker']['cpu_seconds'] for v in done),guard_CPU_s=sum(v['guard_CPU_s'] for v in done),
|
||||
coordinator_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
|
||||
monitor_errors=monitor.errors,contract_sha256=sha(root/'contract.json'),retry=False))
|
||||
return 0 if complete and passed==total else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int);a=p.parse_args()
|
||||
raise SystemExit(guard(a.root,a.guard,a.slot) if a.guard is not None else run(a.root))
|
||||
Reference in new issue
Block a user