Files
MeltMind/tools/run_naive_ordering_queue.py
T

129 lines
7.8 KiB
Python

"""Separate CPU-only full-node custody queue, bounded independently of S3-A."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
import traceback
from freeze_g1_components import sha
from s2_custody_common import read_json, checked
from s2_recovery_supervisor import save, supervise, tree_usage
from s2_recovery_monitoring import disk_usage, Monitor, budget_reason
from run_s2_pilot_preparation import terminate
from naive_ordering_common import contract
def guard(root,index,slot):
c=contract(root);lim=c['limits'];os.sched_setaffinity(0,[c['worker_cpus'][slot]])
dest=root/'parts'/f'part-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
w=supervise([sys.executable,'tools/run_naive_ordering_part.py',str(root),str(index)],dest,'worker',
dict(total_cpu_seconds=lim['part_cpu_seconds']+1,job_cpu_seconds=lim['part_cpu_seconds'],
job_wall_seconds=lim['part_wall_seconds'],tree_rss_bytes=lim['part_rss_bytes'],disk_bytes=lim['part_disk_bytes'],max_processes=1),poll=.2)
result=dest/'output/result.json'
if w['status']=='PASS' and (not result.exists() or read_json(result)['status']!='COMPLETE'):
w.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT')
save(dest/'guard-result.json',dict(index=index,case=c['cases'][index],worker=w,
guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
result_path=str(result) if result.exists() else None,result_sha256=sha(result) if result.exists() else None))
return 0 if w['status']=='PASS' else 1
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return run_locked(root)
def run_locked(root):
c=contract(root);lim=c['limits'];total=len(c['cases'])
if (root/'batch-started.json').exists():raise FileExistsError('Never repeat a custody batch/part')
os.sched_setaffinity(0,c['coordinator_cpus'])
start=time.monotonic();save(root/'batch-started.json',dict(pid=os.getpid(),unix=time.time(),
contract_sha256=sha(root/'contract.json'),code_freeze=read_json(root/'code-freeze.json')))
monitor=Monitor(root/'monitor-errors.jsonl');active={};done=[];cursor=0;reason=None;checked_first=False;accounting=True
live=mem=disk=0
env=dict(os.environ,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
def interrupted(*_):
nonlocal reason
reason='NAIVE_COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
def stop():terminate({k:(v['proc'],v['log']) for k,v in active.items()})
def collect():
nonlocal reason,accounting
for slot,item in list(active.items()):
code=item['proc'].poll()
if code is None:continue
p=root/'parts'/f"part-{item['index']:03d}"/'guard-result.json'
if not p.exists():
reason='MISSING_GUARD_RECEIPT';accounting=False
save(root/'infrastructure-stop.json',dict(index=item['index'],returncode=code,reason=reason,accounting_complete=False))
else:
receipt=read_json(p)
if receipt['index']!=item['index'] or not math.isfinite(receipt['worker']['cpu_seconds']) or receipt['worker']['cpu_seconds']<0:raise ValueError('Receipt mismatch')
if receipt.get('result_path'):checked(receipt['result_path'],receipt['result_sha256'])
done.append(receipt)
print(json.dumps(dict(event='NAIVE_PART_TERMINAL',index=item['index'],status=receipt['worker']['status'],completed=len(done))),flush=True)
item['log'].close();del active[slot]
try:
while active or (cursor<total and reason is None):
collect()
def usage():
samples=[]
for item in active.values():
x=tree_usage(item['proc'].pid);item['cpu_peak']=max(item['cpu_peak'],x[0]);samples.append((item['cpu_peak'],x[1]))
return sum(v[0] for v in samples),sum(v[1] for v in samples)
(live,mem),fresh=monitor.sample('live',usage,(live,mem));disk,ok=monitor.sample('disk',lambda:disk_usage(root),disk);fresh &= ok
if not fresh:reason='GLOBAL_MONITOR_ACCOUNTING_ERROR'
spent=sum(v['worker']['cpu_seconds'] for v in done)
over=budget_reason(0,spent,live,disk,cpu_limit=lim['preparation_cpu_seconds'],disk_limit=lim['disk_bytes'])
if over:reason=over
if not checked_first and len(done)==1 and not active and reason is None:
first=done[0]['worker'];forecast=first['cpu_seconds']*(1+1.25*(total-1))
approved=first['status']=='PASS' and forecast<=lim['preparation_cpu_seconds']
save(root/'first-part-cost-check.json',dict(continue_within_cap=approved,first_worker_CPU_s=first['cpu_seconds'],
forecast_CPU_s=forecast,method='First-part total worker CPU scaled to617 with25percent reserve; heuristic'))
checked_first=True
if not approved:reason='FIRST_PART_FAILURE_OR_COST_OVER_CAP'
if reason:stop()
ceiling=total if checked_first else 1;parallel=lim['workers'] if checked_first else 1
while fresh and reason is None and len(active)<parallel and cursor<ceiling:
if spent+live+(len(active)+1)*lim['part_cpu_seconds']>lim['preparation_cpu_seconds']:
if not active:reason='INSUFFICIENT_REMAINING_PART_CAP'
break
slot=next(k for k in range(lim['workers']) if k not in active)
dest=root/'parts'/f'part-{cursor:03d}';dest.mkdir(parents=True,exist_ok=True);log=(dest/'guard.log').open('x')
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(cursor),'--slot',str(slot)],env=env,
stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[slot]=dict(proc=p,log=log,index=cursor,cpu_peak=0.);cursor+=1
save(root/'progress.json',dict(completed=len(done),active={str(k):v['index'] for k,v in active.items()},next_index=cursor,
worker_CPU_s=spent,live_guard_and_worker_CPU_s=live,RSS=mem,disk_bytes=disk,first_checked=checked_first,
stop_reason=reason,monitor_errors=monitor.errors,fresh_accounting=fresh,elapsed_seconds=time.monotonic()-start))
if active or not fresh:time.sleep(.5)
except Exception as exc:
reason='COORDINATOR_EXCEPTION'
save(root/'coordinator-failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc()))
finally:
stop()
for item in active.values():item['proc'].wait()
collect()
complete=reason is None and len(done)==total and accounting
passed=sum(r['worker']['status']=='PASS' for r in done)
save(root/'batch-complete.json',dict(status='ALL_PARTS_TERMINAL' if complete else 'NAIVE_PREPARATION_INCOMPLETE',
stop_reason=reason,accounting_complete=accounting,jobs=done,passed=passed,failed=len(done)-passed,
not_run_indices=[i for i in range(total) if i not in {r['index'] for r in done}],
worker_CPU_s=sum(v['worker']['cpu_seconds'] for v in done),guard_CPU_s=sum(v['guard_CPU_s'] for v in done),
coordinator_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
monitor_errors=monitor.errors,contract_sha256=sha(root/'contract.json'),retry=False))
return 0 if complete and passed==total else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int);a=p.parse_args()
raise SystemExit(guard(a.root,a.guard,a.slot) if a.guard is not None else run(a.root))