Files
MeltMind/tools/run_s3a_queue.py

235 lines
12 KiB
Python

"""S3-A single-attempt CPU queue: two sequential witnesses then bounded four-way HPO."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
import traceback
from freeze_g1_components import sha
from s2_custody_common import checked, read_json
from s2_recovery_supervisor import save, supervise, tree_usage
from s2_recovery_monitoring import Monitor, disk_usage, budget_reason
from run_s2_pilot_preparation import terminate
from run_s2_r2 import finalize_selections
from s3a_budget import first_fit_projection, can_dispatch
from s3a_runtime import contract
FIRST = [0, 12]
def dispatch_order():
return FIRST + [i for i in range(390) if i not in FIRST]
def guard(root, index, slot):
c = contract(root)
os.sched_setaffinity(0, [c['worker_cpus'][slot]])
limits = c['limits']
dest = root/'jobs'/f'job-{index:03d}'
dest.mkdir(parents=True, exist_ok=True)
w = supervise([sys.executable, 'tools/run_s3a_fit.py', str(root), str(index)], dest, 'worker',
dict(total_cpu_seconds=limits['per_fit_cpu_seconds']+1,
job_cpu_seconds=limits['per_fit_cpu_seconds'], job_wall_seconds=limits['per_fit_wall_seconds'],
tree_rss_bytes=limits['per_fit_tree_rss_bytes'], disk_bytes=limits['per_fit_disk_bytes'], max_processes=1), poll=.2)
path = dest/'output/result.json'
result = read_json(path) if path.exists() else None
if w['status'] == 'PASS' and (not result or result['status'] != 'COMPLETE'):
w.update(status='STOPPED_OR_FAILED', stop_reason='MISSING_COMPLETE_RESULT')
if w['status'] == 'PASS':
checked(dest/'output/model.joblib', result['model_sha256'])
save(dest/'guard-result.json', dict(worker=w, supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
cpu_affinity=sorted(os.sched_getaffinity(0)), result_path=str(path) if path.exists() else None,
result_sha256=sha(path) if path.exists() else None, job=c['jobs'][index]))
return 0 if w['status'] == 'PASS' else 1
def first_check(c, done):
samples = {}
for i in FIRST:
item = next(r for r in done if r['index'] == i)
result = read_json(checked(item['result_path'], item['result_sha256'])) if item.get('result_path') else {}
samples[c['jobs'][i]['arm']] = dict(status=item['worker']['status'],
worker_cpu_seconds=item['worker']['cpu_seconds'], fit_cpu_seconds=result.get('fit_cpu_seconds', 0.))
return first_fit_projection(c, samples)
def freeze_terminal(root, c, done, selections, reason, started, monitor, accounting_complete):
attempts = {r['index']: r for r in done}
finals = []
for job in c['jobs'][360:]:
row = attempts.get(job['index'])
passed = row is not None and row['worker']['status'] == 'PASS'
result = read_json(checked(row['result_path'], row['result_sha256'])) if passed else None
model_path = str(Path(row['result_path']).parent/'model.joblib') if passed else None
if passed:
checked(model_path, result['model_sha256'])
finals.append(dict(cell=job['cell'], arm=job['arm'], fold=job['fold'], pool=job['pool'],
status='PASS' if passed else 'NA',
reason=None if passed else ((row['worker']['stop_reason'] or 'WORKER_EXIT_'+str(row['worker']['returncode'])) if row else 'NOT_RUN_'+str(reason)),
finished_configurations=selections[job['cell']]['finished_configurations'] if selections else None,
model_path=model_path, model_sha256=result['model_sha256'] if result else None))
complete = reason is None and len(attempts) == 390 and accounting_complete
result = dict(status='ALL_S3A_FITS_TERMINAL' if complete else 'S3A_STOPPED_INCOMPLETE',
contract_sha256=sha(root/'contract.json'), stop_reason=reason, accounting_complete=accounting_complete,
final_fits=finals, jobs=done, not_run_indices=[i for i in range(390) if i not in attempts],
worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
disk_bytes=disk_usage(root), wall_seconds=time.monotonic()-started,
monitor_errors=monitor.errors, roster_sha256=c['roster_sha256'],
test_predictions=False, test_scoring=False, retry=False, baseline_refits=0)
save(root/'batch-complete.json', result)
if complete:
save(root/'terminal-freeze.json', dict(status='ALL_S3A_FITS_TERMINAL', contract_sha256=sha(root/'contract.json'),
hpo_terminal_indices=list(range(360)), final_fits=finals,
batch_receipt=str(root/'batch-complete.json'), batch_receipt_sha256=sha(root/'batch-complete.json'),
test_predictions_started=False))
return 0 if complete else 1
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
return run_locked(root)
def run_locked(root):
c = contract(root)
if (root/'batch-started.json').exists():
raise FileExistsError('No restart/retry of an attempted S3-A queue')
os.sched_setaffinity(0, c['coordinator_cpus'])
start = time.monotonic()
save(root/'batch-started.json', dict(pid=os.getpid(), unix=time.time(), contract_sha256=sha(root/'contract.json'),
implementation_freeze=read_json(root/'implementation-freeze.json'), cpu_affinity=sorted(os.sched_getaffinity(0))))
active, done = {}, []
order = dispatch_order()
cursor = 0
reason = None
accounting_complete = True
first_checked = False
selections = None
live_cpu = live_rss = disk = 0
monitor = Monitor(root/'monitor-errors.jsonl')
env = dict(os.environ, OMP_NUM_THREADS='1', MKL_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
def interrupted(*_):
nonlocal reason
reason = 'S3A_COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM, interrupted)
signal.signal(signal.SIGINT, interrupted)
def terminate_active():
terminate({k: (v['process'], v['log']) for k, v in active.items()})
def collect():
nonlocal reason, accounting_complete
for slot, item in list(active.items()):
code = item['process'].poll()
if code is None:
continue
path = root/'jobs'/f"job-{item['index']:03d}"/'guard-result.json'
if not path.exists():
reason = 'MISSING_GUARD_RECEIPT'
accounting_complete = False
save(root/'infrastructure-stop.json', dict(reason=reason, index=item['index'], returncode=code,
last_observed_guard_tree_CPU_s=item.get('cpu_peak', 0.), accounting_complete=False))
else:
row = read_json(path)
if row['job'] != c['jobs'][item['index']] or not math.isfinite(row['worker']['cpu_seconds']) or row['worker']['cpu_seconds'] < 0:
raise ValueError('Invalid job/accounting receipt')
if row.get('result_path'):
checked(row['result_path'], row['result_sha256'])
done.append(dict(index=item['index'], **row))
print(json.dumps(dict(event='S3A_TERMINAL', index=item['index'], arm=row['job']['arm'],
status=row['worker']['status'], completed=len(done))), flush=True)
item['log'].close()
del active[slot]
try:
while active or (cursor < 390 and reason is None):
collect()
def usage():
values = []
for v in active.values():
value = tree_usage(v['process'].pid)
v['cpu_peak'] = max(v.get('cpu_peak', 0.), value[0])
values.append((v['cpu_peak'], value[1]))
return sum(v[0] for v in values), sum(v[1] for v in values)
(live_cpu, live_rss), fresh = monitor.sample('live_usage', usage, (live_cpu, live_rss))
disk, ok = monitor.sample('disk', lambda: disk_usage(root), disk)
fresh &= ok
spent = sum(r['worker']['cpu_seconds'] for r in done)
over = budget_reason(0, spent, live_cpu, disk, cpu_limit=c['limits']['fitting_worker_cpu_seconds'], disk_limit=c['limits']['disk_bytes'])
if over:
reason = over
if not first_checked and not active and done and any(r['worker']['status'] != 'PASS' for r in done):
reason = 'FIRST_TRIAL_FAILED_NO_RETRY'
if not first_checked and len(done) == 2 and not active and reason is None:
check = first_check(c, done)
save(root/'first-fit-cost-check.json', check)
first_checked = True
if not check['continue_within_cap']:
reason = 'FIRST_TRIAL_COST_PROJECTION_OVER_CAP'
if cursor == 360 and not active and len(done) == 360 and selections is None and reason is None:
selections = finalize_selections(root, c, done)
if reason:
terminate_active()
ceiling = (len(done)+1 if len(done) < 2 else 2) if not first_checked else (360 if selections is None else 390)
parallel = 1 if not first_checked else 4
while fresh and reason is None and len(active) < parallel and cursor < ceiling:
i = order[cursor]
job = c['jobs'][i]
if job['kind'] == 'FINAL' and selections[job['cell']]['status'] != 'SELECTED':
done.append(dict(index=i, job=job, worker=dict(status='NA', cpu_seconds=0., returncode=0,
stop_reason='ALL12_HPO_FAILED_NO_FINAL_FIT'), supervisor_cpu_seconds=0., attempted=False, result_path=None, result_sha256=None))
cursor += 1
continue
# Conservatively reserve full caps of active workers plus the next task.
cap = c['limits']['per_fit_cpu_seconds']
if not can_dispatch(c, spent, live_cpu, [cap for _ in active], cap):
if not active:
reason = 'INSUFFICIENT_REMAINING_FIT_CAP_NO_LAUNCH'
break
slot = next(k for k in range(4) if k not in active)
dest = root/'jobs'/f'job-{i:03d}'
dest.mkdir(parents=True, exist_ok=True)
log = (dest/'guard.log').open('x')
proc = subprocess.Popen([sys.executable, __file__, str(root), '--guard', str(i), '--slot', str(slot)],
env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
active[slot] = dict(process=proc, log=log, index=i, cpu_peak=0.)
cursor += 1
save(root/'progress.json', dict(completed=len(done), active={str(k): v['index'] for k,v in active.items()},
dispatch_cursor=cursor, worker_cpu_seconds=spent, live_guard_and_worker_cpu_seconds=live_cpu,
active_tree_rss_bytes=live_rss, disk_bytes=disk, first_fit_checked=first_checked,
selections_frozen=selections is not None, stop_reason=reason, fresh_accounting=fresh,
monitor_errors=monitor.errors, elapsed_seconds=time.monotonic()-start))
if active or not fresh:
time.sleep(.5)
except Exception as exc:
reason = 'COORDINATOR_EXCEPTION'
save(root/'coordinator-failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc()))
finally:
terminate_active()
for item in active.values():
item['process'].wait()
collect()
return freeze_terminal(root, c, done, selections, reason, start, monitor, accounting_complete)
if __name__ == '__main__':
p = argparse.ArgumentParser()
p.add_argument('root', type=Path)
p.add_argument('--guard', type=int)
p.add_argument('--slot', type=int)
a = p.parse_args()
raise SystemExit(guard(a.root, a.guard, a.slot) if a.guard is not None else run(a.root))