Files
MeltMind/tools/run_s3a_recovery_queue.py
T

246 lines
13 KiB
Python

"""Explicit 4+155 infrastructure continuation. Original workers and scientific contract unchanged."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
import traceback
from freeze_g1_components import sha
from s2_custody_common import checked, read_json
from s2_recovery_supervisor import save, supervise, tree_usage
from s2_recovery_monitoring import Monitor, disk_usage, budget_reason
from run_s2_pilot_preparation import terminate
from run_s2_r2 import finalize_selections
from s3a_budget import first_fit_projection, can_dispatch
from s3a_runtime import contract
from s3a_recovery_common import state, canonical_accounting, selection_ready, manifest, ORDER
FIRST = [0, 12]
def dispatch_order():
return list(ORDER)
def guard(root, index, slot):
if index not in ORDER:
raise ValueError('Original successful jobs may not be dispatched again')
c = contract(root)
os.sched_setaffinity(0, [c['worker_cpus'][slot]])
limits = c['limits']
dest = root/'jobs'/f'job-{index:03d}'
dest.mkdir(parents=True, exist_ok=True)
w = supervise([sys.executable, 'tools/run_s3a_fit.py', str(root), str(index)], dest, 'worker',
dict(total_cpu_seconds=limits['per_fit_cpu_seconds']+1,
job_cpu_seconds=limits['per_fit_cpu_seconds'], job_wall_seconds=limits['per_fit_wall_seconds'],
tree_rss_bytes=limits['per_fit_tree_rss_bytes'], disk_bytes=limits['per_fit_disk_bytes'], max_processes=1), poll=.2)
path = dest/'output/result.json'
result = read_json(path) if path.exists() else None
if w['status'] == 'PASS' and (not result or result['status'] != 'COMPLETE'):
w.update(status='STOPPED_OR_FAILED', stop_reason='MISSING_COMPLETE_RESULT')
if w['status'] == 'PASS':
checked(dest/'output/model.joblib', result['model_sha256'])
save(dest/'guard-result.json', dict(worker=w, supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
cpu_affinity=sorted(os.sched_getaffinity(0)), result_path=str(path) if path.exists() else None,
result_sha256=sha(path) if path.exists() else None, job=c['jobs'][index]))
return 0 if w['status'] == 'PASS' else 1
def first_check(c, done):
samples = {}
for i in FIRST:
item = next(r for r in done if r['index'] == i)
result = read_json(checked(item['result_path'], item['result_sha256'])) if item.get('result_path') else {}
samples[c['jobs'][i]['arm']] = dict(status=item['worker']['status'],
worker_cpu_seconds=item['worker']['cpu_seconds'], fit_cpu_seconds=result.get('fit_cpu_seconds', 0.))
return first_fit_projection(c, samples)
def freeze_terminal(root, c, done, selections, reason, started, monitor, accounting_complete):
_, scope, old, _ = state(root)
merged = manifest(root, scope, old, done)
save(root/'two-source-manifest.json', merged)
attempts = {r['index']: r for r in done}
finals = []
for job in c['jobs'][360:]:
row = attempts.get(job['index'])
passed = row is not None and row['worker']['status'] == 'PASS'
result = read_json(checked(row['result_path'], row['result_sha256'])) if passed else None
model_path = str(Path(row['result_path']).parent/'model.joblib') if passed else None
if passed:
checked(model_path, result['model_sha256'])
finals.append(dict(cell=job['cell'], arm=job['arm'], fold=job['fold'], pool=job['pool'],
status='PASS' if passed else 'NA',
reason=None if passed else ((row['worker']['stop_reason'] or 'WORKER_EXIT_'+str(row['worker']['returncode'])) if row else 'NOT_RUN_'+str(reason)),
finished_configurations=selections[job['cell']]['finished_configurations'] if selections else None,
model_path=model_path, model_sha256=result['model_sha256'] if result else None))
complete = reason is None and len(attempts) == 390 and accounting_complete
result = dict(status='ALL_S3A_FITS_TERMINAL' if complete else 'S3A_STOPPED_INCOMPLETE',
contract_sha256=sha(root/'contract.json'), stop_reason=reason, accounting_complete=accounting_complete,
final_fits=finals, jobs=done, not_run_indices=[i for i in range(390) if i not in attempts],
worker_cpu_seconds=canonical_accounting(old, done),
original_worker_cpu_seconds=old['worker_cpu_seconds'],
original_aborted_jobs_retained=[230,232,233,234],
two_source_manifest_sha256=sha(root/'two-source-manifest.json'),
guard_cpu_seconds=old['guard_cpu_seconds']+sum(r['supervisor_cpu_seconds'] for r in done if r['index'] in ORDER),
original_coordinator_cpu_seconds=old['coordinator_cpu_seconds'],
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
disk_bytes=disk_usage(root)+disk_usage(Path(scope['original_root'])), wall_seconds=time.monotonic()-started,
monitor_errors=monitor.errors, roster_sha256=c['roster_sha256'],
test_predictions=False, test_scoring=False, retry=False, baseline_refits=0)
save(root/'batch-complete.json', result)
if complete:
save(root/'terminal-freeze.json', dict(status='ALL_S3A_FITS_TERMINAL', contract_sha256=sha(root/'contract.json'),
hpo_terminal_indices=list(range(360)), final_fits=finals,
batch_receipt=str(root/'batch-complete.json'), batch_receipt_sha256=sha(root/'batch-complete.json'),
test_predictions_started=False))
return 0 if complete else 1
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
return run_locked(root)
def run_locked(root):
c, scope, old, original_passed = state(root)
if (root/'batch-started.json').exists():
raise FileExistsError('No restart/retry of an attempted S3-A queue')
os.sched_setaffinity(0, c['coordinator_cpus'])
start = time.monotonic()
save(root/'batch-started.json', dict(pid=os.getpid(), unix=time.time(), contract_sha256=sha(root/'contract.json'),
implementation_freeze=read_json(root/'implementation-freeze.json'), cpu_affinity=sorted(os.sched_getaffinity(0))))
active, done = {}, list(original_passed)
order = dispatch_order()
cursor = 0
reason = None
accounting_complete = True
first_checked = True
selections = None
live_cpu = live_rss = disk = 0
monitor = Monitor(root/'monitor-errors.jsonl')
env = dict(os.environ, OMP_NUM_THREADS='1', MKL_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
def interrupted(*_):
nonlocal reason
reason = 'S3A_RECOVERY_COORDINATOR_INTERRUPTED'
save(root/'signal-received.json', dict(signal_number=_[0], unix=time.time(), pid=os.getpid()))
signal.signal(signal.SIGTERM, interrupted)
signal.signal(signal.SIGINT, interrupted)
def terminate_active():
terminate({k: (v['process'], v['log']) for k, v in active.items()})
def collect():
nonlocal reason, accounting_complete
for slot, item in list(active.items()):
code = item['process'].poll()
if code is None:
continue
path = root/'jobs'/f"job-{item['index']:03d}"/'guard-result.json'
if not path.exists():
reason = 'MISSING_GUARD_RECEIPT'
accounting_complete = False
save(root/'infrastructure-stop.json', dict(reason=reason, index=item['index'], returncode=code,
last_observed_guard_tree_CPU_s=item.get('cpu_peak', 0.), accounting_complete=False))
else:
row = read_json(path)
if row['job'] != c['jobs'][item['index']] or not math.isfinite(row['worker']['cpu_seconds']) or row['worker']['cpu_seconds'] < 0:
raise ValueError('Invalid job/accounting receipt')
if row.get('result_path'):
checked(row['result_path'], row['result_sha256'])
done.append(dict(index=item['index'], **row))
print(json.dumps(dict(event='S3A_TERMINAL', index=item['index'], arm=row['job']['arm'],
status=row['worker']['status'], completed=len(done))), flush=True)
item['log'].close()
del active[slot]
try:
while active or (cursor < len(order) and reason is None):
collect()
def usage():
values = []
for v in active.values():
value = tree_usage(v['process'].pid)
v['cpu_peak'] = max(v.get('cpu_peak', 0.), value[0])
values.append((v['cpu_peak'], value[1]))
return sum(v[0] for v in values), sum(v[1] for v in values)
(live_cpu, live_rss), fresh = monitor.sample('live_usage', usage, (live_cpu, live_rss))
disk, ok = monitor.sample('disk', lambda: disk_usage(root)+disk_usage(Path(scope['original_root'])), disk)
fresh &= ok
spent = canonical_accounting(old, done)
over = budget_reason(0, spent, live_cpu, disk, cpu_limit=c['limits']['fitting_worker_cpu_seconds'], disk_limit=c['limits']['disk_bytes'])
if over:
reason = over
if not first_checked and not active and done and any(r['worker']['status'] != 'PASS' for r in done):
reason = 'FIRST_TRIAL_FAILED_NO_RETRY'
if not first_checked and len(done) == 2 and not active and reason is None:
check = first_check(c, done)
save(root/'first-fit-cost-check.json', check)
first_checked = True
if not check['continue_within_cap']:
reason = 'FIRST_TRIAL_COST_PROJECTION_OVER_CAP'
if selection_ready(done, active) and selections is None and reason is None:
selections = finalize_selections(root, c, done)
if reason:
terminate_active()
ceiling = len(order)-30 if selections is None else len(order)
parallel = 1 if not first_checked else 4
while fresh and reason is None and len(active) < parallel and cursor < ceiling:
i = order[cursor]
job = c['jobs'][i]
if job['kind'] == 'FINAL' and selections[job['cell']]['status'] != 'SELECTED':
done.append(dict(index=i, job=job, worker=dict(status='NA', cpu_seconds=0., returncode=0,
stop_reason='ALL12_HPO_FAILED_NO_FINAL_FIT'), supervisor_cpu_seconds=0., attempted=False, result_path=None, result_sha256=None))
cursor += 1
continue
# Conservatively reserve full caps of active workers plus the next task.
cap = c['limits']['per_fit_cpu_seconds']
if not can_dispatch(c, spent, live_cpu, [cap for _ in active], cap):
if not active:
reason = 'INSUFFICIENT_REMAINING_FIT_CAP_NO_LAUNCH'
break
slot = next(k for k in range(4) if k not in active)
dest = root/'jobs'/f'job-{i:03d}'
dest.mkdir(parents=True, exist_ok=True)
log = (dest/'guard.log').open('x')
proc = subprocess.Popen([sys.executable, __file__, str(root), '--guard', str(i), '--slot', str(slot)],
env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
active[slot] = dict(process=proc, log=log, index=i, cpu_peak=0.)
cursor += 1
save(root/'progress.json', dict(completed=len(done), active={str(k): v['index'] for k,v in active.items()},
dispatch_cursor=cursor, worker_cpu_seconds=spent, live_guard_and_worker_cpu_seconds=live_cpu,
active_tree_rss_bytes=live_rss, disk_bytes=disk, first_fit_checked=first_checked,
selections_frozen=selections is not None, stop_reason=reason, fresh_accounting=fresh,
monitor_errors=monitor.errors, elapsed_seconds=time.monotonic()-start))
if active or not fresh:
time.sleep(.5)
except Exception as exc:
reason = 'COORDINATOR_EXCEPTION'
save(root/'coordinator-failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc()))
finally:
terminate_active()
for item in active.values():
item['process'].wait()
collect()
return freeze_terminal(root, c, done, selections, reason, start, monitor, accounting_complete)
if __name__ == '__main__':
p = argparse.ArgumentParser()
p.add_argument('root', type=Path)
p.add_argument('--guard', type=int)
p.add_argument('--slot', type=int)
a = p.parse_args()
raise SystemExit(guard(a.root, a.guard, a.slot) if a.guard is not None else run(a.root))