235 lines
12 KiB
Python
235 lines
12 KiB
Python
"""S3-A single-attempt CPU queue: two sequential witnesses then bounded four-way HPO."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
from freeze_g1_components import sha
|
|
from s2_custody_common import checked, read_json
|
|
from s2_recovery_supervisor import save, supervise, tree_usage
|
|
from s2_recovery_monitoring import Monitor, disk_usage, budget_reason
|
|
from run_s2_pilot_preparation import terminate
|
|
from run_s2_r2 import finalize_selections
|
|
from s3a_budget import first_fit_projection, can_dispatch
|
|
from s3a_runtime import contract
|
|
|
|
FIRST = [0, 12]
|
|
|
|
|
|
def dispatch_order():
|
|
return FIRST + [i for i in range(390) if i not in FIRST]
|
|
|
|
|
|
def guard(root, index, slot):
|
|
c = contract(root)
|
|
os.sched_setaffinity(0, [c['worker_cpus'][slot]])
|
|
limits = c['limits']
|
|
dest = root/'jobs'/f'job-{index:03d}'
|
|
dest.mkdir(parents=True, exist_ok=True)
|
|
w = supervise([sys.executable, 'tools/run_s3a_fit.py', str(root), str(index)], dest, 'worker',
|
|
dict(total_cpu_seconds=limits['per_fit_cpu_seconds']+1,
|
|
job_cpu_seconds=limits['per_fit_cpu_seconds'], job_wall_seconds=limits['per_fit_wall_seconds'],
|
|
tree_rss_bytes=limits['per_fit_tree_rss_bytes'], disk_bytes=limits['per_fit_disk_bytes'], max_processes=1), poll=.2)
|
|
path = dest/'output/result.json'
|
|
result = read_json(path) if path.exists() else None
|
|
if w['status'] == 'PASS' and (not result or result['status'] != 'COMPLETE'):
|
|
w.update(status='STOPPED_OR_FAILED', stop_reason='MISSING_COMPLETE_RESULT')
|
|
if w['status'] == 'PASS':
|
|
checked(dest/'output/model.joblib', result['model_sha256'])
|
|
save(dest/'guard-result.json', dict(worker=w, supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
|
cpu_affinity=sorted(os.sched_getaffinity(0)), result_path=str(path) if path.exists() else None,
|
|
result_sha256=sha(path) if path.exists() else None, job=c['jobs'][index]))
|
|
return 0 if w['status'] == 'PASS' else 1
|
|
|
|
|
|
def first_check(c, done):
|
|
samples = {}
|
|
for i in FIRST:
|
|
item = next(r for r in done if r['index'] == i)
|
|
result = read_json(checked(item['result_path'], item['result_sha256'])) if item.get('result_path') else {}
|
|
samples[c['jobs'][i]['arm']] = dict(status=item['worker']['status'],
|
|
worker_cpu_seconds=item['worker']['cpu_seconds'], fit_cpu_seconds=result.get('fit_cpu_seconds', 0.))
|
|
return first_fit_projection(c, samples)
|
|
|
|
|
|
def freeze_terminal(root, c, done, selections, reason, started, monitor, accounting_complete):
|
|
attempts = {r['index']: r for r in done}
|
|
finals = []
|
|
for job in c['jobs'][360:]:
|
|
row = attempts.get(job['index'])
|
|
passed = row is not None and row['worker']['status'] == 'PASS'
|
|
result = read_json(checked(row['result_path'], row['result_sha256'])) if passed else None
|
|
model_path = str(Path(row['result_path']).parent/'model.joblib') if passed else None
|
|
if passed:
|
|
checked(model_path, result['model_sha256'])
|
|
finals.append(dict(cell=job['cell'], arm=job['arm'], fold=job['fold'], pool=job['pool'],
|
|
status='PASS' if passed else 'NA',
|
|
reason=None if passed else ((row['worker']['stop_reason'] or 'WORKER_EXIT_'+str(row['worker']['returncode'])) if row else 'NOT_RUN_'+str(reason)),
|
|
finished_configurations=selections[job['cell']]['finished_configurations'] if selections else None,
|
|
model_path=model_path, model_sha256=result['model_sha256'] if result else None))
|
|
complete = reason is None and len(attempts) == 390 and accounting_complete
|
|
result = dict(status='ALL_S3A_FITS_TERMINAL' if complete else 'S3A_STOPPED_INCOMPLETE',
|
|
contract_sha256=sha(root/'contract.json'), stop_reason=reason, accounting_complete=accounting_complete,
|
|
final_fits=finals, jobs=done, not_run_indices=[i for i in range(390) if i not in attempts],
|
|
worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
|
|
guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),
|
|
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
|
disk_bytes=disk_usage(root), wall_seconds=time.monotonic()-started,
|
|
monitor_errors=monitor.errors, roster_sha256=c['roster_sha256'],
|
|
test_predictions=False, test_scoring=False, retry=False, baseline_refits=0)
|
|
save(root/'batch-complete.json', result)
|
|
if complete:
|
|
save(root/'terminal-freeze.json', dict(status='ALL_S3A_FITS_TERMINAL', contract_sha256=sha(root/'contract.json'),
|
|
hpo_terminal_indices=list(range(360)), final_fits=finals,
|
|
batch_receipt=str(root/'batch-complete.json'), batch_receipt_sha256=sha(root/'batch-complete.json'),
|
|
test_predictions_started=False))
|
|
return 0 if complete else 1
|
|
|
|
|
|
def run(root):
|
|
with (root/'batch.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
return run_locked(root)
|
|
|
|
|
|
def run_locked(root):
|
|
c = contract(root)
|
|
if (root/'batch-started.json').exists():
|
|
raise FileExistsError('No restart/retry of an attempted S3-A queue')
|
|
os.sched_setaffinity(0, c['coordinator_cpus'])
|
|
start = time.monotonic()
|
|
save(root/'batch-started.json', dict(pid=os.getpid(), unix=time.time(), contract_sha256=sha(root/'contract.json'),
|
|
implementation_freeze=read_json(root/'implementation-freeze.json'), cpu_affinity=sorted(os.sched_getaffinity(0))))
|
|
active, done = {}, []
|
|
order = dispatch_order()
|
|
cursor = 0
|
|
reason = None
|
|
accounting_complete = True
|
|
first_checked = False
|
|
selections = None
|
|
live_cpu = live_rss = disk = 0
|
|
monitor = Monitor(root/'monitor-errors.jsonl')
|
|
env = dict(os.environ, OMP_NUM_THREADS='1', MKL_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
|
|
|
|
def interrupted(*_):
|
|
nonlocal reason
|
|
reason = 'S3A_COORDINATOR_INTERRUPTED'
|
|
|
|
signal.signal(signal.SIGTERM, interrupted)
|
|
signal.signal(signal.SIGINT, interrupted)
|
|
|
|
def terminate_active():
|
|
terminate({k: (v['process'], v['log']) for k, v in active.items()})
|
|
|
|
def collect():
|
|
nonlocal reason, accounting_complete
|
|
for slot, item in list(active.items()):
|
|
code = item['process'].poll()
|
|
if code is None:
|
|
continue
|
|
path = root/'jobs'/f"job-{item['index']:03d}"/'guard-result.json'
|
|
if not path.exists():
|
|
reason = 'MISSING_GUARD_RECEIPT'
|
|
accounting_complete = False
|
|
save(root/'infrastructure-stop.json', dict(reason=reason, index=item['index'], returncode=code,
|
|
last_observed_guard_tree_CPU_s=item.get('cpu_peak', 0.), accounting_complete=False))
|
|
else:
|
|
row = read_json(path)
|
|
if row['job'] != c['jobs'][item['index']] or not math.isfinite(row['worker']['cpu_seconds']) or row['worker']['cpu_seconds'] < 0:
|
|
raise ValueError('Invalid job/accounting receipt')
|
|
if row.get('result_path'):
|
|
checked(row['result_path'], row['result_sha256'])
|
|
done.append(dict(index=item['index'], **row))
|
|
print(json.dumps(dict(event='S3A_TERMINAL', index=item['index'], arm=row['job']['arm'],
|
|
status=row['worker']['status'], completed=len(done))), flush=True)
|
|
item['log'].close()
|
|
del active[slot]
|
|
|
|
try:
|
|
while active or (cursor < 390 and reason is None):
|
|
collect()
|
|
|
|
def usage():
|
|
values = []
|
|
for v in active.values():
|
|
value = tree_usage(v['process'].pid)
|
|
v['cpu_peak'] = max(v.get('cpu_peak', 0.), value[0])
|
|
values.append((v['cpu_peak'], value[1]))
|
|
return sum(v[0] for v in values), sum(v[1] for v in values)
|
|
|
|
(live_cpu, live_rss), fresh = monitor.sample('live_usage', usage, (live_cpu, live_rss))
|
|
disk, ok = monitor.sample('disk', lambda: disk_usage(root), disk)
|
|
fresh &= ok
|
|
spent = sum(r['worker']['cpu_seconds'] for r in done)
|
|
over = budget_reason(0, spent, live_cpu, disk, cpu_limit=c['limits']['fitting_worker_cpu_seconds'], disk_limit=c['limits']['disk_bytes'])
|
|
if over:
|
|
reason = over
|
|
if not first_checked and not active and done and any(r['worker']['status'] != 'PASS' for r in done):
|
|
reason = 'FIRST_TRIAL_FAILED_NO_RETRY'
|
|
if not first_checked and len(done) == 2 and not active and reason is None:
|
|
check = first_check(c, done)
|
|
save(root/'first-fit-cost-check.json', check)
|
|
first_checked = True
|
|
if not check['continue_within_cap']:
|
|
reason = 'FIRST_TRIAL_COST_PROJECTION_OVER_CAP'
|
|
if cursor == 360 and not active and len(done) == 360 and selections is None and reason is None:
|
|
selections = finalize_selections(root, c, done)
|
|
if reason:
|
|
terminate_active()
|
|
ceiling = (len(done)+1 if len(done) < 2 else 2) if not first_checked else (360 if selections is None else 390)
|
|
parallel = 1 if not first_checked else 4
|
|
while fresh and reason is None and len(active) < parallel and cursor < ceiling:
|
|
i = order[cursor]
|
|
job = c['jobs'][i]
|
|
if job['kind'] == 'FINAL' and selections[job['cell']]['status'] != 'SELECTED':
|
|
done.append(dict(index=i, job=job, worker=dict(status='NA', cpu_seconds=0., returncode=0,
|
|
stop_reason='ALL12_HPO_FAILED_NO_FINAL_FIT'), supervisor_cpu_seconds=0., attempted=False, result_path=None, result_sha256=None))
|
|
cursor += 1
|
|
continue
|
|
# Conservatively reserve full caps of active workers plus the next task.
|
|
cap = c['limits']['per_fit_cpu_seconds']
|
|
if not can_dispatch(c, spent, live_cpu, [cap for _ in active], cap):
|
|
if not active:
|
|
reason = 'INSUFFICIENT_REMAINING_FIT_CAP_NO_LAUNCH'
|
|
break
|
|
slot = next(k for k in range(4) if k not in active)
|
|
dest = root/'jobs'/f'job-{i:03d}'
|
|
dest.mkdir(parents=True, exist_ok=True)
|
|
log = (dest/'guard.log').open('x')
|
|
proc = subprocess.Popen([sys.executable, __file__, str(root), '--guard', str(i), '--slot', str(slot)],
|
|
env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
|
active[slot] = dict(process=proc, log=log, index=i, cpu_peak=0.)
|
|
cursor += 1
|
|
save(root/'progress.json', dict(completed=len(done), active={str(k): v['index'] for k,v in active.items()},
|
|
dispatch_cursor=cursor, worker_cpu_seconds=spent, live_guard_and_worker_cpu_seconds=live_cpu,
|
|
active_tree_rss_bytes=live_rss, disk_bytes=disk, first_fit_checked=first_checked,
|
|
selections_frozen=selections is not None, stop_reason=reason, fresh_accounting=fresh,
|
|
monitor_errors=monitor.errors, elapsed_seconds=time.monotonic()-start))
|
|
if active or not fresh:
|
|
time.sleep(.5)
|
|
except Exception as exc:
|
|
reason = 'COORDINATOR_EXCEPTION'
|
|
save(root/'coordinator-failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc()))
|
|
finally:
|
|
terminate_active()
|
|
for item in active.values():
|
|
item['process'].wait()
|
|
collect()
|
|
return freeze_terminal(root, c, done, selections, reason, start, monitor, accounting_complete)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('root', type=Path)
|
|
p.add_argument('--guard', type=int)
|
|
p.add_argument('--slot', type=int)
|
|
a = p.parse_args()
|
|
raise SystemExit(guard(a.root, a.guard, a.slot) if a.guard is not None else run(a.root))
|