"""Five original grouped folds on released GPUs; train and freeze only, never score.""" import fcntl import os from pathlib import Path import signal import subprocess import sys import time import psutil from phase2_gate_outcome_review import validate_review from phase2_h1_worker_v2 import checked, save, sha from phase2_home_freeze_io import freeze from s2_recovery_gpu_supervisor import disk_usage, tree_usage from phase3_seed1_continuation_common import config, dispatchable, execution_command, project_cell, project_total, read, available_gpus, idle_eligible_gpus def run(root): c, _ = config(root) validate_review(root) lock = (root / 'queue.lock').open('a') fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) if (root / 'started.json').exists(): raise FileExistsError('One Seed1 attempt; no retry') for p in psutil.process_iter(['pid', 'cmdline']): if p.pid != os.getpid() and execution_command(p.info['cmdline'] or []): raise RuntimeError('Existing Seed1 execution') os.sched_setaffinity(0, c['coordinator_cpus']) os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True' start = time.monotonic() cpu = time.process_time() active, terminal, estimates, last_epoch = {}, {}, {}, {} pending = list(c['jobs']) requested = [] def stop(reason): if not (root / 'global-stop.json').exists(): save(root / 'global-stop.json', dict(reason=reason, unix=time.time(), no_retry=True)) def active_charges(): return {i: c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'] for i, v in active.items()} def charged(): return sum(v['gpu_seconds_charged'] for v in terminal.values()) + sum(active_charges().values()) def terminate(v): if v.get('stop_sent') is None: if v['process'].poll() is None: v['process'].send_signal(signal.SIGTERM) v['stop_sent'] = time.monotonic() elif time.monotonic() - v['stop_sent'] > 20: try: parent = psutil.Process(v['process'].pid) for p in reversed(parent.children(recursive=True)): try: p.kill() except psutil.NoSuchProcess: pass parent.kill() except psutil.NoSuchProcess: pass def collect(i, v): rc = v['process'].poll() if rc is None: return False p = root / 'jobs' / str(i) / 'complete.json' r = read(p) if p.exists() else None observed = c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'] ok = (rc == 0 and r is not None and r['status'] == 'PASS' and not r.get('monitor_errors') and r.get('worker') is not None and not r['worker'].get('monitor_errors')) terminal[i] = dict(id=i, status='PASS' if ok else 'STOPPED_OR_FAILED', returncode=rc, gpu_seconds_charged=max(observed, r['gpu_seconds_charged'] if r else 0), guard_complete_sha256=sha(p) if r else None, gpu_uuid=v['job']['gpu_uuid'], assignment_sha256=sha(p.parent/'assignment.json')) v['log'].close() del active[i] save(root / 'terminal-ledger.json', dict(entries=list(terminal.values()), settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values()))) if not ok: stop('JOB_' + str(i) + '_FAILED') return True def dispatch(): eligible=available_gpus(c)-{v['job']['gpu_uuid'] for v in active.values()} idle,observations=idle_eligible_gpus(eligible) observed_unix=time.time() save(root/'dispatch-idle-observation.json',dict(unix=observed_unix,eligible=sorted(eligible),observations=observations)) for j in dispatchable(pending, [v['job'] for v in active.values()], idle): i = j['id'] d = root / 'jobs' / str(i) d.mkdir(parents=True, exist_ok=False) log = (d / 'entry.log').open('x') begin = time.monotonic() save(d / 'assignment.json', dict(id=i, gpu_uuid=j['gpu_uuid'], contract_sha256=sha(root/'contract.json'), unix=time.time(), release_evidence=sorted(eligible), idle_observations=observations, idle_observed_unix=observed_unix)) cmd = [sys.executable, 'tools/phase3_seed1_continuation_guard.py', str(root), str(i), sha(d/'assignment.json')] try: p = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) except BaseException: log.close() raise active[i] = dict(process=p, log=log, start=begin, stop_sent=None, job=j) pending.remove(c['jobs'][i]) save(d / 'dispatch.json', dict(pid=p.pid, command=cmd, unix=time.time(), gpu_uuid=j['gpu_uuid'])) handlers = {s: signal.signal(s, lambda n, f: requested.append('SIGNAL_' + str(n))) for s in (signal.SIGTERM, signal.SIGINT)} save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), new_fits=5, contract_sha256=sha(root / 'contract.json'), review_sha256=sha(root / 'review.json'), gpu_cap=c['gpu_cap'], A_refit=False, test_targets_read=False)) try: while active or pending: for i, v in list(active.items()): collect(i, v) for reason in requested: stop(reason) amount = charged() _, rss, _ = tree_usage(os.getpid()) if amount >= c['train_gpu_cap'] - 80: stop('CUMULATIVE_TRAIN_GPU_BOUNDARY') if rss > c['aggregate_rss_bytes'] or disk_usage(root) > c['aggregate_disk_bytes']: stop('AGGREGATE_MEMORY_DISK_LIMIT') if not (root / 'global-stop.json').exists(): for i in active: files = sorted((root / 'jobs' / str(i) / 'worker/history').glob('*.json')) rows = [read(p) for p in files] if not rows: continue epoch = rows[-1]['epoch'] # Recalculate every epoch: waiting, setup and observed cost stay charged. if epoch > last_epoch.get(i, 0): observation = dict(epoch=epoch, charged=active_charges()[i], max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows), max_validation_seconds=max(r.get('validation_seconds',0) for r in rows)) cell = project_cell(c['jobs'][i], observation) estimates[i] = cell projection = dict(**project_total(c, estimates, terminal, active_charges()), cell=cell, contract_sha256=sha(root / 'contract.json'), unix=time.time()) name = 'first-epoch-projection.json' if i not in last_epoch else 'latest-cost-projection.json' save(root / 'jobs' / str(i) / name, projection) save(root / 'latest-cost-projection.json', projection) last_epoch[i] = epoch if projection['status'] != 'PASS': stop('PROJECTED_SEED1_BUDGET') if not (root / 'global-stop.json').exists(): dispatch() if (root / 'global-stop.json').exists(): for v in active.values(): terminate(v) save(root / 'live-progress.json', dict(stage='STOPPING' if (root / 'global-stop.json').exists() else 'SEED1_TRAINING', unix=time.time(), active=[dict(id=i, guard_pid=v['process'].pid) for i, v in active.items()], terminal=list(terminal.values()), pending=[j['id'] for j in pending], charged_training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve'])) if (root / 'global-stop.json').exists() and not active: break if active or pending: time.sleep(1) if (root / 'global-stop.json').exists() or len(terminal) != 5 or any(v['status'] != 'PASS' for v in terminal.values()): raise RuntimeError('Seed1 training incomplete') config(root) models = [] for j in c['jobs']: w = root / 'jobs' / str(j['id']) / 'worker' r = read(w / 'result.json') stats = read(checked(w / 'standardization.json', r['standardization_sha256'])) if (r['optimizer_steps'] != j['target_steps'] or r['completed_epochs'] != 200 or stats['parts'] != j['learning'] or r['test_targets_read']): raise ValueError('Terminal model identity') checked(r['checkpoint'], r['checkpoint_sha256']) models.append(dict(job=j['id'], path=r['checkpoint'], sha256=r['checkpoint_sha256'], standardization=str(w / 'standardization.json'), standardization_sha256=r['standardization_sha256'])) if charged() > c['train_gpu_cap']: raise ValueError('Terminal cumulative cost') save(root / 'models-terminal.json', dict(models=models, new_fits=5, A_refit=False, test_prediction_started=False)) freeze(root, 'models', [root / 'contract.json', root / 'models-terminal.json', root / 'terminal-ledger.json', *[Path(m['path']) for m in models], *[Path(m['standardization']) for m in models]]) save(root / 'complete.json', dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING', entries=list(terminal.values()), training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve'], gpu_cap=c['gpu_cap'], coordinator_cpu_seconds=time.process_time() - cpu, wall_seconds=time.monotonic() - start, phase3_prior_gpu_once=c['phase3_prior_gpu_once'], cumulative_phase3_training_gpu_seconds=c['phase3_prior_gpu_once']+charged(), no_retry=True, test_targets_read=False, postfit_started=False)) freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'models-terminal.json']) return 0 except BaseException as exc: stop(type(exc).__name__ + ': ' + str(exc)) while active: for i, v in list(active.items()): try: if not collect(i, v): terminate(v) except Exception: terminate(v) if v['process'].poll() is not None: terminal[i] = dict(id=i, status='MALFORMED_TERMINAL', gpu_seconds_charged=c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']) v['log'].close() del active[i] if active: time.sleep(.5) save(root / 'complete.json', dict(status='STOPPED', reason=str(exc), entries=list(terminal.values()), never_attempted=[j['id'] for j in pending], training_gpu_seconds=charged(), postfit_started=False, no_retry=True)) return 1 finally: for s, handler in handlers.items(): signal.signal(s, handler) if __name__ == '__main__': raise SystemExit(run(Path(sys.argv[1])))