213 lines
11 KiB
Python
213 lines
11 KiB
Python
"""Five original grouped folds on released GPUs; train and freeze only, never score."""
|
|
import fcntl
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
import psutil
|
|
from phase2_gate_outcome_review import validate_review
|
|
from phase2_h1_worker_v2 import checked, save, sha
|
|
from phase2_home_freeze_io import freeze
|
|
from s2_recovery_gpu_supervisor import disk_usage, tree_usage
|
|
from phase3_seed1_common import config, dispatchable, execution_command, project_cell, project_total, read, available_gpus, idle_eligible_gpus
|
|
|
|
|
|
def run(root):
|
|
c, _ = config(root)
|
|
validate_review(root)
|
|
lock = (root / 'queue.lock').open('a')
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if (root / 'started.json').exists():
|
|
raise FileExistsError('One Seed1 attempt; no retry')
|
|
for p in psutil.process_iter(['pid', 'cmdline']):
|
|
if p.pid != os.getpid() and execution_command(p.info['cmdline'] or []):
|
|
raise RuntimeError('Existing Seed1 execution')
|
|
os.sched_setaffinity(0, c['coordinator_cpus'])
|
|
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
|
|
start = time.monotonic()
|
|
cpu = time.process_time()
|
|
active, terminal, estimates, last_epoch = {}, {}, {}, {}
|
|
pending = list(c['jobs'])
|
|
requested = []
|
|
|
|
def stop(reason):
|
|
if not (root / 'global-stop.json').exists():
|
|
save(root / 'global-stop.json', dict(reason=reason, unix=time.time(), no_retry=True))
|
|
|
|
def active_charges():
|
|
return {i: c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
|
|
for i, v in active.items()}
|
|
|
|
def charged():
|
|
return sum(v['gpu_seconds_charged'] for v in terminal.values()) + sum(active_charges().values())
|
|
|
|
def terminate(v):
|
|
if v.get('stop_sent') is None:
|
|
if v['process'].poll() is None:
|
|
v['process'].send_signal(signal.SIGTERM)
|
|
v['stop_sent'] = time.monotonic()
|
|
elif time.monotonic() - v['stop_sent'] > 20:
|
|
try:
|
|
parent = psutil.Process(v['process'].pid)
|
|
for p in reversed(parent.children(recursive=True)):
|
|
try:
|
|
p.kill()
|
|
except psutil.NoSuchProcess:
|
|
pass
|
|
parent.kill()
|
|
except psutil.NoSuchProcess:
|
|
pass
|
|
|
|
def collect(i, v):
|
|
rc = v['process'].poll()
|
|
if rc is None:
|
|
return False
|
|
p = root / 'jobs' / str(i) / 'complete.json'
|
|
r = read(p) if p.exists() else None
|
|
observed = c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
|
|
ok = (rc == 0 and r is not None and r['status'] == 'PASS'
|
|
and not r.get('monitor_errors') and r.get('worker') is not None
|
|
and not r['worker'].get('monitor_errors'))
|
|
terminal[i] = dict(id=i, status='PASS' if ok else 'STOPPED_OR_FAILED', returncode=rc,
|
|
gpu_seconds_charged=max(observed, r['gpu_seconds_charged'] if r else 0),
|
|
guard_complete_sha256=sha(p) if r else None, gpu_uuid=v['job']['gpu_uuid'], assignment_sha256=sha(p.parent/'assignment.json'))
|
|
v['log'].close()
|
|
del active[i]
|
|
save(root / 'terminal-ledger.json', dict(entries=list(terminal.values()),
|
|
settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values())))
|
|
if not ok:
|
|
stop('JOB_' + str(i) + '_FAILED')
|
|
return True
|
|
|
|
def dispatch():
|
|
eligible=available_gpus(c)-{v['job']['gpu_uuid'] for v in active.values()}
|
|
idle,observations=idle_eligible_gpus(eligible)
|
|
observed_unix=time.time()
|
|
save(root/'dispatch-idle-observation.json',dict(unix=observed_unix,eligible=sorted(eligible),observations=observations))
|
|
for j in dispatchable(pending, [v['job'] for v in active.values()], idle):
|
|
i = j['id']
|
|
d = root / 'jobs' / str(i)
|
|
d.mkdir(parents=True, exist_ok=False)
|
|
log = (d / 'entry.log').open('x')
|
|
begin = time.monotonic()
|
|
save(d / 'assignment.json', dict(id=i, gpu_uuid=j['gpu_uuid'], contract_sha256=sha(root/'contract.json'), unix=time.time(), release_evidence=sorted(eligible), idle_observations=observations, idle_observed_unix=observed_unix))
|
|
cmd = [sys.executable, 'tools/phase3_seed1_guard.py', str(root), str(i), sha(d/'assignment.json')]
|
|
try:
|
|
p = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
|
except BaseException:
|
|
log.close()
|
|
raise
|
|
active[i] = dict(process=p, log=log, start=begin, stop_sent=None, job=j)
|
|
pending.remove(c['jobs'][i])
|
|
save(d / 'dispatch.json', dict(pid=p.pid, command=cmd, unix=time.time(), gpu_uuid=j['gpu_uuid']))
|
|
|
|
handlers = {s: signal.signal(s, lambda n, f: requested.append('SIGNAL_' + str(n)))
|
|
for s in (signal.SIGTERM, signal.SIGINT)}
|
|
save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), new_fits=5,
|
|
contract_sha256=sha(root / 'contract.json'), review_sha256=sha(root / 'review.json'),
|
|
gpu_cap=c['gpu_cap'], A_refit=False, test_targets_read=False))
|
|
try:
|
|
while active or pending:
|
|
for i, v in list(active.items()):
|
|
collect(i, v)
|
|
for reason in requested:
|
|
stop(reason)
|
|
amount = charged()
|
|
_, rss, _ = tree_usage(os.getpid())
|
|
if amount >= c['train_gpu_cap'] - 80:
|
|
stop('CUMULATIVE_TRAIN_GPU_BOUNDARY')
|
|
if rss > c['aggregate_rss_bytes'] or disk_usage(root) > c['aggregate_disk_bytes']:
|
|
stop('AGGREGATE_MEMORY_DISK_LIMIT')
|
|
if not (root / 'global-stop.json').exists():
|
|
for i in active:
|
|
files = sorted((root / 'jobs' / str(i) / 'worker/history').glob('*.json'))
|
|
rows = [read(p) for p in files]
|
|
if not rows:
|
|
continue
|
|
epoch = rows[-1]['epoch']
|
|
# Recalculate every epoch: waiting, setup and observed cost stay charged.
|
|
if epoch > last_epoch.get(i, 0):
|
|
observation = dict(epoch=epoch, charged=active_charges()[i],
|
|
max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows),
|
|
max_validation_seconds=max(r.get('validation_seconds',0) for r in rows))
|
|
cell = project_cell(c['jobs'][i], observation)
|
|
estimates[i] = cell
|
|
projection = dict(**project_total(c, estimates, terminal, active_charges()),
|
|
cell=cell, contract_sha256=sha(root / 'contract.json'), unix=time.time())
|
|
name = 'first-epoch-projection.json' if i not in last_epoch else 'latest-cost-projection.json'
|
|
save(root / 'jobs' / str(i) / name, projection)
|
|
save(root / 'latest-cost-projection.json', projection)
|
|
last_epoch[i] = epoch
|
|
if projection['status'] != 'PASS':
|
|
stop('PROJECTED_SEED1_BUDGET')
|
|
if not (root / 'global-stop.json').exists():
|
|
dispatch()
|
|
if (root / 'global-stop.json').exists():
|
|
for v in active.values():
|
|
terminate(v)
|
|
save(root / 'live-progress.json', dict(stage='STOPPING' if (root / 'global-stop.json').exists()
|
|
else 'SEED1_TRAINING', unix=time.time(), active=[dict(id=i, guard_pid=v['process'].pid)
|
|
for i, v in active.items()], terminal=list(terminal.values()), pending=[j['id'] for j in pending],
|
|
charged_training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve']))
|
|
if (root / 'global-stop.json').exists() and not active:
|
|
break
|
|
if active or pending:
|
|
time.sleep(1)
|
|
if (root / 'global-stop.json').exists() or len(terminal) != 5 or any(v['status'] != 'PASS' for v in terminal.values()):
|
|
raise RuntimeError('Seed1 training incomplete')
|
|
config(root)
|
|
models = []
|
|
for j in c['jobs']:
|
|
w = root / 'jobs' / str(j['id']) / 'worker'
|
|
r = read(w / 'result.json')
|
|
stats = read(checked(w / 'standardization.json', r['standardization_sha256']))
|
|
if (r['optimizer_steps'] != j['target_steps'] or r['completed_epochs'] != 200
|
|
or stats['parts'] != j['learning'] or r['test_targets_read']):
|
|
raise ValueError('Terminal model identity')
|
|
checked(r['checkpoint'], r['checkpoint_sha256'])
|
|
models.append(dict(job=j['id'], path=r['checkpoint'], sha256=r['checkpoint_sha256'],
|
|
standardization=str(w / 'standardization.json'),
|
|
standardization_sha256=r['standardization_sha256']))
|
|
if charged() > c['train_gpu_cap']:
|
|
raise ValueError('Terminal cumulative cost')
|
|
save(root / 'models-terminal.json', dict(models=models, new_fits=5, A_refit=False, test_prediction_started=False))
|
|
freeze(root, 'models', [root / 'contract.json', root / 'models-terminal.json', root / 'terminal-ledger.json',
|
|
*[Path(m['path']) for m in models], *[Path(m['standardization']) for m in models]])
|
|
save(root / 'complete.json', dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING', entries=list(terminal.values()),
|
|
training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve'], gpu_cap=c['gpu_cap'],
|
|
coordinator_cpu_seconds=time.process_time() - cpu, wall_seconds=time.monotonic() - start,
|
|
phase3_prior_gpu_once=c['phase3_prior_gpu_once'], cumulative_phase3_training_gpu_seconds=c['phase3_prior_gpu_once']+charged(),
|
|
no_retry=True, test_targets_read=False, postfit_started=False))
|
|
freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'models-terminal.json'])
|
|
return 0
|
|
except BaseException as exc:
|
|
stop(type(exc).__name__ + ': ' + str(exc))
|
|
while active:
|
|
for i, v in list(active.items()):
|
|
try:
|
|
if not collect(i, v):
|
|
terminate(v)
|
|
except Exception:
|
|
terminate(v)
|
|
if v['process'].poll() is not None:
|
|
terminal[i] = dict(id=i, status='MALFORMED_TERMINAL',
|
|
gpu_seconds_charged=c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'])
|
|
v['log'].close()
|
|
del active[i]
|
|
if active:
|
|
time.sleep(.5)
|
|
save(root / 'complete.json', dict(status='STOPPED', reason=str(exc), entries=list(terminal.values()),
|
|
never_attempted=[j['id'] for j in pending], training_gpu_seconds=charged(),
|
|
postfit_started=False, no_retry=True))
|
|
return 1
|
|
finally:
|
|
for s, handler in handlers.items():
|
|
signal.signal(s, handler)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(run(Path(sys.argv[1])))
|