Prepare reviewed-scope Track A H2 training entry and synthetic acceptance
This commit is contained in:
1 parent
1eae27ade8
commit
2cf0a0307d
18 files changed
+43660
No files matched your search
@@ -0,0 +1,205 @@
|
||||
"""Five independent fits on two idle GPUs; train and freeze only, never score."""
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import psutil
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase2_home_freeze_io import freeze
|
||||
from s2_recovery_gpu_supervisor import disk_usage, tree_usage
|
||||
from track_a_h2_common import config, dispatchable, execution_command, project_cell, project_total, read
|
||||
|
||||
|
||||
def run(root):
|
||||
c, _ = config(root)
|
||||
validate_review(root)
|
||||
lock = (root / 'queue.lock').open('a')
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if (root / 'started.json').exists():
|
||||
raise FileExistsError('One Track A attempt; no retry')
|
||||
for p in psutil.process_iter(['pid', 'cmdline']):
|
||||
if p.pid != os.getpid() and execution_command(p.info['cmdline'] or []):
|
||||
raise RuntimeError('Existing Track A execution')
|
||||
os.sched_setaffinity(0, c['coordinator_cpus'])
|
||||
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
|
||||
start = time.monotonic()
|
||||
cpu = time.process_time()
|
||||
active, terminal, estimates, last_epoch = {}, {}, {}, {}
|
||||
pending = list(c['jobs'])
|
||||
requested = []
|
||||
|
||||
def stop(reason):
|
||||
if not (root / 'global-stop.json').exists():
|
||||
save(root / 'global-stop.json', dict(reason=reason, unix=time.time(), no_retry=True))
|
||||
|
||||
def active_charges():
|
||||
return {i: c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
|
||||
for i, v in active.items()}
|
||||
|
||||
def charged():
|
||||
return sum(v['gpu_seconds_charged'] for v in terminal.values()) + sum(active_charges().values())
|
||||
|
||||
def terminate(v):
|
||||
if v.get('stop_sent') is None:
|
||||
if v['process'].poll() is None:
|
||||
v['process'].send_signal(signal.SIGTERM)
|
||||
v['stop_sent'] = time.monotonic()
|
||||
elif time.monotonic() - v['stop_sent'] > 20:
|
||||
try:
|
||||
parent = psutil.Process(v['process'].pid)
|
||||
for p in reversed(parent.children(recursive=True)):
|
||||
try:
|
||||
p.kill()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
parent.kill()
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
|
||||
def collect(i, v):
|
||||
rc = v['process'].poll()
|
||||
if rc is None:
|
||||
return False
|
||||
p = root / 'jobs' / str(i) / 'complete.json'
|
||||
r = read(p) if p.exists() else None
|
||||
observed = c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
|
||||
ok = (rc == 0 and r is not None and r['status'] == 'PASS'
|
||||
and not r.get('monitor_errors') and r.get('worker') is not None
|
||||
and not r['worker'].get('monitor_errors'))
|
||||
terminal[i] = dict(id=i, status='PASS' if ok else 'STOPPED_OR_FAILED', returncode=rc,
|
||||
gpu_seconds_charged=max(observed, r['gpu_seconds_charged'] if r else 0),
|
||||
guard_complete_sha256=sha(p) if r else None)
|
||||
v['log'].close()
|
||||
del active[i]
|
||||
save(root / 'terminal-ledger.json', dict(entries=list(terminal.values()),
|
||||
settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values())))
|
||||
if not ok:
|
||||
stop('JOB_' + str(i) + '_FAILED')
|
||||
return True
|
||||
|
||||
def dispatch():
|
||||
for j in dispatchable(pending, [c['jobs'][i] for i in active]):
|
||||
i = j['id']
|
||||
d = root / 'jobs' / str(i)
|
||||
d.mkdir(parents=True, exist_ok=False)
|
||||
log = (d / 'entry.log').open('x')
|
||||
begin = time.monotonic()
|
||||
cmd = [sys.executable, 'tools/track_a_h2_guard.py', str(root), str(i)]
|
||||
try:
|
||||
p = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
except BaseException:
|
||||
log.close()
|
||||
raise
|
||||
active[i] = dict(process=p, log=log, start=begin, stop_sent=None)
|
||||
pending.remove(j)
|
||||
save(d / 'dispatch.json', dict(pid=p.pid, command=cmd, unix=time.time(), gpu_uuid=j['gpu_uuid']))
|
||||
|
||||
handlers = {s: signal.signal(s, lambda n, f: requested.append('SIGNAL_' + str(n)))
|
||||
for s in (signal.SIGTERM, signal.SIGINT)}
|
||||
save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), new_fits=5,
|
||||
contract_sha256=sha(root / 'contract.json'), review_sha256=sha(root / 'review.json'),
|
||||
gpu_cap=c['gpu_cap'], A_refit=False, test_targets_read=False))
|
||||
try:
|
||||
while active or pending:
|
||||
for i, v in list(active.items()):
|
||||
collect(i, v)
|
||||
for reason in requested:
|
||||
stop(reason)
|
||||
amount = charged()
|
||||
_, rss, _ = tree_usage(os.getpid())
|
||||
if amount >= c['train_gpu_cap'] - 80:
|
||||
stop('CUMULATIVE_TRAIN_GPU_BOUNDARY')
|
||||
if rss > c['aggregate_rss_bytes'] or disk_usage(root) > c['aggregate_disk_bytes']:
|
||||
stop('AGGREGATE_MEMORY_DISK_LIMIT')
|
||||
if not (root / 'global-stop.json').exists():
|
||||
for i in active:
|
||||
files = sorted((root / 'jobs' / str(i) / 'worker/history').glob('*.json'))
|
||||
rows = [read(p) for p in files]
|
||||
if not rows:
|
||||
continue
|
||||
epoch = rows[-1]['epoch']
|
||||
# Recalculate every epoch: waiting, setup and observed cost stay charged.
|
||||
if epoch > last_epoch.get(i, 0):
|
||||
observation = dict(epoch=epoch, charged=active_charges()[i],
|
||||
max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows))
|
||||
cell = project_cell(c['jobs'][i], observation)
|
||||
estimates[i] = cell
|
||||
projection = dict(**project_total(c, estimates, terminal, active_charges()),
|
||||
cell=cell, contract_sha256=sha(root / 'contract.json'), unix=time.time())
|
||||
name = 'first-epoch-projection.json' if i not in last_epoch else 'latest-cost-projection.json'
|
||||
save(root / 'jobs' / str(i) / name, projection)
|
||||
save(root / 'latest-cost-projection.json', projection)
|
||||
last_epoch[i] = epoch
|
||||
if projection['status'] != 'PASS':
|
||||
stop('PROJECTED_TRACK_A_BUDGET')
|
||||
if not (root / 'global-stop.json').exists():
|
||||
dispatch()
|
||||
if (root / 'global-stop.json').exists():
|
||||
for v in active.values():
|
||||
terminate(v)
|
||||
save(root / 'live-progress.json', dict(stage='STOPPING' if (root / 'global-stop.json').exists()
|
||||
else 'TRACK_A_TRAINING', unix=time.time(), active=[dict(id=i, guard_pid=v['process'].pid)
|
||||
for i, v in active.items()], terminal=list(terminal.values()), pending=[j['id'] for j in pending],
|
||||
charged_training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve']))
|
||||
if (root / 'global-stop.json').exists() and not active:
|
||||
break
|
||||
if active:
|
||||
time.sleep(1)
|
||||
if (root / 'global-stop.json').exists() or len(terminal) != 5 or any(v['status'] != 'PASS' for v in terminal.values()):
|
||||
raise RuntimeError('Track A training incomplete')
|
||||
config(root)
|
||||
models = []
|
||||
for j in c['jobs']:
|
||||
w = root / 'jobs' / str(j['id']) / 'worker'
|
||||
r = read(w / 'result.json')
|
||||
stats = read(checked(w / 'standardization.json', r['standardization_sha256']))
|
||||
if (r['optimizer_steps'] != j['target_steps'] or r['completed_epochs'] != 200
|
||||
or stats['parts'] != j['learning'] or r['test_targets_read']):
|
||||
raise ValueError('Terminal model identity')
|
||||
checked(r['checkpoint'], r['checkpoint_sha256'])
|
||||
models.append(dict(job=j['id'], path=r['checkpoint'], sha256=r['checkpoint_sha256'],
|
||||
standardization=str(w / 'standardization.json'),
|
||||
standardization_sha256=r['standardization_sha256']))
|
||||
if charged() > c['train_gpu_cap']:
|
||||
raise ValueError('Terminal cumulative cost')
|
||||
save(root / 'models-terminal.json', dict(models=models, new_fits=5, A_refit=False, test_prediction_started=False))
|
||||
freeze(root, 'models', [root / 'contract.json', root / 'models-terminal.json', root / 'terminal-ledger.json',
|
||||
*[Path(m['path']) for m in models], *[Path(m['standardization']) for m in models]])
|
||||
save(root / 'complete.json', dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING', entries=list(terminal.values()),
|
||||
training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve'], gpu_cap=c['gpu_cap'],
|
||||
coordinator_cpu_seconds=time.process_time() - cpu, wall_seconds=time.monotonic() - start,
|
||||
no_retry=True, test_targets_read=False, postfit_started=False))
|
||||
freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'models-terminal.json'])
|
||||
return 0
|
||||
except BaseException as exc:
|
||||
stop(type(exc).__name__ + ': ' + str(exc))
|
||||
while active:
|
||||
for i, v in list(active.items()):
|
||||
try:
|
||||
if not collect(i, v):
|
||||
terminate(v)
|
||||
except Exception:
|
||||
terminate(v)
|
||||
if v['process'].poll() is not None:
|
||||
terminal[i] = dict(id=i, status='MALFORMED_TERMINAL',
|
||||
gpu_seconds_charged=c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'])
|
||||
v['log'].close()
|
||||
del active[i]
|
||||
if active:
|
||||
time.sleep(.5)
|
||||
save(root / 'complete.json', dict(status='STOPPED', reason=str(exc), entries=list(terminal.values()),
|
||||
never_attempted=[j['id'] for j in pending], training_gpu_seconds=charged(),
|
||||
postfit_started=False, no_retry=True))
|
||||
return 1
|
||||
finally:
|
||||
for s, handler in handlers.items():
|
||||
signal.signal(s, handler)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(run(Path(sys.argv[1])))
|
||||
Reference in new issue
Block a user