Files
MeltMind/tools/phase3_seed1_v2_queue.py
T

227 lines
12 KiB
Python

"""Five original grouped folds on released GPUs; train and freeze only, never score."""
import fcntl
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
import psutil
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, save, sha
from phase2_home_freeze_io import freeze
from s2_recovery_gpu_supervisor import disk_usage, tree_usage
from phase3_seed1_v2_observation import drain_epoch_observations
from phase3_seed1_v2_common import config, dispatchable, execution_command, project_cell, project_total, read, available_gpus, idle_eligible_gpus
def run(root):
c, _ = config(root)
validate_review(root)
lock = (root / 'queue.lock').open('a')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (root / 'started.json').exists():
raise FileExistsError('One Seed1 attempt; no retry')
for p in psutil.process_iter(['pid', 'cmdline']):
if p.pid != os.getpid() and execution_command(p.info['cmdline'] or []):
raise RuntimeError('Existing Seed1 execution')
os.sched_setaffinity(0, c['coordinator_cpus'])
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
start = time.monotonic()
cpu = time.process_time()
active, terminal, estimates, last_epoch = {}, {}, {}, {}
observed_epochs = {}
pending = list(c['jobs'])
requested = []
def stop(reason):
if not (root / 'global-stop.json').exists():
save(root / 'global-stop.json', dict(reason=reason, unix=time.time(), no_retry=True))
def active_charges():
return {i: c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
for i, v in active.items()}
def charged():
return sum(v['gpu_seconds_charged'] for v in terminal.values()) + sum(active_charges().values())
def terminate(v):
if v.get('stop_sent') is None:
if v['process'].poll() is None:
v['process'].send_signal(signal.SIGTERM)
v['stop_sent'] = time.monotonic()
elif time.monotonic() - v['stop_sent'] > 20:
try:
parent = psutil.Process(v['process'].pid)
for p in reversed(parent.children(recursive=True)):
try:
p.kill()
except psutil.NoSuchProcess:
pass
parent.kill()
except psutil.NoSuchProcess:
pass
def observe_completed(i, v, terminal=False, stopping=False):
return drain_epoch_observations(root, i, v['job']['gpu_uuid'],
observed_epochs.setdefault(i, set()), terminal=terminal, stopping=stopping)
def collect(i, v):
rc = v['process'].poll()
if rc is None:
return False
observe_completed(i, v, terminal=True, stopping=(root/'global-stop.json').exists())
p = root / 'jobs' / str(i) / 'complete.json'
r = read(p) if p.exists() else None
observed = c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
ok = (rc == 0 and r is not None and r['status'] == 'PASS'
and not r.get('monitor_errors') and r.get('worker') is not None
and not r['worker'].get('monitor_errors'))
terminal[i] = dict(id=i, status='PASS' if ok else 'STOPPED_OR_FAILED', returncode=rc,
gpu_seconds_charged=max(observed, r['gpu_seconds_charged'] if r else 0),
guard_complete_sha256=sha(p) if r else None, gpu_uuid=v['job']['gpu_uuid'], assignment_sha256=sha(p.parent/'assignment.json'))
v['log'].close()
del active[i]
save(root / 'terminal-ledger.json', dict(entries=list(terminal.values()),
settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values())))
if not ok:
stop('JOB_' + str(i) + '_FAILED')
return True
def dispatch():
eligible=available_gpus(c)-{v['job']['gpu_uuid'] for v in active.values()}
idle,observations=idle_eligible_gpus(eligible)
observed_unix=time.time()
save(root/'dispatch-idle-observation.json',dict(unix=observed_unix,eligible=sorted(eligible),observations=observations))
for j in dispatchable(pending, [v['job'] for v in active.values()], idle):
i = j['id']
d = root / 'jobs' / str(i)
d.mkdir(parents=True, exist_ok=False)
log = (d / 'entry.log').open('x')
begin = time.monotonic()
save(d / 'assignment.json', dict(id=i, gpu_uuid=j['gpu_uuid'], contract_sha256=sha(root/'contract.json'), unix=time.time(), release_evidence=sorted(eligible), idle_observations=observations, idle_observed_unix=observed_unix))
cmd = [sys.executable, 'tools/phase3_seed1_v2_guard.py', str(root), str(i), sha(d/'assignment.json')]
try:
p = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
except BaseException:
log.close()
raise
active[i] = dict(process=p, log=log, start=begin, stop_sent=None, job=j)
pending.remove(c['jobs'][i])
save(d / 'dispatch.json', dict(pid=p.pid, command=cmd, unix=time.time(), gpu_uuid=j['gpu_uuid']))
handlers = {s: signal.signal(s, lambda n, f: requested.append('SIGNAL_' + str(n)))
for s in (signal.SIGTERM, signal.SIGINT)}
save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), new_fits=5,
contract_sha256=sha(root / 'contract.json'), review_sha256=sha(root / 'review.json'),
gpu_cap=c['gpu_cap'], A_refit=False, test_targets_read=False))
try:
while active or pending:
for i, v in list(active.items()):
if not collect(i, v):
observe_completed(i, v, stopping=bool(requested) or (root/'global-stop.json').exists())
for reason in requested:
stop(reason)
amount = charged()
_, rss, _ = tree_usage(os.getpid())
if amount >= c['train_gpu_cap'] - 80:
stop('CUMULATIVE_TRAIN_GPU_BOUNDARY')
if rss > c['aggregate_rss_bytes'] or disk_usage(root) > c['aggregate_disk_bytes']:
stop('AGGREGATE_MEMORY_DISK_LIMIT')
if not (root / 'global-stop.json').exists():
for i in active:
files = sorted((root / 'jobs' / str(i) / 'worker/history').glob('*.json'))
rows = [read(p) for p in files]
if not rows:
continue
epoch = rows[-1]['epoch']
# Recalculate every epoch: waiting, setup and observed cost stay charged.
if epoch > last_epoch.get(i, 0):
observation = dict(epoch=epoch, charged=active_charges()[i],
max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows),
max_validation_seconds=max(r.get('validation_seconds',0) for r in rows))
cell = project_cell(c['jobs'][i], observation)
estimates[i] = cell
projection = dict(**project_total(c, estimates, terminal, active_charges()),
cell=cell, contract_sha256=sha(root / 'contract.json'), unix=time.time())
name = 'first-epoch-projection.json' if i not in last_epoch else 'latest-cost-projection.json'
save(root / 'jobs' / str(i) / name, projection)
save(root / 'latest-cost-projection.json', projection)
last_epoch[i] = epoch
if projection['status'] != 'PASS':
stop('PROJECTED_SEED1_BUDGET')
if not (root / 'global-stop.json').exists():
dispatch()
if (root / 'global-stop.json').exists():
for i, v in active.items():
observe_completed(i, v, stopping=True)
terminate(v)
save(root / 'live-progress.json', dict(stage='STOPPING' if (root / 'global-stop.json').exists()
else 'SEED1_TRAINING', unix=time.time(), active=[dict(id=i, guard_pid=v['process'].pid)
for i, v in active.items()], terminal=list(terminal.values()), pending=[j['id'] for j in pending],
charged_training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve']))
if (root / 'global-stop.json').exists() and not active:
break
if active or pending:
time.sleep(1)
if (root / 'global-stop.json').exists() or len(terminal) != 5 or any(v['status'] != 'PASS' for v in terminal.values()):
raise RuntimeError('Seed1 training incomplete')
config(root)
models = []
for j in c['jobs']:
w = root / 'jobs' / str(j['id']) / 'worker'
r = read(w / 'result.json')
stats = read(checked(w / 'standardization.json', r['standardization_sha256']))
if (r['optimizer_steps'] != j['target_steps'] or r['completed_epochs'] != 200
or stats['parts'] != j['learning'] or r['test_targets_read']):
raise ValueError('Terminal model identity')
checked(r['checkpoint'], r['checkpoint_sha256'])
models.append(dict(job=j['id'], path=r['checkpoint'], sha256=r['checkpoint_sha256'],
standardization=str(w / 'standardization.json'),
standardization_sha256=r['standardization_sha256']))
if charged() > c['train_gpu_cap']:
raise ValueError('Terminal cumulative cost')
save(root / 'models-terminal.json', dict(models=models, new_fits=5, A_refit=False, test_prediction_started=False))
freeze(root, 'models', [root / 'contract.json', root / 'models-terminal.json', root / 'terminal-ledger.json',
*[Path(m['path']) for m in models], *[Path(m['standardization']) for m in models]])
save(root / 'complete.json', dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING', entries=list(terminal.values()),
training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve'], gpu_cap=c['gpu_cap'],
coordinator_cpu_seconds=time.process_time() - cpu, wall_seconds=time.monotonic() - start,
phase3_prior_gpu_once=c['phase3_prior_gpu_once'], cumulative_phase3_training_gpu_seconds=c['phase3_prior_gpu_once']+charged(),
no_retry=True, test_targets_read=False, postfit_started=False))
freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'models-terminal.json'])
return 0
except BaseException as exc:
stop(type(exc).__name__ + ': ' + str(exc))
while active:
for i, v in list(active.items()):
try:
if not collect(i, v):
observe_completed(i, v, stopping=True)
terminate(v)
except Exception:
terminate(v)
if v['process'].poll() is not None:
try:
observe_completed(i, v, terminal=True, stopping=True)
except Exception as observation_error:
save(root/'jobs'/str(i)/'observation-coverage-error.json', dict(status='OBSERVATION_INCOMPLETE', unix=time.time(), reason=str(observation_error)))
terminal[i] = dict(id=i, status='MALFORMED_TERMINAL',
gpu_seconds_charged=c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'])
v['log'].close()
del active[i]
if active:
time.sleep(.5)
save(root / 'complete.json', dict(status='STOPPED', reason=str(exc), entries=list(terminal.values()),
never_attempted=[j['id'] for j in pending], training_gpu_seconds=charged(),
postfit_started=False, no_retry=True))
return 1
finally:
for s, handler in handlers.items():
signal.signal(s, handler)
if __name__ == '__main__':
raise SystemExit(run(Path(sys.argv[1])))