Repair seed1 v2 observation coverage and preserve original BLOCKED review
This commit is contained in:
1 parent
bf7363bf96
commit
2dc2f39e37
79 files changed
+12451
-25
No files matched your search
@@ -12,7 +12,7 @@ from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase2_home_freeze_io import freeze
|
||||
from s2_recovery_gpu_supervisor import disk_usage, tree_usage
|
||||
from phase3_seed1_v2_observation import epoch_host_observation
|
||||
from phase3_seed1_v2_observation import drain_epoch_observations
|
||||
from phase3_seed1_v2_common import config, dispatchable, execution_command, project_cell, project_total, read, available_gpus, idle_eligible_gpus
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ def run(root):
|
||||
start = time.monotonic()
|
||||
cpu = time.process_time()
|
||||
active, terminal, estimates, last_epoch = {}, {}, {}, {}
|
||||
observed_epochs = {}
|
||||
pending = list(c['jobs'])
|
||||
requested = []
|
||||
|
||||
@@ -62,10 +63,15 @@ def run(root):
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
|
||||
def observe_completed(i, v, terminal=False, stopping=False):
|
||||
return drain_epoch_observations(root, i, v['job']['gpu_uuid'],
|
||||
observed_epochs.setdefault(i, set()), terminal=terminal, stopping=stopping)
|
||||
|
||||
def collect(i, v):
|
||||
rc = v['process'].poll()
|
||||
if rc is None:
|
||||
return False
|
||||
observe_completed(i, v, terminal=True, stopping=(root/'global-stop.json').exists())
|
||||
p = root / 'jobs' / str(i) / 'complete.json'
|
||||
r = read(p) if p.exists() else None
|
||||
observed = c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
|
||||
@@ -113,7 +119,8 @@ def run(root):
|
||||
try:
|
||||
while active or pending:
|
||||
for i, v in list(active.items()):
|
||||
collect(i, v)
|
||||
if not collect(i, v):
|
||||
observe_completed(i, v, stopping=bool(requested) or (root/'global-stop.json').exists())
|
||||
for reason in requested:
|
||||
stop(reason)
|
||||
amount = charged()
|
||||
@@ -131,7 +138,6 @@ def run(root):
|
||||
epoch = rows[-1]['epoch']
|
||||
# Recalculate every epoch: waiting, setup and observed cost stay charged.
|
||||
if epoch > last_epoch.get(i, 0):
|
||||
epoch_host_observation(root, i, epoch, active[i]['job']['gpu_uuid'])
|
||||
observation = dict(epoch=epoch, charged=active_charges()[i],
|
||||
max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows),
|
||||
max_validation_seconds=max(r.get('validation_seconds',0) for r in rows))
|
||||
@@ -148,7 +154,8 @@ def run(root):
|
||||
if not (root / 'global-stop.json').exists():
|
||||
dispatch()
|
||||
if (root / 'global-stop.json').exists():
|
||||
for v in active.values():
|
||||
for i, v in active.items():
|
||||
observe_completed(i, v, stopping=True)
|
||||
terminate(v)
|
||||
save(root / 'live-progress.json', dict(stage='STOPPING' if (root / 'global-stop.json').exists()
|
||||
else 'SEED1_TRAINING', unix=time.time(), active=[dict(id=i, guard_pid=v['process'].pid)
|
||||
@@ -191,10 +198,15 @@ def run(root):
|
||||
for i, v in list(active.items()):
|
||||
try:
|
||||
if not collect(i, v):
|
||||
observe_completed(i, v, stopping=True)
|
||||
terminate(v)
|
||||
except Exception:
|
||||
terminate(v)
|
||||
if v['process'].poll() is not None:
|
||||
try:
|
||||
observe_completed(i, v, terminal=True, stopping=True)
|
||||
except Exception as observation_error:
|
||||
save(root/'jobs'/str(i)/'observation-coverage-error.json', dict(status='OBSERVATION_INCOMPLETE', unix=time.time(), reason=str(observation_error)))
|
||||
terminal[i] = dict(id=i, status='MALFORMED_TERMINAL',
|
||||
gpu_seconds_charged=c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'])
|
||||
v['log'].close()
|
||||
|
||||
Reference in new issue
Block a user