Repair seed1 v2 observation coverage and preserve original BLOCKED review

This commit is contained in:
MeltAstra committed 2026-11-18 16:50:51 +08:00
1 parent bf7363bf96
commit 2dc2f39e37
79 files changed
+12451 -25

No files matched your search

+16 -4
View File
@@ -12,7 +12,7 @@ from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, save, sha
from phase2_home_freeze_io import freeze
from s2_recovery_gpu_supervisor import disk_usage, tree_usage
from phase3_seed1_v2_observation import epoch_host_observation
from phase3_seed1_v2_observation import drain_epoch_observations
from phase3_seed1_v2_common import config, dispatchable, execution_command, project_cell, project_total, read, available_gpus, idle_eligible_gpus
@@ -31,6 +31,7 @@ def run(root):
start = time.monotonic()
cpu = time.process_time()
active, terminal, estimates, last_epoch = {}, {}, {}, {}
observed_epochs = {}
pending = list(c['jobs'])
requested = []
@@ -62,10 +63,15 @@ def run(root):
except psutil.NoSuchProcess:
pass
def observe_completed(i, v, terminal=False, stopping=False):
return drain_epoch_observations(root, i, v['job']['gpu_uuid'],
observed_epochs.setdefault(i, set()), terminal=terminal, stopping=stopping)
def collect(i, v):
rc = v['process'].poll()
if rc is None:
return False
observe_completed(i, v, terminal=True, stopping=(root/'global-stop.json').exists())
p = root / 'jobs' / str(i) / 'complete.json'
r = read(p) if p.exists() else None
observed = c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
@@ -113,7 +119,8 @@ def run(root):
try:
while active or pending:
for i, v in list(active.items()):
collect(i, v)
if not collect(i, v):
observe_completed(i, v, stopping=bool(requested) or (root/'global-stop.json').exists())
for reason in requested:
stop(reason)
amount = charged()
@@ -131,7 +138,6 @@ def run(root):
epoch = rows[-1]['epoch']
# Recalculate every epoch: waiting, setup and observed cost stay charged.
if epoch > last_epoch.get(i, 0):
epoch_host_observation(root, i, epoch, active[i]['job']['gpu_uuid'])
observation = dict(epoch=epoch, charged=active_charges()[i],
max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows),
max_validation_seconds=max(r.get('validation_seconds',0) for r in rows))
@@ -148,7 +154,8 @@ def run(root):
if not (root / 'global-stop.json').exists():
dispatch()
if (root / 'global-stop.json').exists():
for v in active.values():
for i, v in active.items():
observe_completed(i, v, stopping=True)
terminate(v)
save(root / 'live-progress.json', dict(stage='STOPPING' if (root / 'global-stop.json').exists()
else 'SEED1_TRAINING', unix=time.time(), active=[dict(id=i, guard_pid=v['process'].pid)
@@ -191,10 +198,15 @@ def run(root):
for i, v in list(active.items()):
try:
if not collect(i, v):
observe_completed(i, v, stopping=True)
terminate(v)
except Exception:
terminate(v)
if v['process'].poll() is not None:
try:
observe_completed(i, v, terminal=True, stopping=True)
except Exception as observation_error:
save(root/'jobs'/str(i)/'observation-coverage-error.json', dict(status='OBSERVATION_INCOMPLETE', unix=time.time(), reason=str(observation_error)))
terminal[i] = dict(id=i, status='MALFORMED_TERMINAL',
gpu_seconds_charged=c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'])
v['log'].close()