Repair seed1 v2 observation coverage and preserve original BLOCKED review
This commit is contained in:
1 parent
bf7363bf96
commit
2dc2f39e37
79 files changed
+12451
-25
No files matched your search
@@ -1,15 +1,23 @@
|
||||
"""Epoch-boundary host observations only; no scheduling or causal decisions."""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
|
||||
def epoch_host_observation(root,fold,epoch,gpu_uuid):
|
||||
def epoch_host_observation(root,fold,epoch,gpu_uuid,receipt=None,incomplete_reason=None):
|
||||
record=dict(unix=time.time(),fold=fold,completed_epoch=epoch,assigned_gpu_uuid=gpu_uuid,observation_only=True,sampling='After the coordinator reads the completed-epoch receipt; not continuous monitoring and not causal proof',gpu_processes=[],errors=[])
|
||||
if receipt is not None:
|
||||
receipt=Path(receipt)
|
||||
record.update(receipt_path=str(receipt),receipt_sha256=sha(receipt),receipt_mtime_unix=receipt.stat().st_mtime,receipt_age_seconds=max(0,record['unix']-receipt.stat().st_mtime))
|
||||
if incomplete_reason is not None:
|
||||
record.update(status='OBSERVATION_INCOMPLETE',incomplete_reason=incomplete_reason,snapshot_started_unix=None,snapshot_finished_unix=None,own_worker_pids=[],historical_gpu_occupancy_observed=False)
|
||||
return write_observation(root,fold,epoch,record)
|
||||
record['snapshot_started_unix']=time.time()
|
||||
try:record['load_average']=list(os.getloadavg())
|
||||
except OSError as exc:record['errors'].append(dict(stage='load_average',type=type(exc).__name__,reason=str(exc)))
|
||||
own=set()
|
||||
@@ -26,9 +34,43 @@ def epoch_host_observation(root,fold,epoch,gpu_uuid):
|
||||
uuid,pid,name,memory=[s.strip() for s in row];pid=int(pid)
|
||||
record['gpu_processes'].append(dict(gpu_uuid=uuid,pid=pid,process_name=name,used_memory_mib=memory,own_v2_worker=pid in own,on_assigned_gpu=uuid==gpu_uuid,foreign_on_assigned_gpu=uuid==gpu_uuid and pid not in own))
|
||||
except Exception as exc:record['errors'].append(dict(stage='gpu_process_snapshot',type=type(exc).__name__,reason=str(exc)))
|
||||
if not any(p['own_v2_worker'] and p['on_assigned_gpu'] for p in record['gpu_processes']):
|
||||
record['errors'].append(dict(stage='epoch_worker_visibility',reason='Assigned v2 worker not visible at snapshot; historical occupancy not observed'))
|
||||
record['own_worker_pids']=sorted(own);record['status']='OBSERVED' if not record['errors'] else 'OBSERVATION_INCOMPLETE'
|
||||
record['snapshot_finished_unix']=time.time()
|
||||
record['historical_gpu_occupancy_observed']=False
|
||||
return write_observation(root,fold,epoch,record)
|
||||
|
||||
|
||||
def write_observation(root,fold,epoch,record):
|
||||
p=root/'jobs'/str(fold)/'epoch-host-observations'/f'{epoch:03d}.json'
|
||||
p.parent.mkdir(parents=True,exist_ok=True)
|
||||
if p.exists():raise FileExistsError('Preserve original epoch observation')
|
||||
save(p,record)
|
||||
return record
|
||||
|
||||
|
||||
def drain_epoch_observations(root,fold,gpu_uuid,covered,terminal=False,stopping=False):
|
||||
"""Account for each completed receipt independently of the cost cursor.
|
||||
|
||||
Terminal, stopping, batched or >5s-old receipts get explicit missing-history
|
||||
records without GPU queries. A fresh live singleton gets a current snapshot;
|
||||
its true times never claim coverage of earlier epoch occupancy.
|
||||
"""
|
||||
history=root/'jobs'/str(fold)/'worker/history'
|
||||
receipts=[]
|
||||
for p in sorted(history.glob('*.json')):
|
||||
row=json.loads(p.read_text());epoch=row['epoch']
|
||||
if type(epoch) is not int or not 1<=epoch<=200 or p.stem!=f'{epoch:03d}':raise ValueError('Completed epoch receipt identity')
|
||||
if epoch not in covered:receipts.append((epoch,p))
|
||||
backlog=len(receipts)>1;new_records=[]
|
||||
for epoch,p in receipts:
|
||||
output=root/'jobs'/str(fold)/'epoch-host-observations'/f'{epoch:03d}.json'
|
||||
if output.exists():
|
||||
record=json.loads(output.read_text())
|
||||
if record['completed_epoch']!=epoch or record['fold']!=fold or record.get('receipt_sha256')!=sha(p):raise ValueError('Existing observation identity')
|
||||
else:
|
||||
reason=('TERMINAL_BEFORE_OBSERVATION' if terminal else 'GLOBAL_STOP_BEFORE_OBSERVATION' if stopping else 'MULTIPLE_COMPLETED_RECEIPTS_BACKLOG' if backlog else 'DELAYED_RECEIPT_OVER_5_SECONDS' if time.time()-p.stat().st_mtime>5 else None)
|
||||
record=epoch_host_observation(root,fold,epoch,gpu_uuid,receipt=p,incomplete_reason=reason)
|
||||
covered.add(epoch);new_records.append(record)
|
||||
return new_records
|
||||
@@ -12,7 +12,7 @@ from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase2_home_freeze_io import freeze
|
||||
from s2_recovery_gpu_supervisor import disk_usage, tree_usage
|
||||
from phase3_seed1_v2_observation import epoch_host_observation
|
||||
from phase3_seed1_v2_observation import drain_epoch_observations
|
||||
from phase3_seed1_v2_common import config, dispatchable, execution_command, project_cell, project_total, read, available_gpus, idle_eligible_gpus
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ def run(root):
|
||||
start = time.monotonic()
|
||||
cpu = time.process_time()
|
||||
active, terminal, estimates, last_epoch = {}, {}, {}, {}
|
||||
observed_epochs = {}
|
||||
pending = list(c['jobs'])
|
||||
requested = []
|
||||
|
||||
@@ -62,10 +63,15 @@ def run(root):
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
|
||||
def observe_completed(i, v, terminal=False, stopping=False):
|
||||
return drain_epoch_observations(root, i, v['job']['gpu_uuid'],
|
||||
observed_epochs.setdefault(i, set()), terminal=terminal, stopping=stopping)
|
||||
|
||||
def collect(i, v):
|
||||
rc = v['process'].poll()
|
||||
if rc is None:
|
||||
return False
|
||||
observe_completed(i, v, terminal=True, stopping=(root/'global-stop.json').exists())
|
||||
p = root / 'jobs' / str(i) / 'complete.json'
|
||||
r = read(p) if p.exists() else None
|
||||
observed = c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
|
||||
@@ -113,7 +119,8 @@ def run(root):
|
||||
try:
|
||||
while active or pending:
|
||||
for i, v in list(active.items()):
|
||||
collect(i, v)
|
||||
if not collect(i, v):
|
||||
observe_completed(i, v, stopping=bool(requested) or (root/'global-stop.json').exists())
|
||||
for reason in requested:
|
||||
stop(reason)
|
||||
amount = charged()
|
||||
@@ -131,7 +138,6 @@ def run(root):
|
||||
epoch = rows[-1]['epoch']
|
||||
# Recalculate every epoch: waiting, setup and observed cost stay charged.
|
||||
if epoch > last_epoch.get(i, 0):
|
||||
epoch_host_observation(root, i, epoch, active[i]['job']['gpu_uuid'])
|
||||
observation = dict(epoch=epoch, charged=active_charges()[i],
|
||||
max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows),
|
||||
max_validation_seconds=max(r.get('validation_seconds',0) for r in rows))
|
||||
@@ -148,7 +154,8 @@ def run(root):
|
||||
if not (root / 'global-stop.json').exists():
|
||||
dispatch()
|
||||
if (root / 'global-stop.json').exists():
|
||||
for v in active.values():
|
||||
for i, v in active.items():
|
||||
observe_completed(i, v, stopping=True)
|
||||
terminate(v)
|
||||
save(root / 'live-progress.json', dict(stage='STOPPING' if (root / 'global-stop.json').exists()
|
||||
else 'SEED1_TRAINING', unix=time.time(), active=[dict(id=i, guard_pid=v['process'].pid)
|
||||
@@ -191,10 +198,15 @@ def run(root):
|
||||
for i, v in list(active.items()):
|
||||
try:
|
||||
if not collect(i, v):
|
||||
observe_completed(i, v, stopping=True)
|
||||
terminate(v)
|
||||
except Exception:
|
||||
terminate(v)
|
||||
if v['process'].poll() is not None:
|
||||
try:
|
||||
observe_completed(i, v, terminal=True, stopping=True)
|
||||
except Exception as observation_error:
|
||||
save(root/'jobs'/str(i)/'observation-coverage-error.json', dict(status='OBSERVATION_INCOMPLETE', unix=time.time(), reason=str(observation_error)))
|
||||
terminal[i] = dict(id=i, status='MALFORMED_TERMINAL',
|
||||
gpu_seconds_charged=c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'])
|
||||
v['log'].close()
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
"""CPU-only ledger/projection/observation tests for the new stopping rule."""
|
||||
import ast
|
||||
import io
|
||||
import os
|
||||
import time
|
||||
import copy
|
||||
import inspect
|
||||
import json
|
||||
@@ -88,4 +92,84 @@ class V2Tests(unittest.TestCase):
|
||||
r=observation.epoch_host_observation(Path(d),1,2,'GPU-A')
|
||||
self.assertEqual(r['status'],'OBSERVATION_INCOMPLETE');self.assertTrue(r['observation_only']);self.assertEqual(r['errors'][0]['stage'],'gpu_process_snapshot')
|
||||
|
||||
class ObservationCoverageTests(unittest.TestCase):
|
||||
def queue_fragments(self,root,exited,covered,last_cost,requested=None):
|
||||
# Execute the actual coordinator's observe/collect and first poll loop.
|
||||
from phase2_h1_worker_v2 import save,sha
|
||||
tree=ast.parse((Path(__file__).resolve().parent/'phase3_seed1_v2_queue.py').read_text())
|
||||
fn=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run')
|
||||
nested=[n for n in fn.body if isinstance(n,ast.FunctionDef) and n.name in ['observe_completed','collect']]
|
||||
main_try=next(n for n in fn.body if isinstance(n,ast.Try))
|
||||
poll_loop=main_try.body[0].body[0]
|
||||
self.assertIsInstance(poll_loop,ast.For)
|
||||
v=dict(process=SimpleNamespace(poll=lambda:0 if exited else None),start=time.monotonic(),log=io.StringIO(),job=dict(gpu_uuid='SYNTHETIC-GPU'))
|
||||
env=dict(root=root,active={0:v},terminal={},observed_epochs={0:set(covered)},last_epoch={0:last_cost},requested=requested or [],c=dict(jobs=[dict(setup_gpu_seconds=60)]),time=time,read=lambda p:json.loads(p.read_text()),save=save,sha=sha,stop=lambda reason:None,drain_epoch_observations=observation.drain_epoch_observations)
|
||||
code=compile(ast.fix_missing_locations(ast.Module(body=nested+[poll_loop],type_ignores=[])),'actual_queue_poll','exec')
|
||||
exec(code,env)
|
||||
return env
|
||||
|
||||
def receipts(self,root,epochs):
|
||||
from phase2_h1_worker_v2 import save
|
||||
d=root/'jobs/0';(d/'worker/history').mkdir(parents=True)
|
||||
save(d/'assignment.json',{})
|
||||
save(d/'complete.json',dict(status='PASS',monitor_errors=[],worker=dict(monitor_errors=0),gpu_seconds_charged=1000))
|
||||
for e in epochs:save(d/'worker/history'/f'{e:03d}.json',dict(epoch=e,train_epoch_seconds=1))
|
||||
return d
|
||||
|
||||
def test_terminal_before_next_poll_keeps_epoch200(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
root=Path(d);job=self.receipts(root,[199,200])
|
||||
with patch.object(observation.subprocess,'run') as query:
|
||||
env=self.queue_fragments(root,True,range(1,200),199)
|
||||
query.assert_not_called()
|
||||
record=json.loads((job/'epoch-host-observations/200.json').read_text())
|
||||
self.assertEqual(record['status'],'OBSERVATION_INCOMPLETE')
|
||||
self.assertEqual(record['incomplete_reason'],'TERMINAL_BEFORE_OBSERVATION')
|
||||
self.assertIsNone(record['snapshot_started_unix']);self.assertEqual(record['gpu_processes'],[])
|
||||
self.assertEqual(env['active'],{});self.assertEqual(env['observed_epochs'][0],set(range(1,201)));self.assertEqual(env['last_epoch'][0],199)
|
||||
|
||||
def test_two_unseen_receipts_each_covered_no_historical_snapshot(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
root=Path(d);job=self.receipts(root,[199,200])
|
||||
with patch.object(observation.subprocess,'run') as query:
|
||||
env=self.queue_fragments(root,False,range(1,199),198)
|
||||
self.assertEqual(observation.drain_epoch_observations(root,0,'SYNTHETIC-GPU',env['observed_epochs'][0]),[])
|
||||
query.assert_not_called()
|
||||
for epoch in [199,200]:
|
||||
record=json.loads((job/'epoch-host-observations'/f'{epoch:03d}.json').read_text())
|
||||
self.assertEqual(record['incomplete_reason'],'MULTIPLE_COMPLETED_RECEIPTS_BACKLOG');self.assertIsNone(record['snapshot_started_unix'])
|
||||
self.assertIn(0,env['active']);self.assertEqual(env['last_epoch'][0],198);self.assertEqual(env['observed_epochs'][0],set(range(1,201)))
|
||||
|
||||
def test_global_stop_poll_covers_completed_receipt(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
root=Path(d);job=self.receipts(root,[12]);(root/'global-stop.json').write_text('{}')
|
||||
with patch.object(observation.subprocess,'run') as query:
|
||||
env=self.queue_fragments(root,False,range(1,12),11)
|
||||
query.assert_not_called()
|
||||
record=json.loads((job/'epoch-host-observations/012.json').read_text())
|
||||
self.assertEqual(record['incomplete_reason'],'GLOBAL_STOP_BEFORE_OBSERVATION');self.assertIn(12,env['observed_epochs'][0])
|
||||
|
||||
def test_delayed_single_receipt_is_incomplete_without_query(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
root=Path(d);job=self.receipts(root,[1]);p=job/'worker/history/001.json';old=time.time()-20;os.utime(p,(old,old))
|
||||
with patch.object(observation.subprocess,'run') as query:
|
||||
records=observation.drain_epoch_observations(root,0,'GPU-A',set())
|
||||
query.assert_not_called();self.assertEqual(records[0]['incomplete_reason'],'DELAYED_RECEIPT_OVER_5_SECONDS');self.assertGreaterEqual(records[0]['receipt_age_seconds'],20)
|
||||
|
||||
def test_fresh_single_receipt_has_real_snapshot_times_and_hash(self):
|
||||
from phase2_h1_worker_v2 import sha
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
root=Path(d);job=self.receipts(root,[1]);start=time.time()
|
||||
own=SimpleNamespace(pid=10,info=dict(cmdline=['python','tools/phase3_seed1_v2_worker.py',d]))
|
||||
with patch.object(observation.psutil,'process_iter',return_value=[own]),patch.object(observation.subprocess,'run',return_value=SimpleNamespace(stdout='GPU-A, 10, python, 800\n')):
|
||||
records=observation.drain_epoch_observations(root,0,'GPU-A',set())
|
||||
r=records[0];self.assertEqual(r['status'],'OBSERVED');self.assertGreaterEqual(r['snapshot_started_unix'],start);self.assertGreaterEqual(r['snapshot_finished_unix'],r['snapshot_started_unix']);self.assertFalse(r['historical_gpu_occupancy_observed']);self.assertEqual(r['receipt_sha256'],sha(job/'worker/history/001.json'))
|
||||
|
||||
def test_worker_disappearing_between_poll_and_snapshot_is_incomplete(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
root=Path(d);self.receipts(root,[200])
|
||||
with patch.object(observation.psutil,'process_iter',return_value=[]),patch.object(observation.subprocess,'run',return_value=SimpleNamespace(stdout='')):
|
||||
records=observation.drain_epoch_observations(root,0,'GPU-A',set())
|
||||
self.assertEqual(records[0]['status'],'OBSERVATION_INCOMPLETE');self.assertEqual(records[0]['errors'][0]['stage'],'epoch_worker_visibility');self.assertIsNotNone(records[0]['snapshot_started_unix'])
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user