Repair seed1 v2 observation coverage and preserve original BLOCKED review

This commit is contained in:
MeltAstra committed 2026-11-18 16:50:51 +08:00
1 parent bf7363bf96
commit 2dc2f39e37
79 files changed
+12451 -25

No files matched your search

+84
View File
@@ -1,4 +1,8 @@
"""CPU-only ledger/projection/observation tests for the new stopping rule."""
import ast
import io
import os
import time
import copy
import inspect
import json
@@ -88,4 +92,84 @@ class V2Tests(unittest.TestCase):
r=observation.epoch_host_observation(Path(d),1,2,'GPU-A')
self.assertEqual(r['status'],'OBSERVATION_INCOMPLETE');self.assertTrue(r['observation_only']);self.assertEqual(r['errors'][0]['stage'],'gpu_process_snapshot')
class ObservationCoverageTests(unittest.TestCase):
def queue_fragments(self,root,exited,covered,last_cost,requested=None):
# Execute the actual coordinator's observe/collect and first poll loop.
from phase2_h1_worker_v2 import save,sha
tree=ast.parse((Path(__file__).resolve().parent/'phase3_seed1_v2_queue.py').read_text())
fn=next(n for n in tree.body if isinstance(n,ast.FunctionDef) and n.name=='run')
nested=[n for n in fn.body if isinstance(n,ast.FunctionDef) and n.name in ['observe_completed','collect']]
main_try=next(n for n in fn.body if isinstance(n,ast.Try))
poll_loop=main_try.body[0].body[0]
self.assertIsInstance(poll_loop,ast.For)
v=dict(process=SimpleNamespace(poll=lambda:0 if exited else None),start=time.monotonic(),log=io.StringIO(),job=dict(gpu_uuid='SYNTHETIC-GPU'))
env=dict(root=root,active={0:v},terminal={},observed_epochs={0:set(covered)},last_epoch={0:last_cost},requested=requested or [],c=dict(jobs=[dict(setup_gpu_seconds=60)]),time=time,read=lambda p:json.loads(p.read_text()),save=save,sha=sha,stop=lambda reason:None,drain_epoch_observations=observation.drain_epoch_observations)
code=compile(ast.fix_missing_locations(ast.Module(body=nested+[poll_loop],type_ignores=[])),'actual_queue_poll','exec')
exec(code,env)
return env
def receipts(self,root,epochs):
from phase2_h1_worker_v2 import save
d=root/'jobs/0';(d/'worker/history').mkdir(parents=True)
save(d/'assignment.json',{})
save(d/'complete.json',dict(status='PASS',monitor_errors=[],worker=dict(monitor_errors=0),gpu_seconds_charged=1000))
for e in epochs:save(d/'worker/history'/f'{e:03d}.json',dict(epoch=e,train_epoch_seconds=1))
return d
def test_terminal_before_next_poll_keeps_epoch200(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d);job=self.receipts(root,[199,200])
with patch.object(observation.subprocess,'run') as query:
env=self.queue_fragments(root,True,range(1,200),199)
query.assert_not_called()
record=json.loads((job/'epoch-host-observations/200.json').read_text())
self.assertEqual(record['status'],'OBSERVATION_INCOMPLETE')
self.assertEqual(record['incomplete_reason'],'TERMINAL_BEFORE_OBSERVATION')
self.assertIsNone(record['snapshot_started_unix']);self.assertEqual(record['gpu_processes'],[])
self.assertEqual(env['active'],{});self.assertEqual(env['observed_epochs'][0],set(range(1,201)));self.assertEqual(env['last_epoch'][0],199)
def test_two_unseen_receipts_each_covered_no_historical_snapshot(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d);job=self.receipts(root,[199,200])
with patch.object(observation.subprocess,'run') as query:
env=self.queue_fragments(root,False,range(1,199),198)
self.assertEqual(observation.drain_epoch_observations(root,0,'SYNTHETIC-GPU',env['observed_epochs'][0]),[])
query.assert_not_called()
for epoch in [199,200]:
record=json.loads((job/'epoch-host-observations'/f'{epoch:03d}.json').read_text())
self.assertEqual(record['incomplete_reason'],'MULTIPLE_COMPLETED_RECEIPTS_BACKLOG');self.assertIsNone(record['snapshot_started_unix'])
self.assertIn(0,env['active']);self.assertEqual(env['last_epoch'][0],198);self.assertEqual(env['observed_epochs'][0],set(range(1,201)))
def test_global_stop_poll_covers_completed_receipt(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d);job=self.receipts(root,[12]);(root/'global-stop.json').write_text('{}')
with patch.object(observation.subprocess,'run') as query:
env=self.queue_fragments(root,False,range(1,12),11)
query.assert_not_called()
record=json.loads((job/'epoch-host-observations/012.json').read_text())
self.assertEqual(record['incomplete_reason'],'GLOBAL_STOP_BEFORE_OBSERVATION');self.assertIn(12,env['observed_epochs'][0])
def test_delayed_single_receipt_is_incomplete_without_query(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d);job=self.receipts(root,[1]);p=job/'worker/history/001.json';old=time.time()-20;os.utime(p,(old,old))
with patch.object(observation.subprocess,'run') as query:
records=observation.drain_epoch_observations(root,0,'GPU-A',set())
query.assert_not_called();self.assertEqual(records[0]['incomplete_reason'],'DELAYED_RECEIPT_OVER_5_SECONDS');self.assertGreaterEqual(records[0]['receipt_age_seconds'],20)
def test_fresh_single_receipt_has_real_snapshot_times_and_hash(self):
from phase2_h1_worker_v2 import sha
with tempfile.TemporaryDirectory() as d:
root=Path(d);job=self.receipts(root,[1]);start=time.time()
own=SimpleNamespace(pid=10,info=dict(cmdline=['python','tools/phase3_seed1_v2_worker.py',d]))
with patch.object(observation.psutil,'process_iter',return_value=[own]),patch.object(observation.subprocess,'run',return_value=SimpleNamespace(stdout='GPU-A, 10, python, 800\n')):
records=observation.drain_epoch_observations(root,0,'GPU-A',set())
r=records[0];self.assertEqual(r['status'],'OBSERVED');self.assertGreaterEqual(r['snapshot_started_unix'],start);self.assertGreaterEqual(r['snapshot_finished_unix'],r['snapshot_started_unix']);self.assertFalse(r['historical_gpu_occupancy_observed']);self.assertEqual(r['receipt_sha256'],sha(job/'worker/history/001.json'))
def test_worker_disappearing_between_poll_and_snapshot_is_incomplete(self):
with tempfile.TemporaryDirectory() as d:
root=Path(d);self.receipts(root,[200])
with patch.object(observation.psutil,'process_iter',return_value=[]),patch.object(observation.subprocess,'run',return_value=SimpleNamespace(stdout='')):
records=observation.drain_epoch_observations(root,0,'GPU-A',set())
self.assertEqual(records[0]['status'],'OBSERVATION_INCOMPLETE');self.assertEqual(records[0]['errors'][0]['stage'],'epoch_worker_visibility');self.assertIsNotNone(records[0]['snapshot_started_unix'])
if __name__=='__main__':unittest.main()