Accept hash-bound merged preparation terminals without retrying failed parts

This commit is contained in:
Frank14f committed 2026-11-15 03:28:26 +08:00
1 parent f771dea144
commit e527daf8f0
8 files changed
+178 -8

No files matched your search

+6 -4
View File
@@ -6,6 +6,7 @@ import numpy as np
from freeze_g1_components import sha
from s2_e1_supervisor import save
from s2_pilot_data import permitted_role
from s2_pilot_terminal import terminal_batch
def quantile_probe(records,roles):
@@ -15,7 +16,7 @@ def quantile_probe(records,roles):
def finalize(source,root,gpu_uuid):
c=json.loads((source/'contract.json').read_text());batch=json.loads((source/'batch-complete.json').read_text())
c=json.loads((source/'contract.json').read_text());batch,terminal_path=terminal_batch(source)
assert batch['status']=='ATTEMPTS_COMPLETE' and batch['stop_reason'] is None
assert len(batch['jobs'])==len(c['cases'])==499 and not batch['not_run_indices']
assert sorted(r['index'] for r in batch['jobs'])==list(range(499))
@@ -25,7 +26,7 @@ def finalize(source,root,gpu_uuid):
for job in sorted(batch['jobs'],key=lambda r:r['index']):
spec=c['cases'][job['index']];case=spec['case'];role=permitted_role(roles,case)
assert job['case']==case
dest=source/'ready'/case
dest=Path(job.get('ready_source_root',source))/'ready'/case
if job['worker']['status']!='PASS':
failed.append(dict(case=case,role=role,component_id=spec['component_id'],guard=job['worker'],
failure=json.loads((dest/'failure.json').read_text()) if (dest/'failure.json').exists() else None))
@@ -46,11 +47,12 @@ def finalize(source,root,gpu_uuid):
probe=quantile_probe(ready,roles)
paths=['tools/prepare_s2_pilot_training.py','tools/run_s2_pilot.py','tools/run_s2_pilot_probe.py',
'tools/run_s2_pilot_training.py','tools/s2_pilot_train.py','tools/s2_pilot_gpu_guard.py','tools/s2_gpu_process_supervisor.py',
'tools/s2_e1_supervisor.py','tools/s2_pilot_data.py','tools/freeze_g1_components.py']
'tools/s2_e1_supervisor.py','tools/s2_pilot_data.py','tools/s2_pilot_terminal.py','tools/freeze_g1_components.py']
paths+=sorted(str(p) for p in Path('tools/s2_models').glob('*.py'))
code={p:sha(p) for p in paths}
root.mkdir();save(root/'success-manifest.json',dict(ready_parts=ready,failed_parts=failed,successful=successful,
preparation_batch_sha256=sha(source/'batch-complete.json'),preparation_contract_sha256=sha(source/'contract.json')))
preparation_batch_sha256=sha(terminal_path),preparation_terminal_manifest=str(terminal_path),
preparation_sources=batch.get('source_provenance',[]),preparation_contract_sha256=sha(source/'contract.json')))
plan=dict(fold=0,pool=100,seed=0,gpu_uuid=gpu_uuid,code_sha256=code,role_manifest=str(source/'role-manifest.json'),
role_manifest_sha256=sha(source/'role-manifest.json'),success_manifest=str(root/'success-manifest.json'),
success_manifest_sha256=sha(root/'success-manifest.json'),probe_cases=probe,
+8 -3
View File
@@ -13,12 +13,14 @@ from freeze_g1_components import sha
from s2_e1_supervisor import save
from prepare_s2_pilot_training import finalize
from run_s2_pilot import run as pilot
from s2_pilot_terminal import terminal_batch,continuation_allowed
def preparation_terminal(source):
path=source/'batch-complete.json'
if not path.exists():return False
batch=json.loads(path.read_text())
batch,_=terminal_batch(source)
if not (source/'merged-terminal.json').exists() and continuation_allowed(source,batch):return False
if batch['status']!='ATTEMPTS_COMPLETE' or batch['stop_reason'] is not None or batch['not_run_indices']:
raise RuntimeError('Preparation stopped/incomplete; no automatic restart, continuation or GPU launch')
if len(batch['jobs'])!=499 or sorted(r['index'] for r in batch['jobs'])!=list(range(499)):
@@ -43,14 +45,17 @@ def run(source,root,watch,gpu_uuid):
try:
while not preparation_terminal(source):
progress=json.loads((source/'progress.json').read_text())
save(watch/'progress.json',dict(stage='WAITING_FOR_EXISTING_PREPARATION',completed=progress['completed'],
stage='WAITING_FOR_ACCEPTED_MERGED_CONTINUATION' if (source/'batch-complete.json').exists() else 'WAITING_FOR_EXISTING_PREPARATION'
save(watch/'progress.json',dict(stage=stage,completed=progress['completed'],
waiting_seconds=time.monotonic()-start,GPU_started=False,original_preparation_not_restarted=True))
time.sleep(10)
assert sha(source/'contract.json')==expected
for p,h in frozen.items():assert sha(p)==h,p
plan=finalize(source,root,gpu_uuid)
terminal,terminal_path=terminal_batch(source)
save(watch/'preparation-accepted.json',dict(status='ALL_499_TERMINAL_MANIFEST_VERIFIED',
batch_sha256=sha(source/'batch-complete.json'),success_manifest_sha256=plan['success_manifest_sha256'],
batch_sha256=sha(terminal_path),terminal_manifest=str(terminal_path),
preparation_sources=terminal.get('source_provenance',[]),success_manifest_sha256=plan['success_manifest_sha256'],
successful=plan['successful'],failed_parts_dropped=499-sum(v['parts'] for v in plan['successful'].values()),
pilot_plan_sha256=sha(root/'pilot-plan.json'),waiting_seconds=time.monotonic()-start))
save(watch/'progress.json',dict(stage='BOUNDED_PROBE_AND_PILOT',pilot_root=str(root),
+78
View File
@@ -0,0 +1,78 @@
"""Validate original or merged preparation terminal evidence without rerunning it."""
import argparse
import json
import os
from pathlib import Path
from freeze_g1_components import sha
from s2_e1_supervisor import disk_usage
def continuation_allowed(source,batch):
if batch['status']!='STOPPED' or batch.get('stop_reason')!='FIRST_50_PROJECTION_OVER_CAP':return False
checkpoint=json.loads((source/'first-50-checkpoint.json').read_text())
return (60*3600<checkpoint['projected_worker_cpu_seconds']<=80*3600
and checkpoint['projected_disk_bytes']<=100*2**30
and len(batch['jobs'])==50 and bool(batch['not_run_indices']))
def combine(source,continuation,proof):
roots=[source,continuation];contracts=[];batches=[]
assert len(proof)==2
for root,entry in zip(roots,proof):
assert str(root)==entry['root']
assert sha(root/'contract.json')==entry['contract_sha256']
assert sha(root/'batch-complete.json')==entry['batch_sha256']
contracts.append(json.loads((root/'contract.json').read_text()))
batches.append(json.loads((root/'batch-complete.json').read_text()))
original,later=contracts;before,after=batches
assert len(original['cases'])==499 and continuation_allowed(source,before)
assert after['status']=='ATTEMPTS_COMPLETE' and after['stop_reason'] is None and not after['not_run_indices']
assert later['continuation_of']==proof[0]
assert later['role_manifest_sha256']==original['role_manifest_sha256']
original_cases={s['case']:i for i,s in enumerate(original['cases'])};assert len(original_cases)==499
pending={original['cases'][i]['case'] for i in before['not_run_indices']}
assert {s['case'] for s in later['cases']}==pending and len(later['cases'])==len(pending)
assert sorted(r['index'] for r in after['jobs'])==list(range(len(later['cases'])))
for spec in later['cases']:
assert spec==original['cases'][original_cases[spec['case']]],'Continuation cannot replace or alter a part specification'
jobs=[];seen=set()
for root,c,b in zip(roots,contracts,batches):
for job in b['jobs']:
case=job['case'];assert c['cases'][job['index']]['case']==case
assert case not in seen,'An attempted part may not be retried by continuation'
seen.add(case);assert job['worker']['cpu_seconds']>=0
assert job['worker']['status']!='GUARD_FAILED_ACCOUNTING_UNKNOWN'
jobs.append(dict(job,index=original_cases[case],source_index=job['index'],ready_source_root=str(root)))
assert seen==set(original_cases) and len(jobs)==499
cpu=sum(r['worker']['cpu_seconds'] for r in jobs);used=sum(disk_usage(root) for root in roots)
assert cpu<=80*3600 and used<=100*2**30,'Merged budget exceeded'
return dict(status='ATTEMPTS_COMPLETE',stop_reason=None,not_run_indices=[],jobs=sorted(jobs,key=lambda r:r['index']),
worker_cpu_seconds=cpu,guard_cpu_seconds=sum(b['guard_cpu_seconds'] for b in batches),
failed=sum(r['worker']['status']!='PASS' for r in jobs),disk_bytes=used,
source_provenance=proof,continuation_ruling='1794683049.057259',retry=False,replacement=False,test_labels_read=False)
def terminal_batch(source):
merged=source/'merged-terminal.json'
if not merged.exists():
path=source/'batch-complete.json';return json.loads(path.read_text()),path
m=json.loads(merged.read_text());proof=m['source_provenance']
batch=combine(source,Path(proof[1]['root']),proof)
assert batch['jobs']==m['jobs'] and batch['worker_cpu_seconds']==m['worker_cpu_seconds']
return batch,merged
def merge(source,continuation):
proof=[dict(root=str(root),contract_sha256=sha(root/'contract.json'),batch_sha256=sha(root/'batch-complete.json')) for root in [source,continuation]]
batch=combine(source,continuation,proof)
pending=source/'merged-terminal.json.pending'
with pending.open('x') as f:json.dump(batch,f,indent=2);f.write('\n');f.flush();os.fsync(f.fileno())
# The waiting entry sees either no merged manifest or the complete file.
# link creates the final name atomically and refuses an existing manifest.
os.link(pending,source/'merged-terminal.json');pending.unlink()
return batch
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('source',type=Path);p.add_argument('continuation',type=Path);a=p.parse_args();r=merge(a.source,a.continuation)
print(json.dumps(dict(status=r['status'],terminal=len(r['jobs']),failed=r['failed'],worker_cpu_seconds=r['worker_cpu_seconds'])))
+40
View File
@@ -4,6 +4,8 @@ import unittest
from pathlib import Path
from unittest.mock import patch
import run_s2_pilot_when_ready as waiter
from s2_pilot_terminal import merge,terminal_batch
from freeze_g1_components import sha
class WaiterTests(unittest.TestCase):
@@ -27,4 +29,42 @@ class WaiterTests(unittest.TestCase):
self.assertTrue((watch/'failure.json').exists());self.assertFalse(root.exists())
class MergedTerminalTests(unittest.TestCase):
def fixture(self,root):
source=root/'original';later=root/'continuation';source.mkdir();later.mkdir()
cases=[dict(case=str(i),tet_nodes=i+1) for i in range(499)]
def job(i,case):return dict(index=i,case=str(case),worker=dict(status='STOPPED_OR_FAILED' if case==0 else 'PASS',cpu_seconds=1.))
original=dict(cases=cases,role_manifest_sha256='synthetic-roles')
before=dict(status='STOPPED',stop_reason='FIRST_50_PROJECTION_OVER_CAP',not_run_indices=list(range(50,499)),jobs=[job(i,i) for i in range(50)],guard_cpu_seconds=1.)
(source/'contract.json').write_text(json.dumps(original));(source/'batch-complete.json').write_text(json.dumps(before))
(source/'first-50-checkpoint.json').write_text(json.dumps(dict(projected_worker_cpu_seconds=70*3600,projected_disk_bytes=10*2**30)))
proof=dict(root=str(source),contract_sha256=sha(source/'contract.json'),batch_sha256=sha(source/'batch-complete.json'))
(later/'contract.json').write_text(json.dumps(dict(cases=cases[50:],role_manifest_sha256='synthetic-roles',continuation_of=proof)))
(later/'batch-complete.json').write_text(json.dumps(dict(status='ATTEMPTS_COMPLETE',stop_reason=None,not_run_indices=[],jobs=[job(i,i+50) for i in range(449)],guard_cpu_seconds=1.)))
return source,later
def test_waits_for_approved_merge_accepts_failed_parts_and_binds_both_sources(self):
with tempfile.TemporaryDirectory() as tmp:
source,later=self.fixture(Path(tmp));self.assertFalse(waiter.preparation_terminal(source))
batch=merge(source,later);self.assertEqual(batch['failed'],1);self.assertEqual(len(batch['jobs']),499)
self.assertEqual(batch['jobs'][49]['ready_source_root'],str(source));self.assertEqual(batch['jobs'][50]['ready_source_root'],str(later))
self.assertEqual(batch['jobs'][50]['source_index'],0);self.assertEqual(batch['worker_cpu_seconds'],499.)
self.assertTrue(waiter.preparation_terminal(source))
fixed=sha(source/'merged-terminal.json')
with self.assertRaises(FileExistsError):merge(source,later)
self.assertEqual(sha(source/'merged-terminal.json'),fixed)
(later/'contract.json').write_text('{}')
with self.assertRaises(AssertionError):terminal_batch(source)
def test_merge_rejects_retries_changed_specs_and_cumulative_overrun(self):
for change in ['retry','spec','cpu']:
with self.subTest(change=change),tempfile.TemporaryDirectory() as tmp:
source,later=self.fixture(Path(tmp))
p=later/('batch-complete.json' if change=='cpu' else 'contract.json');data=json.loads(p.read_text())
if change=='retry':data['cases'][0]['case']='0'
elif change=='spec':data['cases'][0]['tet_nodes']+=1
else:data['jobs'][0]['worker']['cpu_seconds']=81*3600
p.write_text(json.dumps(data))
with self.assertRaises(AssertionError):merge(source,later)
self.assertFalse((source/'merged-terminal.json').exists())
if __name__=='__main__':unittest.main(verbosity=2)