117 lines
8.9 KiB
Python
117 lines
8.9 KiB
Python
"""Freeze the original incident, quarantine partial outputs, and plan 134 parts."""
|
|
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
from freeze_g1_components import sha
|
|
from s2_e1_supervisor import save
|
|
from s2_recovery_monitoring import disk_usage
|
|
|
|
ORIGINAL=Path('artifacts/s2-pilot-inputs-2026-11-15')
|
|
INCIDENT=Path('artifacts/s2-preparation-incident-2026-11-15')
|
|
RECOVERY=Path('artifacts/s2-pilot-recovery-inputs-2026-11-15')
|
|
INFRA=[318,322,349,354,355,356,365,366,370,371,372,373,375,378,379,380]
|
|
WALL=[181,201,211]
|
|
UNATTEMPTED=list(range(381,499))
|
|
INDICES=sorted(INFRA+UNATTEMPTED)
|
|
|
|
|
|
def inventory(path):
|
|
if not path.exists():return []
|
|
result=[]
|
|
for p in sorted(path.rglob('*')):
|
|
assert not p.is_symlink()
|
|
if p.is_file():result.append(dict(path=str(p.relative_to(path)),bytes=p.stat().st_size,sha256=sha(p)))
|
|
return result
|
|
|
|
|
|
def main():
|
|
assert not INCIDENT.exists() and not RECOVERY.exists(),'Never overwrite an incident/recovery plan'
|
|
assert not (ORIGINAL/'batch-complete.json').exists(),'Original incident has no normal batch terminal'
|
|
c=json.loads((ORIGINAL/'contract.json').read_text());roles=json.loads((ORIGINAL/'role-manifest.json').read_text())
|
|
assert sha(ORIGINAL/'role-manifest.json')==c['role_manifest_sha256']
|
|
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
|
progress=json.loads((ORIGINAL/'progress.json').read_text())
|
|
assert progress['completed']==365 and progress['next_index']==381 and progress['active']==INFRA
|
|
started={int(p.parent.name.split('-')[1]) for p in (ORIGINAL/'jobs').glob('*/worker.started.json')}
|
|
completed={int(p.parent.name.split('-')[1]) for p in (ORIGINAL/'jobs').glob('*/guard-result.json')}
|
|
assert started==completed==set(range(381))
|
|
rows=[]
|
|
for i in range(381):
|
|
folder=ORIGINAL/'jobs'/f'job-{i:04d}';r=json.loads((folder/'guard-result.json').read_text());w=r['worker'];spec=c['cases'][i]
|
|
if i in INFRA:
|
|
assert w['returncode']==-9 and w['stop_reason'] is None and w['status']!='PASS'
|
|
classification='INFRA_ABORTED';assert not (ORIGINAL/'ready'/spec['case']/'receipt.json').exists()
|
|
assert not spec.get('reuse_preparation'),'Infrastructure aborts must start from fresh output directories'
|
|
elif i in WALL:
|
|
assert w['returncode']==-9 and w['stop_reason']=='wall_limit';classification='WALL_LIMIT_FAIL'
|
|
else:
|
|
assert w['status']=='PASS' and w['returncode']==0;classification='PASS'
|
|
assert (ORIGINAL/'ready'/spec['case']/'receipt.json').exists()
|
|
rows.append(dict(index=i,case=spec['case'],nodes=spec['tet_nodes'],role='learning' if spec['case'] in roles['learning_parts'] else 'validation',
|
|
classification=classification,worker=r['worker'],supervisor_cpu_seconds=r['supervisor_cpu_seconds'],
|
|
guard_receipt_path=str(folder/'guard-result.json'),guard_receipt_sha256=sha(folder/'guard-result.json'),
|
|
worker_started_sha256=sha(folder/'worker.started.json'),worker_receipt_sha256=sha(folder/'worker.receipt.json')))
|
|
INCIDENT.mkdir();(INCIDENT/'quarantine').mkdir()
|
|
snapshot=dict(status='INFRA_ABORTED',original_root=str(ORIGINAL),original_contract_sha256=sha(ORIGINAL/'contract.json'),
|
|
batch_started_sha256=sha(ORIGINAL/'batch-started.json'),last_progress_sha256=sha(ORIGINAL/'progress.json'),jobs=rows,
|
|
ordinary_pass=362,wall_limit_fail_indices=WALL,infra_aborted_indices=INFRA,never_attempted_indices=UNATTEMPTED,
|
|
cause='Coordinator FileNotFoundError during disk_usage stat of progress.json.tmp; finally terminated 16 active workers',
|
|
original_session=16682,original_exit_code=1,original_batch_complete_created=False)
|
|
save(INCIDENT/'original-batch-infra-aborted.json',snapshot)
|
|
moves=[]
|
|
for i in INFRA:
|
|
case=c['cases'][i]['case']
|
|
for kind in ['parts','ready']:
|
|
source=ORIGINAL/kind/case;dest=INCIDENT/'quarantine'/kind/case
|
|
assert source.exists() and not dest.exists()
|
|
moves.append(dict(original_index=i,kind=kind,from_path=str(source),to_path=str(dest),files=inventory(source)))
|
|
save(INCIDENT/'quarantine-plan.json',dict(moves=moves))
|
|
moved=[]
|
|
for item in moves:
|
|
source=Path(item['from_path']);dest=Path(item['to_path']);dest.parent.mkdir(parents=True,exist_ok=True)
|
|
source.rename(dest);assert inventory(dest)==item['files']
|
|
moved.append(item['to_path']);save(INCIDENT/'quarantine-progress.json',dict(moved=moved,total=len(moves)))
|
|
save(INCIDENT/'quarantine-receipt.json',dict(status='ALL_16_PARTIALS_MOVED_AND_HASH_VERIFIED',moves=moves,reused_by_recovery=False))
|
|
prior=sum(r['worker']['cpu_seconds'] for r in rows);aborted=sum(r['worker']['cpu_seconds'] for r in rows if r['classification']=='INFRA_ABORTED')
|
|
ledger=dict(status='FROZEN_INCIDENT_LEDGER',ruling='1794689807.792819',original_root=str(ORIGINAL),
|
|
original_snapshot=str(INCIDENT/'original-batch-infra-aborted.json'),original_snapshot_sha256=sha(INCIDENT/'original-batch-infra-aborted.json'),
|
|
quarantine_receipt=str(INCIDENT/'quarantine-receipt.json'),quarantine_receipt_sha256=sha(INCIDENT/'quarantine-receipt.json'),
|
|
pass_count=362,wall_limit_indices=WALL,infra_aborted_indices=INFRA,unattempted_indices=UNATTEMPTED,recovery_indices=INDICES,
|
|
prior_worker_cpu_seconds=prior,prior_aborted_worker_cpu_seconds_included=aborted,
|
|
prior_guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in rows),coordinator_cpu_seconds='UNKNOWN_NO_EXIT_ACCOUNTING_RECEIPT',
|
|
accounting_rule='All 381 original worker receipts counted once, including the 16 aborted; add only new recovery worker CPU',
|
|
labelled_recovery_allowance_per_infra_part=1,original_batch_complete_created=False)
|
|
save(INCIDENT/'incident-ledger.json',ledger)
|
|
RECOVERY.mkdir();(RECOVERY/'public').mkdir();shutil.copy2(ORIGINAL/'role-manifest.json',RECOVERY/'role-manifest.json')
|
|
selected=[c['cases'][i] for i in INDICES]
|
|
terminal=[r for r in rows if r['classification']!='INFRA_ABORTED']
|
|
rates={}
|
|
for cached in [False,True]:
|
|
group=[r for r in terminal if bool(c['cases'][r['index']].get('reuse_preparation'))==cached]
|
|
rates[cached]=sum(r['worker']['cpu_seconds'] for r in group)/sum(r['nodes'] for r in group) if group else c['initial_cpu_seconds_per_node']
|
|
projected=prior+1.25*sum(rates[bool(s.get('reuse_preparation'))]*s['tet_nodes'] for s in selected)
|
|
roots=[str(ORIGINAL),str(INCIDENT),str(RECOVERY)];existing=sum(disk_usage(p) for p in roots)
|
|
projected_disk=existing+c['initial_projection_gib']*2**30*sum(s['tet_nodes'] for s in selected)/sum(s['tet_nodes'] for s in c['cases'])
|
|
added=['tools/s2_recovery_monitoring.py','tools/s2_recovery_supervisor.py','tools/run_s2_recovery_preparation.py','tools/prepare_s2_incident_recovery.py']
|
|
contract=dict(c,version='S2-labelled-infrastructure-recovery-v1',cases=selected,original_indices=INDICES,infra_aborted_indices=INFRA,
|
|
original_root=str(ORIGINAL),original_contract_sha256=sha(ORIGINAL/'contract.json'),
|
|
original_snapshot=ledger['original_snapshot'],original_snapshot_sha256=ledger['original_snapshot_sha256'],
|
|
incident_ledger=str(INCIDENT/'incident-ledger.json'),incident_ledger_sha256=sha(INCIDENT/'incident-ledger.json'),
|
|
code_sha256={p:sha(p) for p in list(c['code_sha256'])+added},prior_worker_cpu_seconds=prior,budget_roots=roots,
|
|
projected_cumulative_worker_cpu_hours=projected/3600,projected_cumulative_gib=projected_disk/2**30,
|
|
projection_method='Exclude partial infrastructure aborts from rate estimation, retain their CPU in prior total; successful/true-timeout observed category rates times 134 nodes plus 25%; original maximum-byte/node margin for extra outputs',
|
|
limits=dict(worker_cpu_hours=60,workers=16,disk_bytes=100*2**30,per_part_wall_seconds=1800,per_part_tree_rss_bytes=1342177280,retries=0,replacements=0),
|
|
checkpoint='No new first50 barrier; original accepted checkpoint retained',authorizing_rulings=['1794689807.792819'],
|
|
partial_outputs_policy='Original 16 quarantined by rename and hash verified, never read/reused; recovery uses a new root; original part specs identical')
|
|
save(RECOVERY/'contract.json',contract)
|
|
report=dict(status='READY_WITHIN_CAPS' if projected<=60*3600 and projected_disk<=100*2**30 else 'PROJECTION_OVER_CAP_REPORT_BEFORE_LAUNCH',
|
|
original_pass=362,original_wall_fail=3,labelled_recovery=16,never_attempted=118,recovery_parts=134,
|
|
prior_worker_cpu_hours=prior/3600,prior_aborted_cpu_hours_already_included=aborted/3600,
|
|
projected_cumulative_worker_cpu_hours=projected/3600,projected_cumulative_gib=projected_disk/2**30,
|
|
contract_sha256=sha(RECOVERY/'contract.json'),incident_ledger_sha256=sha(INCIDENT/'incident-ledger.json'),
|
|
worker_numerical_code_unchanged=True,workers_started=False,GPU_started=False)
|
|
save(RECOVERY/'public/prelaunch-projection.json',report);print(json.dumps(report))
|
|
|
|
|
|
if __name__=='__main__':main()
|