"""Freeze the original incident, quarantine partial outputs, and plan 134 parts.""" import json from pathlib import Path import shutil from freeze_g1_components import sha from s2_e1_supervisor import save from s2_recovery_monitoring import disk_usage ORIGINAL=Path('artifacts/s2-pilot-inputs-2026-11-15') INCIDENT=Path('artifacts/s2-preparation-incident-2026-11-15') RECOVERY=Path('artifacts/s2-pilot-recovery-inputs-2026-11-15') INFRA=[318,322,349,354,355,356,365,366,370,371,372,373,375,378,379,380] WALL=[181,201,211] UNATTEMPTED=list(range(381,499)) INDICES=sorted(INFRA+UNATTEMPTED) def inventory(path): if not path.exists():return [] result=[] for p in sorted(path.rglob('*')): assert not p.is_symlink() if p.is_file():result.append(dict(path=str(p.relative_to(path)),bytes=p.stat().st_size,sha256=sha(p))) return result def main(): assert not INCIDENT.exists() and not RECOVERY.exists(),'Never overwrite an incident/recovery plan' assert not (ORIGINAL/'batch-complete.json').exists(),'Original incident has no normal batch terminal' c=json.loads((ORIGINAL/'contract.json').read_text());roles=json.loads((ORIGINAL/'role-manifest.json').read_text()) assert sha(ORIGINAL/'role-manifest.json')==c['role_manifest_sha256'] for p,h in c['code_sha256'].items():assert sha(p)==h,p progress=json.loads((ORIGINAL/'progress.json').read_text()) assert progress['completed']==365 and progress['next_index']==381 and progress['active']==INFRA started={int(p.parent.name.split('-')[1]) for p in (ORIGINAL/'jobs').glob('*/worker.started.json')} completed={int(p.parent.name.split('-')[1]) for p in (ORIGINAL/'jobs').glob('*/guard-result.json')} assert started==completed==set(range(381)) rows=[] for i in range(381): folder=ORIGINAL/'jobs'/f'job-{i:04d}';r=json.loads((folder/'guard-result.json').read_text());w=r['worker'];spec=c['cases'][i] if i in INFRA: assert w['returncode']==-9 and w['stop_reason'] is None and w['status']!='PASS' classification='INFRA_ABORTED';assert not (ORIGINAL/'ready'/spec['case']/'receipt.json').exists() assert not spec.get('reuse_preparation'),'Infrastructure aborts must start from fresh output directories' elif i in WALL: assert w['returncode']==-9 and w['stop_reason']=='wall_limit';classification='WALL_LIMIT_FAIL' else: assert w['status']=='PASS' and w['returncode']==0;classification='PASS' assert (ORIGINAL/'ready'/spec['case']/'receipt.json').exists() rows.append(dict(index=i,case=spec['case'],nodes=spec['tet_nodes'],role='learning' if spec['case'] in roles['learning_parts'] else 'validation', classification=classification,worker=r['worker'],supervisor_cpu_seconds=r['supervisor_cpu_seconds'], guard_receipt_path=str(folder/'guard-result.json'),guard_receipt_sha256=sha(folder/'guard-result.json'), worker_started_sha256=sha(folder/'worker.started.json'),worker_receipt_sha256=sha(folder/'worker.receipt.json'))) INCIDENT.mkdir();(INCIDENT/'quarantine').mkdir() snapshot=dict(status='INFRA_ABORTED',original_root=str(ORIGINAL),original_contract_sha256=sha(ORIGINAL/'contract.json'), batch_started_sha256=sha(ORIGINAL/'batch-started.json'),last_progress_sha256=sha(ORIGINAL/'progress.json'),jobs=rows, ordinary_pass=362,wall_limit_fail_indices=WALL,infra_aborted_indices=INFRA,never_attempted_indices=UNATTEMPTED, cause='Coordinator FileNotFoundError during disk_usage stat of progress.json.tmp; finally terminated 16 active workers', original_session=16682,original_exit_code=1,original_batch_complete_created=False) save(INCIDENT/'original-batch-infra-aborted.json',snapshot) moves=[] for i in INFRA: case=c['cases'][i]['case'] for kind in ['parts','ready']: source=ORIGINAL/kind/case;dest=INCIDENT/'quarantine'/kind/case assert source.exists() and not dest.exists() moves.append(dict(original_index=i,kind=kind,from_path=str(source),to_path=str(dest),files=inventory(source))) save(INCIDENT/'quarantine-plan.json',dict(moves=moves)) moved=[] for item in moves: source=Path(item['from_path']);dest=Path(item['to_path']);dest.parent.mkdir(parents=True,exist_ok=True) source.rename(dest);assert inventory(dest)==item['files'] moved.append(item['to_path']);save(INCIDENT/'quarantine-progress.json',dict(moved=moved,total=len(moves))) save(INCIDENT/'quarantine-receipt.json',dict(status='ALL_16_PARTIALS_MOVED_AND_HASH_VERIFIED',moves=moves,reused_by_recovery=False)) prior=sum(r['worker']['cpu_seconds'] for r in rows);aborted=sum(r['worker']['cpu_seconds'] for r in rows if r['classification']=='INFRA_ABORTED') ledger=dict(status='FROZEN_INCIDENT_LEDGER',ruling='1794689807.792819',original_root=str(ORIGINAL), original_snapshot=str(INCIDENT/'original-batch-infra-aborted.json'),original_snapshot_sha256=sha(INCIDENT/'original-batch-infra-aborted.json'), quarantine_receipt=str(INCIDENT/'quarantine-receipt.json'),quarantine_receipt_sha256=sha(INCIDENT/'quarantine-receipt.json'), pass_count=362,wall_limit_indices=WALL,infra_aborted_indices=INFRA,unattempted_indices=UNATTEMPTED,recovery_indices=INDICES, prior_worker_cpu_seconds=prior,prior_aborted_worker_cpu_seconds_included=aborted, prior_guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in rows),coordinator_cpu_seconds='UNKNOWN_NO_EXIT_ACCOUNTING_RECEIPT', accounting_rule='All 381 original worker receipts counted once, including the 16 aborted; add only new recovery worker CPU', labelled_recovery_allowance_per_infra_part=1,original_batch_complete_created=False) save(INCIDENT/'incident-ledger.json',ledger) RECOVERY.mkdir();(RECOVERY/'public').mkdir();shutil.copy2(ORIGINAL/'role-manifest.json',RECOVERY/'role-manifest.json') selected=[c['cases'][i] for i in INDICES] terminal=[r for r in rows if r['classification']!='INFRA_ABORTED'] rates={} for cached in [False,True]: group=[r for r in terminal if bool(c['cases'][r['index']].get('reuse_preparation'))==cached] rates[cached]=sum(r['worker']['cpu_seconds'] for r in group)/sum(r['nodes'] for r in group) if group else c['initial_cpu_seconds_per_node'] projected=prior+1.25*sum(rates[bool(s.get('reuse_preparation'))]*s['tet_nodes'] for s in selected) roots=[str(ORIGINAL),str(INCIDENT),str(RECOVERY)];existing=sum(disk_usage(p) for p in roots) projected_disk=existing+c['initial_projection_gib']*2**30*sum(s['tet_nodes'] for s in selected)/sum(s['tet_nodes'] for s in c['cases']) added=['tools/s2_recovery_monitoring.py','tools/s2_recovery_supervisor.py','tools/run_s2_recovery_preparation.py','tools/prepare_s2_incident_recovery.py'] contract=dict(c,version='S2-labelled-infrastructure-recovery-v1',cases=selected,original_indices=INDICES,infra_aborted_indices=INFRA, original_root=str(ORIGINAL),original_contract_sha256=sha(ORIGINAL/'contract.json'), original_snapshot=ledger['original_snapshot'],original_snapshot_sha256=ledger['original_snapshot_sha256'], incident_ledger=str(INCIDENT/'incident-ledger.json'),incident_ledger_sha256=sha(INCIDENT/'incident-ledger.json'), code_sha256={p:sha(p) for p in list(c['code_sha256'])+added},prior_worker_cpu_seconds=prior,budget_roots=roots, projected_cumulative_worker_cpu_hours=projected/3600,projected_cumulative_gib=projected_disk/2**30, projection_method='Exclude partial infrastructure aborts from rate estimation, retain their CPU in prior total; successful/true-timeout observed category rates times 134 nodes plus 25%; original maximum-byte/node margin for extra outputs', limits=dict(worker_cpu_hours=60,workers=16,disk_bytes=100*2**30,per_part_wall_seconds=1800,per_part_tree_rss_bytes=1342177280,retries=0,replacements=0), checkpoint='No new first50 barrier; original accepted checkpoint retained',authorizing_rulings=['1794689807.792819'], partial_outputs_policy='Original 16 quarantined by rename and hash verified, never read/reused; recovery uses a new root; original part specs identical') save(RECOVERY/'contract.json',contract) report=dict(status='READY_WITHIN_CAPS' if projected<=60*3600 and projected_disk<=100*2**30 else 'PROJECTION_OVER_CAP_REPORT_BEFORE_LAUNCH', original_pass=362,original_wall_fail=3,labelled_recovery=16,never_attempted=118,recovery_parts=134, prior_worker_cpu_hours=prior/3600,prior_aborted_cpu_hours_already_included=aborted/3600, projected_cumulative_worker_cpu_hours=projected/3600,projected_cumulative_gib=projected_disk/2**30, contract_sha256=sha(RECOVERY/'contract.json'),incident_ledger_sha256=sha(INCIDENT/'incident-ledger.json'), worker_numerical_code_unchanged=True,workers_started=False,GPU_started=False) save(RECOVERY/'public/prelaunch-projection.json',report);print(json.dumps(report)) if __name__=='__main__':main()