import json from pathlib import Path import tempfile import unittest from freeze_g1_components import sha from s2_incident_recovery_manifest import create_manifest,load_manifest,verify from prepare_s2_incident_recovery import INFRA,WALL,UNATTEMPTED,INDICES def put(path,value):path.parent.mkdir(parents=True,exist_ok=True);path.write_text(json.dumps(value)) class RecoveryMergeTests(unittest.TestCase): def fixture(self,base): original=base/'original';incident=base/'incident';recovery=base/'recovery' for p in [original,incident,recovery]:p.mkdir() cases=[dict(case=f'case-{i}',tet_nodes=i+1) for i in range(499)] for root in [original,recovery]:put(root/'role-manifest.json',dict(learning_parts=[s['case'] for s in cases[:387]],validation_parts=[s['case'] for s in cases[387:]],sealed_test_parts=['sealed'])) old=dict(cases=cases,code_sha256={},role_manifest_sha256=sha(original/'role-manifest.json')) put(original/'contract.json',old);put(original/'batch-started.json',{});put(original/'progress.json',dict(completed=365,active=INFRA,next_index=381)) rows=[] for i in range(381): status='INFRA_ABORTED' if i in INFRA else 'WALL_LIMIT_FAIL' if i in WALL else 'PASS' w=dict(status='PASS' if status=='PASS' else 'STOPPED_OR_FAILED',returncode=0 if status=='PASS' else -9, stop_reason='wall_limit' if i in WALL else None,cpu_seconds=1.,gpu_visible=False) folder=original/'jobs'/f'job-{i:04d}';put(folder/'guard-result.json',dict(worker=w,supervisor_cpu_seconds=.01)) put(folder/'worker.started.json',{});put(folder/'worker.receipt.json',w) rows.append(dict(index=i,case=cases[i]['case'],nodes=i+1,classification=status,worker=w,supervisor_cpu_seconds=.01, guard_receipt_path=str(folder/'guard-result.json'),guard_receipt_sha256=sha(folder/'guard-result.json'), worker_started_sha256=sha(folder/'worker.started.json'),worker_receipt_sha256=sha(folder/'worker.receipt.json'))) snapshot=incident/'original-batch-infra-aborted.json' put(snapshot,dict(status='INFRA_ABORTED',original_root=str(original),original_contract_sha256=sha(original/'contract.json'),batch_started_sha256=sha(original/'batch-started.json'),last_progress_sha256=sha(original/'progress.json'),jobs=rows)) quarantine=incident/'quarantine-receipt.json' put(quarantine,dict(status='ALL_16_PARTIALS_MOVED_AND_HASH_VERIFIED',reused_by_recovery=False,moves=[dict(original_index=i,kind=k,from_path=str(original/k/cases[i]['case']),to_path=str(incident/'quarantine'/k/cases[i]['case']),files=[]) for i in INFRA for k in ['parts','ready']])) ledger=incident/'incident-ledger.json' put(ledger,dict(original_snapshot=str(snapshot),original_snapshot_sha256=sha(snapshot),quarantine_receipt=str(quarantine),quarantine_receipt_sha256=sha(quarantine),recovery_indices=INDICES,infra_aborted_indices=INFRA,wall_limit_indices=WALL,unattempted_indices=UNATTEMPTED,prior_worker_cpu_seconds=381.,prior_aborted_worker_cpu_seconds_included=16.)) c=dict(cases=[cases[i] for i in INDICES],original_indices=INDICES,infra_aborted_indices=INFRA,original_root=str(original),original_contract_sha256=sha(original/'contract.json'),incident_ledger=str(ledger),incident_ledger_sha256=sha(ledger),original_snapshot=str(snapshot),original_snapshot_sha256=sha(snapshot),role_manifest_sha256=old['role_manifest_sha256'],code_sha256={},prior_worker_cpu_seconds=381.,budget_roots=[str(original),str(incident),str(recovery)],limits=dict(worker_cpu_hours=60,workers=16,disk_bytes=100*2**30,per_part_wall_seconds=1800,per_part_tree_rss_bytes=1342177280,retries=0,replacements=0)) put(recovery/'contract.json',c);jobs=[] for local,i in enumerate(INDICES): w=dict(status='PASS',returncode=0,stop_reason=None,cpu_seconds=1.,gpu_visible=False) r=dict(worker=w,supervisor_cpu_seconds=.01,original_index=i,labelled_infrastructure_recovery=i in INFRA) put(recovery/'jobs'/f'job-{local:04d}'/'guard-result.json',r);jobs.append(dict(index=local,case=cases[i]['case'],**r)) put(recovery/'batch-complete.json',dict(status='ATTEMPTS_COMPLETE',stop_reason=None,not_run_indices=[],jobs=jobs,worker_cpu_seconds=134.,cumulative_worker_cpu_seconds=515.)) return original,incident,recovery def test_three_sources_499_outcomes_keep_abort_cpu_once_and_no_fake_original_terminal(self): with tempfile.TemporaryDirectory() as tmp: original,incident,recovery=self.fixture(Path(tmp));path=create_manifest(incident,recovery);r=load_manifest(path) self.assertEqual(len(r['jobs']),499);self.assertEqual(r['failed'],3);self.assertEqual(r['worker_cpu_seconds'],515.) self.assertEqual(sum(j['worker']['cpu_seconds'] for j in r['jobs']),499.) self.assertEqual(r['original_infra_abort_CPU_already_included'],16.) self.assertIsNotNone(r['jobs'][INFRA[0]]['original_infra_abort']);self.assertFalse((original/'batch-complete.json').exists()) with self.assertRaises(FileExistsError):create_manifest(incident,recovery) def test_reject_wrong_index_set_changed_specs_or_duplicate_budget(self): for kind in ['index','spec','double_count','missing_terminal','fake_original','global_stop']: with self.subTest(kind=kind),tempfile.TemporaryDirectory() as tmp: original,incident,recovery=self.fixture(Path(tmp)) if kind=='fake_original':put(original/'batch-complete.json',{}) elif kind in ['index','spec']: path=recovery/'contract.json';c=json.loads(path.read_text()) if kind=='index':c['original_indices'][0]=0 else:c['cases'][0]['tet_nodes']+=1 put(path,c) else: path=recovery/'batch-complete.json';b=json.loads(path.read_text()) if kind=='double_count':b['cumulative_worker_cpu_seconds']+=16. elif kind=='global_stop':b.update(status='STOPPED',stop_reason='GLOBAL_CPU_LIMIT') else:b['jobs'].pop() put(path,b) with self.assertRaises(AssertionError):create_manifest(incident,recovery) def test_hash_bound_source_mutation_rejected(self): with tempfile.TemporaryDirectory() as tmp: original,incident,recovery=self.fixture(Path(tmp));path=create_manifest(incident,recovery) put(original/'jobs/job-0000/guard-result.json',{}) with self.assertRaises(AssertionError):load_manifest(path) if __name__=='__main__':unittest.main(verbosity=2)