"""Three-source merge for this explicitly authorized infrastructure incident.""" import json import math import os from pathlib import Path from freeze_g1_components import sha from s2_recovery_monitoring import disk_usage from prepare_s2_incident_recovery import INFRA,WALL,UNATTEMPTED,INDICES def read_bound(entry): p=Path(entry['path']);assert sha(p)==entry['sha256'];return json.loads(p.read_text()) def verify(proof): assert set(proof)=={'original_batch','recovery_batch','incident_ledger'} original=read_bound(proof['original_batch']);recovered=read_bound(proof['recovery_batch']);ledger=read_bound(proof['incident_ledger']) root=Path(original['original_root']);recovery=Path(proof['recovery_batch']['root']) assert original['status']=='INFRA_ABORTED' and not (root/'batch-complete.json').exists() assert sha(root/'contract.json')==original['original_contract_sha256'] assert sha(root/'batch-started.json')==original['batch_started_sha256'] assert sha(root/'progress.json')==original['last_progress_sha256'] old=json.loads((root/'contract.json').read_text());c=json.loads((recovery/'contract.json').read_text()) assert sha(recovery/'contract.json')==proof['recovery_batch']['contract_sha256'] assert c['original_root']==str(root) and c['original_contract_sha256']==original['original_contract_sha256'] assert c['incident_ledger_sha256']==proof['incident_ledger']['sha256'] assert c['incident_ledger']==proof['incident_ledger']['path'] assert c['original_snapshot']==ledger['original_snapshot']==proof['original_batch']['path'] assert c['original_snapshot_sha256']==proof['original_batch']['sha256']==ledger['original_snapshot_sha256'] assert ledger['recovery_indices']==c['original_indices']==INDICES assert ledger['infra_aborted_indices']==c['infra_aborted_indices']==INFRA assert ledger['wall_limit_indices']==WALL and ledger['unattempted_indices']==UNATTEMPTED assert c['role_manifest_sha256']==old['role_manifest_sha256']==sha(root/'role-manifest.json')==sha(recovery/'role-manifest.json') assert len(old['cases'])==499 and len(c['cases'])==134 assert c['cases']==[old['cases'][i] for i in INDICES] assert c['limits']==dict(worker_cpu_hours=60,workers=16,disk_bytes=100*2**30,per_part_wall_seconds=1800,per_part_tree_rss_bytes=1342177280,retries=0,replacements=0) for p,h in old['code_sha256'].items():assert c['code_sha256'][p]==h and sha(p)==h for p,h in c['code_sha256'].items():assert sha(p)==h assert sha(ledger['quarantine_receipt'])==ledger['quarantine_receipt_sha256'] quarantine=json.loads(Path(ledger['quarantine_receipt']).read_text()) assert quarantine['status']=='ALL_16_PARTIALS_MOVED_AND_HASH_VERIFIED' and quarantine['reused_by_recovery'] is False assert {(m['original_index'],m['kind']) for m in quarantine['moves']}=={(i,k) for i in INFRA for k in ['parts','ready']} for move in quarantine['moves']: assert not Path(move['from_path']).exists() for f in move['files']:assert sha(Path(move['to_path'])/f['path'])==f['sha256'] old_jobs=original['jobs'];assert [r['index'] for r in old_jobs]==list(range(381)) for r in old_jobs: i=r['index'];assert r['case']==old['cases'][i]['case'] and r['nodes']==old['cases'][i]['tet_nodes'] path=Path(r['guard_receipt_path']);assert sha(path)==r['guard_receipt_sha256'] raw=json.loads(path.read_text());assert r['worker']==raw['worker'] and r['supervisor_cpu_seconds']==raw['supervisor_cpu_seconds'] assert sha(path.parent/'worker.started.json')==r['worker_started_sha256'] and sha(path.parent/'worker.receipt.json')==r['worker_receipt_sha256'] expected='INFRA_ABORTED' if i in INFRA else 'WALL_LIMIT_FAIL' if i in WALL else 'PASS' assert r['classification']==expected if i in INFRA:assert r['worker']['returncode']==-9 and r['worker']['stop_reason'] is None elif i in WALL:assert r['worker']['stop_reason']=='wall_limit' else:assert r['worker']['status']=='PASS' and r['worker']['returncode']==0 prior=sum(r['worker']['cpu_seconds'] for r in old_jobs) assert math.isclose(prior,ledger['prior_worker_cpu_seconds'],rel_tol=1e-12) assert math.isclose(prior,c['prior_worker_cpu_seconds'],rel_tol=1e-12) assert math.isclose(sum(r['worker']['cpu_seconds'] for r in old_jobs if r['index'] in INFRA),ledger['prior_aborted_worker_cpu_seconds_included'],rel_tol=1e-12) assert recovered['status']=='ATTEMPTS_COMPLETE' and recovered['stop_reason'] is None and not recovered['not_run_indices'] assert sorted(r['index'] for r in recovered['jobs'])==list(range(134)) jobs=[dict(r,ready_source_root=str(root)) for r in old_jobs if r['index'] not in INFRA] for r in recovered['jobs']: local=r['index'];i=INDICES[local];spec=old['cases'][i] assert r['case']==spec['case'] and r['original_index']==i assert r['labelled_infrastructure_recovery']==(i in INFRA) path=recovery/'jobs'/f'job-{local:04d}'/'guard-result.json';raw=json.loads(path.read_text()) assert r['worker']==raw['worker'] and r['supervisor_cpu_seconds']==raw['supervisor_cpu_seconds'] assert raw['original_index']==i and raw['labelled_infrastructure_recovery']==(i in INFRA) w=r['worker'];assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0 and not w['gpu_visible'] assert not (w['returncode']<0 and w['stop_reason'] is None),'Unexplained recovery signal is not a scientific failure' jobs.append(dict(r,index=i,source_index=local,ready_source_root=str(recovery),guard_receipt_sha256=sha(path), original_infra_abort=old_jobs[i] if i in INFRA else None)) assert sorted(r['index'] for r in jobs)==list(range(499)) extra=sum(r['worker']['cpu_seconds'] for r in recovered['jobs']);cpu=prior+extra assert math.isclose(extra,recovered['worker_cpu_seconds'],rel_tol=1e-12) assert math.isclose(cpu,recovered['cumulative_worker_cpu_seconds'],rel_tol=1e-12) assert len(c['budget_roots'])==len(set(c['budget_roots']))==3 assert set(c['budget_roots'])=={str(root),str(recovery),str(Path(proof['incident_ledger']['path']).parent)} used=sum(disk_usage(p) for p in c['budget_roots']) assert cpu<=60*3600 and used<=100*2**30 return dict(status='ATTEMPTS_COMPLETE',stop_reason=None,jobs=sorted(jobs,key=lambda r:r['index']),not_run_indices=[], source_provenance=proof,prior_worker_cpu_seconds=prior,recovery_worker_cpu_seconds=extra,worker_cpu_seconds=cpu, original_infra_abort_CPU_already_included=ledger['prior_aborted_worker_cpu_seconds_included'],disk_bytes=used, failed=sum(r['worker']['status']!='PASS' for r in jobs),labelled_infrastructure_recoveries=16, scientific_retries=0,test_labels_read=False,ruling='1794689807.792819') def load_manifest(path): saved=json.loads(Path(path).read_text());batch=verify(saved['source_provenance']) assert batch['jobs']==saved['jobs'] and math.isclose(batch['worker_cpu_seconds'],saved['worker_cpu_seconds'],rel_tol=1e-12) return batch def create_manifest(incident,recovery): proof=dict(original_batch=dict(path=str(incident/'original-batch-infra-aborted.json'),sha256=sha(incident/'original-batch-infra-aborted.json')), incident_ledger=dict(path=str(incident/'incident-ledger.json'),sha256=sha(incident/'incident-ledger.json')), recovery_batch=dict(path=str(recovery/'batch-complete.json'),sha256=sha(recovery/'batch-complete.json'),root=str(recovery),contract_sha256=sha(recovery/'contract.json'))) batch=verify(proof);dest=incident/'merged-recovery-terminal.json';pending=dest.with_suffix('.json.pending') with pending.open('x') as f:json.dump(batch,f,indent=2);f.write('\n');f.flush();os.fsync(f.fileno()) os.link(pending,dest);pending.unlink();return dest