Files
MeltMind/tools/s2_incident_recovery_manifest.py
T

103 lines
7.7 KiB
Python

"""Three-source merge for this explicitly authorized infrastructure incident."""
import json
import math
import os
from pathlib import Path
from freeze_g1_components import sha
from s2_recovery_monitoring import disk_usage
from prepare_s2_incident_recovery import INFRA,WALL,UNATTEMPTED,INDICES
def read_bound(entry):
p=Path(entry['path']);assert sha(p)==entry['sha256'];return json.loads(p.read_text())
def verify(proof):
assert set(proof)=={'original_batch','recovery_batch','incident_ledger'}
original=read_bound(proof['original_batch']);recovered=read_bound(proof['recovery_batch']);ledger=read_bound(proof['incident_ledger'])
root=Path(original['original_root']);recovery=Path(proof['recovery_batch']['root'])
assert original['status']=='INFRA_ABORTED' and not (root/'batch-complete.json').exists()
assert sha(root/'contract.json')==original['original_contract_sha256']
assert sha(root/'batch-started.json')==original['batch_started_sha256']
assert sha(root/'progress.json')==original['last_progress_sha256']
old=json.loads((root/'contract.json').read_text());c=json.loads((recovery/'contract.json').read_text())
assert sha(recovery/'contract.json')==proof['recovery_batch']['contract_sha256']
assert c['original_root']==str(root) and c['original_contract_sha256']==original['original_contract_sha256']
assert c['incident_ledger_sha256']==proof['incident_ledger']['sha256']
assert c['incident_ledger']==proof['incident_ledger']['path']
assert c['original_snapshot']==ledger['original_snapshot']==proof['original_batch']['path']
assert c['original_snapshot_sha256']==proof['original_batch']['sha256']==ledger['original_snapshot_sha256']
assert ledger['recovery_indices']==c['original_indices']==INDICES
assert ledger['infra_aborted_indices']==c['infra_aborted_indices']==INFRA
assert ledger['wall_limit_indices']==WALL and ledger['unattempted_indices']==UNATTEMPTED
assert c['role_manifest_sha256']==old['role_manifest_sha256']==sha(root/'role-manifest.json')==sha(recovery/'role-manifest.json')
assert len(old['cases'])==499 and len(c['cases'])==134
assert c['cases']==[old['cases'][i] for i in INDICES]
assert c['limits']==dict(worker_cpu_hours=60,workers=16,disk_bytes=100*2**30,per_part_wall_seconds=1800,per_part_tree_rss_bytes=1342177280,retries=0,replacements=0)
for p,h in old['code_sha256'].items():assert c['code_sha256'][p]==h and sha(p)==h
for p,h in c['code_sha256'].items():assert sha(p)==h
assert sha(ledger['quarantine_receipt'])==ledger['quarantine_receipt_sha256']
quarantine=json.loads(Path(ledger['quarantine_receipt']).read_text())
assert quarantine['status']=='ALL_16_PARTIALS_MOVED_AND_HASH_VERIFIED' and quarantine['reused_by_recovery'] is False
assert {(m['original_index'],m['kind']) for m in quarantine['moves']}=={(i,k) for i in INFRA for k in ['parts','ready']}
for move in quarantine['moves']:
assert not Path(move['from_path']).exists()
for f in move['files']:assert sha(Path(move['to_path'])/f['path'])==f['sha256']
old_jobs=original['jobs'];assert [r['index'] for r in old_jobs]==list(range(381))
for r in old_jobs:
i=r['index'];assert r['case']==old['cases'][i]['case'] and r['nodes']==old['cases'][i]['tet_nodes']
path=Path(r['guard_receipt_path']);assert sha(path)==r['guard_receipt_sha256']
raw=json.loads(path.read_text());assert r['worker']==raw['worker'] and r['supervisor_cpu_seconds']==raw['supervisor_cpu_seconds']
assert sha(path.parent/'worker.started.json')==r['worker_started_sha256'] and sha(path.parent/'worker.receipt.json')==r['worker_receipt_sha256']
expected='INFRA_ABORTED' if i in INFRA else 'WALL_LIMIT_FAIL' if i in WALL else 'PASS'
assert r['classification']==expected
if i in INFRA:assert r['worker']['returncode']==-9 and r['worker']['stop_reason'] is None
elif i in WALL:assert r['worker']['stop_reason']=='wall_limit'
else:assert r['worker']['status']=='PASS' and r['worker']['returncode']==0
prior=sum(r['worker']['cpu_seconds'] for r in old_jobs)
assert math.isclose(prior,ledger['prior_worker_cpu_seconds'],rel_tol=1e-12)
assert math.isclose(prior,c['prior_worker_cpu_seconds'],rel_tol=1e-12)
assert math.isclose(sum(r['worker']['cpu_seconds'] for r in old_jobs if r['index'] in INFRA),ledger['prior_aborted_worker_cpu_seconds_included'],rel_tol=1e-12)
assert recovered['status']=='ATTEMPTS_COMPLETE' and recovered['stop_reason'] is None and not recovered['not_run_indices']
assert sorted(r['index'] for r in recovered['jobs'])==list(range(134))
jobs=[dict(r,ready_source_root=str(root)) for r in old_jobs if r['index'] not in INFRA]
for r in recovered['jobs']:
local=r['index'];i=INDICES[local];spec=old['cases'][i]
assert r['case']==spec['case'] and r['original_index']==i
assert r['labelled_infrastructure_recovery']==(i in INFRA)
path=recovery/'jobs'/f'job-{local:04d}'/'guard-result.json';raw=json.loads(path.read_text())
assert r['worker']==raw['worker'] and r['supervisor_cpu_seconds']==raw['supervisor_cpu_seconds']
assert raw['original_index']==i and raw['labelled_infrastructure_recovery']==(i in INFRA)
w=r['worker'];assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0 and not w['gpu_visible']
assert not (w['returncode']<0 and w['stop_reason'] is None),'Unexplained recovery signal is not a scientific failure'
jobs.append(dict(r,index=i,source_index=local,ready_source_root=str(recovery),guard_receipt_sha256=sha(path),
original_infra_abort=old_jobs[i] if i in INFRA else None))
assert sorted(r['index'] for r in jobs)==list(range(499))
extra=sum(r['worker']['cpu_seconds'] for r in recovered['jobs']);cpu=prior+extra
assert math.isclose(extra,recovered['worker_cpu_seconds'],rel_tol=1e-12)
assert math.isclose(cpu,recovered['cumulative_worker_cpu_seconds'],rel_tol=1e-12)
assert len(c['budget_roots'])==len(set(c['budget_roots']))==3
assert set(c['budget_roots'])=={str(root),str(recovery),str(Path(proof['incident_ledger']['path']).parent)}
used=sum(disk_usage(p) for p in c['budget_roots'])
assert cpu<=60*3600 and used<=100*2**30
return dict(status='ATTEMPTS_COMPLETE',stop_reason=None,jobs=sorted(jobs,key=lambda r:r['index']),not_run_indices=[],
source_provenance=proof,prior_worker_cpu_seconds=prior,recovery_worker_cpu_seconds=extra,worker_cpu_seconds=cpu,
original_infra_abort_CPU_already_included=ledger['prior_aborted_worker_cpu_seconds_included'],disk_bytes=used,
failed=sum(r['worker']['status']!='PASS' for r in jobs),labelled_infrastructure_recoveries=16,
scientific_retries=0,test_labels_read=False,ruling='1794689807.792819')
def load_manifest(path):
saved=json.loads(Path(path).read_text());batch=verify(saved['source_provenance'])
assert batch['jobs']==saved['jobs'] and math.isclose(batch['worker_cpu_seconds'],saved['worker_cpu_seconds'],rel_tol=1e-12)
return batch
def create_manifest(incident,recovery):
proof=dict(original_batch=dict(path=str(incident/'original-batch-infra-aborted.json'),sha256=sha(incident/'original-batch-infra-aborted.json')),
incident_ledger=dict(path=str(incident/'incident-ledger.json'),sha256=sha(incident/'incident-ledger.json')),
recovery_batch=dict(path=str(recovery/'batch-complete.json'),sha256=sha(recovery/'batch-complete.json'),root=str(recovery),contract_sha256=sha(recovery/'contract.json')))
batch=verify(proof);dest=incident/'merged-recovery-terminal.json';pending=dest.with_suffix('.json.pending')
with pending.open('x') as f:json.dump(batch,f,indent=2);f.write('\n');f.flush();os.fsync(f.fileno())
os.link(pending,dest);pending.unlink();return dest