103 lines
7.7 KiB
Python
103 lines
7.7 KiB
Python
"""Three-source merge for this explicitly authorized infrastructure incident."""
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
from freeze_g1_components import sha
|
|
from s2_recovery_monitoring import disk_usage
|
|
from prepare_s2_incident_recovery import INFRA,WALL,UNATTEMPTED,INDICES
|
|
|
|
|
|
def read_bound(entry):
|
|
p=Path(entry['path']);assert sha(p)==entry['sha256'];return json.loads(p.read_text())
|
|
|
|
|
|
def verify(proof):
|
|
assert set(proof)=={'original_batch','recovery_batch','incident_ledger'}
|
|
original=read_bound(proof['original_batch']);recovered=read_bound(proof['recovery_batch']);ledger=read_bound(proof['incident_ledger'])
|
|
root=Path(original['original_root']);recovery=Path(proof['recovery_batch']['root'])
|
|
assert original['status']=='INFRA_ABORTED' and not (root/'batch-complete.json').exists()
|
|
assert sha(root/'contract.json')==original['original_contract_sha256']
|
|
assert sha(root/'batch-started.json')==original['batch_started_sha256']
|
|
assert sha(root/'progress.json')==original['last_progress_sha256']
|
|
old=json.loads((root/'contract.json').read_text());c=json.loads((recovery/'contract.json').read_text())
|
|
assert sha(recovery/'contract.json')==proof['recovery_batch']['contract_sha256']
|
|
assert c['original_root']==str(root) and c['original_contract_sha256']==original['original_contract_sha256']
|
|
assert c['incident_ledger_sha256']==proof['incident_ledger']['sha256']
|
|
assert c['incident_ledger']==proof['incident_ledger']['path']
|
|
assert c['original_snapshot']==ledger['original_snapshot']==proof['original_batch']['path']
|
|
assert c['original_snapshot_sha256']==proof['original_batch']['sha256']==ledger['original_snapshot_sha256']
|
|
assert ledger['recovery_indices']==c['original_indices']==INDICES
|
|
assert ledger['infra_aborted_indices']==c['infra_aborted_indices']==INFRA
|
|
assert ledger['wall_limit_indices']==WALL and ledger['unattempted_indices']==UNATTEMPTED
|
|
assert c['role_manifest_sha256']==old['role_manifest_sha256']==sha(root/'role-manifest.json')==sha(recovery/'role-manifest.json')
|
|
assert len(old['cases'])==499 and len(c['cases'])==134
|
|
assert c['cases']==[old['cases'][i] for i in INDICES]
|
|
assert c['limits']==dict(worker_cpu_hours=60,workers=16,disk_bytes=100*2**30,per_part_wall_seconds=1800,per_part_tree_rss_bytes=1342177280,retries=0,replacements=0)
|
|
for p,h in old['code_sha256'].items():assert c['code_sha256'][p]==h and sha(p)==h
|
|
for p,h in c['code_sha256'].items():assert sha(p)==h
|
|
assert sha(ledger['quarantine_receipt'])==ledger['quarantine_receipt_sha256']
|
|
quarantine=json.loads(Path(ledger['quarantine_receipt']).read_text())
|
|
assert quarantine['status']=='ALL_16_PARTIALS_MOVED_AND_HASH_VERIFIED' and quarantine['reused_by_recovery'] is False
|
|
assert {(m['original_index'],m['kind']) for m in quarantine['moves']}=={(i,k) for i in INFRA for k in ['parts','ready']}
|
|
for move in quarantine['moves']:
|
|
assert not Path(move['from_path']).exists()
|
|
for f in move['files']:assert sha(Path(move['to_path'])/f['path'])==f['sha256']
|
|
old_jobs=original['jobs'];assert [r['index'] for r in old_jobs]==list(range(381))
|
|
for r in old_jobs:
|
|
i=r['index'];assert r['case']==old['cases'][i]['case'] and r['nodes']==old['cases'][i]['tet_nodes']
|
|
path=Path(r['guard_receipt_path']);assert sha(path)==r['guard_receipt_sha256']
|
|
raw=json.loads(path.read_text());assert r['worker']==raw['worker'] and r['supervisor_cpu_seconds']==raw['supervisor_cpu_seconds']
|
|
assert sha(path.parent/'worker.started.json')==r['worker_started_sha256'] and sha(path.parent/'worker.receipt.json')==r['worker_receipt_sha256']
|
|
expected='INFRA_ABORTED' if i in INFRA else 'WALL_LIMIT_FAIL' if i in WALL else 'PASS'
|
|
assert r['classification']==expected
|
|
if i in INFRA:assert r['worker']['returncode']==-9 and r['worker']['stop_reason'] is None
|
|
elif i in WALL:assert r['worker']['stop_reason']=='wall_limit'
|
|
else:assert r['worker']['status']=='PASS' and r['worker']['returncode']==0
|
|
prior=sum(r['worker']['cpu_seconds'] for r in old_jobs)
|
|
assert math.isclose(prior,ledger['prior_worker_cpu_seconds'],rel_tol=1e-12)
|
|
assert math.isclose(prior,c['prior_worker_cpu_seconds'],rel_tol=1e-12)
|
|
assert math.isclose(sum(r['worker']['cpu_seconds'] for r in old_jobs if r['index'] in INFRA),ledger['prior_aborted_worker_cpu_seconds_included'],rel_tol=1e-12)
|
|
assert recovered['status']=='ATTEMPTS_COMPLETE' and recovered['stop_reason'] is None and not recovered['not_run_indices']
|
|
assert sorted(r['index'] for r in recovered['jobs'])==list(range(134))
|
|
jobs=[dict(r,ready_source_root=str(root)) for r in old_jobs if r['index'] not in INFRA]
|
|
for r in recovered['jobs']:
|
|
local=r['index'];i=INDICES[local];spec=old['cases'][i]
|
|
assert r['case']==spec['case'] and r['original_index']==i
|
|
assert r['labelled_infrastructure_recovery']==(i in INFRA)
|
|
path=recovery/'jobs'/f'job-{local:04d}'/'guard-result.json';raw=json.loads(path.read_text())
|
|
assert r['worker']==raw['worker'] and r['supervisor_cpu_seconds']==raw['supervisor_cpu_seconds']
|
|
assert raw['original_index']==i and raw['labelled_infrastructure_recovery']==(i in INFRA)
|
|
w=r['worker'];assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0 and not w['gpu_visible']
|
|
assert not (w['returncode']<0 and w['stop_reason'] is None),'Unexplained recovery signal is not a scientific failure'
|
|
jobs.append(dict(r,index=i,source_index=local,ready_source_root=str(recovery),guard_receipt_sha256=sha(path),
|
|
original_infra_abort=old_jobs[i] if i in INFRA else None))
|
|
assert sorted(r['index'] for r in jobs)==list(range(499))
|
|
extra=sum(r['worker']['cpu_seconds'] for r in recovered['jobs']);cpu=prior+extra
|
|
assert math.isclose(extra,recovered['worker_cpu_seconds'],rel_tol=1e-12)
|
|
assert math.isclose(cpu,recovered['cumulative_worker_cpu_seconds'],rel_tol=1e-12)
|
|
assert len(c['budget_roots'])==len(set(c['budget_roots']))==3
|
|
assert set(c['budget_roots'])=={str(root),str(recovery),str(Path(proof['incident_ledger']['path']).parent)}
|
|
used=sum(disk_usage(p) for p in c['budget_roots'])
|
|
assert cpu<=60*3600 and used<=100*2**30
|
|
return dict(status='ATTEMPTS_COMPLETE',stop_reason=None,jobs=sorted(jobs,key=lambda r:r['index']),not_run_indices=[],
|
|
source_provenance=proof,prior_worker_cpu_seconds=prior,recovery_worker_cpu_seconds=extra,worker_cpu_seconds=cpu,
|
|
original_infra_abort_CPU_already_included=ledger['prior_aborted_worker_cpu_seconds_included'],disk_bytes=used,
|
|
failed=sum(r['worker']['status']!='PASS' for r in jobs),labelled_infrastructure_recoveries=16,
|
|
scientific_retries=0,test_labels_read=False,ruling='1794689807.792819')
|
|
|
|
|
|
def load_manifest(path):
|
|
saved=json.loads(Path(path).read_text());batch=verify(saved['source_provenance'])
|
|
assert batch['jobs']==saved['jobs'] and math.isclose(batch['worker_cpu_seconds'],saved['worker_cpu_seconds'],rel_tol=1e-12)
|
|
return batch
|
|
|
|
|
|
def create_manifest(incident,recovery):
|
|
proof=dict(original_batch=dict(path=str(incident/'original-batch-infra-aborted.json'),sha256=sha(incident/'original-batch-infra-aborted.json')),
|
|
incident_ledger=dict(path=str(incident/'incident-ledger.json'),sha256=sha(incident/'incident-ledger.json')),
|
|
recovery_batch=dict(path=str(recovery/'batch-complete.json'),sha256=sha(recovery/'batch-complete.json'),root=str(recovery),contract_sha256=sha(recovery/'contract.json')))
|
|
batch=verify(proof);dest=incident/'merged-recovery-terminal.json';pending=dest.with_suffix('.json.pending')
|
|
with pending.open('x') as f:json.dump(batch,f,indent=2);f.write('\n');f.flush();os.fsync(f.fileno())
|
|
os.link(pending,dest);pending.unlink();return dest
|