78 lines
6.6 KiB
Python
78 lines
6.6 KiB
Python
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from freeze_g1_components import sha
|
|
from s2_incident_recovery_manifest import create_manifest,load_manifest,verify
|
|
from prepare_s2_incident_recovery import INFRA,WALL,UNATTEMPTED,INDICES
|
|
|
|
|
|
def put(path,value):path.parent.mkdir(parents=True,exist_ok=True);path.write_text(json.dumps(value))
|
|
|
|
|
|
class RecoveryMergeTests(unittest.TestCase):
|
|
def fixture(self,base):
|
|
original=base/'original';incident=base/'incident';recovery=base/'recovery'
|
|
for p in [original,incident,recovery]:p.mkdir()
|
|
cases=[dict(case=f'case-{i}',tet_nodes=i+1) for i in range(499)]
|
|
for root in [original,recovery]:put(root/'role-manifest.json',dict(learning_parts=[s['case'] for s in cases[:387]],validation_parts=[s['case'] for s in cases[387:]],sealed_test_parts=['sealed']))
|
|
old=dict(cases=cases,code_sha256={},role_manifest_sha256=sha(original/'role-manifest.json'))
|
|
put(original/'contract.json',old);put(original/'batch-started.json',{});put(original/'progress.json',dict(completed=365,active=INFRA,next_index=381))
|
|
rows=[]
|
|
for i in range(381):
|
|
status='INFRA_ABORTED' if i in INFRA else 'WALL_LIMIT_FAIL' if i in WALL else 'PASS'
|
|
w=dict(status='PASS' if status=='PASS' else 'STOPPED_OR_FAILED',returncode=0 if status=='PASS' else -9,
|
|
stop_reason='wall_limit' if i in WALL else None,cpu_seconds=1.,gpu_visible=False)
|
|
folder=original/'jobs'/f'job-{i:04d}';put(folder/'guard-result.json',dict(worker=w,supervisor_cpu_seconds=.01))
|
|
put(folder/'worker.started.json',{});put(folder/'worker.receipt.json',w)
|
|
rows.append(dict(index=i,case=cases[i]['case'],nodes=i+1,classification=status,worker=w,supervisor_cpu_seconds=.01,
|
|
guard_receipt_path=str(folder/'guard-result.json'),guard_receipt_sha256=sha(folder/'guard-result.json'),
|
|
worker_started_sha256=sha(folder/'worker.started.json'),worker_receipt_sha256=sha(folder/'worker.receipt.json')))
|
|
snapshot=incident/'original-batch-infra-aborted.json'
|
|
put(snapshot,dict(status='INFRA_ABORTED',original_root=str(original),original_contract_sha256=sha(original/'contract.json'),batch_started_sha256=sha(original/'batch-started.json'),last_progress_sha256=sha(original/'progress.json'),jobs=rows))
|
|
quarantine=incident/'quarantine-receipt.json'
|
|
put(quarantine,dict(status='ALL_16_PARTIALS_MOVED_AND_HASH_VERIFIED',reused_by_recovery=False,moves=[dict(original_index=i,kind=k,from_path=str(original/k/cases[i]['case']),to_path=str(incident/'quarantine'/k/cases[i]['case']),files=[]) for i in INFRA for k in ['parts','ready']]))
|
|
ledger=incident/'incident-ledger.json'
|
|
put(ledger,dict(original_snapshot=str(snapshot),original_snapshot_sha256=sha(snapshot),quarantine_receipt=str(quarantine),quarantine_receipt_sha256=sha(quarantine),recovery_indices=INDICES,infra_aborted_indices=INFRA,wall_limit_indices=WALL,unattempted_indices=UNATTEMPTED,prior_worker_cpu_seconds=381.,prior_aborted_worker_cpu_seconds_included=16.))
|
|
c=dict(cases=[cases[i] for i in INDICES],original_indices=INDICES,infra_aborted_indices=INFRA,original_root=str(original),original_contract_sha256=sha(original/'contract.json'),incident_ledger=str(ledger),incident_ledger_sha256=sha(ledger),original_snapshot=str(snapshot),original_snapshot_sha256=sha(snapshot),role_manifest_sha256=old['role_manifest_sha256'],code_sha256={},prior_worker_cpu_seconds=381.,budget_roots=[str(original),str(incident),str(recovery)],limits=dict(worker_cpu_hours=60,workers=16,disk_bytes=100*2**30,per_part_wall_seconds=1800,per_part_tree_rss_bytes=1342177280,retries=0,replacements=0))
|
|
put(recovery/'contract.json',c);jobs=[]
|
|
for local,i in enumerate(INDICES):
|
|
w=dict(status='PASS',returncode=0,stop_reason=None,cpu_seconds=1.,gpu_visible=False)
|
|
r=dict(worker=w,supervisor_cpu_seconds=.01,original_index=i,labelled_infrastructure_recovery=i in INFRA)
|
|
put(recovery/'jobs'/f'job-{local:04d}'/'guard-result.json',r);jobs.append(dict(index=local,case=cases[i]['case'],**r))
|
|
put(recovery/'batch-complete.json',dict(status='ATTEMPTS_COMPLETE',stop_reason=None,not_run_indices=[],jobs=jobs,worker_cpu_seconds=134.,cumulative_worker_cpu_seconds=515.))
|
|
return original,incident,recovery
|
|
def test_three_sources_499_outcomes_keep_abort_cpu_once_and_no_fake_original_terminal(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
original,incident,recovery=self.fixture(Path(tmp));path=create_manifest(incident,recovery);r=load_manifest(path)
|
|
self.assertEqual(len(r['jobs']),499);self.assertEqual(r['failed'],3);self.assertEqual(r['worker_cpu_seconds'],515.)
|
|
self.assertEqual(sum(j['worker']['cpu_seconds'] for j in r['jobs']),499.)
|
|
self.assertEqual(r['original_infra_abort_CPU_already_included'],16.)
|
|
self.assertIsNotNone(r['jobs'][INFRA[0]]['original_infra_abort']);self.assertFalse((original/'batch-complete.json').exists())
|
|
with self.assertRaises(FileExistsError):create_manifest(incident,recovery)
|
|
def test_reject_wrong_index_set_changed_specs_or_duplicate_budget(self):
|
|
for kind in ['index','spec','double_count','missing_terminal','fake_original','global_stop']:
|
|
with self.subTest(kind=kind),tempfile.TemporaryDirectory() as tmp:
|
|
original,incident,recovery=self.fixture(Path(tmp))
|
|
if kind=='fake_original':put(original/'batch-complete.json',{})
|
|
elif kind in ['index','spec']:
|
|
path=recovery/'contract.json';c=json.loads(path.read_text())
|
|
if kind=='index':c['original_indices'][0]=0
|
|
else:c['cases'][0]['tet_nodes']+=1
|
|
put(path,c)
|
|
else:
|
|
path=recovery/'batch-complete.json';b=json.loads(path.read_text())
|
|
if kind=='double_count':b['cumulative_worker_cpu_seconds']+=16.
|
|
elif kind=='global_stop':b.update(status='STOPPED',stop_reason='GLOBAL_CPU_LIMIT')
|
|
else:b['jobs'].pop()
|
|
put(path,b)
|
|
with self.assertRaises(AssertionError):create_manifest(incident,recovery)
|
|
def test_hash_bound_source_mutation_rejected(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
original,incident,recovery=self.fixture(Path(tmp));path=create_manifest(incident,recovery)
|
|
put(original/'jobs/job-0000/guard-result.json',{})
|
|
with self.assertRaises(AssertionError):load_manifest(path)
|
|
|
|
|
|
if __name__=='__main__':unittest.main(verbosity=2)
|