Files
MeltMind/tools/s3a_recovery_common.py

88 lines
4.6 KiB
Python

"""Explicit two-source S3-A infrastructure continuation; original sources unchanged."""
from pathlib import Path
from freeze_g1_components import sha
from s2_custody_common import checked, read_json
from s3a_runtime import contract
RECOVER = [230, 232, 233, 234]
NEVER = list(range(235, 390))
ORDER = RECOVER + NEVER
def validate_scope(c, old, scope):
if scope['recovery_indices'] != RECOVER or scope['never_attempted_indices'] != NEVER:
raise ValueError('Only four explicit recoveries and 155 never attempted are authorized')
if old['status'] != 'S3A_STOPPED_INCOMPLETE' or old['stop_reason'] != 'S3A_COORDINATOR_INTERRUPTED' or not old['accounting_complete']:
raise ValueError('Unexpected original terminal')
attempted = {r['index']: r for r in old['jobs']}
if len(old['jobs']) != 235 or set(attempted) != set(range(235)) or old['not_run_indices'] != NEVER:
raise ValueError('Original exact attempt set differs')
passed=[]
for i,row in attempted.items():
if row['job'] != c['jobs'][i]:raise ValueError('Original specification changed')
if i in RECOVER:
if row['worker']['status'] != 'STOPPED_OR_FAILED' or row['worker']['returncode'] != -9:
raise ValueError('Expected aborted original worker')
else:
if row['worker']['status'] != 'PASS':raise ValueError('Cannot expand recovery to another failed job')
passed.append(row)
if scope['original_worker_CPU_s'] != old['worker_cpu_seconds']:
raise ValueError('Prior CPU charge changed')
if abs(sum(r['worker']['cpu_seconds'] for r in old['jobs'])-old['worker_cpu_seconds']) > 1e-6:
raise ValueError('Incomplete original ledger')
return sorted(passed,key=lambda r:r['index'])
def canonical_accounting(old, done):
ids=[r['index'] for r in done]
if len(ids)!=len(set(ids)):raise ValueError('Duplicate canonical index')
original_good={r['index']:r for r in old['jobs'] if r['worker']['status']=='PASS'}
if not set(original_good)<=set(ids):raise ValueError('Dropped original success')
new=[]
for row in done:
i=row['index']
if i in original_good:
if row!=original_good[i]:raise ValueError('Original success modified')
elif i not in ORDER:raise ValueError('Unauthorized new index')
else:new.append(row)
return old['worker_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in new)
def selection_ready(done, active):
return not active and {r['index'] for r in done}==set(range(360))
def state(root, require_code_freeze=True):
root=Path(root);c=contract(root);scope=read_json(root/'recovery-scope.json')
original=Path(scope['original_root'])
checked(original/'contract.json',sha(root/'contract.json'))
old=read_json(checked(original/'batch-complete.json',scope['original_batch_sha256']))
passed=validate_scope(c,old,scope)
for path,digest in scope['original_evidence_sha256'].items():checked(path,digest)
for row in passed:
i=row['index'];oldpath=original/'jobs'/f'job-{i:03d}'/'guard-result.json'
checked(root/'jobs'/f'job-{i:03d}'/'guard-result.json',sha(oldpath))
if require_code_freeze:
f=read_json(root/'recovery-code-freeze.json')
if not f['git_commit'] or f['scope_sha256']!=sha(root/'recovery-scope.json'):
raise ValueError('Recovery must be fixed before execution')
for p,digest in f['source_sha256'].items():checked(p,digest)
return c,scope,old,passed
def manifest(root,scope,old,done):
original=Path(scope['original_root']);oldgood={r['index'] for r in old['jobs'] if r['worker']['status']=='PASS'}
canonical_accounting(old,done)
records=[]
for row in sorted(done,key=lambda r:r['index']):
i=row['index'];origin=original if i in oldgood else Path(root)
guard=origin/'jobs'/f'job-{i:03d}'/'guard-result.json'
# A final fit with all HPO trials failed has a logical NA, not a worker attempt.
records.append(dict(index=i,job=row['job'],origin='original_success' if i in oldgood else
('explicit_infrastructure_recovery' if i in RECOVER else 'never_attempted_continuation'),
guard_path=str(guard) if guard.exists() else None,guard_sha256=sha(guard) if guard.exists() else None,
result_path=row.get('result_path'),result_sha256=row.get('result_sha256'),status=row['worker']['status']))
return dict(original_batch_sha256=scope['original_batch_sha256'],original_aborted_indices_retained=RECOVER,
original_root=str(original),recovery_root=str(root),records=records,
original_worker_CPU_s=old['worker_cpu_seconds'],cumulative_worker_CPU_s=canonical_accounting(old,done))