88 lines
4.6 KiB
Python
88 lines
4.6 KiB
Python
"""Explicit two-source S3-A infrastructure continuation; original sources unchanged."""
|
|
from pathlib import Path
|
|
from freeze_g1_components import sha
|
|
from s2_custody_common import checked, read_json
|
|
from s3a_runtime import contract
|
|
|
|
RECOVER = [230, 232, 233, 234]
|
|
NEVER = list(range(235, 390))
|
|
ORDER = RECOVER + NEVER
|
|
|
|
|
|
def validate_scope(c, old, scope):
|
|
if scope['recovery_indices'] != RECOVER or scope['never_attempted_indices'] != NEVER:
|
|
raise ValueError('Only four explicit recoveries and 155 never attempted are authorized')
|
|
if old['status'] != 'S3A_STOPPED_INCOMPLETE' or old['stop_reason'] != 'S3A_COORDINATOR_INTERRUPTED' or not old['accounting_complete']:
|
|
raise ValueError('Unexpected original terminal')
|
|
attempted = {r['index']: r for r in old['jobs']}
|
|
if len(old['jobs']) != 235 or set(attempted) != set(range(235)) or old['not_run_indices'] != NEVER:
|
|
raise ValueError('Original exact attempt set differs')
|
|
passed=[]
|
|
for i,row in attempted.items():
|
|
if row['job'] != c['jobs'][i]:raise ValueError('Original specification changed')
|
|
if i in RECOVER:
|
|
if row['worker']['status'] != 'STOPPED_OR_FAILED' or row['worker']['returncode'] != -9:
|
|
raise ValueError('Expected aborted original worker')
|
|
else:
|
|
if row['worker']['status'] != 'PASS':raise ValueError('Cannot expand recovery to another failed job')
|
|
passed.append(row)
|
|
if scope['original_worker_CPU_s'] != old['worker_cpu_seconds']:
|
|
raise ValueError('Prior CPU charge changed')
|
|
if abs(sum(r['worker']['cpu_seconds'] for r in old['jobs'])-old['worker_cpu_seconds']) > 1e-6:
|
|
raise ValueError('Incomplete original ledger')
|
|
return sorted(passed,key=lambda r:r['index'])
|
|
|
|
|
|
def canonical_accounting(old, done):
|
|
ids=[r['index'] for r in done]
|
|
if len(ids)!=len(set(ids)):raise ValueError('Duplicate canonical index')
|
|
original_good={r['index']:r for r in old['jobs'] if r['worker']['status']=='PASS'}
|
|
if not set(original_good)<=set(ids):raise ValueError('Dropped original success')
|
|
new=[]
|
|
for row in done:
|
|
i=row['index']
|
|
if i in original_good:
|
|
if row!=original_good[i]:raise ValueError('Original success modified')
|
|
elif i not in ORDER:raise ValueError('Unauthorized new index')
|
|
else:new.append(row)
|
|
return old['worker_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in new)
|
|
|
|
|
|
def selection_ready(done, active):
|
|
return not active and {r['index'] for r in done}==set(range(360))
|
|
|
|
|
|
def state(root, require_code_freeze=True):
|
|
root=Path(root);c=contract(root);scope=read_json(root/'recovery-scope.json')
|
|
original=Path(scope['original_root'])
|
|
checked(original/'contract.json',sha(root/'contract.json'))
|
|
old=read_json(checked(original/'batch-complete.json',scope['original_batch_sha256']))
|
|
passed=validate_scope(c,old,scope)
|
|
for path,digest in scope['original_evidence_sha256'].items():checked(path,digest)
|
|
for row in passed:
|
|
i=row['index'];oldpath=original/'jobs'/f'job-{i:03d}'/'guard-result.json'
|
|
checked(root/'jobs'/f'job-{i:03d}'/'guard-result.json',sha(oldpath))
|
|
if require_code_freeze:
|
|
f=read_json(root/'recovery-code-freeze.json')
|
|
if not f['git_commit'] or f['scope_sha256']!=sha(root/'recovery-scope.json'):
|
|
raise ValueError('Recovery must be fixed before execution')
|
|
for p,digest in f['source_sha256'].items():checked(p,digest)
|
|
return c,scope,old,passed
|
|
|
|
|
|
def manifest(root,scope,old,done):
|
|
original=Path(scope['original_root']);oldgood={r['index'] for r in old['jobs'] if r['worker']['status']=='PASS'}
|
|
canonical_accounting(old,done)
|
|
records=[]
|
|
for row in sorted(done,key=lambda r:r['index']):
|
|
i=row['index'];origin=original if i in oldgood else Path(root)
|
|
guard=origin/'jobs'/f'job-{i:03d}'/'guard-result.json'
|
|
# A final fit with all HPO trials failed has a logical NA, not a worker attempt.
|
|
records.append(dict(index=i,job=row['job'],origin='original_success' if i in oldgood else
|
|
('explicit_infrastructure_recovery' if i in RECOVER else 'never_attempted_continuation'),
|
|
guard_path=str(guard) if guard.exists() else None,guard_sha256=sha(guard) if guard.exists() else None,
|
|
result_path=row.get('result_path'),result_sha256=row.get('result_sha256'),status=row['worker']['status']))
|
|
return dict(original_batch_sha256=scope['original_batch_sha256'],original_aborted_indices_retained=RECOVER,
|
|
original_root=str(original),recovery_root=str(root),records=records,
|
|
original_worker_CPU_s=old['worker_cpu_seconds'],cumulative_worker_CPU_s=canonical_accounting(old,done))
|