Preserve S3-A interruption and freeze authorized 159-job continuation
This commit is contained in:
1 parent
087f63b057
commit
259d1d685b
253 files changed
+51609
No files matched your search
@@ -0,0 +1,80 @@
|
||||
"""Claude1794736668: exact-case descriptive join of already frozen results only."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
from s2_recovery_supervisor import save, supervise
|
||||
|
||||
|
||||
def read_checked(path, digest):
|
||||
raw=Path(path).read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest()!=digest:raise ValueError('Source changed')
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def worker(root):
|
||||
c=json.loads((root/'contract.json').read_text());start=time.process_time()
|
||||
src={p:read_checked(p,h) for p,h in c['source_sha256'].items()}
|
||||
logs=next(v for p,v in src.items() if p.endswith('/index.json'))
|
||||
scores=next(v for p,v in src.items() if p.endswith('/part-scores.json'))
|
||||
by_case={Path(v['member']).stem:v for v in logs}
|
||||
if len(by_case)!=624 or len(logs)!=624:raise ValueError('Duplicate/missing LOG keys')
|
||||
arms=c['arms'];parts={}
|
||||
for row in scores:
|
||||
if row['pool']!=100 or row['arm'] not in arms:continue
|
||||
if row['seed'] is not None:raise ValueError('Oracle should be deterministic')
|
||||
case=row['case'];d=parts.setdefault(case,dict(case=case,fold=row['fold'],component_id=row['component_id'],oracle_relative_l2={}))
|
||||
if row['arm'] in d['oracle_relative_l2']:raise ValueError('Duplicate oracle case')
|
||||
d['oracle_relative_l2'][row['arm']]=row['relative_l2']
|
||||
if len(parts)!=617:raise ValueError('Oracle part count')
|
||||
for case,row in parts.items():
|
||||
if set(row['oracle_relative_l2'])!=set(arms):raise ValueError('Incomplete oracle arms')
|
||||
log=by_case[case];d=log['diagnostic']
|
||||
if not d['quantitative_defined']:raise ValueError('Undefined LOG diagnostic')
|
||||
row.update(log_report=log['report'],log_report_sha256=log['report_sha256'],
|
||||
log_max_linearity_deviation_pp=d['linearity']['max_deviation_percentage_points'],
|
||||
log_within_original_1pp=d['linearity']['within_fixed_tolerance'],
|
||||
log_first_printed_time_s=d['first_row']['time_s'],
|
||||
log_first_printed_fill_percent=d['first_row']['fill_percent'],
|
||||
log_printed_100_time_s=d['last_row']['time_s'],
|
||||
log_explicit_switch_rows=d['observed_switch_rows'],
|
||||
log_equal_printed_time_intervals=d['equal_printed_time_intervals'])
|
||||
unmatched=sorted(set(by_case)-set(parts))
|
||||
if len(unmatched)!=7:raise ValueError('LOG-only count')
|
||||
data=[parts[k] for k in sorted(parts)]
|
||||
save(root/'joined-parts.json',data);save(root/'log-only-parts.json',unmatched)
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
fig,axes=plt.subplots(1,3,figsize=(12,3.7),layout='constrained')
|
||||
for ax,arm in zip(axes,arms):
|
||||
ax.scatter([r['log_max_linearity_deviation_pp'] for r in data],
|
||||
[r['oracle_relative_l2'][arm] for r in data],s=10,alpha=.5)
|
||||
ax.set(xlabel='LOG maximum deviation (percentage points)',ylabel='Per-part relative L2',title=arm)
|
||||
fig.suptitle('617 exact-case joins; descriptive only, no fitted relation or significance test')
|
||||
fig.savefig(root/'oracle-log-join.png',dpi=170);plt.close(fig)
|
||||
save(root/'result.json',dict(status='EXACT_CASE_JOIN_COMPLETE',parts=len(data),log_only_parts=unmatched,
|
||||
new_decodes=0,new_fits=0,new_predictions=0,new_significance_tests=0,
|
||||
explanatory_conclusion='No causal or universal time-map conclusion drawn from the scatter.',
|
||||
worker_analysis_CPU_s=time.process_time()-start,
|
||||
output_sha256={p.name:hashlib.sha256(p.read_bytes()).hexdigest() for p in root.iterdir() if p.name in ('joined-parts.json','log-only-parts.json','oracle-log-join.png')}))
|
||||
return 0
|
||||
|
||||
|
||||
def guard(root):
|
||||
c=json.loads((root/'contract.json').read_text());lim=c['limits'];os.sched_setaffinity(0,[24])
|
||||
r=supervise([sys.executable,__file__,str(root),'--worker'],root,'worker',dict(
|
||||
total_cpu_seconds=lim['CPU_s'],job_cpu_seconds=lim['CPU_s'],job_wall_seconds=lim['wall_s'],
|
||||
tree_rss_bytes=lim['RSS_bytes'],disk_bytes=lim['disk_bytes'],max_processes=1),poll=.2)
|
||||
previous=json.loads(Path('artifacts/moldflow-time-map-2026-11-15/report-receipt.json').read_text())
|
||||
save(root/'guard-result.json',dict(worker=r,prior_LOG_task_CPU_s=previous['cumulative_task_CPU_s'],
|
||||
cumulative_LOG_task_CPU_s=previous['cumulative_task_CPU_s']+r['cpu_seconds'],LOG_task_CPU_cap=3600))
|
||||
return 0 if r['status']=='PASS' else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--worker',action='store_true');a=p.parse_args()
|
||||
raise SystemExit(worker(a.root) if a.worker else guard(a.root))
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Explicit 4+155 infrastructure continuation. Original workers and scientific contract unchanged."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from freeze_g1_components import sha
|
||||
from s2_custody_common import checked, read_json
|
||||
from s2_recovery_supervisor import save, supervise, tree_usage
|
||||
from s2_recovery_monitoring import Monitor, disk_usage, budget_reason
|
||||
from run_s2_pilot_preparation import terminate
|
||||
from run_s2_r2 import finalize_selections
|
||||
from s3a_budget import first_fit_projection, can_dispatch
|
||||
from s3a_runtime import contract
|
||||
from s3a_recovery_common import state, canonical_accounting, selection_ready, manifest, ORDER
|
||||
|
||||
FIRST = [0, 12]
|
||||
|
||||
|
||||
def dispatch_order():
|
||||
return list(ORDER)
|
||||
|
||||
|
||||
def guard(root, index, slot):
|
||||
if index not in ORDER:
|
||||
raise ValueError('Original successful jobs may not be dispatched again')
|
||||
c = contract(root)
|
||||
os.sched_setaffinity(0, [c['worker_cpus'][slot]])
|
||||
limits = c['limits']
|
||||
dest = root/'jobs'/f'job-{index:03d}'
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
w = supervise([sys.executable, 'tools/run_s3a_fit.py', str(root), str(index)], dest, 'worker',
|
||||
dict(total_cpu_seconds=limits['per_fit_cpu_seconds']+1,
|
||||
job_cpu_seconds=limits['per_fit_cpu_seconds'], job_wall_seconds=limits['per_fit_wall_seconds'],
|
||||
tree_rss_bytes=limits['per_fit_tree_rss_bytes'], disk_bytes=limits['per_fit_disk_bytes'], max_processes=1), poll=.2)
|
||||
path = dest/'output/result.json'
|
||||
result = read_json(path) if path.exists() else None
|
||||
if w['status'] == 'PASS' and (not result or result['status'] != 'COMPLETE'):
|
||||
w.update(status='STOPPED_OR_FAILED', stop_reason='MISSING_COMPLETE_RESULT')
|
||||
if w['status'] == 'PASS':
|
||||
checked(dest/'output/model.joblib', result['model_sha256'])
|
||||
save(dest/'guard-result.json', dict(worker=w, supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
cpu_affinity=sorted(os.sched_getaffinity(0)), result_path=str(path) if path.exists() else None,
|
||||
result_sha256=sha(path) if path.exists() else None, job=c['jobs'][index]))
|
||||
return 0 if w['status'] == 'PASS' else 1
|
||||
|
||||
|
||||
def first_check(c, done):
|
||||
samples = {}
|
||||
for i in FIRST:
|
||||
item = next(r for r in done if r['index'] == i)
|
||||
result = read_json(checked(item['result_path'], item['result_sha256'])) if item.get('result_path') else {}
|
||||
samples[c['jobs'][i]['arm']] = dict(status=item['worker']['status'],
|
||||
worker_cpu_seconds=item['worker']['cpu_seconds'], fit_cpu_seconds=result.get('fit_cpu_seconds', 0.))
|
||||
return first_fit_projection(c, samples)
|
||||
|
||||
|
||||
def freeze_terminal(root, c, done, selections, reason, started, monitor, accounting_complete):
|
||||
_, scope, old, _ = state(root)
|
||||
merged = manifest(root, scope, old, done)
|
||||
save(root/'two-source-manifest.json', merged)
|
||||
attempts = {r['index']: r for r in done}
|
||||
finals = []
|
||||
for job in c['jobs'][360:]:
|
||||
row = attempts.get(job['index'])
|
||||
passed = row is not None and row['worker']['status'] == 'PASS'
|
||||
result = read_json(checked(row['result_path'], row['result_sha256'])) if passed else None
|
||||
model_path = str(Path(row['result_path']).parent/'model.joblib') if passed else None
|
||||
if passed:
|
||||
checked(model_path, result['model_sha256'])
|
||||
finals.append(dict(cell=job['cell'], arm=job['arm'], fold=job['fold'], pool=job['pool'],
|
||||
status='PASS' if passed else 'NA',
|
||||
reason=None if passed else ((row['worker']['stop_reason'] or 'WORKER_EXIT_'+str(row['worker']['returncode'])) if row else 'NOT_RUN_'+str(reason)),
|
||||
finished_configurations=selections[job['cell']]['finished_configurations'] if selections else None,
|
||||
model_path=model_path, model_sha256=result['model_sha256'] if result else None))
|
||||
complete = reason is None and len(attempts) == 390 and accounting_complete
|
||||
result = dict(status='ALL_S3A_FITS_TERMINAL' if complete else 'S3A_STOPPED_INCOMPLETE',
|
||||
contract_sha256=sha(root/'contract.json'), stop_reason=reason, accounting_complete=accounting_complete,
|
||||
final_fits=finals, jobs=done, not_run_indices=[i for i in range(390) if i not in attempts],
|
||||
worker_cpu_seconds=canonical_accounting(old, done),
|
||||
original_worker_cpu_seconds=old['worker_cpu_seconds'],
|
||||
original_aborted_jobs_retained=[230,232,233,234],
|
||||
two_source_manifest_sha256=sha(root/'two-source-manifest.json'),
|
||||
guard_cpu_seconds=old['guard_cpu_seconds']+sum(r['supervisor_cpu_seconds'] for r in done if r['index'] in ORDER),
|
||||
original_coordinator_cpu_seconds=old['coordinator_cpu_seconds'],
|
||||
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
disk_bytes=disk_usage(root)+disk_usage(Path(scope['original_root'])), wall_seconds=time.monotonic()-started,
|
||||
monitor_errors=monitor.errors, roster_sha256=c['roster_sha256'],
|
||||
test_predictions=False, test_scoring=False, retry=False, baseline_refits=0)
|
||||
save(root/'batch-complete.json', result)
|
||||
if complete:
|
||||
save(root/'terminal-freeze.json', dict(status='ALL_S3A_FITS_TERMINAL', contract_sha256=sha(root/'contract.json'),
|
||||
hpo_terminal_indices=list(range(360)), final_fits=finals,
|
||||
batch_receipt=str(root/'batch-complete.json'), batch_receipt_sha256=sha(root/'batch-complete.json'),
|
||||
test_predictions_started=False))
|
||||
return 0 if complete else 1
|
||||
|
||||
|
||||
def run(root):
|
||||
with (root/'batch.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
return run_locked(root)
|
||||
|
||||
|
||||
def run_locked(root):
|
||||
c, scope, old, original_passed = state(root)
|
||||
if (root/'batch-started.json').exists():
|
||||
raise FileExistsError('No restart/retry of an attempted S3-A queue')
|
||||
os.sched_setaffinity(0, c['coordinator_cpus'])
|
||||
start = time.monotonic()
|
||||
save(root/'batch-started.json', dict(pid=os.getpid(), unix=time.time(), contract_sha256=sha(root/'contract.json'),
|
||||
implementation_freeze=read_json(root/'implementation-freeze.json'), cpu_affinity=sorted(os.sched_getaffinity(0))))
|
||||
active, done = {}, list(original_passed)
|
||||
order = dispatch_order()
|
||||
cursor = 0
|
||||
reason = None
|
||||
accounting_complete = True
|
||||
first_checked = True
|
||||
selections = None
|
||||
live_cpu = live_rss = disk = 0
|
||||
monitor = Monitor(root/'monitor-errors.jsonl')
|
||||
env = dict(os.environ, OMP_NUM_THREADS='1', MKL_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
|
||||
|
||||
def interrupted(*_):
|
||||
nonlocal reason
|
||||
reason = 'S3A_RECOVERY_COORDINATOR_INTERRUPTED'
|
||||
save(root/'signal-received.json', dict(signal_number=_[0], unix=time.time(), pid=os.getpid()))
|
||||
|
||||
signal.signal(signal.SIGTERM, interrupted)
|
||||
signal.signal(signal.SIGINT, interrupted)
|
||||
|
||||
def terminate_active():
|
||||
terminate({k: (v['process'], v['log']) for k, v in active.items()})
|
||||
|
||||
def collect():
|
||||
nonlocal reason, accounting_complete
|
||||
for slot, item in list(active.items()):
|
||||
code = item['process'].poll()
|
||||
if code is None:
|
||||
continue
|
||||
path = root/'jobs'/f"job-{item['index']:03d}"/'guard-result.json'
|
||||
if not path.exists():
|
||||
reason = 'MISSING_GUARD_RECEIPT'
|
||||
accounting_complete = False
|
||||
save(root/'infrastructure-stop.json', dict(reason=reason, index=item['index'], returncode=code,
|
||||
last_observed_guard_tree_CPU_s=item.get('cpu_peak', 0.), accounting_complete=False))
|
||||
else:
|
||||
row = read_json(path)
|
||||
if row['job'] != c['jobs'][item['index']] or not math.isfinite(row['worker']['cpu_seconds']) or row['worker']['cpu_seconds'] < 0:
|
||||
raise ValueError('Invalid job/accounting receipt')
|
||||
if row.get('result_path'):
|
||||
checked(row['result_path'], row['result_sha256'])
|
||||
done.append(dict(index=item['index'], **row))
|
||||
print(json.dumps(dict(event='S3A_TERMINAL', index=item['index'], arm=row['job']['arm'],
|
||||
status=row['worker']['status'], completed=len(done))), flush=True)
|
||||
item['log'].close()
|
||||
del active[slot]
|
||||
|
||||
try:
|
||||
while active or (cursor < len(order) and reason is None):
|
||||
collect()
|
||||
|
||||
def usage():
|
||||
values = []
|
||||
for v in active.values():
|
||||
value = tree_usage(v['process'].pid)
|
||||
v['cpu_peak'] = max(v.get('cpu_peak', 0.), value[0])
|
||||
values.append((v['cpu_peak'], value[1]))
|
||||
return sum(v[0] for v in values), sum(v[1] for v in values)
|
||||
|
||||
(live_cpu, live_rss), fresh = monitor.sample('live_usage', usage, (live_cpu, live_rss))
|
||||
disk, ok = monitor.sample('disk', lambda: disk_usage(root)+disk_usage(Path(scope['original_root'])), disk)
|
||||
fresh &= ok
|
||||
spent = canonical_accounting(old, done)
|
||||
over = budget_reason(0, spent, live_cpu, disk, cpu_limit=c['limits']['fitting_worker_cpu_seconds'], disk_limit=c['limits']['disk_bytes'])
|
||||
if over:
|
||||
reason = over
|
||||
if not first_checked and not active and done and any(r['worker']['status'] != 'PASS' for r in done):
|
||||
reason = 'FIRST_TRIAL_FAILED_NO_RETRY'
|
||||
if not first_checked and len(done) == 2 and not active and reason is None:
|
||||
check = first_check(c, done)
|
||||
save(root/'first-fit-cost-check.json', check)
|
||||
first_checked = True
|
||||
if not check['continue_within_cap']:
|
||||
reason = 'FIRST_TRIAL_COST_PROJECTION_OVER_CAP'
|
||||
if selection_ready(done, active) and selections is None and reason is None:
|
||||
selections = finalize_selections(root, c, done)
|
||||
if reason:
|
||||
terminate_active()
|
||||
ceiling = len(order)-30 if selections is None else len(order)
|
||||
parallel = 1 if not first_checked else 4
|
||||
while fresh and reason is None and len(active) < parallel and cursor < ceiling:
|
||||
i = order[cursor]
|
||||
job = c['jobs'][i]
|
||||
if job['kind'] == 'FINAL' and selections[job['cell']]['status'] != 'SELECTED':
|
||||
done.append(dict(index=i, job=job, worker=dict(status='NA', cpu_seconds=0., returncode=0,
|
||||
stop_reason='ALL12_HPO_FAILED_NO_FINAL_FIT'), supervisor_cpu_seconds=0., attempted=False, result_path=None, result_sha256=None))
|
||||
cursor += 1
|
||||
continue
|
||||
# Conservatively reserve full caps of active workers plus the next task.
|
||||
cap = c['limits']['per_fit_cpu_seconds']
|
||||
if not can_dispatch(c, spent, live_cpu, [cap for _ in active], cap):
|
||||
if not active:
|
||||
reason = 'INSUFFICIENT_REMAINING_FIT_CAP_NO_LAUNCH'
|
||||
break
|
||||
slot = next(k for k in range(4) if k not in active)
|
||||
dest = root/'jobs'/f'job-{i:03d}'
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
log = (dest/'guard.log').open('x')
|
||||
proc = subprocess.Popen([sys.executable, __file__, str(root), '--guard', str(i), '--slot', str(slot)],
|
||||
env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
active[slot] = dict(process=proc, log=log, index=i, cpu_peak=0.)
|
||||
cursor += 1
|
||||
save(root/'progress.json', dict(completed=len(done), active={str(k): v['index'] for k,v in active.items()},
|
||||
dispatch_cursor=cursor, worker_cpu_seconds=spent, live_guard_and_worker_cpu_seconds=live_cpu,
|
||||
active_tree_rss_bytes=live_rss, disk_bytes=disk, first_fit_checked=first_checked,
|
||||
selections_frozen=selections is not None, stop_reason=reason, fresh_accounting=fresh,
|
||||
monitor_errors=monitor.errors, elapsed_seconds=time.monotonic()-start))
|
||||
if active or not fresh:
|
||||
time.sleep(.5)
|
||||
except Exception as exc:
|
||||
reason = 'COORDINATOR_EXCEPTION'
|
||||
save(root/'coordinator-failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc()))
|
||||
finally:
|
||||
terminate_active()
|
||||
for item in active.values():
|
||||
item['process'].wait()
|
||||
collect()
|
||||
return freeze_terminal(root, c, done, selections, reason, start, monitor, accounting_complete)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('root', type=Path)
|
||||
p.add_argument('--guard', type=int)
|
||||
p.add_argument('--slot', type=int)
|
||||
a = p.parse_args()
|
||||
raise SystemExit(guard(a.root, a.guard, a.slot) if a.guard is not None else run(a.root))
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Explicit two-source S3-A infrastructure continuation; original sources unchanged."""
|
||||
from pathlib import Path
|
||||
from freeze_g1_components import sha
|
||||
from s2_custody_common import checked, read_json
|
||||
from s3a_runtime import contract
|
||||
|
||||
RECOVER = [230, 232, 233, 234]
|
||||
NEVER = list(range(235, 390))
|
||||
ORDER = RECOVER + NEVER
|
||||
|
||||
|
||||
def validate_scope(c, old, scope):
|
||||
if scope['recovery_indices'] != RECOVER or scope['never_attempted_indices'] != NEVER:
|
||||
raise ValueError('Only four explicit recoveries and 155 never attempted are authorized')
|
||||
if old['status'] != 'S3A_STOPPED_INCOMPLETE' or old['stop_reason'] != 'S3A_COORDINATOR_INTERRUPTED' or not old['accounting_complete']:
|
||||
raise ValueError('Unexpected original terminal')
|
||||
attempted = {r['index']: r for r in old['jobs']}
|
||||
if len(old['jobs']) != 235 or set(attempted) != set(range(235)) or old['not_run_indices'] != NEVER:
|
||||
raise ValueError('Original exact attempt set differs')
|
||||
passed=[]
|
||||
for i,row in attempted.items():
|
||||
if row['job'] != c['jobs'][i]:raise ValueError('Original specification changed')
|
||||
if i in RECOVER:
|
||||
if row['worker']['status'] != 'STOPPED_OR_FAILED' or row['worker']['returncode'] != -9:
|
||||
raise ValueError('Expected aborted original worker')
|
||||
else:
|
||||
if row['worker']['status'] != 'PASS':raise ValueError('Cannot expand recovery to another failed job')
|
||||
passed.append(row)
|
||||
if scope['original_worker_CPU_s'] != old['worker_cpu_seconds']:
|
||||
raise ValueError('Prior CPU charge changed')
|
||||
if abs(sum(r['worker']['cpu_seconds'] for r in old['jobs'])-old['worker_cpu_seconds']) > 1e-6:
|
||||
raise ValueError('Incomplete original ledger')
|
||||
return sorted(passed,key=lambda r:r['index'])
|
||||
|
||||
|
||||
def canonical_accounting(old, done):
|
||||
ids=[r['index'] for r in done]
|
||||
if len(ids)!=len(set(ids)):raise ValueError('Duplicate canonical index')
|
||||
original_good={r['index']:r for r in old['jobs'] if r['worker']['status']=='PASS'}
|
||||
if not set(original_good)<=set(ids):raise ValueError('Dropped original success')
|
||||
new=[]
|
||||
for row in done:
|
||||
i=row['index']
|
||||
if i in original_good:
|
||||
if row!=original_good[i]:raise ValueError('Original success modified')
|
||||
elif i not in ORDER:raise ValueError('Unauthorized new index')
|
||||
else:new.append(row)
|
||||
return old['worker_cpu_seconds']+sum(r['worker']['cpu_seconds'] for r in new)
|
||||
|
||||
|
||||
def selection_ready(done, active):
|
||||
return not active and {r['index'] for r in done}==set(range(360))
|
||||
|
||||
|
||||
def state(root, require_code_freeze=True):
|
||||
root=Path(root);c=contract(root);scope=read_json(root/'recovery-scope.json')
|
||||
original=Path(scope['original_root'])
|
||||
checked(original/'contract.json',sha(root/'contract.json'))
|
||||
old=read_json(checked(original/'batch-complete.json',scope['original_batch_sha256']))
|
||||
passed=validate_scope(c,old,scope)
|
||||
for path,digest in scope['original_evidence_sha256'].items():checked(path,digest)
|
||||
for row in passed:
|
||||
i=row['index'];oldpath=original/'jobs'/f'job-{i:03d}'/'guard-result.json'
|
||||
checked(root/'jobs'/f'job-{i:03d}'/'guard-result.json',sha(oldpath))
|
||||
if require_code_freeze:
|
||||
f=read_json(root/'recovery-code-freeze.json')
|
||||
if not f['git_commit'] or f['scope_sha256']!=sha(root/'recovery-scope.json'):
|
||||
raise ValueError('Recovery must be fixed before execution')
|
||||
for p,digest in f['source_sha256'].items():checked(p,digest)
|
||||
return c,scope,old,passed
|
||||
|
||||
|
||||
def manifest(root,scope,old,done):
|
||||
original=Path(scope['original_root']);oldgood={r['index'] for r in old['jobs'] if r['worker']['status']=='PASS'}
|
||||
canonical_accounting(old,done)
|
||||
records=[]
|
||||
for row in sorted(done,key=lambda r:r['index']):
|
||||
i=row['index'];origin=original if i in oldgood else Path(root)
|
||||
guard=origin/'jobs'/f'job-{i:03d}'/'guard-result.json'
|
||||
# A final fit with all HPO trials failed has a logical NA, not a worker attempt.
|
||||
records.append(dict(index=i,job=row['job'],origin='original_success' if i in oldgood else
|
||||
('explicit_infrastructure_recovery' if i in RECOVER else 'never_attempted_continuation'),
|
||||
guard_path=str(guard) if guard.exists() else None,guard_sha256=sha(guard) if guard.exists() else None,
|
||||
result_path=row.get('result_path'),result_sha256=row.get('result_sha256'),status=row['worker']['status']))
|
||||
return dict(original_batch_sha256=scope['original_batch_sha256'],original_aborted_indices_retained=RECOVER,
|
||||
original_root=str(original),recovery_root=str(root),records=records,
|
||||
original_worker_CPU_s=old['worker_cpu_seconds'],cumulative_worker_CPU_s=canonical_accounting(old,done))
|
||||
@@ -0,0 +1,55 @@
|
||||
import copy
|
||||
import unittest
|
||||
from s3a_recovery_common import RECOVER, NEVER, ORDER, validate_scope, canonical_accounting, selection_ready
|
||||
|
||||
|
||||
def fixture():
|
||||
c=dict(jobs=[dict(index=i,kind='HPO' if i<360 else 'FINAL') for i in range(390)])
|
||||
jobs=[dict(index=i,job=c['jobs'][i],worker=dict(status='STOPPED_OR_FAILED' if i in RECOVER else 'PASS',
|
||||
returncode=-9 if i in RECOVER else 0,cpu_seconds=2. if i in RECOVER else 1.)) for i in range(235)]
|
||||
old=dict(status='S3A_STOPPED_INCOMPLETE',stop_reason='S3A_COORDINATOR_INTERRUPTED',accounting_complete=True,
|
||||
jobs=jobs,not_run_indices=NEVER,worker_cpu_seconds=239.)
|
||||
scope=dict(recovery_indices=RECOVER,never_attempted_indices=NEVER,original_worker_CPU_s=239.)
|
||||
return c,old,scope
|
||||
|
||||
|
||||
class Recovery(unittest.TestCase):
|
||||
def test_exact_159_partition(self):
|
||||
c,old,s=fixture();passed=validate_scope(c,old,s)
|
||||
self.assertEqual(len(passed),231);self.assertEqual(len(ORDER),159)
|
||||
self.assertFalse(set(r['index'] for r in passed)&set(ORDER))
|
||||
self.assertEqual({r['index'] for r in passed}|set(ORDER),set(range(390)))
|
||||
|
||||
def test_changed_scope_or_spec_rejected(self):
|
||||
for field in ('recovery_indices','never_attempted_indices'):
|
||||
c,old,s=fixture();s[field]=s[field]+[1]
|
||||
with self.assertRaises(ValueError):validate_scope(c,old,s)
|
||||
c,old,s=fixture();old['jobs'][0]['job']=dict(index=999)
|
||||
with self.assertRaises(ValueError):validate_scope(c,old,s)
|
||||
|
||||
def test_prior_charge_once_including_aborted(self):
|
||||
c,old,s=fixture();p=validate_scope(c,old,s)
|
||||
self.assertEqual(canonical_accounting(old,p),239.)
|
||||
new=copy.deepcopy(old['jobs'][230]);new['worker']=dict(status='PASS',returncode=0,cpu_seconds=7.)
|
||||
self.assertEqual(canonical_accounting(old,p+[new]),246.)
|
||||
|
||||
def test_original_success_cannot_change_or_repeat(self):
|
||||
c,old,s=fixture();p=validate_scope(c,old,s)
|
||||
with self.assertRaises(ValueError):canonical_accounting(old,p+[p[0]])
|
||||
p=copy.deepcopy(p);p[0]['worker']['cpu_seconds']=0
|
||||
with self.assertRaises(ValueError):canonical_accounting(old,p)
|
||||
|
||||
def test_selection_after_360_not_390(self):
|
||||
rows=[dict(index=i) for i in range(360)]
|
||||
self.assertTrue(selection_ready(rows,{}))
|
||||
self.assertFalse(selection_ready(rows,{0:'active'}))
|
||||
self.assertFalse(selection_ready(rows[:-1],{}))
|
||||
self.assertEqual([i for i in ORDER if i<360],RECOVER+list(range(235,360)))
|
||||
self.assertEqual(ORDER[-30:],list(range(360,390)))
|
||||
|
||||
def test_incomplete_original_ledger_rejected(self):
|
||||
c,old,s=fixture();old['worker_cpu_seconds']=1
|
||||
with self.assertRaises(ValueError):validate_scope(c,old,s)
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user