Files
MeltMind/tools/phase3_item1_resume_common.py
T

121 lines
6.2 KiB
Python

"""One labelled infrastructure continuation; original scientific modules unchanged."""
import math
from pathlib import Path
from phase2_h1_worker_v2 import checked, sha
from phase2_home_freeze_io import verify
from phase3_item1_postfit_common import config as original_config, read, terminal_inputs
ALLOWED_GPUS = {'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36'}
def validate_scope(c, old):
if c['recovery_approval'] != '1794980643.205679' or c['resume_indices'] != list(range(1, 7)):
raise PermissionError('Exactly one recovery for jobs1 through6')
if (c['gpu_uuid'] not in ALLOWED_GPUS or c['recovery_attempt'] != 1
or c['idle_wait_seconds'] != 20 or c['idle_wait_approval'] != '1794981566.532619'):
raise PermissionError('Reserved Track A GPU or extra recovery')
for key in ('jobs', 'rows', 'training_root', 'training_contract', 'comparison_contract_sha256',
'postfit_gpu_cap', 'postfit_cpu_cap', 'item_gpu_cap', 'new_fits', 'new_decodes'):
if c[key] != old[key]:
raise ValueError('Original science/budget differs: ' + key)
if c['postfit_root'] == old['postfit_root']:
raise ValueError('Original stopped root must remain untouched')
for p, h in old['code_sha256'].items():
if c['code_sha256'].get(p) != h:
raise ValueError('Original pinned source changed')
def validate_stop(done):
records = done['records']
if done['status'] != 'STOPPED' or len(records) != 2 or [x['index'] for x in records] != [0, 1]:
raise ValueError('Unexpected original stop scope')
first, failed = [x['guard'] for x in records]
if first['status'] != 'PASS' or first['monitor_errors'] or not first['worker'] or first['worker']['status'] != 'PASS' or first['worker'].get('monitor_errors') != 0:
raise ValueError('Predict0 was not a clean PASS')
if failed['status'] != 'STOPPED' or failed['worker'] is not None or 'RuntimeError: SELECTED_GPU_BUSY_NO_RETRY' not in failed['reasons']:
raise ValueError('Not the authorized zero-worker infrastructure stop')
for key in ('prediction_gpu_seconds', 'postfit_cpu_seconds', 'training_gpu_seconds_once'):
if not math.isfinite(done[key]) or done[key] < 0:
raise ValueError('Invalid original ledger')
def six_receipts(c):
source = Path(c['original_root']) / 'tron-six-training-receipts-check.json'
check = read(source)
if check['status'] != 'PASS' or [r['job'] for r in check['rows']] != list(range(6)):
raise ValueError('Original six-receipt check missing')
for row in check['rows']:
p = Path(c['training_root']) / 'jobs' / str(row['job']) / 'guards/train.receipt.json'
if str(p) != row['path']:
raise ValueError('Original training receipt path mismatch')
receipt = read(checked(p, row['sha256']))
if type(receipt.get('monitor_errors')) is not int or receipt['monitor_errors'] != 0 or receipt['status'] != 'PASS' or receipt['returncode'] != 0:
raise ValueError('Training monitoring receipt not clean')
guard = read(p.parent.parent / 'complete.json')
if guard['worker'] != receipt:
raise ValueError('Embedded training receipt differs')
return check
def config(root):
c, tc, pilot = original_config(root)
original = Path(c['original_root'])
old, _, _ = original_config(original)
validate_scope(c, old)
done = read(checked(original / 'complete.json', c['original_complete_sha256']))
validate_stop(done)
if any((original / n).exists() for n in ('score', 'predictions-manifest.json')):
raise ValueError('Original scoring or full prediction barrier already exists')
for k, name in [('prior_postfit_gpu', 'prediction_gpu_seconds'), ('prior_postfit_cpu', 'postfit_cpu_seconds'),
('training_gpu_once', 'training_gpu_seconds_once')]:
if c[k] != done[name]:
raise ValueError('Original cost must be retained exactly once')
six_receipts(c)
return c, tc, pilot
def adoption(c, tc):
current = terminal_inputs(c, tc)
original = Path(c['original_root'])
verify(original, 'models')
old = read(original / 'models-adopted.json')
if old != current or current['training_gpu_seconds_once'] != c['training_gpu_once']:
raise ValueError('Predict0 and resumed jobs must use identical frozen models')
return current
def reuse_entries(c, tc, *, verify_bytes=True):
original = Path(c['original_root'])
manifest = read(checked(original / 'jobs/0/prediction-records/manifest.json', c['predict0_manifest_sha256']))
result = read(original / 'jobs/0/prediction-records/result.json')
if result['status'] != 'COMPLETE' or result['targets_read'] or result['parts'] != c['jobs'][0]['expected_parts']:
raise ValueError('Predict0 result mismatch')
entries = manifest['entries']
f = tc['pools']['25']
domains = {'learning': 'own_learning', 'validation': 'common_validation', 'test': 'common_test'}
expected = {(arm, case): domain for role, domain in domains.items() for case in f[role]
for arm in ('epoch25', 'c25F')}
if len(entries) != len(expected) or {(e['arm'], e['case']) for e in entries} != set(expected):
raise ValueError('Complete predict0 model and clock coverage required')
if manifest['targets_read'] or result['entries'] != len(entries):
raise ValueError('Predict0 targets/entry count')
for e in entries:
p = original / 'predictions' / e['arm'] / (e['case'] + '.npz')
if (e['path'] != str(p) or e['pool'] != 25 or e['domain'] != expected[(e['arm'], e['case'])]
or e['component_id'] != c['rows'][e['case']]['component_id']
or e['origin'] != 'ITEM1_FROZEN_ENDPOINT_OR_CLOCK'):
raise ValueError('Predict0 original identity')
if verify_bytes:
checked(p, e['sha256'])
return entries
def cumulative_cost(c, new_gpu, new_cpu):
if any(not math.isfinite(v) or v < 0 for v in [new_gpu, new_cpu]):
raise ValueError('Invalid new cost ledger')
gpu = c['prior_postfit_gpu'] + new_gpu
return dict(prediction_gpu_seconds=gpu, postfit_cpu_seconds=c['prior_postfit_cpu'] + new_cpu,
training_gpu_seconds_once=c['training_gpu_once'],
cumulative_item_gpu_seconds=c['training_gpu_once'] + gpu)