Archive both protective stops and publish the reviewed-scope Track A four-split binding
This commit is contained in:
1 parent
18398c2b2f
commit
482c490afb
35 files changed
+229770
No files matched your search
@@ -0,0 +1,88 @@
|
||||
"""Freeze the authorized four-split Track A binding without opening targets or models."""
|
||||
|
||||
import fcntl
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from phase2_h1_worker_v2 import checked, save, sha
|
||||
from phase2_home_freeze_io import freeze, verify
|
||||
from track_a_h2_prepare import local_code_closure
|
||||
from track_a_postfit_common import config as original_config
|
||||
from track_a_postfit_rebind_common import (
|
||||
ORIGINAL, ORIGINAL_POSTFIT, RECOVERY, ROOT, RULING, config, read, scope, terminal,
|
||||
)
|
||||
|
||||
BASE = Path(__file__).resolve().parents[1]
|
||||
RECORDS = BASE / 'artifacts/track-a-postfit-rebind-2026-11-19'
|
||||
|
||||
|
||||
def source_files():
|
||||
return [
|
||||
*(ORIGINAL_POSTFIT / name for name in ('contract.json', 'expected-review-binding.json',
|
||||
'implementation-git-freeze.json')),
|
||||
*(ORIGINAL / name for name in ('complete.json', 'infrastructure-stop-git-freeze.json',
|
||||
'infrastructure-stop-authorization.json')),
|
||||
*(RECOVERY / name for name in ('contract.json', 'complete.json', 'terminal-ledger.json',
|
||||
'terminal-git-freeze.json', 'terminal-authorization.json',
|
||||
'available-models-incomplete.json', 'incomplete-models-git-freeze.json',
|
||||
'incomplete-models-authorization.json')),
|
||||
RECOVERY / 'jobs/3/stop.json',
|
||||
RECOVERY / 'jobs/3/first-epoch-projection.json',
|
||||
RECOVERY / 'jobs/4/worker/result.json',
|
||||
*(RECOVERY / f'jobs/{i}/complete.json' for i in (3, 4)),
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
start = time.process_time()
|
||||
old, training, _ = original_config(ORIGINAL_POSTFIT)
|
||||
if ROOT.exists():
|
||||
raise FileExistsError('The rebind root already exists')
|
||||
sources = {str(p): sha(p) for p in source_files()}
|
||||
if set(sources).intersection(old['source_sha256']):
|
||||
for p in set(sources).intersection(old['source_sha256']):
|
||||
checked(p, old['source_sha256'][p])
|
||||
models = terminal(training)
|
||||
codes = local_code_closure([Path(__file__), *(BASE / 'tools' / name for name in (
|
||||
'track_a_postfit_rebind.py', 'track_a_postfit_rebind_worker.py',
|
||||
'track_a_postfit_rebind_guard.py', 'test_track_a_postfit_rebind.py',
|
||||
))])
|
||||
contract = dict(old, **scope(old, training), version='track-a-postfit-four-split',
|
||||
rebind_ruling=RULING, training_evidence=models,
|
||||
source_sha256={**old['source_sha256'], **sources}, code_sha256=codes)
|
||||
del contract['all_five_training_terminals_before_prediction']
|
||||
contract['all_selected_training_terminals_before_prediction'] = True
|
||||
if (contract['expected_pair_visits'], contract['expected_unique_target_reads']) != (1228, 571):
|
||||
raise ValueError('Four-split overlap/domain differs from the original metadata')
|
||||
with Path('/tmp/meltmind-single-writer.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
ROOT.mkdir()
|
||||
save(ROOT / 'contract.json', contract)
|
||||
save(ROOT / 'preparation-receipt.json', dict(
|
||||
status='PREPARED_PENDING_INDEPENDENT_PRE_SCORE_REVIEW', rebind_ruling=RULING,
|
||||
prior_postfit_implementation=verify(ORIGINAL_POSTFIT, 'implementation')['commit'],
|
||||
evaluated_splits=[x['id'] for x in contract['jobs']], excluded_split=contract['excluded_split'],
|
||||
pair_visits=contract['expected_pair_visits'], unique_parts=contract['expected_unique_target_reads'],
|
||||
training_gpu_seconds_once=models['training_gpu_seconds_once'],
|
||||
source_pins=len(contract['source_sha256']), code_pins=len(codes),
|
||||
bootstrap=contract['bootstrap'], opaque_checkpoint_hashes_only=True,
|
||||
metadata_cpu_seconds=time.process_time()-start, review_adopted=False,
|
||||
launched=False, predicted=False, scored=False, test_targets_read=False,
|
||||
))
|
||||
freeze(ROOT, 'implementation', [ROOT / 'contract.json', ROOT / 'preparation-receipt.json',
|
||||
*(Path(p) for p in codes), *source_files(),
|
||||
BASE / 'docs/track-a-four-split-postfit-2026-11-19.md',
|
||||
RECORDS / 'acceptance.log', RECORDS / 'protective-stops.json'])
|
||||
proof = verify(ROOT, 'implementation')
|
||||
frozen, _, _ = config(ROOT)
|
||||
save(ROOT / 'expected-review-binding.json', dict(
|
||||
reviewed_commit=proof['commit'], contract_sha256=sha(ROOT / 'contract.json'),
|
||||
implementation_proof_sha256=sha(ROOT / 'implementation-git-freeze.json'),
|
||||
comparison_contract_sha256=frozen['comparison_contract_sha256'], code_sha256=codes))
|
||||
receipt = read(ROOT / 'preparation-receipt.json')
|
||||
print(dict(receipt, implementation=proof['commit'], contract=sha(ROOT / 'contract.json'),
|
||||
binding_sha256=sha(ROOT / 'expected-review-binding.json')))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Synthetic checks of the four-split boundary and unchanged paired estimator."""
|
||||
|
||||
from pathlib import Path
|
||||
from unittest import TestCase, main
|
||||
from unittest.mock import patch
|
||||
|
||||
from track_a_postfit_rebind_common import OMITTED, SPLITS, scope, selected_training
|
||||
from track_a_postfit_rebind_statistics import paired_uncertainty, summarize
|
||||
from track_a_postfit_statistics import paired_uncertainty as original_uncertainty
|
||||
|
||||
|
||||
class RebindTests(TestCase):
|
||||
def setUp(self):
|
||||
self.jobs = [dict(id=i, test=['a', str(i)], missing_test=[]) for i in range(5)]
|
||||
self.rows = [dict(split=i, case=case, A=1., H2=2.+i)
|
||||
for i in SPLITS for case in self.jobs[i]['test']]
|
||||
|
||||
def test_exact_split_ids_and_visit_counts(self):
|
||||
original = dict(A_predictions={str(i): {str(i): {'sha256': str(i)}} for i in range(5)})
|
||||
result = scope(original, {'jobs': self.jobs})
|
||||
self.assertEqual([job['id'] for job in result['jobs']], list(SPLITS))
|
||||
self.assertEqual(set(result['A_predictions']), {'0', '1', '2', '4'})
|
||||
self.assertEqual(result['excluded_split'], OMITTED)
|
||||
self.assertEqual(result['expected_pair_visits'], 8)
|
||||
self.assertEqual(result['expected_unique_target_reads'], 5)
|
||||
|
||||
def test_four_split_summary_uses_original_part_bootstrap(self):
|
||||
outcome = summarize(self.rows, selected_training({'jobs': self.jobs})['jobs'])
|
||||
self.assertIs(paired_uncertainty, original_uncertainty)
|
||||
self.assertEqual(outcome['descriptive_paired_uncertainty'], original_uncertainty(self.rows))
|
||||
self.assertEqual(outcome['mean_of_four_split_summaries']['H2']['median'], 3.75)
|
||||
self.assertEqual(outcome['excluded_split']['split'], 3)
|
||||
self.assertNotIn('mean_of_five_split_summaries', outcome)
|
||||
|
||||
def test_incomplete_duplicate_or_extra_visits_rejected(self):
|
||||
jobs = selected_training({'jobs': self.jobs})['jobs']
|
||||
for rows in (self.rows[:-1], self.rows+[self.rows[0]],
|
||||
self.rows+[dict(split=3, case='a', A=1., H2=4.)]):
|
||||
with self.assertRaises(ValueError):
|
||||
summarize(rows, jobs)
|
||||
|
||||
def test_mixed_source_checkpoint_identity(self):
|
||||
from track_a_postfit_common import checkpoint_identity
|
||||
for i in SPLITS:
|
||||
training_sha = 'recovery' if i == 4 else 'original'
|
||||
blob = dict(seed=0, fold=i, completed_epochs=200, optimizer_steps=62000,
|
||||
contract_sha256=training_sha, role='last_epoch_only')
|
||||
checkpoint_identity(blob, dict(id=i, target_steps=62000), training_sha)
|
||||
with self.assertRaises(ValueError):
|
||||
checkpoint_identity(blob, dict(id=i, target_steps=62000), 'other')
|
||||
|
||||
def test_scoring_requires_new_review_before_target_access(self):
|
||||
from track_a_postfit_rebind_worker import score
|
||||
with patch('track_a_postfit_rebind_worker.config', return_value=({}, {}, {})), \
|
||||
patch('track_a_postfit_rebind_worker.validate_review', side_effect=PermissionError), \
|
||||
patch('phase2_h2_data.cached') as targets:
|
||||
with self.assertRaises(PermissionError):
|
||||
score(Path('/synthetic'))
|
||||
targets.assert_not_called()
|
||||
|
||||
def test_scoring_requires_prediction_freeze_before_target_access(self):
|
||||
from track_a_postfit_rebind_worker import score
|
||||
with patch('track_a_postfit_rebind_worker.config', return_value=({}, {}, {})), \
|
||||
patch('track_a_postfit_rebind_worker.validate_review'), \
|
||||
patch('track_a_postfit_rebind_worker.adopted'), \
|
||||
patch('track_a_postfit_rebind_worker.verify', side_effect=PermissionError), \
|
||||
patch('phase2_h2_data.cached') as targets:
|
||||
with self.assertRaises(PermissionError):
|
||||
score(Path('/synthetic'))
|
||||
targets.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Four-split Track A postfit after exact mixed-source review and training terminal verification."""
|
||||
import fcntl,json,os,resource,sys,time
|
||||
from pathlib import Path
|
||||
import psutil
|
||||
from track_a_postfit_common import prediction_identity,budget
|
||||
from track_a_postfit_rebind_common import config,read,terminal,selected_training,SPLITS,RECOVERY,ENTRYPOINTS
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_home_freeze_io import freeze,verify
|
||||
from phase2_h1_worker_v2 import save,sha,checked
|
||||
from r4bc_guard import supervise
|
||||
|
||||
def run(root):
|
||||
gpu_clock=time.monotonic();start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c,tc,entry=config(root);validate_review(root);adopted=terminal(tc);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if adopted!=c['training_evidence']:raise ValueError('Training terminal changed since rebind')
|
||||
if (root/'started.json').exists():raise FileExistsError('One Track A postfit attempt, no retry')
|
||||
for p in psutil.process_iter(['pid','cmdline']):
|
||||
if p.pid!=os.getpid() and any(Path(x).name in ENTRYPOINTS for x in p.info['cmdline'] or []):raise RuntimeError('Existing postfit process')
|
||||
gpu_lock=Path('/tmp/meltmind-track-a-postfit-'+c['gpu_uuid']+'.lock').open('a');fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True';gpu=0.;charged=0.;records=[];prior=adopted['training_gpu_seconds_once'];prediction_start=None
|
||||
def cpu():
|
||||
u=resource.getrusage(resource.RUSAGE_CHILDREN);return 180+time.process_time()-start+max(charged,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),training_gpu_seconds_once=prior,postfit_gpu_cap=7200,track_a_gpu_cap=162000,review_sha256=sha(root/'review.json'),cpu_start_counters=dict(self_process_time=start,children_user=u0.ru_utime,children_system=u0.ru_stime),cpu_setup_allowance=180,new_fits=0,new_decodes=0))
|
||||
try:
|
||||
budget(prior,gpu,cpu(),len(SPLITS));save(root/'models-adopted.json',adopted);train=Path(c['training_root']);freeze(root,'models',[root/'contract.json',root/'models-adopted.json',train/'complete.json',RECOVERY/'complete.json',RECOVERY/'available-models-incomplete.json',RECOVERY/'incomplete-models-git-freeze.json',RECOVERY/'terminal-git-freeze.json',*[Path(m['path']) for m in adopted['models']],*[Path(m['standardization']) for m in adopted['models']]])
|
||||
entries=[]
|
||||
from track_a_postfit_rebind_guard import run_prediction
|
||||
for position,index in enumerate(SPLITS):
|
||||
gpu=max(gpu,60+time.monotonic()-gpu_clock);budget(prior,gpu,cpu(),len(SPLITS)-position);(root/'jobs'/str(index)).mkdir(parents=True,exist_ok=False);prediction_start=time.monotonic();g=run_prediction(root,c,tc,index,prior+gpu,300);observed=60+time.monotonic()-prediction_start;prediction_start=None;cost=max(observed,g['gpu_seconds_charged']);gpu+=cost;charged+=g['worker']['cpu_seconds'] if g['worker'] else 0.;records.append(dict(mode='predict',index=index,charged_gpu_seconds=cost,guard=g));save(root/'resource-progress.json',dict(records=records,prediction_gpu_seconds=gpu,postfit_cpu_seconds=cpu(),cumulative_track_a_gpu_seconds=prior+gpu))
|
||||
if g['status']!='PASS' or cost>900 or gpu>7200 or prior+gpu>162000:raise RuntimeError('Prediction/resource guard failure')
|
||||
m=read(root/'jobs'/str(index)/'prediction-records/manifest.json');entries.extend(m['entries'])
|
||||
prediction_identity(root,c,selected_training(tc),entries);save(root/'predictions-manifest.json',dict(entries=entries,A_predictions_reused=c['expected_pair_visits'],evaluated_splits=list(SPLITS),excluded_split=c['excluded_split'],new_fits=0,targets_read=False));freeze(root,'predictions',[root/'contract.json',root/'models-adopted.json',root/'predictions-manifest.json',root/'resource-progress.json',*[Path(p) for p in c['code_sha256']]])
|
||||
gpu=max(gpu,60+time.monotonic()-gpu_clock);budget(prior,gpu,cpu(),0);validate_review(root);verify(root,'models');verify(root,'predictions')
|
||||
g=supervise([sys.executable,'tools/track_a_postfit_rebind_worker.py','score',str(root)],root/'score-guard',dict(cpu_seconds=900,wall_seconds=1200,rss_bytes=16*2**30,disk_bytes=20*2**30));charged+=g['cpu_seconds'];records.append(dict(mode='score',guard=g))
|
||||
if g['status']!='PASS' or cpu()+120>3600:raise RuntimeError('Score/resource failure')
|
||||
save(root/'complete.json',dict(status='COMPLETE_FOUR_OF_FIVE',evaluated_splits=list(SPLITS),excluded_split=c['excluded_split'],records=records,prediction_gpu_seconds=gpu,postfit_cpu_seconds=cpu(),training_gpu_seconds_once=prior,cumulative_track_a_gpu_seconds=prior+gpu,track_a_gpu_cap=162000,new_fits=0,new_decodes=0,future_training_started=False,no_retry=True));freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',root/'review.json'])
|
||||
final=cpu();save(root/'post-terminal-cost.json',dict(postfit_cpu_seconds=final,prediction_gpu_seconds=gpu,cumulative_track_a_gpu_seconds=prior+gpu,within_cap=final<=3600 and gpu<=7200 and prior+gpu<=162000,cpu_start_counters=read(root/'started.json')['cpu_start_counters']))
|
||||
if final>3600:raise RuntimeError('Final CPU cap after freeze')
|
||||
except BaseException as exc:
|
||||
if prediction_start is not None:gpu+=60+time.monotonic()-prediction_start
|
||||
if (root/'complete.json').exists():save(root/'pre-failure-complete.json',read(root/'complete.json'))
|
||||
save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),records=records,postfit_cpu_seconds=cpu(),prediction_gpu_seconds=gpu,training_gpu_seconds_once=prior,cumulative_track_a_gpu_seconds=prior+gpu,new_fits=0,no_retry=True));raise
|
||||
if __name__=='__main__':run(Path(sys.argv[1]))
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Identity and custody for the four completed splits of the Track A rebind."""
|
||||
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from phase2_h1_worker_v2 import checked, sha
|
||||
from phase2_home_freeze_io import verify
|
||||
from track_a_h2_common import read
|
||||
from track_a_postfit_common import config as original_postfit_config
|
||||
from track_a_recovery_common import ORIGINAL, ROOT as RECOVERY, accepted_guard, execution_command, reuse_identity
|
||||
|
||||
|
||||
RULING = '1795022093.308479'
|
||||
ROOT = ORIGINAL.with_name('track-a-postfit-four-split-2026-11-19')
|
||||
ORIGINAL_POSTFIT = ORIGINAL.with_name('track-a-postfit-2026-11-18')
|
||||
SPLITS = (0, 1, 2, 4)
|
||||
OMITTED = dict(split=3, status='NOT_TRAINED_TO_COMPLETION_BUDGET_STOP',
|
||||
reason='OWN_PROJECTED_CAP', recovery_completed_epochs=1)
|
||||
ENTRYPOINTS = {
|
||||
'track_a_postfit.py', 'track_a_postfit_worker.py',
|
||||
'track_a_postfit_rebind.py', 'track_a_postfit_rebind_worker.py',
|
||||
}
|
||||
|
||||
|
||||
def selected_training(training):
|
||||
return dict(training, jobs=[training['jobs'][split] for split in SPLITS])
|
||||
|
||||
|
||||
def scope(original, training):
|
||||
jobs = selected_training(training)['jobs']
|
||||
if tuple(job['id'] for job in jobs) != SPLITS:
|
||||
raise ValueError('Original split identities changed')
|
||||
return dict(
|
||||
jobs=[dict(id=j['id'], expected_parts=len(j['test'])) for j in jobs],
|
||||
A_predictions={str(split): original['A_predictions'][str(split)] for split in SPLITS},
|
||||
expected_pair_visits=sum(len(j['test']) for j in jobs),
|
||||
expected_unique_target_reads=len({part for j in jobs for part in j['test']}),
|
||||
excluded_split=OMITTED,
|
||||
)
|
||||
|
||||
|
||||
def config(root):
|
||||
if root.resolve() != ROOT:
|
||||
raise PermissionError('Only the four-split rebind root is approved')
|
||||
contract = read(root / 'contract.json')
|
||||
verify(root, 'implementation')
|
||||
original, training, entry = original_postfit_config(ORIGINAL_POSTFIT)
|
||||
expected = scope(original, training)
|
||||
for key in ('approval', 'budget_approval', 'estimator_approval', 'training_root',
|
||||
'training_contract_sha256', 'comparison_contract', 'comparison_contract_sha256',
|
||||
'gpu_uuid', 'worker_cpus', 'coordinator_cpus', 'gpu_cap', 'postfit_gpu_cap',
|
||||
'postfit_cpu_cap', 'idle_wait_seconds', 'no_retry', 'bootstrap',
|
||||
'all_predictions_frozen_before_targets', 'A_refit', 'H2_refit', 'new_decodes',
|
||||
'metric', 'original_failed'):
|
||||
expected[key] = original[key]
|
||||
if (contract['rebind_ruling'] != RULING or contract['version'] != 'track-a-postfit-four-split'
|
||||
or not contract['no_auto_launch'] or any(contract[k] != v for k, v in expected.items())
|
||||
or (contract['expected_pair_visits'], contract['expected_unique_target_reads']) != (1228, 571)):
|
||||
raise PermissionError('Frozen four-of-five postfit scope')
|
||||
for p, h in {**contract['source_sha256'], **contract['code_sha256']}.items():
|
||||
checked(p, h)
|
||||
return contract, training, entry
|
||||
|
||||
|
||||
def terminal(training):
|
||||
import psutil
|
||||
|
||||
for process in psutil.process_iter(['pid', 'cmdline']):
|
||||
args = process.info['cmdline'] or []
|
||||
if execution_command(args) or (process.pid != os.getpid()
|
||||
and any(Path(a).name in ENTRYPOINTS for a in args)):
|
||||
raise RuntimeError('Track A training or another postfit process still present')
|
||||
verify(RECOVERY, 'terminal')
|
||||
model_proof = verify(RECOVERY, 'incomplete-models')
|
||||
recovered = read(RECOVERY / 'complete.json')
|
||||
incomplete = read(RECOVERY / 'available-models-incomplete.json')
|
||||
recovery = read(RECOVERY / 'contract.json')
|
||||
originals = reuse_identity(recovery)
|
||||
if (recovered['status'] != 'RECOVERY_INCOMPLETE_NO_FURTHER_RECOVERY'
|
||||
or recovered['available_folds'] != list(SPLITS) or not recovered['no_further_recovery']
|
||||
or recovered['postfit_started'] or recovered['test_targets_read']
|
||||
or incomplete['all_five_complete'] or incomplete['postfit_started']
|
||||
or (RECOVERY / 'global-stop.json').exists() or (ORIGINAL_POSTFIT / 'started.json').exists()):
|
||||
raise ValueError('Track A terminal scope differs')
|
||||
entries = {e['id']: e for e in recovered['entries']}
|
||||
if len(recovered['entries']) != 2 or set(entries) != {3, 4}:
|
||||
raise ValueError('Exact two recovery receipts required')
|
||||
fold3, fold4 = (RECOVERY / 'jobs' / str(i) for i in (3, 4))
|
||||
stopped = read(checked(fold3 / 'complete.json', entries[3]['guard_complete_sha256']))
|
||||
stop = read(fold3 / 'stop.json')
|
||||
projection = read(fold3 / 'first-epoch-projection.json')
|
||||
if (entries[3]['status'] != 'STOPPED_OR_FAILED' or 'OWN_FOLD_STOP' not in stopped['reasons']
|
||||
or stop['id'] != 3 or stop['reason'] != OMITTED['reason']
|
||||
or projection['cell']['id'] != 3 or projection['cell']['within_cap']
|
||||
or projection['cell']['projected_gpu_seconds'] <= recovery['jobs'][3]['gpu_seconds']
|
||||
or (fold3 / 'worker' / 'result.json').exists()):
|
||||
raise ValueError('Split 3 budget stop identity')
|
||||
g = read(checked(fold4 / 'complete.json', entries[4]['guard_complete_sha256']))
|
||||
if (entries[4]['status'] != 'PASS' or not accepted_guard(entries[4]['returncode'], g)
|
||||
or g['worker'] != read(fold4 / 'guards' / 'train.receipt.json')
|
||||
or g['worker']['monitor_errors'] != 0 or g['monitor_errors']):
|
||||
raise ValueError('Split 4 clean recovery PASS required')
|
||||
result = read(checked(fold4 / 'worker' / 'result.json', g['worker_result_sha256']))
|
||||
job = training['jobs'][4]
|
||||
if (result['status'] != 'COMPLETE' or result['fold'] != 4 or result['job'] != 4
|
||||
or result['completed_epochs'] != 200 or result['optimizer_steps'] != job['target_steps']
|
||||
or result['test_targets_read'] or result['test_prediction_started']
|
||||
or result['validation_used_for_selection']):
|
||||
raise ValueError('Split 4 last epoch identity')
|
||||
model = dict(job=4, path=result['checkpoint'], sha256=result['checkpoint_sha256'],
|
||||
standardization=str(fold4 / 'worker' / 'standardization.json'),
|
||||
standardization_sha256=result['standardization_sha256'],
|
||||
origin='RECOVERY_FRESH_SEED0', training_contract_sha256=sha(RECOVERY / 'contract.json'),
|
||||
guard_complete_sha256=entries[4]['guard_complete_sha256'])
|
||||
models = originals + [model]
|
||||
if models != incomplete['models'] or tuple(m['job'] for m in models) != SPLITS:
|
||||
raise ValueError('Mixed-source manifest differs from recovered models')
|
||||
for m in models:
|
||||
for p, h in ((m['path'], m['sha256']), (m['standardization'], m['standardization_sha256'])):
|
||||
checked(p, h)
|
||||
if model_proof['files'].get(str(Path(p).resolve())) != h:
|
||||
raise ValueError('Model or normalization is missing from the recovery freeze')
|
||||
stats = read(m['standardization'])
|
||||
j = training['jobs'][m['job']]
|
||||
if stats['parts'] != j['learning'] or stats['nodes'] != j['learning_nodes']:
|
||||
raise ValueError('Learning-only normalization identity')
|
||||
old = read(ORIGINAL / 'complete.json')['training_gpu_seconds']
|
||||
new = sum(e['gpu_seconds_charged'] for e in entries.values())
|
||||
prior = recovered['cumulative_training_gpu_seconds']
|
||||
if (old != recovered['old_gpu_seconds_once'] or new != recovered['new_training_gpu_seconds']
|
||||
or not math.isclose(prior, old + new, rel_tol=0, abs_tol=1e-8) or not 0 <= prior <= 154800):
|
||||
raise ValueError('Old and recovery training charges must be included exactly once')
|
||||
return dict(models=models, excluded_split=OMITTED, training_gpu_seconds_once=prior,
|
||||
original_gpu_seconds_once=old, recovery_gpu_seconds_once=new,
|
||||
original_terminal_sha256=sha(ORIGINAL / 'complete.json'),
|
||||
recovery_terminal_sha256=sha(RECOVERY / 'complete.json'))
|
||||
|
||||
|
||||
def adopted(root, contract):
|
||||
verify(root, 'models')
|
||||
receipt = read(root / 'models-adopted.json')
|
||||
if receipt != contract['training_evidence']:
|
||||
raise ValueError('Adopted evidence differs from the frozen binding')
|
||||
return receipt
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Track A postfit guard: bounded selected-GPU wait, no retries."""
|
||||
import json,os,signal,sys,threading,time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_precision_continuation_guard import device_usage
|
||||
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
|
||||
def final_reasons(reasons,receipt,result,tracker,gpu_seconds,cpu_seconds,cpu_cap,expected_parts):
|
||||
out=list(reasons)
|
||||
def add(s):
|
||||
if s not in out:out.append(s)
|
||||
if not receipt or receipt['status']!='PASS':add('WORKER_NOT_PASS')
|
||||
if receipt and receipt.get('monitor_errors'):add('WORKER_MONITOR_ERRORS')
|
||||
if not result or result.get('status')!='COMPLETE' or result.get('parts')!=expected_parts:add('INCOMPLETE_FOLD')
|
||||
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):add('MISSING_PEAKS')
|
||||
if any(v>15*2**30 for v in tracker.maxima.values()):add('RECORDED_VRAM_LIMIT')
|
||||
if gpu_seconds>900 or cpu_seconds>cpu_cap:add('FINAL_RESOURCE_LIMIT')
|
||||
return out
|
||||
|
||||
def run_prediction(queue,c,pilot,fold,prior_gpu,cpu_cap):
|
||||
root=queue/'jobs'/str(fold);gpu_uuid=c['gpu_uuid'];expected_parts=next(j['expected_parts'] for j in c['jobs'] if j['id']==fold)
|
||||
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
|
||||
def elapsed():return 60+time.monotonic()-start
|
||||
def reason(x):
|
||||
if x not in reasons:reasons.append(x)
|
||||
command=[sys.executable,'tools/track_a_postfit_rebind_worker.py','predict',str(queue),str(fold)]
|
||||
def kill():
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
def scan():
|
||||
if tracker.scan(root/'prediction-records'):reason('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();mem,util=device_usage(gpu_uuid);_,rss,_=tree_usage(os.getpid())
|
||||
if mem>15*2**30:reason('DEVICE_VRAM_LIMIT')
|
||||
if rss>16*2**30:reason('TREE_RSS_LIMIT')
|
||||
if disk_usage(queue)>20*2**30:reason('DISK_LIMIT')
|
||||
if elapsed()>=895:reason('PREDICTION_TIME_LIMIT')
|
||||
for x in requested:reason(x)
|
||||
save(root/'prediction-live.json',dict(gpu_seconds_charged=elapsed(),device_bytes=mem,device_utilization=util,rss_bytes=rss,peaks=tracker.maxima,reasons=reasons,unix=time.time()))
|
||||
except Exception as exc:reason('MONITOR_ERROR');errors.append(str(exc))
|
||||
if reasons:kill()
|
||||
done.wait(.5)
|
||||
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGTERM,signal.SIGINT]};thread=None
|
||||
try:
|
||||
wait_start=time.monotonic();waiting=[]
|
||||
while True:
|
||||
mem,util=device_usage(gpu_uuid);waiting.append(dict(unix=time.time(),bytes=mem,utilization=util))
|
||||
save(root/'recovery-idle-wait.json',dict(samples=waiting,limit_seconds=c['idle_wait_seconds'],worker_started=False))
|
||||
if requested:raise RuntimeError('Signal received during idle wait')
|
||||
duration=time.monotonic()-wait_start
|
||||
if duration>c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
|
||||
if mem<=128*2**20 and util==0:break
|
||||
if duration>=c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
|
||||
time.sleep(min(.5,max(0.,c['idle_wait_seconds']-(time.monotonic()-wait_start))))
|
||||
for i in range(3):
|
||||
mem,util=device_usage(gpu_uuid);samples.append(dict(unix=time.time(),bytes=mem,utilization=util))
|
||||
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY_NO_RETRY')
|
||||
if i<2:time.sleep(2)
|
||||
save(root/'prediction-idle-polls.json',samples)
|
||||
if requested:raise RuntimeError('Signal received during setup')
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
remaining=900-elapsed()-10
|
||||
if remaining<=0:raise RuntimeError('Setup exhausted prediction budget')
|
||||
receipt=supervise(command,root/'prediction-guard','predict',dict(total_cpu_seconds=cpu_cap-60,job_cpu_seconds=cpu_cap-60,job_wall_seconds=remaining,tree_rss_bytes=16*2**30,disk_bytes=20*2**30,max_processes=1),poll=.2,gpu_uuid=gpu_uuid)
|
||||
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill()
|
||||
finally:
|
||||
done.set()
|
||||
if thread:thread.join(timeout=4)
|
||||
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
|
||||
try:scan()
|
||||
except Exception as exc:reason('FINAL_SCAN_ERROR');errors.append(str(exc))
|
||||
for s,h in old.items():signal.signal(s,h)
|
||||
result=None;p=root/'prediction-records/result.json'
|
||||
try:
|
||||
if p.exists():result=json.loads(p.read_text())
|
||||
except Exception as exc:reason('FINAL_RESULT_READ_ERROR');errors.append(str(exc))
|
||||
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
|
||||
duration=elapsed();reasons=final_reasons(reasons,receipt,result,tracker,duration,totalcpu,cpu_cap,expected_parts)
|
||||
out=dict(status='PASS' if not reasons else 'STOPPED',reasons=reasons,monitor_errors=errors,worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=duration,cpu_seconds_charged=totalcpu,cumulative_track_a_gpu_seconds=prior_gpu+duration,no_retry=True)
|
||||
save(root/'prediction-guard.json',out);return out
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Frozen full-domain RMSE/max(truth) summaries on identical paired support."""
|
||||
import numpy as np
|
||||
|
||||
from track_a_postfit_rebind_common import OMITTED, SPLITS
|
||||
from track_a_postfit_statistics import paired_uncertainty
|
||||
|
||||
|
||||
def summarize(rows, jobs):
|
||||
if tuple(j['id'] for j in jobs) != SPLITS or {r['split'] for r in rows} != set(SPLITS):
|
||||
raise ValueError('Exactly the four approved original splits required')
|
||||
per = []
|
||||
for j in jobs:
|
||||
chosen = [r for r in rows if r['split'] == j['id']]
|
||||
if {r['case'] for r in chosen} != set(j['test']) or len(chosen) != len(j['test']):
|
||||
raise ValueError('Complete paired prediction support required')
|
||||
valid = [r for r in chosen if r['A'] is not None and r['H2'] is not None]
|
||||
summaries = {}
|
||||
for method in ['A', 'H2']:
|
||||
values = np.asarray([r[method] for r in valid], np.float64)
|
||||
if not np.isfinite(values).all() or np.any(values < 0):
|
||||
raise ValueError('Invalid paired metric')
|
||||
summaries[method] = {name: float(fn(values)) if len(values) else None
|
||||
for name, fn in [('min',np.min),('median',np.median),('max',np.max)]}
|
||||
delta = np.asarray([r['H2']-r['A'] for r in valid],np.float64)
|
||||
per.append(dict(split=j['id'],requested_test_parts=315,paired_predictions=len(chosen),
|
||||
scored_pairs=len(valid),missing_original=list(j['missing_test']),
|
||||
undefined=[r['case'] for r in chosen if r['A'] is None or r['H2'] is None],
|
||||
summaries=summaries,paired_mean_H2_minus_A=float(delta.mean()) if len(delta) else None,
|
||||
paired_median_H2_minus_A=float(np.median(delta)) if len(delta) else None))
|
||||
means = {method: {name: float(np.mean([r['summaries'][method][name] for r in per]))
|
||||
if all(r['summaries'][method][name] is not None for r in per) else None
|
||||
for name in ['min','median','max']} for method in ['A','H2']}
|
||||
return dict(per_split=per,mean_of_four_split_summaries=means,
|
||||
evaluated_splits=list(SPLITS), excluded_split=OMITTED,
|
||||
direction='Negative paired H2-minus-A favours H2; descriptive only',
|
||||
descriptive_paired_uncertainty=paired_uncertainty(rows),significance_gate=False,metric='RMSE / max(full truth)',
|
||||
full_requested_315_coverage=all(not r['missing_original'] for r in per))
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Four fixed test split predictions then independently gated paired full-node scoring."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
import numpy as np
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_h1_worker_v2 import checked, sha, save
|
||||
from phase2_home_freeze_io import verify
|
||||
from track_a_postfit_common import checkpoint_identity, prediction_identity
|
||||
from track_a_postfit_rebind_common import config, read, adopted, selected_training, SPLITS
|
||||
from track_a_postfit_rebind_statistics import summarize
|
||||
from phase2_residual_math import decode_seconds
|
||||
|
||||
|
||||
def predict(root,index):
|
||||
import torch
|
||||
from phase2_h2_model import H2Geo
|
||||
from phase2_h2_data import to_device,arguments
|
||||
from phase3_item1_postfit_data import load_evaluation_part
|
||||
c,tc,entry=config(root);validate_review(root);a=adopted(root,c)
|
||||
if index not in SPLITS:raise ValueError('Split outside approved four-of-five support')
|
||||
j=tc['jobs'][index];m=next(m for m in a['models'] if m['job']==index)
|
||||
out=root/'jobs'/str(index)/'prediction-records';out.mkdir(exist_ok=False)
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU allocator identity')
|
||||
if torch.__version__!=tc['versions']['torch'] or np.__version__!=tc['versions']['numpy']:raise ValueError('Library identity')
|
||||
os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False;torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats()
|
||||
def peaks():return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
|
||||
parts=0;entries=[];start=time.monotonic();cpu=time.process_time()
|
||||
try:
|
||||
blob=torch.load(checked(m['path'],m['sha256']),map_location='cpu',weights_only=False)
|
||||
checkpoint_identity(blob,j,m['training_contract_sha256']);model=H2Geo().cuda();model.load_state_dict(blob['model'],strict=True);model.eval();del blob
|
||||
stats=read(checked(m['standardization'],m['standardization_sha256']));view=dict(tc,folds=[j])
|
||||
with torch.no_grad():
|
||||
for case in j['test']:
|
||||
data=to_device(load_evaluation_part(view,case,stats,'test'),'cuda')
|
||||
residual=model(*arguments(data),checkpoint_blocks=False)
|
||||
if not bool(torch.isfinite(residual).all()):raise ValueError('Nonfinite test residual')
|
||||
seconds=decode_seconds(residual.cpu().numpy(),data['F'],j['c'],data['t_set']);torch.cuda.synchronize()
|
||||
if seconds.shape!=(tc['parts'][case]['nodes'],) or not np.isfinite(seconds).all():raise ValueError('Full-node prediction invalid')
|
||||
p=root/'predictions'/str(index)/(case+'.npz');p.parent.mkdir(parents=True,exist_ok=True)
|
||||
if p.exists():raise FileExistsError('No prediction overwrite/retry')
|
||||
np.savez_compressed(p,seconds=seconds,raw_ids=data['raw_ids'])
|
||||
entries.append(dict(split=index,case=case,path=str(p),sha256=sha(p),nodes=len(seconds),
|
||||
raw_ids_sha256=tc['parts'][case]['output_sha256']['raw_ids.npy'],negative_predictions=int(np.sum(seconds<0))))
|
||||
parts+=1;save(out/'progress.json',dict(stage='TEST_PREDICTION_NO_TARGETS',parts=parts,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
del data,residual,seconds
|
||||
save(out/'manifest.json',dict(entries=entries,test_targets_read=False))
|
||||
save(out/'result.json',dict(status='COMPLETE',parts=parts,checkpoint_sha256=m['sha256'],test_targets_read=False,**peaks()))
|
||||
except BaseException as exc:
|
||||
save(out/'failure.json',dict(status='FAILED',parts=parts,reason=type(exc).__name__+': '+str(exc),**peaks()));raise
|
||||
|
||||
|
||||
def score(root):
|
||||
import json
|
||||
from phase2_h2_data import cached
|
||||
from gre24_features_v2 import per_geometry_metrics
|
||||
c,tc,entry=config(root);validate_review(root);adopted(root,c);verify(root,'predictions')
|
||||
selected=selected_training(tc)
|
||||
entries=read(root/'predictions-manifest.json')['entries'];prediction_identity(root,c,selected,entries)
|
||||
out=root/'score';out.mkdir(exist_ok=False);bycase={}
|
||||
save(out/'review-consumed.json',dict(review_sha256=sha(root/'review.json'),prediction_manifest_sha256=sha(root/'predictions-manifest.json'),before_target_values=True))
|
||||
for e in entries:bycase.setdefault(e['case'],[]).append(e)
|
||||
expected=set().union(*(set(j['test']) for j in selected['jobs']))
|
||||
if set(bycase)!=expected or len(expected)!=c['expected_unique_target_reads']:raise ValueError('Frozen unique test domain')
|
||||
rows=[];count=0
|
||||
for case in sorted(bycase):
|
||||
raw=cached(tc,case,'raw_ids')
|
||||
with (out/'target-access.jsonl').open('a') as log:
|
||||
log.write(json.dumps(dict(case=case,role='test',source_sha256=tc['parts'][case]['output_sha256']['seconds.npy'],unix=time.time()))+'\n')
|
||||
truth=cached(tc,case,'seconds');count+=1
|
||||
if truth.shape!=raw.shape:raise ValueError('Target native node domain')
|
||||
for e in bycase[case]:
|
||||
ref=c['A_predictions'][str(e['split'])][case]
|
||||
A=np.load(checked(ref['path'],ref['sha256']),allow_pickle=False)
|
||||
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z:
|
||||
if not np.array_equal(z['raw_ids'],raw):raise ValueError('H2 native raw-ID ordering')
|
||||
H2=z['seconds'].copy()
|
||||
am=per_geometry_metrics(truth,A);hm=per_geometry_metrics(truth,H2)
|
||||
if am['nodes']!=e['nodes'] or hm['nodes']!=e['nodes'] or int(np.sum(H2<0))!=e['negative_predictions']:raise ValueError('Paired full-node coverage')
|
||||
av=am['rrmse_maximum'];hv=hm['rrmse_maximum']
|
||||
rows.append(dict(split=e['split'],case=case,nodes=e['nodes'],A=av,H2=hv,
|
||||
delta_H2_minus_A=hv-av if av is not None and hv is not None else None,
|
||||
truth_maximum=am['denominators']['maximum'],undefined=av is None or hv is None,
|
||||
A_rmse=am['rmse'],H2_rmse=hm['rmse'],A_negative=int(np.sum(A<0)),H2_negative=e['negative_predictions']))
|
||||
del truth
|
||||
save(out/'progress.json',dict(unique_test_target_reads=count,paired_rows=len(rows)))
|
||||
if len(rows)!=c['expected_pair_visits'] or count!=c['expected_unique_target_reads']:raise ValueError('Complete paired scoring required')
|
||||
save(out/'result.json',dict(status='COMPLETE_FOUR_OF_FIVE',rows=rows,summary=summarize(rows,selected['jobs']),
|
||||
evaluated_splits=list(SPLITS),excluded_split=c['excluded_split'],
|
||||
unique_test_target_reads=count,paired_visits=len(rows),original_failed=tc['original_failed'],
|
||||
A_refit=False,H2_refit=False,clipped=False,new_decodes=0,comparison_contract_sha256=c['comparison_contract_sha256'],
|
||||
review_sha256=sha(root/'review.json'),limitations=['Four of five fixed splits; split 3 has no completed training checkpoint after the budget stop',
|
||||
'Protocol-level re-creation; original author split IDs unavailable',
|
||||
'Random-part splits can share geometry families','Original14 preparation failures are retained, not resampled',
|
||||
'RMSE/max(full truth) is the frozen documented proxy; paper numbers are context only']))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
if sys.argv[1]=='predict':predict(Path(sys.argv[2]),int(sys.argv[3]))
|
||||
elif sys.argv[1]=='score' and len(sys.argv)==3:score(Path(sys.argv[2]))
|
||||
else:raise ValueError('predict root index or score root')
|
||||
Reference in new issue
Block a user