Prepare reviewed-scope Track A H2 training entry and synthetic acceptance

This commit is contained in:
MeltAstra committed 2026-11-18 12:18:33 +08:00
1 parent 1eae27ade8
commit 2cf0a0307d
18 files changed
+43660

No files matched your search

+200
View File
@@ -0,0 +1,200 @@
"""CPU synthetic regression checks. No real feature/target arrays, CUDA or fits."""
import copy
from contextlib import ExitStack
import json
from pathlib import Path
import tempfile
import time
import unittest
from unittest.mock import patch
from phase2_h1_worker_v2 import save
from phase2_h2_data import load_part
from track_a_h2_common import (dispatchable, execution_command, first_epoch_release,
project_cell, project_total, read, validate_roles)
import track_a_h2_guard as guard
ROOT = Path('/home/frank14f/MeltMind-runs/track-a-h2-2026-11-18')
class EntryTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.c = read(ROOT / 'contract.json')
cls.entry = read(cls.c['reuse_entry'])
def test_01_exact_original_roles(self):
validate_roles(self.c, self.entry)
self.assertEqual([len(j['learning']) for j in self.c['jobs']], [311, 304, 307, 306, 310])
def test_02_test_to_train_rejected(self):
c = copy.deepcopy(self.c)
c['jobs'][0]['learning'][0] = c['jobs'][0]['test'][0]
with self.assertRaises(ValueError):
validate_roles(c, self.entry)
def test_03_extra_validation_rejected(self):
c = copy.deepcopy(self.c)
c['jobs'][0]['validation'] = [c['jobs'][0]['learning'][0]]
with self.assertRaises(ValueError):
validate_roles(c, self.entry)
def test_04_test_loader_rejects_before_io(self):
with patch('phase2_h2_data.np.load', side_effect=AssertionError('Unexpected array read')):
with self.assertRaises(PermissionError):
load_part(dict(folds=[dict(learning=['a'], validation=[])]), 'b', {}, 'test', Path('/tmp'))
def test_05_clock_cannot_be_changed(self):
c = copy.deepcopy(self.c)
c['jobs'][0]['c'] += .01
with self.assertRaises(ValueError):
validate_roles(c, self.entry)
def test_06_two_slots_do_not_require_five_first_epochs(self):
jobs = self.c['jobs']
self.assertEqual([j['id'] for j in dispatchable(jobs, [])], [0, 1])
self.assertEqual([j['id'] for j in dispatchable(jobs[2:], [jobs[1]])], [2])
def test_07_item1_gpu_protected(self):
j = copy.deepcopy(self.c['jobs'][0])
j['gpu_uuid'] = 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c'
with self.assertRaises(ValueError):
dispatchable([j], [])
def test_08_existing_item1_is_not_duplicate_track_a(self):
self.assertFalse(execution_command(['python', 'tools/phase3_data_sufficiency_v2_worker.py']))
self.assertTrue(execution_command(['python', 'tools/track_a_h2_queue.py']))
def test_09_pending_caps_reserved(self):
p = project_total(self.c, {}, {})
self.assertEqual(p['projected_gpu_seconds'], 162000)
self.assertEqual(p['status'], 'PASS')
def test_10_terminal_replaces_estimate_once(self):
p = project_total(self.c, {0: dict(projected_gpu_seconds=30000)},
{0: dict(gpu_seconds_charged=10000)})
self.assertEqual(p['projected_gpu_seconds'], 162000 - 30960 + 10000)
def test_11_active_waits_cannot_disappear(self):
p = project_total(self.c, {0: dict(projected_gpu_seconds=100)}, {}, {0: 31000})
self.assertEqual(p['status'], 'STOP_BUDGET_PROJECTION')
def test_12_slow_first_epoch_stops(self):
j = self.c['jobs'][0]
self.assertFalse(project_cell(j, dict(epoch=1, max_epoch_seconds=200, charged=300))['within_cap'])
self.assertTrue(project_cell(j, dict(epoch=1, max_epoch_seconds=50, charged=100))['within_cap'])
def test_13_invalid_projection_rejected(self):
for d in [float('nan'), -1, 0, float('inf')]:
with self.assertRaises(ValueError):
project_cell(self.c['jobs'][0], dict(epoch=1, max_epoch_seconds=d, charged=100))
def test_14_first_epoch_global_stop(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
self.assertFalse(first_epoch_release(root, 0))
save(root / 'global-stop.json', dict(reason='synthetic'))
with self.assertRaises(RuntimeError):
first_epoch_release(root, 0)
def guard_case(self, *, monitor_errors=None, peak=1024, worker_status='PASS'):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / 'jobs/0').mkdir(parents=True)
c = copy.deepcopy(self.c)
# No GPU query/process is used; preserve real guard decision paths.
save(root / 'contract.json', c)
def fake_supervise(*args, **kwargs):
w = root / 'jobs/0/worker'
w.mkdir()
save(w / 'result.json', dict(status='COMPLETE', job=0,
optimizer_steps=c['jobs'][0]['target_steps'], completed_epochs=200,
peak_allocated_bytes=1024, peak_reserved_bytes=1024))
if peak > 1024:
save(w / 'failure.json', dict(peak_reserved_bytes=peak))
return dict(status=worker_status, cpu_seconds=0,
monitor_errors=monitor_errors or [])
with patch.object(guard, 'config', return_value=(c, c)), \
patch.object(guard, 'device_usage', return_value=(0, 0)), \
patch.object(guard, 'supervise', side_effect=fake_supervise), \
patch.object(guard.os, 'sched_setaffinity'), patch.object(guard.time, 'sleep'):
rc = guard.run(root, 0)
return rc, read(root / 'jobs/0/complete.json')
def test_15_guard_success_path(self):
rc, r = self.guard_case()
self.assertEqual(rc, 0)
self.assertEqual(r['status'], 'PASS')
def test_16_worker_monitor_error_overrides_pass(self):
rc, r = self.guard_case(monitor_errors=[dict(reason='synthetic monitor error')])
self.assertEqual(rc, 1)
self.assertIn('WORKER_MONITOR_ERRORS', r['reasons'])
def test_17_failure_peak_remains_enforced(self):
rc, r = self.guard_case(peak=16 * 2**30, worker_status='FAILED')
self.assertEqual(rc, 1)
self.assertIn('RECORDED_VRAM_LIMIT', r['reasons'])
def test_18_foreign_ledger_job_rejected(self):
with self.assertRaises(ValueError):
project_total(self.c, {5: dict(projected_gpu_seconds=1)}, {})
def test_19_geometry_cannot_mix_versions(self):
c = copy.deepcopy(self.c)
case = next(iter(c['geometry']))
c['geometry'][case]['protocol'] = 'grouped'
with self.assertRaises(ValueError):
validate_roles(c, self.entry)
def test_20_worker_all_epochs_no_validation_or_test(self):
import track_a_h2_worker as worker
import os
class Model:
def cuda(self): return self
def parameters(self): return []
def state_dict(self): return {}
class Optimizer:
param_groups = [{'lr': worker.lr_epoch(1)}]
def state_dict(self): return {}
job = dict(id=0, learning=['a', 'b'], validation=[], test=['sealed'], c=1.,
learning_nodes=5, clock_sha256='synthetic', worker_cpus=[0], gpu_uuid='synthetic')
c = dict(jobs=[job])
accesses = []
def load(view, case, stats, role, out):
accesses.append((case, role))
if case not in ['a', 'b'] or role != 'learning':
raise AssertionError('Test/validation accessed')
return dict(nodes=2 if case == 'a' else 3)
with tempfile.TemporaryDirectory() as tmp, ExitStack() as stack:
root = Path(tmp)
(root / 'jobs/0').mkdir(parents=True)
save(root / 'contract.json', c)
stack.enter_context(patch.dict(os.environ, CUDA_VISIBLE_DEVICES='synthetic',
PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True'))
replacements = dict(config=lambda _: (c, c), standardization=lambda view, out: {},
H2Geo=Model, optimizer=lambda _: Optimizer(), seed=lambda: None,
peaks=lambda: dict(peak_allocated_bytes=0, peak_reserved_bytes=0),
load_part=load, to_device=lambda data, device: data,
train_step=lambda *a: dict(loss=0., gradient_norm_before_clip=0.),
first_epoch_release=lambda *a: True)
for name, value in replacements.items():
stack.enter_context(patch.object(worker, name, value))
for name in ['set_device', 'init', 'reset_peak_memory_stats']:
stack.enter_context(patch.object(worker.torch.cuda, name))
stack.enter_context(patch.object(worker.torch.cuda.memory, '_snapshot', return_value={}))
stack.enter_context(patch.object(worker.os, 'sched_setaffinity'))
stack.enter_context(patch.object(worker.torch, 'set_num_threads'))
stack.enter_context(patch.object(worker.torch, 'set_num_interop_threads'))
stack.enter_context(patch.object(worker.torch, 'save', side_effect=lambda v, p: p.write_bytes(b'synthetic')))
worker.run(root, 0)
result = read(root / 'jobs/0/worker/result.json')
self.assertEqual(result['completed_epochs'], 200)
self.assertEqual(result['optimizer_steps'], 400)
self.assertEqual(accesses.count(('a', 'learning')), 200)
self.assertEqual(accesses.count(('b', 'learning')), 200)
self.assertFalse(result['test_targets_read'])
if __name__ == '__main__':
unittest.main()
+144
View File
@@ -0,0 +1,144 @@
"""Track A random-split identity and cost gates; no array or target reads."""
import json
import math
from pathlib import Path
from phase2_h1_worker_v2 import checked, sha
from phase2_home_freeze_io import verify
GPU_SECONDS = 162000
PROTECTED_GPUS = {
'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c',
'GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
}
def read(path):
return json.loads(Path(path).read_text())
def validate_roles(c, entry):
if len(c['parts']) != 615 or len(c['jobs']) != 5:
raise ValueError('Five original splits and 615 v4 parts required')
if c['original_failed'] != entry['original_failed']:
raise ValueError('Original failures changed')
for case, original in entry['H2_inputs'].items():
f = original['features_and_targets']
expected = dict(case=case, output=f['output'], nodes=f['nodes'],
t_set=f['t_scale_seconds'], output_sha256=f['output_sha256'])
if c['parts'].get(case) != expected or c['geometry'].get(case) != original['geometry']:
raise ValueError('Frozen native v4 input identity differs')
for j, original in zip(c['jobs'], entry['jobs']):
if j['id'] != original['split'] or j['split_seed'] != original['split_seed']:
raise ValueError('Original split identity changed')
for role in ('learning', 'test'):
if j[role] != original[role] or len(set(j[role])) != len(j[role]):
raise ValueError('Original membership/order changed: ' + role)
if j['validation'] or set(j['learning']) & set(j['test']):
raise ValueError('No new holdout or test leakage')
clock = read(checked(j['clock_path'], j['clock_sha256']))
if set(clock['parts']) != set(j['learning']) or len(clock['parts']) != len(j['learning']):
raise ValueError('Clock learning membership differs')
if clock['groups'] != ['part:' + x for x in clock['parts']]:
raise ValueError('Original v4 equal-part clock groups differ')
if j['c'] != clock['c'] or not math.isfinite(j['c']) or j['c'] <= 0:
raise ValueError('Clock value differs')
if j['learning_nodes'] != sum(c['parts'][x]['nodes'] for x in j['learning']):
raise ValueError('Learning node count differs')
if j['gpu_uuid'] in PROTECTED_GPUS or j['target_steps'] != 200 * len(j['learning']):
raise ValueError('Protected GPU or epoch count')
def config(root):
root = Path(root)
c = read(root / 'contract.json')
verify(root, 'implementation')
if (c['approval'] != '1794973932.596749' or c['gpu_cap'] != GPU_SECONDS
or c['new_fits'] != 5 or c['seed'] != 0 or c['epochs'] != 200
or not c['no_retry'] or c['train_gpu_cap'] + c['postfit_gpu_reserve'] != GPU_SECONDS):
raise PermissionError('Frozen Track A scope differs')
if c['protocol_ruling'] != '1794975093.432709':
raise PermissionError('Actual protocol ruling differs')
for p, h in {**c['source_sha256'], **c['code_sha256']}.items():
checked(p, h)
if sum(j['gpu_seconds'] for j in c['jobs']) != c['train_gpu_cap']:
raise ValueError('Five job budgets must partition training cap')
entry = read(c['reuse_entry'])
validate_roles(c, entry)
import numpy as np
import torch
if np.__version__ != c['versions']['numpy'] or torch.__version__ != c['versions']['torch']:
raise ValueError('Original library versions differ')
return c, c
def view_for(c, job):
# The shared loader rejects every case outside this split's learning role.
return dict(c, folds=[job])
def project_cell(job, observation):
epoch = observation['epoch']
duration = observation['max_epoch_seconds']
charged = observation['charged']
if (not isinstance(epoch, int) or not 1 <= epoch <= 200
or not math.isfinite(duration) or duration <= 0
or not math.isfinite(charged) or charged < 0):
raise ValueError('Invalid full-epoch cost evidence')
projected = charged + 1.25 * (200 - epoch) * duration + 120
return dict(id=job['id'], projected_gpu_seconds=projected,
within_cap=projected <= job['gpu_seconds'])
def project_total(c, estimates, terminal, active_charges=None):
"""Pending jobs retain full reserved caps; terminal costs replace estimates."""
active_charges = active_charges or {}
valid = {j['id'] for j in c['jobs']}
if (set(estimates) | set(terminal) | set(active_charges)) - valid:
raise ValueError('Unknown job in ledger')
total = c['postfit_gpu_reserve']
rows = []
for j in c['jobs']:
i = j['id']
value = (terminal[i]['gpu_seconds_charged'] if i in terminal else
max(active_charges.get(i, 0), estimates.get(i, {}).get('projected_gpu_seconds', j['gpu_seconds'])))
if not math.isfinite(value) or value < 0:
raise ValueError('Invalid ledger value')
total += value
rows.append(dict(id=i, projected_gpu_seconds=value, within_cap=value <= j['gpu_seconds']))
return dict(status='PASS' if total <= c['gpu_cap'] and all(r['within_cap'] for r in rows)
else 'STOP_BUDGET_PROJECTION', rows=rows, projected_gpu_seconds=total,
postfit_reserve=c['postfit_gpu_reserve'])
def first_epoch_release(root, jobid):
if (root / 'global-stop.json').exists():
raise RuntimeError('Track A global stop')
p = root / 'jobs' / str(jobid) / 'first-epoch-projection.json'
if not p.exists():
return False
r = read(p)
if r['status'] != 'PASS' or r['contract_sha256'] != sha(root / 'contract.json'):
raise RuntimeError('First-epoch cost gate rejected')
return True
def dispatchable(pending, active):
occupied = {j['gpu_uuid'] for j in active}
if len(occupied) != len(active) or len(active) > 2:
raise ValueError('Active GPU collision')
selected = []
for j in pending:
if len(active) + len(selected) == 2:
break
if j['gpu_uuid'] in PROTECTED_GPUS:
raise ValueError('Item1 GPU reservation')
if j['gpu_uuid'] not in occupied:
selected.append(j)
occupied.add(j['gpu_uuid'])
return selected
def execution_command(args):
return any(Path(a).name in {'track_a_h2_queue.py', 'track_a_h2_guard.py',
'track_a_h2_worker.py'} for a in args)
+88
View File
@@ -0,0 +1,88 @@
"""Independent bounded TRACK_A fold guard; shared global stop and immutable S1."""
import fcntl
import json
import os
from pathlib import Path
import signal
import sys
import threading
import time
import psutil
from track_a_h2_common import config
from phase2_h1_worker_v2 import save,sha
from s2_main_memory_guard import PeakRecords
from s2_precision_continuation_guard import device_usage
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
def run(queue,fold_id):
c,_=config(queue);job=c['jobs'][fold_id];c=dict(c,**job);root=queue/'jobs'/str(fold_id);os.sched_setaffinity(0,c['guard_cpus'])
with (root/'fold.lock').open('a') as lock, Path('/tmp/meltmind-track-a-'+c['gpu_uuid']+'.lock').open('a') as gpu_lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('Single TRACK_A fold launch, no retry')
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(queue/'contract.json'),gpu_uuid=c['gpu_uuid'],setup_gpu_seconds=c['setup_gpu_seconds']))
def elapsed():return c['setup_gpu_seconds']+time.monotonic()-start
def reason(x):
if x not in reasons:reasons.append(x)
command=[sys.executable,'tools/track_a_h2_worker.py',str(queue),str(fold_id)]
def kill_worker():
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:stop_group(p.pid)
except psutil.NoSuchProcess:pass
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]}
def scan():
if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT')
def watch():
while not done.is_set():
try:
scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid())
if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT')
if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT')
if elapsed()>=c['gpu_seconds']-5:reason('TRACK_A_FOLD_TIME_LIMIT')
if (queue/'global-stop.json').exists():reason('TRACK_A_GLOBAL_STOP')
for x in requested:reason(x)
save(root/'live-progress.json',dict(stage='TRACK_A_TRAINING',fold=fold_id,elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time()))
except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc)))
if reasons:kill_worker()
done.wait(.5)
thread=None
try:
for i in range(3):
mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util))
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY')
if i<2:time.sleep(2)
save(root/'idle-polls.json',samples)
if requested or (queue/'global-stop.json').exists():raise RuntimeError('STOP_BEFORE_WORKER_START')
thread=threading.Thread(target=watch,daemon=True);thread.start()
remaining=c['gpu_seconds']-elapsed()-10
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid'])
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker()
finally:
done.set()
if thread:thread.join(timeout=4)
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
try:scan()
except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc)))
for s,handler in old.items():signal.signal(s,handler)
value=None;p=root/'worker/result.json'
try:
if p.exists():value=json.loads(p.read_text())
except Exception as exc:reason('RESULT_READ_ERROR');errors.append(dict(reason=str(exc)))
if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS')
if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS')
if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT')
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS')
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
try:config(queue)
except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc))
if value and (value.get('optimizer_steps')!=c['target_steps'] or value.get('job')!=fold_id or value.get('completed_epochs')!=200):reason('INCOMPLETE_EPOCH_OR_FOLD_COUNT')
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'TRACK_A_FOLD_INCOMPLETE',reasons=reasons,monitor_errors=errors,
worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),fold=fold_id,cpu_seconds_charged=totalcpu,
worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,track_a_started=True,no_retry=True))
return 0 if not reasons else 1
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2])))
+108
View File
@@ -0,0 +1,108 @@
"""Build the independent Track A contract using JSON metadata only. Never launch."""
import ast
import json
from pathlib import Path
import sys
import time
from phase2_h1_worker_v2 import checked, save, sha
from track_a_h2_common import read, validate_roles
BASE = Path(__file__).resolve().parents[1]
ENTRY = BASE / 'artifacts/track-a-existing-input-audit-2026-11-18/reuse-entry.json'
ENTRY_SHA = '8f0f3c1088d899b85aa3f578b7552f04cb409fd7a5dcfca3e17eaa3d7e25d0d6'
RULING = '1794975093.432709'
def local_code_closure(paths):
found = set()
pending = list(paths)
while pending:
p = Path(pending.pop()).resolve()
if p in found:
continue
found.add(p)
for node in ast.walk(ast.parse(p.read_text())):
names = ([node.module] if isinstance(node, ast.ImportFrom) and node.module else
[a.name for a in node.names] if isinstance(node, ast.Import) else [])
for name in names:
q = BASE / 'tools' / (name.split('.')[0] + '.py')
if q.exists():
pending.append(q)
return {str(p): sha(p) for p in sorted(found)}
def prepare(root):
cpu = time.process_time()
entry = read(checked(ENTRY, ENTRY_SHA))
sources = {str(ENTRY): ENTRY_SHA}
for key in ('A_models_freeze', 'A_predictions_freeze', 'split_manifest',
'preparation_manifest', 'geometry_manifest', 'source_h2_recipe'):
ref = entry[key]
checked(ref['path'], ref['sha256'])
sources[ref['path']] = ref['sha256']
pilot = read(entry['source_h2_recipe']['path'])
h1root = BASE / 'artifacts/phase2-h1-labelled-recovery-2026-11-17'
h1models = read(h1root / 'models-terminal.json')['models']
sources[str(h1root / 'models-terminal.json')] = sha(h1root / 'models-terminal.json')
parts = {}
geometry = {}
for case, row in entry['H2_inputs'].items():
f = row['features_and_targets']
parts[case] = dict(case=case, output=f['output'], nodes=f['nodes'],
t_set=f['t_scale_seconds'], output_sha256=f['output_sha256'])
geometry[case] = row['geometry']
if geometry[case]['protocol'] != 'v4_random' or geometry[case]['nodes'] != f['nodes']:
raise ValueError('Native v4 geometry metadata mismatch')
jobs = []
gpus = ['GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76', 'GPU-8b71b032-1b75-2891-935c-ce106a53034a']
for original in entry['jobs']:
i = original['split']
model = next(m for m in h1models if m['index'] == original['existing_H1_v4_job'])
if model['status'] != 'PASS':
raise ValueError('Original H1 clock unavailable')
clock = read(checked(model['clock_path'], model['clock_sha256']))
sources[model['clock_path']] = model['clock_sha256']
jobs.append(dict(id=i, split_seed=original['split_seed'], fold=i,
learning=original['learning'], test=original['test'], validation=[],
missing_train=original['missing_train'], missing_test=original['missing_test'],
clock_path=model['clock_path'], clock_sha256=model['clock_sha256'], c=clock['c'],
learning_nodes=sum(parts[x]['nodes'] for x in original['learning']),
learning_parts=len(original['learning']), validation_nodes=0,
target_steps=200 * len(original['learning']), gpu_uuid=gpus[i % 2],
worker_cpus=[8 + 2 * (i % 2)], guard_cpus=[28 + (i % 2)],
gpu_seconds=30960, setup_gpu_seconds=60, cpu_seconds=61920,
vram_bytes=15 * 2**30, rss_bytes=16 * 2**30, disk_bytes=8 * 2**30))
code = local_code_closure([*[BASE / 'tools' / ('track_a_h2_' + name + '.py')
for name in ('common', 'worker', 'guard', 'queue', 'prepare')],
BASE / 'tools/test_track_a_h2_entry.py'])
# Pin original scientific functions against the already frozen H2 recipe.
for name in ('phase2_h2_data.py', 'phase2_h2_model.py', 'phase2_h2_pilot_worker.py', 'phase2_residual_math.py'):
p = BASE / 'tools' / name
pin = next(h for path, h in pilot['code_sha256'].items() if Path(path).name == name)
checked(p, pin)
c = dict(version='track-a-h2-v1', approval='1794973932.596749', protocol_ruling=RULING,
reuse_entry=str(ENTRY), comparison_contract=str(ENTRY), comparison_contract_sha256=ENTRY_SHA,
new_fits=5, A_refit=False, seed=0, epochs=200, no_retry=True, new_decodes=0,
model=pilot['model'], versions=pilot['versions'], parts=parts, geometry=geometry, jobs=jobs,
original_failed=entry['original_failed'], source_sha256=sources, code_sha256=code,
gpu_cap=162000, train_gpu_cap=154800, postfit_gpu_reserve=7200,
coordinator_cpus=[30, 31], aggregate_rss_bytes=40 * 2**30, aggregate_disk_bytes=48 * 2**30,
normalization='Per-split learning-only original 19 columns, FP64 population mean/std; separate FP64 Vf skip',
validation='None; no checkpoint selection; no test access in this training entry',
selection='Last epoch 200 only', clock_policy='Reuse frozen v4 H1 equal successful train-part clock',
postfit='Separate exact Assist review required before any test access; training never starts postfit',
metric=entry['metric'], expected_pairing_counts_provisional=[304, 311, 308, 309, 305])
validate_roles(c, entry)
root.mkdir(parents=True, exist_ok=False)
save(root / 'contract.json', c)
save(root / 'preparation-receipt.json', dict(status='ENTRY_PREPARED_NOT_EXECUTED',
contract_sha256=sha(root / 'contract.json'), code_files=len(code),
clock_values_recomputed=False, scientific_arrays_read=False, new_fits_started=False,
prior_A_audit_rerun=False, learning_counts=[len(j['learning']) for j in jobs],
cpu_seconds=time.process_time() - cpu, protocol_ruling=RULING))
print(json.dumps(read(root / 'preparation-receipt.json')))
if __name__ == '__main__':
prepare(Path(sys.argv[1]).resolve())
+205
View File
@@ -0,0 +1,205 @@
"""Five independent fits on two idle GPUs; train and freeze only, never score."""
import fcntl
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
import psutil
from phase2_gate_outcome_review import validate_review
from phase2_h1_worker_v2 import checked, save, sha
from phase2_home_freeze_io import freeze
from s2_recovery_gpu_supervisor import disk_usage, tree_usage
from track_a_h2_common import config, dispatchable, execution_command, project_cell, project_total, read
def run(root):
c, _ = config(root)
validate_review(root)
lock = (root / 'queue.lock').open('a')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (root / 'started.json').exists():
raise FileExistsError('One Track A attempt; no retry')
for p in psutil.process_iter(['pid', 'cmdline']):
if p.pid != os.getpid() and execution_command(p.info['cmdline'] or []):
raise RuntimeError('Existing Track A execution')
os.sched_setaffinity(0, c['coordinator_cpus'])
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
start = time.monotonic()
cpu = time.process_time()
active, terminal, estimates, last_epoch = {}, {}, {}, {}
pending = list(c['jobs'])
requested = []
def stop(reason):
if not (root / 'global-stop.json').exists():
save(root / 'global-stop.json', dict(reason=reason, unix=time.time(), no_retry=True))
def active_charges():
return {i: c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
for i, v in active.items()}
def charged():
return sum(v['gpu_seconds_charged'] for v in terminal.values()) + sum(active_charges().values())
def terminate(v):
if v.get('stop_sent') is None:
if v['process'].poll() is None:
v['process'].send_signal(signal.SIGTERM)
v['stop_sent'] = time.monotonic()
elif time.monotonic() - v['stop_sent'] > 20:
try:
parent = psutil.Process(v['process'].pid)
for p in reversed(parent.children(recursive=True)):
try:
p.kill()
except psutil.NoSuchProcess:
pass
parent.kill()
except psutil.NoSuchProcess:
pass
def collect(i, v):
rc = v['process'].poll()
if rc is None:
return False
p = root / 'jobs' / str(i) / 'complete.json'
r = read(p) if p.exists() else None
observed = c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start']
ok = (rc == 0 and r is not None and r['status'] == 'PASS'
and not r.get('monitor_errors') and r.get('worker') is not None
and not r['worker'].get('monitor_errors'))
terminal[i] = dict(id=i, status='PASS' if ok else 'STOPPED_OR_FAILED', returncode=rc,
gpu_seconds_charged=max(observed, r['gpu_seconds_charged'] if r else 0),
guard_complete_sha256=sha(p) if r else None)
v['log'].close()
del active[i]
save(root / 'terminal-ledger.json', dict(entries=list(terminal.values()),
settled_gpu_seconds=sum(x['gpu_seconds_charged'] for x in terminal.values())))
if not ok:
stop('JOB_' + str(i) + '_FAILED')
return True
def dispatch():
for j in dispatchable(pending, [c['jobs'][i] for i in active]):
i = j['id']
d = root / 'jobs' / str(i)
d.mkdir(parents=True, exist_ok=False)
log = (d / 'entry.log').open('x')
begin = time.monotonic()
cmd = [sys.executable, 'tools/track_a_h2_guard.py', str(root), str(i)]
try:
p = subprocess.Popen(cmd, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
except BaseException:
log.close()
raise
active[i] = dict(process=p, log=log, start=begin, stop_sent=None)
pending.remove(j)
save(d / 'dispatch.json', dict(pid=p.pid, command=cmd, unix=time.time(), gpu_uuid=j['gpu_uuid']))
handlers = {s: signal.signal(s, lambda n, f: requested.append('SIGNAL_' + str(n)))
for s in (signal.SIGTERM, signal.SIGINT)}
save(root / 'started.json', dict(pid=os.getpid(), unix=time.time(), new_fits=5,
contract_sha256=sha(root / 'contract.json'), review_sha256=sha(root / 'review.json'),
gpu_cap=c['gpu_cap'], A_refit=False, test_targets_read=False))
try:
while active or pending:
for i, v in list(active.items()):
collect(i, v)
for reason in requested:
stop(reason)
amount = charged()
_, rss, _ = tree_usage(os.getpid())
if amount >= c['train_gpu_cap'] - 80:
stop('CUMULATIVE_TRAIN_GPU_BOUNDARY')
if rss > c['aggregate_rss_bytes'] or disk_usage(root) > c['aggregate_disk_bytes']:
stop('AGGREGATE_MEMORY_DISK_LIMIT')
if not (root / 'global-stop.json').exists():
for i in active:
files = sorted((root / 'jobs' / str(i) / 'worker/history').glob('*.json'))
rows = [read(p) for p in files]
if not rows:
continue
epoch = rows[-1]['epoch']
# Recalculate every epoch: waiting, setup and observed cost stay charged.
if epoch > last_epoch.get(i, 0):
observation = dict(epoch=epoch, charged=active_charges()[i],
max_epoch_seconds=max(r['train_epoch_seconds'] for r in rows))
cell = project_cell(c['jobs'][i], observation)
estimates[i] = cell
projection = dict(**project_total(c, estimates, terminal, active_charges()),
cell=cell, contract_sha256=sha(root / 'contract.json'), unix=time.time())
name = 'first-epoch-projection.json' if i not in last_epoch else 'latest-cost-projection.json'
save(root / 'jobs' / str(i) / name, projection)
save(root / 'latest-cost-projection.json', projection)
last_epoch[i] = epoch
if projection['status'] != 'PASS':
stop('PROJECTED_TRACK_A_BUDGET')
if not (root / 'global-stop.json').exists():
dispatch()
if (root / 'global-stop.json').exists():
for v in active.values():
terminate(v)
save(root / 'live-progress.json', dict(stage='STOPPING' if (root / 'global-stop.json').exists()
else 'TRACK_A_TRAINING', unix=time.time(), active=[dict(id=i, guard_pid=v['process'].pid)
for i, v in active.items()], terminal=list(terminal.values()), pending=[j['id'] for j in pending],
charged_training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve']))
if (root / 'global-stop.json').exists() and not active:
break
if active:
time.sleep(1)
if (root / 'global-stop.json').exists() or len(terminal) != 5 or any(v['status'] != 'PASS' for v in terminal.values()):
raise RuntimeError('Track A training incomplete')
config(root)
models = []
for j in c['jobs']:
w = root / 'jobs' / str(j['id']) / 'worker'
r = read(w / 'result.json')
stats = read(checked(w / 'standardization.json', r['standardization_sha256']))
if (r['optimizer_steps'] != j['target_steps'] or r['completed_epochs'] != 200
or stats['parts'] != j['learning'] or r['test_targets_read']):
raise ValueError('Terminal model identity')
checked(r['checkpoint'], r['checkpoint_sha256'])
models.append(dict(job=j['id'], path=r['checkpoint'], sha256=r['checkpoint_sha256'],
standardization=str(w / 'standardization.json'),
standardization_sha256=r['standardization_sha256']))
if charged() > c['train_gpu_cap']:
raise ValueError('Terminal cumulative cost')
save(root / 'models-terminal.json', dict(models=models, new_fits=5, A_refit=False, test_prediction_started=False))
freeze(root, 'models', [root / 'contract.json', root / 'models-terminal.json', root / 'terminal-ledger.json',
*[Path(m['path']) for m in models], *[Path(m['standardization']) for m in models]])
save(root / 'complete.json', dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING', entries=list(terminal.values()),
training_gpu_seconds=charged(), postfit_gpu_reserve=c['postfit_gpu_reserve'], gpu_cap=c['gpu_cap'],
coordinator_cpu_seconds=time.process_time() - cpu, wall_seconds=time.monotonic() - start,
no_retry=True, test_targets_read=False, postfit_started=False))
freeze(root, 'terminal', [root / 'contract.json', root / 'complete.json', root / 'models-terminal.json'])
return 0
except BaseException as exc:
stop(type(exc).__name__ + ': ' + str(exc))
while active:
for i, v in list(active.items()):
try:
if not collect(i, v):
terminate(v)
except Exception:
terminate(v)
if v['process'].poll() is not None:
terminal[i] = dict(id=i, status='MALFORMED_TERMINAL',
gpu_seconds_charged=c['jobs'][i]['setup_gpu_seconds'] + time.monotonic() - v['start'])
v['log'].close()
del active[i]
if active:
time.sleep(.5)
save(root / 'complete.json', dict(status='STOPPED', reason=str(exc), entries=list(terminal.values()),
never_attempted=[j['id'] for j in pending], training_gpu_seconds=charged(),
postfit_started=False, no_retry=True))
return 1
finally:
for s, handler in handlers.items():
signal.signal(s, handler)
if __name__ == '__main__':
raise SystemExit(run(Path(sys.argv[1])))
+55
View File
@@ -0,0 +1,55 @@
"""Track A v4 random split; same H2 seed0/200epochs, learning-only, no test access."""
import json,os,sys,time,traceback
from pathlib import Path
import numpy as np
import torch
from phase2_h2_s1_common import selected_checkpoint
from track_a_h2_common import config,view_for,first_epoch_release
from phase2_h2_data import standardization
from phase2_h2_data import load_part,to_device,arguments,cached
from phase2_h2_model import H2Geo,optimizer,lr_epoch
from phase2_h2_pilot_worker import seed,peaks,train_step
from phase2_residual_math import residual_target
from phase2_h1_worker_v2 import save,sha
def run(root,fold_id):
c,_=config(root);job=c['jobs'][fold_id];old=view_for(c,job);out=root/'jobs'/str(fold_id)/'worker';out.mkdir(exist_ok=False);(out/'history').mkdir()
os.sched_setaffinity(0,job['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
start=time.monotonic();cpu=time.process_time();stage='SETUP';epoch=0;steps=0
def record(item,epoch_file=False):
item=dict(fold=fold_id,stage=stage,epoch=epoch,optimizer_steps=steps,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**item,**peaks())
with (out/'history.jsonl').open('a') as f:f.write(json.dumps(item,allow_nan=False)+'\n')
save(out/'progress.json',item)
if epoch_file:save(out/'history'/f'{epoch:03d}.json',item)
return item
try:
if os.environ.get('CUDA_VISIBLE_DEVICES')!=job['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':raise ValueError('GPU/allocator identity')
stage='LEARNING_ONLY_NORMALIZATION';normal=out/'normalization';normal.mkdir();stats=standardization(old,normal);save(out/'standardization.json',stats);save(out/'fold-identity.json',dict(actual_fold=fold_id,learning=old['folds'][0]['learning'],validation=old['folds'][0]['validation'],test=old['folds'][0]['test'],clock_sha256=old['folds'][0]['clock_sha256']))
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats();seed()
model=H2Geo().cuda();opt=optimizer(model);fold=old['folds'][0];parts=len(fold['learning']);total=fold['learning_nodes']
save(out/'environment.json',dict(torch=torch.__version__,gpu_uuid=job['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,grad_scaler=False,fresh_seed=0,pilot_weights_reused=False,parameter_count=sum(p.numel() for p in model.parameters())))
for epoch in range(1,201):
stage='TRAIN';begin=time.monotonic();seen=0;nodes=0;loss_sum=0.
if any(abs(g['lr']-lr_epoch(epoch))>1e-15 for g in opt.param_groups):raise ValueError('Learning rate epoch mismatch')
order=np.random.default_rng(np.random.SeedSequence([0,epoch])).permutation(parts)
for index in order:
case=fold['learning'][int(index)];data=to_device(load_part(old,case,stats,'learning',out),'cuda')
result=train_step(model,opt,data,parts,total);seen+=1;steps+=1;nodes+=data['nodes'];loss_sum+=result['loss']
record(dict(case=case,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch),loss=result['loss'],gradient_norm_before_clip=result['gradient_norm_before_clip']))
del data
if seen!=parts or nodes!=total:raise ValueError('Incomplete full-node learning epoch')
metrics=dict(train_mean_weighted_loss=loss_sum/parts,train_epoch_seconds=time.monotonic()-begin,epoch_parts=seen,epoch_nodes=nodes,lr=lr_epoch(epoch))
stage='EPOCH_COMPLETE';record(metrics,epoch_file=True)
if epoch==1:
stage='WAIT_OWN_FIRST_EPOCH_COST_GATE';record(dict(epoch_parts=seen,epoch_nodes=nodes))
while not first_epoch_release(root,fold_id):time.sleep(.5)
if epoch<200:
for group in opt.param_groups:group['lr']=lr_epoch(epoch+1)
stage='FINAL_CHECKPOINT';name=selected_checkpoint(epoch)
torch.save(dict(model=model.state_dict(),optimizer=opt.state_dict(),completed_epochs=epoch,optimizer_steps=steps,seed=0,fold=fold_id,contract_sha256=sha(root/'contract.json'),role='last_epoch_only'),out/name)
save(out/'result.json',dict(status='COMPLETE',job=fold_id,fold=fold_id,completed_epochs=epoch,optimizer_steps=steps,learning_parts=parts,learning_nodes_per_epoch=total,checkpoint=str(out/name),checkpoint_sha256=sha(out/name),standardization_sha256=sha(out/'standardization.json'),test_targets_read=False,test_prediction_started=False,validation_used_for_selection=False,worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**peaks()))
except BaseException as exc:
memory=peaks() if torch.cuda.is_initialized() else dict(peak_allocated_bytes=0,peak_reserved_bytes=0)
save(out/'failure.json',dict(status='FAILED',stage=stage,epoch=epoch,optimizer_steps=steps,type=type(exc).__name__,reason=str(exc),traceback=traceback.format_exc(),worker_wall_seconds=time.monotonic()-start,worker_cpu_seconds=time.process_time()-cpu,**memory));raise
if __name__=='__main__':run(Path(sys.argv[1]),int(sys.argv[2]))