Prepare exact seed1 postfit review binding without execution

This commit is contained in:
MeltAstra committed 2026-11-18 17:15:17 +08:00
1 parent 38f0044299
commit e49851ba0b
20 files changed
+83890

No files matched your search

+41
View File
@@ -0,0 +1,41 @@
"""Seed1 postfit only after five frozen training terminals; exact review before test access."""
import fcntl,json,os,resource,sys,time
from pathlib import Path
import psutil
from phase3_seed1_postfit_common import config,read,terminal,prediction_identity,budget,OLD,PHASE3_PRIOR
from phase2_gate_outcome_review import validate_review
from phase2_home_freeze_io import freeze,verify
from phase2_h1_worker_v2 import save,sha,checked
from r4bc_guard import supervise
def run(root):
gpu_clock=time.monotonic();start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c,tc,pilot=config(root);validate_review(root);adopted=terminal(root,c,tc);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('One Seed1 postfit attempt, no retry')
for p in psutil.process_iter(['pid','cmdline']):
if p.pid!=os.getpid() and any(Path(x).name in {'phase3_seed1_postfit.py','phase3_seed1_postfit_worker.py'} for x in p.info['cmdline'] or []):raise RuntimeError('Existing postfit process')
gpu_lock=Path('/tmp/meltmind-seed1-postfit-'+c['gpu_uuid']+'.lock').open('a');fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True';gpu=0.;charged=0.;records=[];prior=adopted['new_training_gpu_seconds_once'];prediction_start=None
def cpu():
u=resource.getrusage(resource.RUSAGE_CHILDREN);return 180+time.process_time()-start+max(charged,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),new_training_gpu_seconds_once=prior,old_seed1_gpu_seconds_once=OLD,phase3_prior_gpu_once=PHASE3_PRIOR,postfit_gpu_cap=7200,seed1_gpu_cap=167200,phase3_gpu_cap=612000,review_sha256=sha(root/'review.json'),cpu_start_counters=dict(self_process_time=start,children_user=u0.ru_utime,children_system=u0.ru_stime),cpu_setup_allowance=180,new_fits=0,new_decodes=0))
try:
budget(prior,gpu,cpu(),5);save(root/'models-adopted.json',adopted);train=Path(c['training_root']);freeze(root,'models',[root/'contract.json',root/'models-adopted.json',train/'complete.json',train/'models-terminal.json',train/'models-git-freeze.json',train/'terminal-git-freeze.json',*[Path(m['path']) for m in adopted['models']],*[Path(m['standardization']) for m in adopted['models']]])
entries=[]
from phase3_seed1_postfit_guard import run_prediction
for index in range(5):
gpu=max(gpu,60+time.monotonic()-gpu_clock);budget(prior,gpu,cpu(),5-index);(root/'jobs'/str(index)).mkdir(parents=True,exist_ok=False);prediction_start=time.monotonic();g=run_prediction(root,c,tc,index,OLD+prior+gpu,300);observed=60+time.monotonic()-prediction_start;prediction_start=None;cost=max(observed,g['gpu_seconds_charged']);gpu+=cost;charged+=g['worker']['cpu_seconds'] if g['worker'] else 0.;records.append(dict(mode='predict',index=index,charged_gpu_seconds=cost,guard=g));save(root/'resource-progress.json',dict(records=records,prediction_gpu_seconds=gpu,postfit_cpu_seconds=cpu(),cumulative_seed1_gpu_seconds=OLD+prior+gpu,cumulative_phase3_gpu_seconds=PHASE3_PRIOR+prior+gpu))
if g['status']!='PASS' or cost>900 or gpu>7200 or OLD+prior+gpu>167200 or PHASE3_PRIOR+prior+gpu>612000:raise RuntimeError('Prediction/resource guard failure')
m=read(root/'jobs'/str(index)/'prediction-records/manifest.json');entries.extend(m['entries'])
prediction_identity(root,c,tc,entries);save(root/'predictions-manifest.json',dict(entries=entries,seed0_predictions_reused=552,new_fits=0,targets_read=False));freeze(root,'predictions',[root/'contract.json',root/'models-adopted.json',root/'predictions-manifest.json',root/'resource-progress.json',*[Path(p) for p in c['code_sha256']]])
gpu=max(gpu,60+time.monotonic()-gpu_clock);budget(prior,gpu,cpu(),0);validate_review(root);verify(root,'models');verify(root,'predictions')
g=supervise([sys.executable,'tools/phase3_seed1_postfit_worker.py','score',str(root)],root/'score-guard',dict(cpu_seconds=900,wall_seconds=1200,rss_bytes=16*2**30,disk_bytes=20*2**30));charged+=g['cpu_seconds'];records.append(dict(mode='score',guard=g))
if g['status']!='PASS' or g['returncode']!=0 or g['monitor_errors'] or cpu()+120>3600:raise RuntimeError('Score/resource failure')
save(root/'complete.json',dict(status='COMPLETE',records=records,prediction_gpu_seconds=gpu,postfit_cpu_seconds=cpu(),new_training_gpu_seconds_once=prior,old_seed1_gpu_seconds_once=OLD,phase3_prior_gpu_once=PHASE3_PRIOR,cumulative_seed1_gpu_seconds=OLD+prior+gpu,cumulative_phase3_gpu_seconds=PHASE3_PRIOR+prior+gpu,seed1_gpu_cap=167200,phase3_gpu_cap=612000,new_fits=0,new_decodes=0,future_training_started=False,no_retry=True));freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',root/'review.json'])
final=cpu();save(root/'post-terminal-cost.json',dict(postfit_cpu_seconds=final,prediction_gpu_seconds=gpu,cumulative_seed1_gpu_seconds=OLD+prior+gpu,cumulative_phase3_gpu_seconds=PHASE3_PRIOR+prior+gpu,within_cap=final<=3600 and gpu<=7200 and OLD+prior+gpu<=167200 and PHASE3_PRIOR+prior+gpu<=612000,cpu_start_counters=read(root/'started.json')['cpu_start_counters']))
if final>3600:raise RuntimeError('Final CPU cap after freeze')
except BaseException as exc:
if prediction_start is not None:gpu+=60+time.monotonic()-prediction_start
gpu=max(gpu,60+time.monotonic()-gpu_clock) if not any(r['mode']=='score' for r in records) else gpu
if (root/'complete.json').exists():save(root/'pre-failure-complete.json',read(root/'complete.json'))
save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),records=records,postfit_cpu_seconds=cpu(),prediction_gpu_seconds=gpu,new_training_gpu_seconds_once=prior,old_seed1_gpu_seconds_once=OLD,phase3_prior_gpu_once=PHASE3_PRIOR,cumulative_seed1_gpu_seconds=OLD+prior+gpu,cumulative_phase3_gpu_seconds=PHASE3_PRIOR+prior+gpu,new_fits=0,no_retry=True));raise
if __name__=='__main__':run(Path(sys.argv[1]))
+149
View File
@@ -0,0 +1,149 @@
"""Seed1 custody barriers; same S2 test roles and no performance decision gate."""
import math
from pathlib import Path
import numpy as np
from phase2_h1_worker_v2 import checked, sha
from phase2_home_freeze_io import verify
from phase3_seed1_v2_common import config as training_config, execution_command
from track_a_h2_common import read
from phase2_h2_s2_postfit_common import roster
OLD = 8482.313407897018
PHASE3_PRIOR = 80903.62940728385
GPU = 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c'
TRAINING = '8603801c4644068db5102227878e1af28c3b5f5e'
BOOTSTRAP = dict(replicates=2000, seed=2026111408, confidence=.95,
unit='component_equal_mean_of_part_differences', interval='percentile_linear')
def config(root):
c = read(root / 'contract.json'); verify(root, 'implementation')
if (c['approval'] != '1794992262.078969' or c['gpu_cap'] != 167200
or c['postfit_gpu_cap'] != 7200 or c['postfit_cpu_cap'] != 3600
or c['phase3_gpu_cap'] != 612000 or c['old_seed1_gpu_once'] != OLD
or c['phase3_prior_gpu_once'] != PHASE3_PRIOR or c['gpu_uuid'] != GPU
or c['idle_wait_seconds'] != 20 or not c['no_retry'] or c['performance_gate'] is not None
or c['new_fits'] != 0 or c['new_decodes'] != 0 or c['bootstrap'] != BOOTSTRAP):
raise PermissionError('Fixed seed1 postfit scope')
for p, h in {**c['source_sha256'], **c['code_sha256']}.items(): checked(p, h)
train = Path(c['training_root'])
if verify(train, 'implementation')['commit'] != TRAINING: raise ValueError('Training implementation differs')
tc, pilot = training_config(train)
if sha(train / 'contract.json') != c['training_contract_sha256']: raise ValueError('Training contract differs')
if c['jobs'] != [dict(id=j['id'], expected_parts=len(j['test'])) for j in tc['jobs']]:
raise ValueError('All original five test lists')
for name in ['predictions', 'terminal']: verify(Path(c['seed0_root']), name)
by = roster(c, pilot)
seed0 = read(c['seed0_manifest'])['entries']
if len(seed0) != 552 or len({e['case'] for e in seed0}) != 552 or c['seed0_predictions'] != {e['case']:e for e in seed0}:
raise ValueError('Original seed0 manifest identity')
for case, e in c['seed0_predictions'].items():
row = by[case]
if (e['fold'] != next(a['fold'] for a in row['assignments'] if a['role']=='test')
or e['component_id'] != row['component_id'] or e['nodes'] != row['nodes']):
raise ValueError('Seed0 same test roles')
return c, tc, pilot
def terminal_ledger(tc, done, manifest):
entries = done.get('entries', []); models = manifest.get('models', [])
if (done.get('status') != 'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING'
or done.get('postfit_started') is not False or done.get('test_targets_read') is not False
or len(entries) != 5 or {e['id'] for e in entries} != set(range(5))
or any(e['status'] != 'PASS' or e['returncode'] != 0 for e in entries)
or len(models) != 5 or {m['job'] for m in models} != set(range(5))
or manifest.get('new_fits') != 5 or manifest.get('test_prediction_started') is not False):
raise ValueError('All five original terminal checkpoints required')
costs = [e['gpu_seconds_charged'] for e in entries]
value = done['training_gpu_seconds']
if (any(not math.isfinite(x) or x < 0 or x > 40000 for x in costs)
or not math.isfinite(value) or abs(value-sum(costs)) > 1e-8
or value > tc['train_gpu_cap'] or done['phase3_prior_gpu_once'] != PHASE3_PRIOR
or abs(done['cumulative_phase3_training_gpu_seconds']-PHASE3_PRIOR-value)>1e-8):
raise ValueError('Training ledger must include old costs exactly once')
budget(value, 0, 180, 5)
return value
def normal_identity(stats, job):
if (stats['parts'] != job['learning'] or stats['nodes'] != job['learning_nodes']
or len(stats['mean']) != 19 or len(stats['scale']) != 19
or not np.isfinite(stats['mean']).all() or not np.isfinite(stats['scale']).all()
or min(stats['scale']) <= 0): raise ValueError('Learning-only 19-column normalization')
def terminal(root, c, tc):
import psutil
train = Path(c['training_root'])
tp = verify(train, 'terminal'); mp = verify(train, 'models')
if (train/'global-stop.json').exists(): raise PermissionError('Training global stop')
done = read(train/'complete.json'); manifest = read(train/'models-terminal.json')
value = terminal_ledger(tc, done, manifest)
for p in psutil.process_iter(['cmdline']):
if execution_command(p.info['cmdline'] or []): raise RuntimeError('Seed1 training process still alive')
for j in tc['jobs']:
i=j['id']; w=train/'jobs'/str(i)/'worker'; e=next(e for e in done['entries'] if e['id']==i)
assignment=read(checked(w.parent/'assignment.json',e['assignment_sha256']))
if assignment['id']!=i or assignment['contract_sha256']!=c['training_contract_sha256'] or assignment['gpu_uuid']!=e['gpu_uuid']:
raise ValueError('Original assignment identity')
g=read(checked(w.parent/'complete.json',e['guard_complete_sha256']))
receipt=read(w.parent/'guards/train.receipt.json')
if (g['status']!='PASS' or g['reasons'] or g['monitor_errors'] or g['worker']!=receipt
or type(receipt.get('monitor_errors')) is not int or receipt['monitor_errors']!=0
or receipt['status']!='PASS' or receipt['returncode']!=0): raise ValueError('Training monitoring failure')
r=read(checked(w/'result.json',g['worker_result_sha256'])); m=next(m for m in manifest['models'] if m['job']==i)
if (r['status']!='COMPLETE' or r['fold']!=i or r['completed_epochs']!=200
or r['optimizer_steps']!=j['target_steps'] or r['learning_parts']!=len(j['learning'])
or r['learning_nodes_per_epoch']!=j['learning_nodes'] or r['test_targets_read'] is not False
or r['test_prediction_started'] is not False or r['validation_used_for_selection'] is not False
or m['path']!=str(w/'last-epoch200.pt') or m['path']!=r['checkpoint']
or m['sha256']!=r['checkpoint_sha256'] or m['standardization']!=str(w/'standardization.json')
or m['standardization_sha256']!=r['standardization_sha256']): raise ValueError('Final checkpoint/roles differ')
for p,h in [(m['path'],m['sha256']),(m['standardization'],m['standardization_sha256'])]:
checked(p,h)
if mp['files'].get(str(Path(p).resolve()))!=h: raise ValueError('Model omitted from freeze')
normal_identity(read(m['standardization']),j)
return dict(models=manifest['models'],new_training_gpu_seconds_once=value,
old_seed1_gpu_seconds_once=OLD,cumulative_seed1_training_gpu_seconds=OLD+value,
cumulative_phase3_training_gpu_seconds=PHASE3_PRIOR+value,
training_contract_sha256=c['training_contract_sha256'],training_terminal_sha256=sha(train/'complete.json'),
training_terminal_commit=tp['commit'],training_models_commit=mp['commit'])
def adopted(root, c, tc):
verify(root,'models'); d=read(root/'models-adopted.json')
if d!=terminal(root,c,tc): raise ValueError('Adopted model/terminal identity differs')
return d
def checkpoint_identity(blob, j, training_sha):
if (blob.get('seed')!=1 or blob.get('fold')!=j['id'] or blob.get('completed_epochs')!=200
or blob.get('optimizer_steps')!=j['target_steps'] or blob.get('contract_sha256')!=training_sha
or blob.get('role')!='last_epoch_only'): raise ValueError('Seed1 final checkpoint identity')
def prediction_identity(root, c, tc, entries):
expected={(j['id'],case) for j in tc['jobs'] for case in j['test']}
keys=[(e['fold'],e['case']) for e in entries]
if len(keys)!=552 or len(set(keys))!=552 or set(keys)!=expected: raise ValueError('All552 distinct test predictions required')
rows={r['case']:r for r in c['rows']}
for e in entries:
row=rows[e['case']]; old=c['seed0_predictions'][e['case']]
if (e['path']!=str(root/'jobs'/str(e['fold'])/'prediction-records'/(e['case']+'.npz'))
or e['nodes']!=row['nodes'] or e['query_nodes']!=old['query_nodes']
or e['component_id']!=row['component_id'] or e['seed0_sha256']!=old['sha256']
or e['raw_ids_sha256']!=row['v5_cache']['output_sha256']['raw_ids.npy']):
raise ValueError('Prediction path/query/pair identity')
for k,limit in [('negative_full_predictions',e['nodes']),('negative_query_predictions',e['query_nodes'])]:
if type(e[k]) is not int or not 0<=e[k]<=limit: raise ValueError('Invalid negative counts')
checked(e['path'],e['sha256']); checked(old['path'],old['sha256'])
return rows
def budget(new_train, gpu, cpu, remaining):
if any(not math.isfinite(x) or x<0 for x in [new_train,gpu,cpu]) or type(remaining) is not int or remaining not in range(6):
raise ValueError('Invalid ledger')
reserve=gpu+900*remaining
if new_train>160000-OLD or reserve>7200 or OLD+new_train+reserve>167200 or PHASE3_PRIOR+new_train+reserve>612000:
raise RuntimeError('Seed1/phase3 cumulative GPU reserve exhausted')
if cpu+300*remaining+900+120>3600: raise RuntimeError('Postfit CPU reserve exhausted')
+85
View File
@@ -0,0 +1,85 @@
"""Seed1 postfit guard: bounded selected-GPU wait, no retries."""
import json,os,signal,sys,threading,time
import psutil
from phase2_h1_worker_v2 import save
from s2_main_memory_guard import PeakRecords
from s2_precision_continuation_guard import device_usage
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
def final_reasons(reasons,receipt,result,tracker,gpu_seconds,cpu_seconds,cpu_cap,expected_parts):
out=list(reasons)
def add(s):
if s not in out:out.append(s)
if not receipt or receipt['status']!='PASS':add('WORKER_NOT_PASS')
if receipt and (type(receipt.get('monitor_errors')) is not int or receipt['monitor_errors'] != 0 or receipt.get('returncode') != 0):add('WORKER_MONITOR_ERRORS')
if not result or result.get('status')!='COMPLETE' or result.get('parts')!=expected_parts:add('INCOMPLETE_FOLD')
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):add('MISSING_PEAKS')
if any(v>15*2**30 for v in tracker.maxima.values()):add('RECORDED_VRAM_LIMIT')
if gpu_seconds>900 or cpu_seconds>cpu_cap:add('FINAL_RESOURCE_LIMIT')
return out
def run_prediction(queue,c,pilot,fold,prior_gpu,cpu_cap):
root=queue/'jobs'/str(fold);gpu_uuid=c['gpu_uuid'];expected_parts=c['jobs'][fold]['expected_parts']
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
def elapsed():return 60+time.monotonic()-start
def reason(x):
if x not in reasons:reasons.append(x)
command=[sys.executable,'tools/phase3_seed1_postfit_worker.py','predict',str(queue),str(fold)]
def kill():
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:stop_group(p.pid)
except psutil.NoSuchProcess:pass
def scan():
if tracker.scan(root/'prediction-records'):reason('RECORDED_VRAM_LIMIT')
def watch():
while not done.is_set():
try:
scan();mem,util=device_usage(gpu_uuid);_,rss,_=tree_usage(os.getpid())
if mem>15*2**30:reason('DEVICE_VRAM_LIMIT')
if rss>16*2**30:reason('TREE_RSS_LIMIT')
if disk_usage(queue)>20*2**30:reason('DISK_LIMIT')
if elapsed()>=895:reason('PREDICTION_TIME_LIMIT')
for x in requested:reason(x)
save(root/'prediction-live.json',dict(gpu_seconds_charged=elapsed(),device_bytes=mem,device_utilization=util,rss_bytes=rss,peaks=tracker.maxima,reasons=reasons,unix=time.time()))
except Exception as exc:reason('MONITOR_ERROR');errors.append(str(exc))
if reasons:kill()
done.wait(.5)
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGTERM,signal.SIGINT]};thread=None
try:
wait_start=time.monotonic();waiting=[]
while True:
mem,util=device_usage(gpu_uuid);waiting.append(dict(unix=time.time(),bytes=mem,utilization=util))
save(root/'recovery-idle-wait.json',dict(samples=waiting,limit_seconds=c['idle_wait_seconds'],worker_started=False))
if requested:raise RuntimeError('Signal received during idle wait')
duration=time.monotonic()-wait_start
if duration>c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
if mem<=128*2**20 and util==0:break
if duration>=c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
time.sleep(min(.5,max(0.,c['idle_wait_seconds']-(time.monotonic()-wait_start))))
for i in range(3):
mem,util=device_usage(gpu_uuid);samples.append(dict(unix=time.time(),bytes=mem,utilization=util))
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY_NO_RETRY')
if i<2:time.sleep(2)
save(root/'prediction-idle-polls.json',samples)
if requested:raise RuntimeError('Signal received during setup')
thread=threading.Thread(target=watch,daemon=True);thread.start()
remaining=900-elapsed()-10
if remaining<=0:raise RuntimeError('Setup exhausted prediction budget')
receipt=supervise(command,root/'prediction-guard','predict',dict(total_cpu_seconds=cpu_cap-60,job_cpu_seconds=cpu_cap-60,job_wall_seconds=remaining,tree_rss_bytes=16*2**30,disk_bytes=20*2**30,max_processes=1),poll=.2,gpu_uuid=gpu_uuid)
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill()
finally:
done.set()
if thread:thread.join(timeout=4)
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
try:scan()
except Exception as exc:reason('FINAL_SCAN_ERROR');errors.append(str(exc))
for s,h in old.items():signal.signal(s,h)
result=None;p=root/'prediction-records/result.json'
try:
if p.exists():result=json.loads(p.read_text())
except Exception as exc:reason('FINAL_RESULT_READ_ERROR');errors.append(str(exc))
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
duration=elapsed();reasons=final_reasons(reasons,receipt,result,tracker,duration,totalcpu,cpu_cap,expected_parts)
out=dict(status='PASS' if not reasons else 'STOPPED',reasons=reasons,monitor_errors=errors,worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=duration,cpu_seconds_charged=totalcpu,cumulative_seed1_gpu_seconds=prior_gpu+duration,no_retry=True)
save(root/'prediction-guard.json',out);return out
+26
View File
@@ -0,0 +1,26 @@
"""Original query metrics and paired component bootstrap, seed1 minus seed0."""
from phase2_b13_e1b import pooled
METRICS = ['relative_l2','last_location','kendall_tau_b']
def comparisons(rows):
raw=pooled([dict(component_id=r['component_id'],new=r['seed1'],A=r['seed0']) for r in rows])
out={}
for name,m in raw.items():
undefined={arm:[r['case'] for r in rows if r[arm][name]['value'] is None] for arm in ['seed0','seed1']}
out[name]=dict(means={'seed1':m['means']['new'],'seed0':m['means']['A']},
components={'seed1':m['components']['new'],'seed0':m['components']['A']},
seed1_minus_seed0=m['new_minus_A'],undefined_parts=undefined,
part_counts={arm:len(rows)-len(v) for arm,v in undefined.items()})
return out
def summary(rows):
if len(rows)!=552 or len({r['case'] for r in rows})!=552 or len({r['component_id'] for r in rows})!=204 or {r['fold'] for r in rows}!=set(range(5)):
raise ValueError('Full552/204 support required; no partial cohort inference')
return dict(parts=552,components=204,pooled=comparisons(rows),
per_fold={str(f):dict(parts=sum(r['fold']==f for r in rows),components=len({r['component_id'] for r in rows if r['fold']==f}),metrics=comparisons([r for r in rows if r['fold']==f])) for f in range(5)},
performance_gate=None,descriptive_only=True,
interval_interpretation='Fixed-model component sampling variability, not a population distribution over training seeds',
direction='seed1 minus seed0; negative field/last_location means lower error; positive tau means higher rank agreement')
+104
View File
@@ -0,0 +1,104 @@
"""Predict seed1 without targets; score once only after all552 predictions freeze."""
import os, sys, time
from pathlib import Path
import numpy as np
from phase2_h1_worker_v2 import save, sha, checked
from phase2_home_freeze_io import verify
from phase2_gate_outcome_review import validate_review
from phase3_seed1_postfit_common import config, adopted, checkpoint_identity, prediction_identity, read
from phase3_seed1_postfit_statistics import summary
from phase3_seed1_v2_common import fold_view
from phase2_h2_s1_postfit import query_index
from phase2_residual_math import decode_seconds
from phase2_b1213_features import cached
from s2_custody_statistics import part_metrics
from s2_custody_common import prediction_inputs
def paired_query(raw, q, seed0):
original={k:seed0['query_raw_ids' if k=='raw_ids' else k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']}
ix=query_index(raw,q,original)
if (not np.array_equal(seed0['raw_ids'],raw) or not np.array_equal(seed0['pred'],seed0['seconds'][ix])
or not np.isfinite(seed0['seconds']).all()): raise ValueError('Seed0 full/query identity')
return ix
def predict(root, fold):
c,tc,pilot=config(root); validate_review(root); models=adopted(root,c,tc)
view=fold_view(pilot,fold); f=view['folds'][0]; job=tc['jobs'][fold]
m=next(m for m in models['models'] if m['job']==fold); stats=read(m['standardization'])
import torch
from phase2_h2_model import H2Geo
from phase2_h2_data import to_device, arguments
from phase2_h2_s1_test_data import load_test_part
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':
raise ValueError('Selected GPU/allocator identity')
os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
dest=root/'jobs'/str(fold)/'prediction-records';dest.mkdir(exist_ok=False)
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats()
start=time.monotonic();cpu=time.process_time();entries=[];by={r['case']:r for r in c['rows']}
def peaks():return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
try:
blob=torch.load(checked(m['path'],m['sha256']),map_location='cpu',weights_only=False)
checkpoint_identity(blob,job,c['training_contract_sha256']);model=H2Geo().cuda();model.load_state_dict(blob['model'],strict=True);model.eval();del blob
save(dest/'environment.json',dict(fold=fold,seed=1,torch=torch.__version__,gpu_uuid=c['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,checkpoint_sha256=m['sha256']))
with torch.no_grad():
for case in f['test']:
row=by[case];data=to_device(load_test_part(view,case,stats),'cuda');residual=model(*arguments(data),checkpoint_blocks=False)
if not bool(torch.isfinite(residual).all()):raise ValueError('Nonfinite residual')
seconds=decode_seconds(residual.cpu().numpy(),data['F'],f['c'],data['t_set']);torch.cuda.synchronize()
q=prediction_inputs(row);old=c['seed0_predictions'][case]
with np.load(checked(old['path'],old['sha256']),allow_pickle=False) as z:
ix=paired_query(data['raw_ids'],q,z)
if len(seconds)!=row['nodes'] or not np.isfinite(seconds).all() or len(ix)!=old['query_nodes']:raise ValueError('Prediction coverage')
path=dest/(case+'.npz')
np.savez_compressed(path,seconds=seconds,raw_ids=data['raw_ids'],pred=seconds[ix],query_raw_ids=q['raw_ids'],counts=q['counts'],coordinates_over_Rg=q['coordinates_over_Rg'],near_wall=q['near_wall'])
entries.append(dict(case=case,component_id=row['component_id'],fold=fold,path=str(path),sha256=sha(path),nodes=row['nodes'],query_nodes=len(ix),seed0_sha256=old['sha256'],raw_ids_sha256=row['v5_cache']['output_sha256']['raw_ids.npy'],negative_full_predictions=int(np.sum(seconds<0)),negative_query_predictions=int(np.sum(seconds[ix]<0))))
save(dest/'progress.json',dict(stage='TEST_PREDICTION_NO_TARGETS',fold=fold,parts=len(entries),wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
del data,residual,seconds
save(dest/'manifest.json',dict(entries=entries,test_targets_read=False,new_fits=0,full_nodes=sum(e['nodes'] for e in entries)))
save(dest/'result.json',dict(status='COMPLETE',fold=fold,parts=len(entries),test_targets_read=False,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
except BaseException as exc:
save(dest/'failure.json',dict(status='FAILED',fold=fold,reason=type(exc).__name__+': '+str(exc),completed_parts=len(entries),wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()));raise
def score(root):
c,tc,pilot=config(root);validate_review(root);adopted(root,c,tc);verify(root,'predictions')
entries=read(root/'predictions-manifest.json')['entries'];by=prediction_identity(root,c,tc,entries)
dest=root/'score';dest.mkdir(exist_ok=False)
save(dest/'review-consumed.json',dict(unix=time.time(),review_sha256=sha(root/'review.json'),before_old_score_or_target_values=True))
frozen=read(checked(c['seed0_score'],c['source_sha256'][c['seed0_score']]))
olds=frozen['rows'];oldrows={r['case']:r for r in olds}
if frozen['status']!='COMPLETE' or len(olds)!=552 or len(oldrows)!=552 or set(oldrows)!=set(by):raise ValueError('Seed0 score support')
rows=[];access=[]
for e in entries:
case=e['case'];row=by[case];old=oldrows[case];base=c['seed0_predictions'][case]
if old['fold']!=e['fold'] or old['component_id']!=e['component_id']:raise ValueError('Seed0 score pairing')
access.append(dict(case=case,fold=e['fold'],role='test',target_sha256=row['v5_cache']['output_sha256']['seconds.npy']))
save(dest/'target-access.json',access)
truth=cached(row,'seconds');raw=cached(row,'raw_ids')
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z, np.load(checked(base['path'],base['sha256']),allow_pickle=False) as original:
q={k:z['query_raw_ids' if k=='raw_ids' else k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']}
ix=paired_query(raw,q,original)
if (not np.array_equal(z['raw_ids'],raw) or not np.array_equal(z['pred'],z['seconds'][ix])
or not np.isfinite(z['seconds']).all() or len(z['seconds'])!=e['nodes'] or len(ix)!=e['query_nodes']
or int(np.sum(z['seconds']<0))!=e['negative_full_predictions']
or int(np.sum(z['pred']<0))!=e['negative_query_predictions']):raise ValueError('Seed1 full/query/count identity')
metric=part_metrics(z['pred'],truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0]))
rows.append(dict(case=case,component_id=e['component_id'],fold=e['fold'],seed1=metric,seed0=old['H2']))
save(dest/'progress.json',dict(parts=len(rows),unique_test_target_reads=len(access)))
if len(access)!=552 or len({x['case'] for x in access})!=552:raise ValueError('Exactly552 unique existing target-cache reads')
save(dest/'result.json',dict(status='COMPLETE',**summary(rows),rows=rows,seed0_metrics_reused=552,
seed0_score_sha256=c['source_sha256'][c['seed0_score']],seed0_predictions_reused=552,
unique_test_target_reads=552,new_decodes=0,new_fits=0,review_sha256=sha(root/'review.json'),
negative_predictions={arm:dict(full_nodes=sum(e['negative_full_predictions'] for e in es),query_nodes=sum(e['negative_query_predictions'] for e in es),clipped=False) for arm,es in [('seed1',entries),('seed0',list(c['seed0_predictions'].values()))]},
full_nodes=sum(e['nodes'] for e in entries),eligible_unique_query_nodes=sum(e['query_nodes'] for e in entries),
later_stages_started=False,seed2_started=False,ensemble_started=False))
if __name__=='__main__':
action=sys.argv[1];root=Path(sys.argv[2])
if action=='predict' and len(sys.argv)==4:predict(root,int(sys.argv[3]))
elif action=='score' and len(sys.argv)==3:score(root)
else:raise ValueError('Use predict root fold or score root')
+75
View File
@@ -0,0 +1,75 @@
"""Metadata-only seed1 postfit preparation; never predict, score, or decode targets."""
from pathlib import Path
import time
from phase2_h1_worker_v2 import sha, checked, save
from phase2_home_freeze_io import verify
from phase3_seed1_v2_common import config as training_config
from phase3_seed1_postfit_common import OLD, PHASE3_PRIOR, GPU, BOOTSTRAP, TRAINING, read, roster
from track_a_h2_prepare import local_code_closure
B=Path(__file__).resolve().parents[1]
T=Path('/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18')
S=T.parent/'phase2-h2-s2-postfit-2026-11-17'
R=T.parent/'phase3-seed1-postfit-2026-11-18'
def main():
start=time.process_time()
if R.exists():raise FileExistsError('Existing postfit entry; inspect rather than duplicate')
tc,pilot=training_config(T)
if verify(T,'implementation')['commit']!=TRAINING:raise ValueError('Exact active training pin')
old=read(S/'contract.json')
for name in ['implementation','predictions','terminal']:verify(S,name)
by=roster(old,pilot);manifest=read(S/'predictions-manifest.json')['entries']
seed0={e['case']:e for e in manifest}
if len(manifest)!=552 or len(seed0)!=552 or set(seed0)!=set(by):raise ValueError('Original prediction support')
for case,e in seed0.items():
row=by[case]
if e['fold']!=next(a['fold'] for a in row['assignments'] if a['role']=='test') or e['nodes']!=row['nodes'] or e['component_id']!=row['component_id']:
raise ValueError('Original seed0 test role identity')
sources=dict(tc['source_sha256'])
for p,h in {**old['source_sha256'],**old['code_sha256'],**tc['code_sha256']}.items():
if p in sources and sources[p]!=h:raise ValueError('Conflicting source pins')
sources[p]=h
for folder,names in [(T,['contract.json','implementation-git-freeze.json','implementation-authorization.json','review.json']),
(S,['contract.json','predictions-manifest.json','predictions-git-freeze.json','predictions-authorization.json','terminal-git-freeze.json','terminal-authorization.json','score/result.json'])]:
for name in names:sources[str(folder/name)]=sha(folder/name)
spec=B/'references/2026-11-18/phase3-seed-sensitivity/mold-seed-ensemble-spec.md'
sources[str(spec)]=sha(spec)
comparison=dict(approval='1794992262.078969',training_comparison_sha256=tc['comparison_contract_sha256'],
parts=552,components=204,roles='same original S2 five test folds',seeds=[0,1],new_fits=0,
seed0_predictions='hash reuse all552',seed0_metrics='reuse original S2 per-part H2 metrics; no recomputation',
metrics=['relative_l2','last_location','kendall_tau_b'],difference='seed1 minus seed0',
query='original eligible query/raw IDs/multiplicity/Rg/tie/undefined rules unchanged',
aggregation='within-component part means, then equal components; paired complete component intersection',
bootstrap=BOOTSTRAP,report=['each_fold','pooled','undefined_and_counts','negative_unclipped_nodes'],
performance_gate=None,descriptive_only=True,new_decodes=0,unique_test_target_cache_reads=552,
prior_three_seed_proposal_superseded=True,seed2=False,ensemble=False,
interpretation='Noise reference for later 1b/c/b discussion; no automatic later-stage launch; CI describes fixed-model component variability only')
codes=local_code_closure([B/'tools'/n for n in ['phase3_seed1_postfit.py','phase3_seed1_postfit_common.py',
'phase3_seed1_postfit_worker.py','phase3_seed1_postfit_guard.py','phase3_seed1_postfit_statistics.py',
'prepare_phase3_seed1_postfit.py','test_phase3_seed1_postfit.py']]+list((B/'tools/s2_models').glob('*.py')))
for p,h in tc['code_sha256'].items():checked(p,h)
R.mkdir();save(R/'comparison-contract.json',comparison)
c=dict(version='phase3-seed1-postfit-v1',approval='1794992262.078969',training_root=str(T),
training_contract_sha256=sha(T/'contract.json'),training_implementation=TRAINING,
comparison_contract=str(R/'comparison-contract.json'),comparison_contract_sha256=sha(R/'comparison-contract.json'),
seed0_root=str(S),seed0_manifest=str(S/'predictions-manifest.json'),seed0_score=str(S/'score/result.json'),
seed0_predictions=seed0,rows=old['rows'],baseline_predictions=old['baseline_predictions'],
source_sha256=sources,code_sha256=codes,jobs=[dict(id=j['id'],expected_parts=len(j['test'])) for j in tc['jobs']],
gpu_uuid=GPU,worker_cpus=[0,1],coordinator_cpus=[14,15],gpu_cap=167200,postfit_gpu_cap=7200,
postfit_cpu_cap=3600,phase3_gpu_cap=612000,old_seed1_gpu_once=OLD,phase3_prior_gpu_once=PHASE3_PRIOR,
prediction_per_fold_gpu_seconds=900,prediction_per_fold_cpu_seconds=300,score_cpu_seconds=900,
score_wall_seconds=1200,setup_cpu_seconds=180,terminal_cpu_reserve=120,
vram_bytes=15*2**30,rss_bytes=16*2**30,disk_bytes=20*2**30,idle_wait_seconds=20,no_retry=True,
bootstrap=BOOTSTRAP,performance_gate=None,no_auto_launch=True,all_five_training_terminals_before_prediction=True,
all_predictions_frozen_before_targets=True,new_fits=0,new_decodes=0,seed2_started=False,ensemble_started=False)
save(R/'contract.json',c)
save(R/'preparation-receipt.json',dict(status='PREPARED_PENDING_EXACT_PRE_SCORE_REVIEW',
contract_sha256=sha(R/'contract.json'),code_files=len(codes),parts=552,components=204,
metadata_cpu_seconds=time.process_time()-start,source_training_unchanged=True,model_weights_read=False,
scientific_arrays_decoded=False,target_values_read=False,old_scores_parsed=False,new_predictions=False,
training_complete_exists=(T/'complete.json').exists(),launched=False))
print(read(R/'preparation-receipt.json'))
if __name__=='__main__':main()
+146
View File
@@ -0,0 +1,146 @@
"""CPU-only synthetic regression checks, no project data or CUDA execution."""
import copy
from pathlib import Path
import unittest
from unittest.mock import patch
import numpy as np
from phase3_seed1_postfit_common import checkpoint_identity, terminal_ledger, budget, normal_identity, prediction_identity, OLD, PHASE3_PRIOR
from phase3_seed1_postfit_statistics import comparisons, summary
from phase3_seed1_postfit_worker import paired_query, predict, score
from phase3_seed1_postfit_guard import final_reasons
from phase2_h2_s1_test_data import load_test_part
from s2_custody_statistics import part_metrics
def metric(v):return {k:dict(value=v,reason=None if v is not None else 'undefined') for k in ['relative_l2','last_location','kendall_tau_b']}
class Seed1PostfitTests(unittest.TestCase):
def test_checkpoint_rejects_wrong_seed_fold_epoch_selection(self):
j=dict(id=2,target_steps=60000);b=dict(seed=1,fold=2,completed_epochs=200,optimizer_steps=60000,contract_sha256='fixed',role='last_epoch_only')
checkpoint_identity(b,j,'fixed')
for k,v in [('seed',0),('fold',3),('completed_epochs',199),('optimizer_steps',59999),('contract_sha256','other'),('role','best_validation')]:
with self.assertRaises(ValueError):checkpoint_identity(dict(b,**{k:v}),j,'fixed')
def fixture(self):
tc=dict(train_gpu_cap=160000-OLD)
d=dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING',postfit_started=False,test_targets_read=False,
entries=[dict(id=i,status='PASS',returncode=0,gpu_seconds_charged=100.) for i in range(5)],
training_gpu_seconds=500.,phase3_prior_gpu_once=PHASE3_PRIOR,cumulative_phase3_training_gpu_seconds=PHASE3_PRIOR+500.)
m=dict(models=[dict(job=i) for i in range(5)],new_fits=5,test_prediction_started=False)
return tc,d,m
def test_all_five_terminal_no_partial_adoption(self):
tc,d,m=self.fixture();self.assertEqual(terminal_ledger(tc,d,m),500.)
d['entries'].pop()
with self.assertRaises(ValueError):terminal_ledger(tc,d,m)
def test_failed_or_duplicate_terminal_rejected(self):
for key,val in [('status','STOPPED'),('returncode',1),('id',1)]:
tc,d,m=self.fixture();d['entries'][0][key]=val
with self.assertRaises(ValueError):terminal_ledger(tc,d,m)
def test_ledger_double_old_charge_rejected(self):
tc,d,m=self.fixture();d['training_gpu_seconds']+=OLD
with self.assertRaises(ValueError):terminal_ledger(tc,d,m)
tc,d,m=self.fixture();d['cumulative_phase3_training_gpu_seconds']+=OLD
with self.assertRaises(ValueError):terminal_ledger(tc,d,m)
def test_exact_cumulative_budgets_and_final_caps(self):
budget(160000-OLD,7200,2000,0)
for args in [(160000-OLD,7200.01,180,0),(160000-OLD+.01,0,180,5),(100,0,1081,5),(float('nan'),0,0,5),(100,0,0,True)]:
with self.assertRaises((ValueError,RuntimeError)):budget(*args)
def test_learning_only_normalization(self):
j=dict(learning=['a'],learning_nodes=10);s=dict(parts=['a'],nodes=10,mean=[0.]*19,scale=[1.]*19)
normal_identity(s,j)
for change in [dict(parts=['test']),dict(nodes=11),dict(scale=[0.]*19),dict(mean=[0.]*18)]:
with self.assertRaises(ValueError):normal_identity(dict(s,**change),j)
def query_fixture(self):
raw=np.array([1,3,8]);q=dict(raw_ids=np.array([1,8]),counts=np.array([1,2]),coordinates_over_Rg=np.array([[0.,0.,0.],[1.,0.,0.]]),near_wall=np.array([True,False]))
z=dict(raw_ids=raw,seconds=np.array([1.,2.,3.]),pred=np.array([1.,3.]),query_raw_ids=q['raw_ids'],counts=q['counts'],coordinates_over_Rg=q['coordinates_over_Rg'],near_wall=q['near_wall'])
return raw,q,z
def test_query_full_ids_distinct_and_multiplicity_exact(self):
raw,q,z=self.query_fixture();np.testing.assert_array_equal(paired_query(raw,q,z),[0,2])
for change in [dict(counts=np.array([1,1])),dict(pred=np.array([1.,2.])),dict(raw_ids=np.array([1,4,8])),dict(query_raw_ids=np.array([1,3]))]:
with self.assertRaises(ValueError):paired_query(raw,q,dict(z,**change))
def test_test_loader_denies_foreign_role_before_array_io(self):
with patch('phase2_h2_s1_test_data.np.load',side_effect=AssertionError('Array access')) as io:
with self.assertRaises(PermissionError):load_test_part(dict(folds=[dict(test=['a'])]),'learning',{})
io.assert_not_called()
def test_predict_needs_actual_review_before_model_or_inputs(self):
with patch('phase3_seed1_postfit_worker.config',return_value=({}, {}, {})),patch('phase3_seed1_postfit_worker.validate_review',side_effect=PermissionError('No review')),patch('phase3_seed1_postfit_worker.adopted') as a:
with self.assertRaises(PermissionError):predict(Path('/synthetic'),0)
a.assert_not_called()
def test_predict_needs_all_training_terminals_before_input_loader(self):
with patch('phase3_seed1_postfit_worker.config',return_value=({}, {}, {})),patch('phase3_seed1_postfit_worker.validate_review'),patch('phase3_seed1_postfit_worker.adopted',side_effect=ValueError('Incomplete')),patch('phase2_h2_s1_test_data.load_test_part') as io:
with self.assertRaises(ValueError):predict(Path('/synthetic'),0)
io.assert_not_called()
def test_score_no_targets_or_score_reads_before_review(self):
with patch('phase3_seed1_postfit_worker.config',return_value=({}, {}, {})),patch('phase3_seed1_postfit_worker.validate_review',side_effect=PermissionError('No review')),patch('phase3_seed1_postfit_worker.cached') as io,patch('phase3_seed1_postfit_worker.read') as rd:
with self.assertRaises(PermissionError):score(Path('/synthetic'))
io.assert_not_called();rd.assert_not_called()
def test_score_no_targets_before_prediction_freeze(self):
with patch('phase3_seed1_postfit_worker.config',return_value=({}, {}, {})),patch('phase3_seed1_postfit_worker.validate_review'),patch('phase3_seed1_postfit_worker.adopted'),patch('phase3_seed1_postfit_worker.verify',side_effect=PermissionError('No prediction freeze')),patch('phase3_seed1_postfit_worker.cached') as io,patch('phase3_seed1_postfit_worker.read') as rd:
with self.assertRaises(PermissionError):score(Path('/synthetic'))
io.assert_not_called();rd.assert_not_called()
def test_sign_and_equal_component_weight(self):
rows=[dict(case='a',component_id='x',seed0=metric(1),seed1=metric(2)),dict(case='b',component_id='x',seed0=metric(1),seed1=metric(4)),dict(case='c',component_id='y',seed0=metric(1),seed1=metric(9))]
s=comparisons(rows)['relative_l2'];self.assertEqual(s['seed1_minus_seed0']['value'],5.)
self.assertEqual(s['means'],dict(seed1=6.,seed0=1.));self.assertEqual(s['seed1_minus_seed0']['replicates'],2000)
self.assertEqual(comparisons(rows),comparisons(rows[::-1]))
def test_undefined_component_not_silently_reweighted(self):
rows=[dict(case='a',component_id='x',seed0=metric(1),seed1=metric(None)),dict(case='b',component_id='x',seed0=metric(1),seed1=metric(2)),dict(case='c',component_id='y',seed0=metric(1),seed1=metric(3))]
s=comparisons(rows)['kendall_tau_b'];self.assertEqual(s['components'],dict(seed1=1,seed0=2));self.assertEqual(s['seed1_minus_seed0']['components'],1)
self.assertEqual(s['undefined_parts']['seed1'],['a']);self.assertEqual(s['seed1_minus_seed0']['value'],2.)
def test_original_query_metric_ties_and_undefined(self):
m=part_metrics(np.zeros(3),np.zeros(3),np.ones(3),np.ones(3,bool),np.array([3,1,2]),np.array([[0.,0,0],[1.,0,0],[2.,0,0]]),6)
self.assertIsNone(m['relative_l2']['value']);self.assertIsNone(m['kendall_tau_b']['value']);self.assertEqual(m['last_location']['value'],0.)
def test_no_partial_cohort_summary(self):
with self.assertRaises(ValueError):summary([dict(case='a',component_id='x',fold=0,seed0=metric(1),seed1=metric(2))])
with self.assertRaises(ValueError):prediction_identity(Path('/synthetic'),{},dict(jobs=[]),[])
def test_manifest_requires_exact_complete_roles_and_hashes(self):
root=Path('/synthetic');rows=[];entries=[];seed0={};jobs=[dict(id=f,test=[]) for f in range(5)]
for i in range(552):
case=str(i);f=i%5;group='g'+str(i%204);jobs[f]['test'].append(case)
rows.append(dict(case=case,nodes=3,component_id=group,v5_cache=dict(output_sha256={'raw_ids.npy':'raw'})))
seed0[case]=dict(path='/old/'+case,sha256='old',query_nodes=2)
entries.append(dict(case=case,fold=f,path=str(root/'jobs'/str(f)/'prediction-records'/(case+'.npz')),sha256='new',nodes=3,query_nodes=2,component_id=group,seed0_sha256='old',raw_ids_sha256='raw',negative_full_predictions=0,negative_query_predictions=0))
c=dict(rows=rows,seed0_predictions=seed0);tc=dict(jobs=jobs)
with patch('phase3_seed1_postfit_common.checked'):
self.assertEqual(len(prediction_identity(root,c,tc,entries)),552)
for change in [dict(fold=4),dict(component_id='wrong'),dict(raw_ids_sha256='foreign'),dict(seed0_sha256='other'),dict(query_nodes=3)]:
altered=copy.deepcopy(entries);altered[0].update(change)
with self.assertRaises(ValueError):prediction_identity(root,c,tc,altered)
with self.assertRaises(ValueError):prediction_identity(root,c,tc,entries[:-1]+[entries[0]])
with patch('phase3_seed1_postfit_common.checked',side_effect=ValueError('Changed bytes')):
with self.assertRaises(ValueError):prediction_identity(root,c,tc,entries)
def test_complete_summary_has_no_performance_gate(self):
rows=[dict(case=str(i),component_id='g'+str(i%204),fold=(i%204)%5,seed0=metric(1.),seed1=metric(2.)) for i in range(552)]
r=summary(rows);self.assertIsNone(r['performance_gate']);self.assertTrue(r['descriptive_only'])
self.assertEqual(set(r['per_fold']),set(map(str,range(5))))
self.assertEqual(r['pooled']['relative_l2']['seed1_minus_seed0']['value'],1.)
def test_failed_worker_recorded_peaks_and_monitor_type(self):
class Peaks:
witnesses={'peak_allocated_bytes':1,'peak_reserved_bytes':1}
maxima={'peak_allocated_bytes':1,'peak_reserved_bytes':16*2**30}
r=dict(status='PASS',returncode=0,monitor_errors=0);result=dict(status='COMPLETE',parts=2)
self.assertIn('RECORDED_VRAM_LIMIT',final_reasons([],r,result,Peaks(),100,100,300,2))
self.assertIn('WORKER_MONITOR_ERRORS',final_reasons([],dict(r,monitor_errors=[]),result,Peaks(),100,100,300,2))
self.assertIn('INCOMPLETE_FOLD',final_reasons([],r,None,Peaks(),100,100,300,2))
if __name__=='__main__':unittest.main(verbosity=2)