Prepare exact seed1 postfit review binding without execution
This commit is contained in:
1 parent
38f0044299
commit
e49851ba0b
20 files changed
+83890
No files matched your search
@@ -0,0 +1,41 @@
|
||||
"""Seed1 postfit only after five frozen training terminals; exact review before test access."""
|
||||
import fcntl,json,os,resource,sys,time
|
||||
from pathlib import Path
|
||||
import psutil
|
||||
from phase3_seed1_postfit_common import config,read,terminal,prediction_identity,budget,OLD,PHASE3_PRIOR
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase2_home_freeze_io import freeze,verify
|
||||
from phase2_h1_worker_v2 import save,sha,checked
|
||||
from r4bc_guard import supervise
|
||||
|
||||
def run(root):
|
||||
gpu_clock=time.monotonic();start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);c,tc,pilot=config(root);validate_review(root);adopted=terminal(root,c,tc);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'started.json').exists():raise FileExistsError('One Seed1 postfit attempt, no retry')
|
||||
for p in psutil.process_iter(['pid','cmdline']):
|
||||
if p.pid!=os.getpid() and any(Path(x).name in {'phase3_seed1_postfit.py','phase3_seed1_postfit_worker.py'} for x in p.info['cmdline'] or []):raise RuntimeError('Existing postfit process')
|
||||
gpu_lock=Path('/tmp/meltmind-seed1-postfit-'+c['gpu_uuid']+'.lock').open('a');fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
os.sched_setaffinity(0,c['coordinator_cpus']);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True';gpu=0.;charged=0.;records=[];prior=adopted['new_training_gpu_seconds_once'];prediction_start=None
|
||||
def cpu():
|
||||
u=resource.getrusage(resource.RUSAGE_CHILDREN);return 180+time.process_time()-start+max(charged,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)
|
||||
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),new_training_gpu_seconds_once=prior,old_seed1_gpu_seconds_once=OLD,phase3_prior_gpu_once=PHASE3_PRIOR,postfit_gpu_cap=7200,seed1_gpu_cap=167200,phase3_gpu_cap=612000,review_sha256=sha(root/'review.json'),cpu_start_counters=dict(self_process_time=start,children_user=u0.ru_utime,children_system=u0.ru_stime),cpu_setup_allowance=180,new_fits=0,new_decodes=0))
|
||||
try:
|
||||
budget(prior,gpu,cpu(),5);save(root/'models-adopted.json',adopted);train=Path(c['training_root']);freeze(root,'models',[root/'contract.json',root/'models-adopted.json',train/'complete.json',train/'models-terminal.json',train/'models-git-freeze.json',train/'terminal-git-freeze.json',*[Path(m['path']) for m in adopted['models']],*[Path(m['standardization']) for m in adopted['models']]])
|
||||
entries=[]
|
||||
from phase3_seed1_postfit_guard import run_prediction
|
||||
for index in range(5):
|
||||
gpu=max(gpu,60+time.monotonic()-gpu_clock);budget(prior,gpu,cpu(),5-index);(root/'jobs'/str(index)).mkdir(parents=True,exist_ok=False);prediction_start=time.monotonic();g=run_prediction(root,c,tc,index,OLD+prior+gpu,300);observed=60+time.monotonic()-prediction_start;prediction_start=None;cost=max(observed,g['gpu_seconds_charged']);gpu+=cost;charged+=g['worker']['cpu_seconds'] if g['worker'] else 0.;records.append(dict(mode='predict',index=index,charged_gpu_seconds=cost,guard=g));save(root/'resource-progress.json',dict(records=records,prediction_gpu_seconds=gpu,postfit_cpu_seconds=cpu(),cumulative_seed1_gpu_seconds=OLD+prior+gpu,cumulative_phase3_gpu_seconds=PHASE3_PRIOR+prior+gpu))
|
||||
if g['status']!='PASS' or cost>900 or gpu>7200 or OLD+prior+gpu>167200 or PHASE3_PRIOR+prior+gpu>612000:raise RuntimeError('Prediction/resource guard failure')
|
||||
m=read(root/'jobs'/str(index)/'prediction-records/manifest.json');entries.extend(m['entries'])
|
||||
prediction_identity(root,c,tc,entries);save(root/'predictions-manifest.json',dict(entries=entries,seed0_predictions_reused=552,new_fits=0,targets_read=False));freeze(root,'predictions',[root/'contract.json',root/'models-adopted.json',root/'predictions-manifest.json',root/'resource-progress.json',*[Path(p) for p in c['code_sha256']]])
|
||||
gpu=max(gpu,60+time.monotonic()-gpu_clock);budget(prior,gpu,cpu(),0);validate_review(root);verify(root,'models');verify(root,'predictions')
|
||||
g=supervise([sys.executable,'tools/phase3_seed1_postfit_worker.py','score',str(root)],root/'score-guard',dict(cpu_seconds=900,wall_seconds=1200,rss_bytes=16*2**30,disk_bytes=20*2**30));charged+=g['cpu_seconds'];records.append(dict(mode='score',guard=g))
|
||||
if g['status']!='PASS' or g['returncode']!=0 or g['monitor_errors'] or cpu()+120>3600:raise RuntimeError('Score/resource failure')
|
||||
save(root/'complete.json',dict(status='COMPLETE',records=records,prediction_gpu_seconds=gpu,postfit_cpu_seconds=cpu(),new_training_gpu_seconds_once=prior,old_seed1_gpu_seconds_once=OLD,phase3_prior_gpu_once=PHASE3_PRIOR,cumulative_seed1_gpu_seconds=OLD+prior+gpu,cumulative_phase3_gpu_seconds=PHASE3_PRIOR+prior+gpu,seed1_gpu_cap=167200,phase3_gpu_cap=612000,new_fits=0,new_decodes=0,future_training_started=False,no_retry=True));freeze(root,'terminal',[root/'contract.json',root/'complete.json',root/'score/result.json',root/'review.json'])
|
||||
final=cpu();save(root/'post-terminal-cost.json',dict(postfit_cpu_seconds=final,prediction_gpu_seconds=gpu,cumulative_seed1_gpu_seconds=OLD+prior+gpu,cumulative_phase3_gpu_seconds=PHASE3_PRIOR+prior+gpu,within_cap=final<=3600 and gpu<=7200 and OLD+prior+gpu<=167200 and PHASE3_PRIOR+prior+gpu<=612000,cpu_start_counters=read(root/'started.json')['cpu_start_counters']))
|
||||
if final>3600:raise RuntimeError('Final CPU cap after freeze')
|
||||
except BaseException as exc:
|
||||
if prediction_start is not None:gpu+=60+time.monotonic()-prediction_start
|
||||
gpu=max(gpu,60+time.monotonic()-gpu_clock) if not any(r['mode']=='score' for r in records) else gpu
|
||||
if (root/'complete.json').exists():save(root/'pre-failure-complete.json',read(root/'complete.json'))
|
||||
save(root/'complete.json',dict(status='STOPPED',reason=type(exc).__name__+': '+str(exc),records=records,postfit_cpu_seconds=cpu(),prediction_gpu_seconds=gpu,new_training_gpu_seconds_once=prior,old_seed1_gpu_seconds_once=OLD,phase3_prior_gpu_once=PHASE3_PRIOR,cumulative_seed1_gpu_seconds=OLD+prior+gpu,cumulative_phase3_gpu_seconds=PHASE3_PRIOR+prior+gpu,new_fits=0,no_retry=True));raise
|
||||
if __name__=='__main__':run(Path(sys.argv[1]))
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Seed1 custody barriers; same S2 test roles and no performance decision gate."""
|
||||
import math
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
from phase2_h1_worker_v2 import checked, sha
|
||||
from phase2_home_freeze_io import verify
|
||||
from phase3_seed1_v2_common import config as training_config, execution_command
|
||||
from track_a_h2_common import read
|
||||
from phase2_h2_s2_postfit_common import roster
|
||||
|
||||
OLD = 8482.313407897018
|
||||
PHASE3_PRIOR = 80903.62940728385
|
||||
GPU = 'GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c'
|
||||
TRAINING = '8603801c4644068db5102227878e1af28c3b5f5e'
|
||||
BOOTSTRAP = dict(replicates=2000, seed=2026111408, confidence=.95,
|
||||
unit='component_equal_mean_of_part_differences', interval='percentile_linear')
|
||||
|
||||
|
||||
def config(root):
|
||||
c = read(root / 'contract.json'); verify(root, 'implementation')
|
||||
if (c['approval'] != '1794992262.078969' or c['gpu_cap'] != 167200
|
||||
or c['postfit_gpu_cap'] != 7200 or c['postfit_cpu_cap'] != 3600
|
||||
or c['phase3_gpu_cap'] != 612000 or c['old_seed1_gpu_once'] != OLD
|
||||
or c['phase3_prior_gpu_once'] != PHASE3_PRIOR or c['gpu_uuid'] != GPU
|
||||
or c['idle_wait_seconds'] != 20 or not c['no_retry'] or c['performance_gate'] is not None
|
||||
or c['new_fits'] != 0 or c['new_decodes'] != 0 or c['bootstrap'] != BOOTSTRAP):
|
||||
raise PermissionError('Fixed seed1 postfit scope')
|
||||
for p, h in {**c['source_sha256'], **c['code_sha256']}.items(): checked(p, h)
|
||||
train = Path(c['training_root'])
|
||||
if verify(train, 'implementation')['commit'] != TRAINING: raise ValueError('Training implementation differs')
|
||||
tc, pilot = training_config(train)
|
||||
if sha(train / 'contract.json') != c['training_contract_sha256']: raise ValueError('Training contract differs')
|
||||
if c['jobs'] != [dict(id=j['id'], expected_parts=len(j['test'])) for j in tc['jobs']]:
|
||||
raise ValueError('All original five test lists')
|
||||
for name in ['predictions', 'terminal']: verify(Path(c['seed0_root']), name)
|
||||
by = roster(c, pilot)
|
||||
seed0 = read(c['seed0_manifest'])['entries']
|
||||
if len(seed0) != 552 or len({e['case'] for e in seed0}) != 552 or c['seed0_predictions'] != {e['case']:e for e in seed0}:
|
||||
raise ValueError('Original seed0 manifest identity')
|
||||
for case, e in c['seed0_predictions'].items():
|
||||
row = by[case]
|
||||
if (e['fold'] != next(a['fold'] for a in row['assignments'] if a['role']=='test')
|
||||
or e['component_id'] != row['component_id'] or e['nodes'] != row['nodes']):
|
||||
raise ValueError('Seed0 same test roles')
|
||||
return c, tc, pilot
|
||||
|
||||
|
||||
def terminal_ledger(tc, done, manifest):
|
||||
entries = done.get('entries', []); models = manifest.get('models', [])
|
||||
if (done.get('status') != 'TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING'
|
||||
or done.get('postfit_started') is not False or done.get('test_targets_read') is not False
|
||||
or len(entries) != 5 or {e['id'] for e in entries} != set(range(5))
|
||||
or any(e['status'] != 'PASS' or e['returncode'] != 0 for e in entries)
|
||||
or len(models) != 5 or {m['job'] for m in models} != set(range(5))
|
||||
or manifest.get('new_fits') != 5 or manifest.get('test_prediction_started') is not False):
|
||||
raise ValueError('All five original terminal checkpoints required')
|
||||
costs = [e['gpu_seconds_charged'] for e in entries]
|
||||
value = done['training_gpu_seconds']
|
||||
if (any(not math.isfinite(x) or x < 0 or x > 40000 for x in costs)
|
||||
or not math.isfinite(value) or abs(value-sum(costs)) > 1e-8
|
||||
or value > tc['train_gpu_cap'] or done['phase3_prior_gpu_once'] != PHASE3_PRIOR
|
||||
or abs(done['cumulative_phase3_training_gpu_seconds']-PHASE3_PRIOR-value)>1e-8):
|
||||
raise ValueError('Training ledger must include old costs exactly once')
|
||||
budget(value, 0, 180, 5)
|
||||
return value
|
||||
|
||||
|
||||
def normal_identity(stats, job):
|
||||
if (stats['parts'] != job['learning'] or stats['nodes'] != job['learning_nodes']
|
||||
or len(stats['mean']) != 19 or len(stats['scale']) != 19
|
||||
or not np.isfinite(stats['mean']).all() or not np.isfinite(stats['scale']).all()
|
||||
or min(stats['scale']) <= 0): raise ValueError('Learning-only 19-column normalization')
|
||||
|
||||
|
||||
def terminal(root, c, tc):
|
||||
import psutil
|
||||
train = Path(c['training_root'])
|
||||
tp = verify(train, 'terminal'); mp = verify(train, 'models')
|
||||
if (train/'global-stop.json').exists(): raise PermissionError('Training global stop')
|
||||
done = read(train/'complete.json'); manifest = read(train/'models-terminal.json')
|
||||
value = terminal_ledger(tc, done, manifest)
|
||||
for p in psutil.process_iter(['cmdline']):
|
||||
if execution_command(p.info['cmdline'] or []): raise RuntimeError('Seed1 training process still alive')
|
||||
for j in tc['jobs']:
|
||||
i=j['id']; w=train/'jobs'/str(i)/'worker'; e=next(e for e in done['entries'] if e['id']==i)
|
||||
assignment=read(checked(w.parent/'assignment.json',e['assignment_sha256']))
|
||||
if assignment['id']!=i or assignment['contract_sha256']!=c['training_contract_sha256'] or assignment['gpu_uuid']!=e['gpu_uuid']:
|
||||
raise ValueError('Original assignment identity')
|
||||
g=read(checked(w.parent/'complete.json',e['guard_complete_sha256']))
|
||||
receipt=read(w.parent/'guards/train.receipt.json')
|
||||
if (g['status']!='PASS' or g['reasons'] or g['monitor_errors'] or g['worker']!=receipt
|
||||
or type(receipt.get('monitor_errors')) is not int or receipt['monitor_errors']!=0
|
||||
or receipt['status']!='PASS' or receipt['returncode']!=0): raise ValueError('Training monitoring failure')
|
||||
r=read(checked(w/'result.json',g['worker_result_sha256'])); m=next(m for m in manifest['models'] if m['job']==i)
|
||||
if (r['status']!='COMPLETE' or r['fold']!=i or r['completed_epochs']!=200
|
||||
or r['optimizer_steps']!=j['target_steps'] or r['learning_parts']!=len(j['learning'])
|
||||
or r['learning_nodes_per_epoch']!=j['learning_nodes'] or r['test_targets_read'] is not False
|
||||
or r['test_prediction_started'] is not False or r['validation_used_for_selection'] is not False
|
||||
or m['path']!=str(w/'last-epoch200.pt') or m['path']!=r['checkpoint']
|
||||
or m['sha256']!=r['checkpoint_sha256'] or m['standardization']!=str(w/'standardization.json')
|
||||
or m['standardization_sha256']!=r['standardization_sha256']): raise ValueError('Final checkpoint/roles differ')
|
||||
for p,h in [(m['path'],m['sha256']),(m['standardization'],m['standardization_sha256'])]:
|
||||
checked(p,h)
|
||||
if mp['files'].get(str(Path(p).resolve()))!=h: raise ValueError('Model omitted from freeze')
|
||||
normal_identity(read(m['standardization']),j)
|
||||
return dict(models=manifest['models'],new_training_gpu_seconds_once=value,
|
||||
old_seed1_gpu_seconds_once=OLD,cumulative_seed1_training_gpu_seconds=OLD+value,
|
||||
cumulative_phase3_training_gpu_seconds=PHASE3_PRIOR+value,
|
||||
training_contract_sha256=c['training_contract_sha256'],training_terminal_sha256=sha(train/'complete.json'),
|
||||
training_terminal_commit=tp['commit'],training_models_commit=mp['commit'])
|
||||
|
||||
|
||||
def adopted(root, c, tc):
|
||||
verify(root,'models'); d=read(root/'models-adopted.json')
|
||||
if d!=terminal(root,c,tc): raise ValueError('Adopted model/terminal identity differs')
|
||||
return d
|
||||
|
||||
|
||||
def checkpoint_identity(blob, j, training_sha):
|
||||
if (blob.get('seed')!=1 or blob.get('fold')!=j['id'] or blob.get('completed_epochs')!=200
|
||||
or blob.get('optimizer_steps')!=j['target_steps'] or blob.get('contract_sha256')!=training_sha
|
||||
or blob.get('role')!='last_epoch_only'): raise ValueError('Seed1 final checkpoint identity')
|
||||
|
||||
|
||||
def prediction_identity(root, c, tc, entries):
|
||||
expected={(j['id'],case) for j in tc['jobs'] for case in j['test']}
|
||||
keys=[(e['fold'],e['case']) for e in entries]
|
||||
if len(keys)!=552 or len(set(keys))!=552 or set(keys)!=expected: raise ValueError('All552 distinct test predictions required')
|
||||
rows={r['case']:r for r in c['rows']}
|
||||
for e in entries:
|
||||
row=rows[e['case']]; old=c['seed0_predictions'][e['case']]
|
||||
if (e['path']!=str(root/'jobs'/str(e['fold'])/'prediction-records'/(e['case']+'.npz'))
|
||||
or e['nodes']!=row['nodes'] or e['query_nodes']!=old['query_nodes']
|
||||
or e['component_id']!=row['component_id'] or e['seed0_sha256']!=old['sha256']
|
||||
or e['raw_ids_sha256']!=row['v5_cache']['output_sha256']['raw_ids.npy']):
|
||||
raise ValueError('Prediction path/query/pair identity')
|
||||
for k,limit in [('negative_full_predictions',e['nodes']),('negative_query_predictions',e['query_nodes'])]:
|
||||
if type(e[k]) is not int or not 0<=e[k]<=limit: raise ValueError('Invalid negative counts')
|
||||
checked(e['path'],e['sha256']); checked(old['path'],old['sha256'])
|
||||
return rows
|
||||
|
||||
|
||||
def budget(new_train, gpu, cpu, remaining):
|
||||
if any(not math.isfinite(x) or x<0 for x in [new_train,gpu,cpu]) or type(remaining) is not int or remaining not in range(6):
|
||||
raise ValueError('Invalid ledger')
|
||||
reserve=gpu+900*remaining
|
||||
if new_train>160000-OLD or reserve>7200 or OLD+new_train+reserve>167200 or PHASE3_PRIOR+new_train+reserve>612000:
|
||||
raise RuntimeError('Seed1/phase3 cumulative GPU reserve exhausted')
|
||||
if cpu+300*remaining+900+120>3600: raise RuntimeError('Postfit CPU reserve exhausted')
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Seed1 postfit guard: bounded selected-GPU wait, no retries."""
|
||||
import json,os,signal,sys,threading,time
|
||||
import psutil
|
||||
from phase2_h1_worker_v2 import save
|
||||
from s2_main_memory_guard import PeakRecords
|
||||
from s2_precision_continuation_guard import device_usage
|
||||
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||||
|
||||
def final_reasons(reasons,receipt,result,tracker,gpu_seconds,cpu_seconds,cpu_cap,expected_parts):
|
||||
out=list(reasons)
|
||||
def add(s):
|
||||
if s not in out:out.append(s)
|
||||
if not receipt or receipt['status']!='PASS':add('WORKER_NOT_PASS')
|
||||
if receipt and (type(receipt.get('monitor_errors')) is not int or receipt['monitor_errors'] != 0 or receipt.get('returncode') != 0):add('WORKER_MONITOR_ERRORS')
|
||||
if not result or result.get('status')!='COMPLETE' or result.get('parts')!=expected_parts:add('INCOMPLETE_FOLD')
|
||||
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):add('MISSING_PEAKS')
|
||||
if any(v>15*2**30 for v in tracker.maxima.values()):add('RECORDED_VRAM_LIMIT')
|
||||
if gpu_seconds>900 or cpu_seconds>cpu_cap:add('FINAL_RESOURCE_LIMIT')
|
||||
return out
|
||||
|
||||
def run_prediction(queue,c,pilot,fold,prior_gpu,cpu_cap):
|
||||
root=queue/'jobs'/str(fold);gpu_uuid=c['gpu_uuid'];expected_parts=c['jobs'][fold]['expected_parts']
|
||||
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
|
||||
def elapsed():return 60+time.monotonic()-start
|
||||
def reason(x):
|
||||
if x not in reasons:reasons.append(x)
|
||||
command=[sys.executable,'tools/phase3_seed1_postfit_worker.py','predict',str(queue),str(fold)]
|
||||
def kill():
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
def scan():
|
||||
if tracker.scan(root/'prediction-records'):reason('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();mem,util=device_usage(gpu_uuid);_,rss,_=tree_usage(os.getpid())
|
||||
if mem>15*2**30:reason('DEVICE_VRAM_LIMIT')
|
||||
if rss>16*2**30:reason('TREE_RSS_LIMIT')
|
||||
if disk_usage(queue)>20*2**30:reason('DISK_LIMIT')
|
||||
if elapsed()>=895:reason('PREDICTION_TIME_LIMIT')
|
||||
for x in requested:reason(x)
|
||||
save(root/'prediction-live.json',dict(gpu_seconds_charged=elapsed(),device_bytes=mem,device_utilization=util,rss_bytes=rss,peaks=tracker.maxima,reasons=reasons,unix=time.time()))
|
||||
except Exception as exc:reason('MONITOR_ERROR');errors.append(str(exc))
|
||||
if reasons:kill()
|
||||
done.wait(.5)
|
||||
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGTERM,signal.SIGINT]};thread=None
|
||||
try:
|
||||
wait_start=time.monotonic();waiting=[]
|
||||
while True:
|
||||
mem,util=device_usage(gpu_uuid);waiting.append(dict(unix=time.time(),bytes=mem,utilization=util))
|
||||
save(root/'recovery-idle-wait.json',dict(samples=waiting,limit_seconds=c['idle_wait_seconds'],worker_started=False))
|
||||
if requested:raise RuntimeError('Signal received during idle wait')
|
||||
duration=time.monotonic()-wait_start
|
||||
if duration>c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
|
||||
if mem<=128*2**20 and util==0:break
|
||||
if duration>=c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
|
||||
time.sleep(min(.5,max(0.,c['idle_wait_seconds']-(time.monotonic()-wait_start))))
|
||||
for i in range(3):
|
||||
mem,util=device_usage(gpu_uuid);samples.append(dict(unix=time.time(),bytes=mem,utilization=util))
|
||||
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY_NO_RETRY')
|
||||
if i<2:time.sleep(2)
|
||||
save(root/'prediction-idle-polls.json',samples)
|
||||
if requested:raise RuntimeError('Signal received during setup')
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
remaining=900-elapsed()-10
|
||||
if remaining<=0:raise RuntimeError('Setup exhausted prediction budget')
|
||||
receipt=supervise(command,root/'prediction-guard','predict',dict(total_cpu_seconds=cpu_cap-60,job_cpu_seconds=cpu_cap-60,job_wall_seconds=remaining,tree_rss_bytes=16*2**30,disk_bytes=20*2**30,max_processes=1),poll=.2,gpu_uuid=gpu_uuid)
|
||||
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill()
|
||||
finally:
|
||||
done.set()
|
||||
if thread:thread.join(timeout=4)
|
||||
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
|
||||
try:scan()
|
||||
except Exception as exc:reason('FINAL_SCAN_ERROR');errors.append(str(exc))
|
||||
for s,h in old.items():signal.signal(s,h)
|
||||
result=None;p=root/'prediction-records/result.json'
|
||||
try:
|
||||
if p.exists():result=json.loads(p.read_text())
|
||||
except Exception as exc:reason('FINAL_RESULT_READ_ERROR');errors.append(str(exc))
|
||||
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
|
||||
duration=elapsed();reasons=final_reasons(reasons,receipt,result,tracker,duration,totalcpu,cpu_cap,expected_parts)
|
||||
out=dict(status='PASS' if not reasons else 'STOPPED',reasons=reasons,monitor_errors=errors,worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=duration,cpu_seconds_charged=totalcpu,cumulative_seed1_gpu_seconds=prior_gpu+duration,no_retry=True)
|
||||
save(root/'prediction-guard.json',out);return out
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Original query metrics and paired component bootstrap, seed1 minus seed0."""
|
||||
from phase2_b13_e1b import pooled
|
||||
|
||||
METRICS = ['relative_l2','last_location','kendall_tau_b']
|
||||
|
||||
|
||||
def comparisons(rows):
|
||||
raw=pooled([dict(component_id=r['component_id'],new=r['seed1'],A=r['seed0']) for r in rows])
|
||||
out={}
|
||||
for name,m in raw.items():
|
||||
undefined={arm:[r['case'] for r in rows if r[arm][name]['value'] is None] for arm in ['seed0','seed1']}
|
||||
out[name]=dict(means={'seed1':m['means']['new'],'seed0':m['means']['A']},
|
||||
components={'seed1':m['components']['new'],'seed0':m['components']['A']},
|
||||
seed1_minus_seed0=m['new_minus_A'],undefined_parts=undefined,
|
||||
part_counts={arm:len(rows)-len(v) for arm,v in undefined.items()})
|
||||
return out
|
||||
|
||||
|
||||
def summary(rows):
|
||||
if len(rows)!=552 or len({r['case'] for r in rows})!=552 or len({r['component_id'] for r in rows})!=204 or {r['fold'] for r in rows}!=set(range(5)):
|
||||
raise ValueError('Full552/204 support required; no partial cohort inference')
|
||||
return dict(parts=552,components=204,pooled=comparisons(rows),
|
||||
per_fold={str(f):dict(parts=sum(r['fold']==f for r in rows),components=len({r['component_id'] for r in rows if r['fold']==f}),metrics=comparisons([r for r in rows if r['fold']==f])) for f in range(5)},
|
||||
performance_gate=None,descriptive_only=True,
|
||||
interval_interpretation='Fixed-model component sampling variability, not a population distribution over training seeds',
|
||||
direction='seed1 minus seed0; negative field/last_location means lower error; positive tau means higher rank agreement')
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Predict seed1 without targets; score once only after all552 predictions freeze."""
|
||||
import os, sys, time
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
from phase2_h1_worker_v2 import save, sha, checked
|
||||
from phase2_home_freeze_io import verify
|
||||
from phase2_gate_outcome_review import validate_review
|
||||
from phase3_seed1_postfit_common import config, adopted, checkpoint_identity, prediction_identity, read
|
||||
from phase3_seed1_postfit_statistics import summary
|
||||
from phase3_seed1_v2_common import fold_view
|
||||
from phase2_h2_s1_postfit import query_index
|
||||
from phase2_residual_math import decode_seconds
|
||||
from phase2_b1213_features import cached
|
||||
from s2_custody_statistics import part_metrics
|
||||
from s2_custody_common import prediction_inputs
|
||||
|
||||
|
||||
def paired_query(raw, q, seed0):
|
||||
original={k:seed0['query_raw_ids' if k=='raw_ids' else k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']}
|
||||
ix=query_index(raw,q,original)
|
||||
if (not np.array_equal(seed0['raw_ids'],raw) or not np.array_equal(seed0['pred'],seed0['seconds'][ix])
|
||||
or not np.isfinite(seed0['seconds']).all()): raise ValueError('Seed0 full/query identity')
|
||||
return ix
|
||||
|
||||
|
||||
def predict(root, fold):
|
||||
c,tc,pilot=config(root); validate_review(root); models=adopted(root,c,tc)
|
||||
view=fold_view(pilot,fold); f=view['folds'][0]; job=tc['jobs'][fold]
|
||||
m=next(m for m in models['models'] if m['job']==fold); stats=read(m['standardization'])
|
||||
import torch
|
||||
from phase2_h2_model import H2Geo
|
||||
from phase2_h2_data import to_device, arguments
|
||||
from phase2_h2_s1_test_data import load_test_part
|
||||
if os.environ.get('CUDA_VISIBLE_DEVICES')!=c['gpu_uuid'] or os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!='expandable_segments:True':
|
||||
raise ValueError('Selected GPU/allocator identity')
|
||||
os.sched_setaffinity(0,c['worker_cpus']);torch.set_num_threads(1);torch.set_num_interop_threads(1)
|
||||
torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
|
||||
dest=root/'jobs'/str(fold)/'prediction-records';dest.mkdir(exist_ok=False)
|
||||
torch.cuda.set_device(0);torch.cuda.init();torch.cuda.reset_peak_memory_stats()
|
||||
start=time.monotonic();cpu=time.process_time();entries=[];by={r['case']:r for r in c['rows']}
|
||||
def peaks():return dict(peak_allocated_bytes=int(torch.cuda.max_memory_allocated()),peak_reserved_bytes=int(torch.cuda.max_memory_reserved()))
|
||||
try:
|
||||
blob=torch.load(checked(m['path'],m['sha256']),map_location='cpu',weights_only=False)
|
||||
checkpoint_identity(blob,job,c['training_contract_sha256']);model=H2Geo().cuda();model.load_state_dict(blob['model'],strict=True);model.eval();del blob
|
||||
save(dest/'environment.json',dict(fold=fold,seed=1,torch=torch.__version__,gpu_uuid=c['gpu_uuid'],allocator_settings=str(torch.cuda.memory._snapshot().get('allocator_settings',{})),model_dtype='FP32',autocast=False,checkpoint_sha256=m['sha256']))
|
||||
with torch.no_grad():
|
||||
for case in f['test']:
|
||||
row=by[case];data=to_device(load_test_part(view,case,stats),'cuda');residual=model(*arguments(data),checkpoint_blocks=False)
|
||||
if not bool(torch.isfinite(residual).all()):raise ValueError('Nonfinite residual')
|
||||
seconds=decode_seconds(residual.cpu().numpy(),data['F'],f['c'],data['t_set']);torch.cuda.synchronize()
|
||||
q=prediction_inputs(row);old=c['seed0_predictions'][case]
|
||||
with np.load(checked(old['path'],old['sha256']),allow_pickle=False) as z:
|
||||
ix=paired_query(data['raw_ids'],q,z)
|
||||
if len(seconds)!=row['nodes'] or not np.isfinite(seconds).all() or len(ix)!=old['query_nodes']:raise ValueError('Prediction coverage')
|
||||
path=dest/(case+'.npz')
|
||||
np.savez_compressed(path,seconds=seconds,raw_ids=data['raw_ids'],pred=seconds[ix],query_raw_ids=q['raw_ids'],counts=q['counts'],coordinates_over_Rg=q['coordinates_over_Rg'],near_wall=q['near_wall'])
|
||||
entries.append(dict(case=case,component_id=row['component_id'],fold=fold,path=str(path),sha256=sha(path),nodes=row['nodes'],query_nodes=len(ix),seed0_sha256=old['sha256'],raw_ids_sha256=row['v5_cache']['output_sha256']['raw_ids.npy'],negative_full_predictions=int(np.sum(seconds<0)),negative_query_predictions=int(np.sum(seconds[ix]<0))))
|
||||
save(dest/'progress.json',dict(stage='TEST_PREDICTION_NO_TARGETS',fold=fold,parts=len(entries),wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
del data,residual,seconds
|
||||
save(dest/'manifest.json',dict(entries=entries,test_targets_read=False,new_fits=0,full_nodes=sum(e['nodes'] for e in entries)))
|
||||
save(dest/'result.json',dict(status='COMPLETE',fold=fold,parts=len(entries),test_targets_read=False,wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()))
|
||||
except BaseException as exc:
|
||||
save(dest/'failure.json',dict(status='FAILED',fold=fold,reason=type(exc).__name__+': '+str(exc),completed_parts=len(entries),wall_seconds=time.monotonic()-start,cpu_seconds=time.process_time()-cpu,**peaks()));raise
|
||||
|
||||
|
||||
def score(root):
|
||||
c,tc,pilot=config(root);validate_review(root);adopted(root,c,tc);verify(root,'predictions')
|
||||
entries=read(root/'predictions-manifest.json')['entries'];by=prediction_identity(root,c,tc,entries)
|
||||
dest=root/'score';dest.mkdir(exist_ok=False)
|
||||
save(dest/'review-consumed.json',dict(unix=time.time(),review_sha256=sha(root/'review.json'),before_old_score_or_target_values=True))
|
||||
frozen=read(checked(c['seed0_score'],c['source_sha256'][c['seed0_score']]))
|
||||
olds=frozen['rows'];oldrows={r['case']:r for r in olds}
|
||||
if frozen['status']!='COMPLETE' or len(olds)!=552 or len(oldrows)!=552 or set(oldrows)!=set(by):raise ValueError('Seed0 score support')
|
||||
rows=[];access=[]
|
||||
for e in entries:
|
||||
case=e['case'];row=by[case];old=oldrows[case];base=c['seed0_predictions'][case]
|
||||
if old['fold']!=e['fold'] or old['component_id']!=e['component_id']:raise ValueError('Seed0 score pairing')
|
||||
access.append(dict(case=case,fold=e['fold'],role='test',target_sha256=row['v5_cache']['output_sha256']['seconds.npy']))
|
||||
save(dest/'target-access.json',access)
|
||||
truth=cached(row,'seconds');raw=cached(row,'raw_ids')
|
||||
with np.load(checked(e['path'],e['sha256']),allow_pickle=False) as z, np.load(checked(base['path'],base['sha256']),allow_pickle=False) as original:
|
||||
q={k:z['query_raw_ids' if k=='raw_ids' else k] for k in ['raw_ids','counts','coordinates_over_Rg','near_wall']}
|
||||
ix=paired_query(raw,q,original)
|
||||
if (not np.array_equal(z['raw_ids'],raw) or not np.array_equal(z['pred'],z['seconds'][ix])
|
||||
or not np.isfinite(z['seconds']).all() or len(z['seconds'])!=e['nodes'] or len(ix)!=e['query_nodes']
|
||||
or int(np.sum(z['seconds']<0))!=e['negative_full_predictions']
|
||||
or int(np.sum(z['pred']<0))!=e['negative_query_predictions']):raise ValueError('Seed1 full/query/count identity')
|
||||
metric=part_metrics(z['pred'],truth[ix],q['counts'],q['near_wall'],q['raw_ids'],q['coordinates_over_Rg'],int(case.split('_')[0]))
|
||||
rows.append(dict(case=case,component_id=e['component_id'],fold=e['fold'],seed1=metric,seed0=old['H2']))
|
||||
save(dest/'progress.json',dict(parts=len(rows),unique_test_target_reads=len(access)))
|
||||
if len(access)!=552 or len({x['case'] for x in access})!=552:raise ValueError('Exactly552 unique existing target-cache reads')
|
||||
save(dest/'result.json',dict(status='COMPLETE',**summary(rows),rows=rows,seed0_metrics_reused=552,
|
||||
seed0_score_sha256=c['source_sha256'][c['seed0_score']],seed0_predictions_reused=552,
|
||||
unique_test_target_reads=552,new_decodes=0,new_fits=0,review_sha256=sha(root/'review.json'),
|
||||
negative_predictions={arm:dict(full_nodes=sum(e['negative_full_predictions'] for e in es),query_nodes=sum(e['negative_query_predictions'] for e in es),clipped=False) for arm,es in [('seed1',entries),('seed0',list(c['seed0_predictions'].values()))]},
|
||||
full_nodes=sum(e['nodes'] for e in entries),eligible_unique_query_nodes=sum(e['query_nodes'] for e in entries),
|
||||
later_stages_started=False,seed2_started=False,ensemble_started=False))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
action=sys.argv[1];root=Path(sys.argv[2])
|
||||
if action=='predict' and len(sys.argv)==4:predict(root,int(sys.argv[3]))
|
||||
elif action=='score' and len(sys.argv)==3:score(root)
|
||||
else:raise ValueError('Use predict root fold or score root')
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Metadata-only seed1 postfit preparation; never predict, score, or decode targets."""
|
||||
from pathlib import Path
|
||||
import time
|
||||
from phase2_h1_worker_v2 import sha, checked, save
|
||||
from phase2_home_freeze_io import verify
|
||||
from phase3_seed1_v2_common import config as training_config
|
||||
from phase3_seed1_postfit_common import OLD, PHASE3_PRIOR, GPU, BOOTSTRAP, TRAINING, read, roster
|
||||
from track_a_h2_prepare import local_code_closure
|
||||
|
||||
B=Path(__file__).resolve().parents[1]
|
||||
T=Path('/home/frank14f/MeltMind-runs/phase3-seed1-v2-2026-11-18')
|
||||
S=T.parent/'phase2-h2-s2-postfit-2026-11-17'
|
||||
R=T.parent/'phase3-seed1-postfit-2026-11-18'
|
||||
|
||||
|
||||
def main():
|
||||
start=time.process_time()
|
||||
if R.exists():raise FileExistsError('Existing postfit entry; inspect rather than duplicate')
|
||||
tc,pilot=training_config(T)
|
||||
if verify(T,'implementation')['commit']!=TRAINING:raise ValueError('Exact active training pin')
|
||||
old=read(S/'contract.json')
|
||||
for name in ['implementation','predictions','terminal']:verify(S,name)
|
||||
by=roster(old,pilot);manifest=read(S/'predictions-manifest.json')['entries']
|
||||
seed0={e['case']:e for e in manifest}
|
||||
if len(manifest)!=552 or len(seed0)!=552 or set(seed0)!=set(by):raise ValueError('Original prediction support')
|
||||
for case,e in seed0.items():
|
||||
row=by[case]
|
||||
if e['fold']!=next(a['fold'] for a in row['assignments'] if a['role']=='test') or e['nodes']!=row['nodes'] or e['component_id']!=row['component_id']:
|
||||
raise ValueError('Original seed0 test role identity')
|
||||
sources=dict(tc['source_sha256'])
|
||||
for p,h in {**old['source_sha256'],**old['code_sha256'],**tc['code_sha256']}.items():
|
||||
if p in sources and sources[p]!=h:raise ValueError('Conflicting source pins')
|
||||
sources[p]=h
|
||||
for folder,names in [(T,['contract.json','implementation-git-freeze.json','implementation-authorization.json','review.json']),
|
||||
(S,['contract.json','predictions-manifest.json','predictions-git-freeze.json','predictions-authorization.json','terminal-git-freeze.json','terminal-authorization.json','score/result.json'])]:
|
||||
for name in names:sources[str(folder/name)]=sha(folder/name)
|
||||
spec=B/'references/2026-11-18/phase3-seed-sensitivity/mold-seed-ensemble-spec.md'
|
||||
sources[str(spec)]=sha(spec)
|
||||
comparison=dict(approval='1794992262.078969',training_comparison_sha256=tc['comparison_contract_sha256'],
|
||||
parts=552,components=204,roles='same original S2 five test folds',seeds=[0,1],new_fits=0,
|
||||
seed0_predictions='hash reuse all552',seed0_metrics='reuse original S2 per-part H2 metrics; no recomputation',
|
||||
metrics=['relative_l2','last_location','kendall_tau_b'],difference='seed1 minus seed0',
|
||||
query='original eligible query/raw IDs/multiplicity/Rg/tie/undefined rules unchanged',
|
||||
aggregation='within-component part means, then equal components; paired complete component intersection',
|
||||
bootstrap=BOOTSTRAP,report=['each_fold','pooled','undefined_and_counts','negative_unclipped_nodes'],
|
||||
performance_gate=None,descriptive_only=True,new_decodes=0,unique_test_target_cache_reads=552,
|
||||
prior_three_seed_proposal_superseded=True,seed2=False,ensemble=False,
|
||||
interpretation='Noise reference for later 1b/c/b discussion; no automatic later-stage launch; CI describes fixed-model component variability only')
|
||||
codes=local_code_closure([B/'tools'/n for n in ['phase3_seed1_postfit.py','phase3_seed1_postfit_common.py',
|
||||
'phase3_seed1_postfit_worker.py','phase3_seed1_postfit_guard.py','phase3_seed1_postfit_statistics.py',
|
||||
'prepare_phase3_seed1_postfit.py','test_phase3_seed1_postfit.py']]+list((B/'tools/s2_models').glob('*.py')))
|
||||
for p,h in tc['code_sha256'].items():checked(p,h)
|
||||
R.mkdir();save(R/'comparison-contract.json',comparison)
|
||||
c=dict(version='phase3-seed1-postfit-v1',approval='1794992262.078969',training_root=str(T),
|
||||
training_contract_sha256=sha(T/'contract.json'),training_implementation=TRAINING,
|
||||
comparison_contract=str(R/'comparison-contract.json'),comparison_contract_sha256=sha(R/'comparison-contract.json'),
|
||||
seed0_root=str(S),seed0_manifest=str(S/'predictions-manifest.json'),seed0_score=str(S/'score/result.json'),
|
||||
seed0_predictions=seed0,rows=old['rows'],baseline_predictions=old['baseline_predictions'],
|
||||
source_sha256=sources,code_sha256=codes,jobs=[dict(id=j['id'],expected_parts=len(j['test'])) for j in tc['jobs']],
|
||||
gpu_uuid=GPU,worker_cpus=[0,1],coordinator_cpus=[14,15],gpu_cap=167200,postfit_gpu_cap=7200,
|
||||
postfit_cpu_cap=3600,phase3_gpu_cap=612000,old_seed1_gpu_once=OLD,phase3_prior_gpu_once=PHASE3_PRIOR,
|
||||
prediction_per_fold_gpu_seconds=900,prediction_per_fold_cpu_seconds=300,score_cpu_seconds=900,
|
||||
score_wall_seconds=1200,setup_cpu_seconds=180,terminal_cpu_reserve=120,
|
||||
vram_bytes=15*2**30,rss_bytes=16*2**30,disk_bytes=20*2**30,idle_wait_seconds=20,no_retry=True,
|
||||
bootstrap=BOOTSTRAP,performance_gate=None,no_auto_launch=True,all_five_training_terminals_before_prediction=True,
|
||||
all_predictions_frozen_before_targets=True,new_fits=0,new_decodes=0,seed2_started=False,ensemble_started=False)
|
||||
save(R/'contract.json',c)
|
||||
save(R/'preparation-receipt.json',dict(status='PREPARED_PENDING_EXACT_PRE_SCORE_REVIEW',
|
||||
contract_sha256=sha(R/'contract.json'),code_files=len(codes),parts=552,components=204,
|
||||
metadata_cpu_seconds=time.process_time()-start,source_training_unchanged=True,model_weights_read=False,
|
||||
scientific_arrays_decoded=False,target_values_read=False,old_scores_parsed=False,new_predictions=False,
|
||||
training_complete_exists=(T/'complete.json').exists(),launched=False))
|
||||
print(read(R/'preparation-receipt.json'))
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,146 @@
|
||||
"""CPU-only synthetic regression checks, no project data or CUDA execution."""
|
||||
import copy
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import numpy as np
|
||||
from phase3_seed1_postfit_common import checkpoint_identity, terminal_ledger, budget, normal_identity, prediction_identity, OLD, PHASE3_PRIOR
|
||||
from phase3_seed1_postfit_statistics import comparisons, summary
|
||||
from phase3_seed1_postfit_worker import paired_query, predict, score
|
||||
from phase3_seed1_postfit_guard import final_reasons
|
||||
from phase2_h2_s1_test_data import load_test_part
|
||||
from s2_custody_statistics import part_metrics
|
||||
|
||||
|
||||
def metric(v):return {k:dict(value=v,reason=None if v is not None else 'undefined') for k in ['relative_l2','last_location','kendall_tau_b']}
|
||||
|
||||
|
||||
class Seed1PostfitTests(unittest.TestCase):
|
||||
def test_checkpoint_rejects_wrong_seed_fold_epoch_selection(self):
|
||||
j=dict(id=2,target_steps=60000);b=dict(seed=1,fold=2,completed_epochs=200,optimizer_steps=60000,contract_sha256='fixed',role='last_epoch_only')
|
||||
checkpoint_identity(b,j,'fixed')
|
||||
for k,v in [('seed',0),('fold',3),('completed_epochs',199),('optimizer_steps',59999),('contract_sha256','other'),('role','best_validation')]:
|
||||
with self.assertRaises(ValueError):checkpoint_identity(dict(b,**{k:v}),j,'fixed')
|
||||
|
||||
def fixture(self):
|
||||
tc=dict(train_gpu_cap=160000-OLD)
|
||||
d=dict(status='TRAIN_COMPLETE_POSTFIT_REVIEW_PENDING',postfit_started=False,test_targets_read=False,
|
||||
entries=[dict(id=i,status='PASS',returncode=0,gpu_seconds_charged=100.) for i in range(5)],
|
||||
training_gpu_seconds=500.,phase3_prior_gpu_once=PHASE3_PRIOR,cumulative_phase3_training_gpu_seconds=PHASE3_PRIOR+500.)
|
||||
m=dict(models=[dict(job=i) for i in range(5)],new_fits=5,test_prediction_started=False)
|
||||
return tc,d,m
|
||||
|
||||
def test_all_five_terminal_no_partial_adoption(self):
|
||||
tc,d,m=self.fixture();self.assertEqual(terminal_ledger(tc,d,m),500.)
|
||||
d['entries'].pop()
|
||||
with self.assertRaises(ValueError):terminal_ledger(tc,d,m)
|
||||
|
||||
def test_failed_or_duplicate_terminal_rejected(self):
|
||||
for key,val in [('status','STOPPED'),('returncode',1),('id',1)]:
|
||||
tc,d,m=self.fixture();d['entries'][0][key]=val
|
||||
with self.assertRaises(ValueError):terminal_ledger(tc,d,m)
|
||||
|
||||
def test_ledger_double_old_charge_rejected(self):
|
||||
tc,d,m=self.fixture();d['training_gpu_seconds']+=OLD
|
||||
with self.assertRaises(ValueError):terminal_ledger(tc,d,m)
|
||||
tc,d,m=self.fixture();d['cumulative_phase3_training_gpu_seconds']+=OLD
|
||||
with self.assertRaises(ValueError):terminal_ledger(tc,d,m)
|
||||
|
||||
def test_exact_cumulative_budgets_and_final_caps(self):
|
||||
budget(160000-OLD,7200,2000,0)
|
||||
for args in [(160000-OLD,7200.01,180,0),(160000-OLD+.01,0,180,5),(100,0,1081,5),(float('nan'),0,0,5),(100,0,0,True)]:
|
||||
with self.assertRaises((ValueError,RuntimeError)):budget(*args)
|
||||
|
||||
def test_learning_only_normalization(self):
|
||||
j=dict(learning=['a'],learning_nodes=10);s=dict(parts=['a'],nodes=10,mean=[0.]*19,scale=[1.]*19)
|
||||
normal_identity(s,j)
|
||||
for change in [dict(parts=['test']),dict(nodes=11),dict(scale=[0.]*19),dict(mean=[0.]*18)]:
|
||||
with self.assertRaises(ValueError):normal_identity(dict(s,**change),j)
|
||||
|
||||
def query_fixture(self):
|
||||
raw=np.array([1,3,8]);q=dict(raw_ids=np.array([1,8]),counts=np.array([1,2]),coordinates_over_Rg=np.array([[0.,0.,0.],[1.,0.,0.]]),near_wall=np.array([True,False]))
|
||||
z=dict(raw_ids=raw,seconds=np.array([1.,2.,3.]),pred=np.array([1.,3.]),query_raw_ids=q['raw_ids'],counts=q['counts'],coordinates_over_Rg=q['coordinates_over_Rg'],near_wall=q['near_wall'])
|
||||
return raw,q,z
|
||||
|
||||
def test_query_full_ids_distinct_and_multiplicity_exact(self):
|
||||
raw,q,z=self.query_fixture();np.testing.assert_array_equal(paired_query(raw,q,z),[0,2])
|
||||
for change in [dict(counts=np.array([1,1])),dict(pred=np.array([1.,2.])),dict(raw_ids=np.array([1,4,8])),dict(query_raw_ids=np.array([1,3]))]:
|
||||
with self.assertRaises(ValueError):paired_query(raw,q,dict(z,**change))
|
||||
|
||||
def test_test_loader_denies_foreign_role_before_array_io(self):
|
||||
with patch('phase2_h2_s1_test_data.np.load',side_effect=AssertionError('Array access')) as io:
|
||||
with self.assertRaises(PermissionError):load_test_part(dict(folds=[dict(test=['a'])]),'learning',{})
|
||||
io.assert_not_called()
|
||||
|
||||
def test_predict_needs_actual_review_before_model_or_inputs(self):
|
||||
with patch('phase3_seed1_postfit_worker.config',return_value=({}, {}, {})),patch('phase3_seed1_postfit_worker.validate_review',side_effect=PermissionError('No review')),patch('phase3_seed1_postfit_worker.adopted') as a:
|
||||
with self.assertRaises(PermissionError):predict(Path('/synthetic'),0)
|
||||
a.assert_not_called()
|
||||
|
||||
def test_predict_needs_all_training_terminals_before_input_loader(self):
|
||||
with patch('phase3_seed1_postfit_worker.config',return_value=({}, {}, {})),patch('phase3_seed1_postfit_worker.validate_review'),patch('phase3_seed1_postfit_worker.adopted',side_effect=ValueError('Incomplete')),patch('phase2_h2_s1_test_data.load_test_part') as io:
|
||||
with self.assertRaises(ValueError):predict(Path('/synthetic'),0)
|
||||
io.assert_not_called()
|
||||
|
||||
def test_score_no_targets_or_score_reads_before_review(self):
|
||||
with patch('phase3_seed1_postfit_worker.config',return_value=({}, {}, {})),patch('phase3_seed1_postfit_worker.validate_review',side_effect=PermissionError('No review')),patch('phase3_seed1_postfit_worker.cached') as io,patch('phase3_seed1_postfit_worker.read') as rd:
|
||||
with self.assertRaises(PermissionError):score(Path('/synthetic'))
|
||||
io.assert_not_called();rd.assert_not_called()
|
||||
|
||||
def test_score_no_targets_before_prediction_freeze(self):
|
||||
with patch('phase3_seed1_postfit_worker.config',return_value=({}, {}, {})),patch('phase3_seed1_postfit_worker.validate_review'),patch('phase3_seed1_postfit_worker.adopted'),patch('phase3_seed1_postfit_worker.verify',side_effect=PermissionError('No prediction freeze')),patch('phase3_seed1_postfit_worker.cached') as io,patch('phase3_seed1_postfit_worker.read') as rd:
|
||||
with self.assertRaises(PermissionError):score(Path('/synthetic'))
|
||||
io.assert_not_called();rd.assert_not_called()
|
||||
|
||||
def test_sign_and_equal_component_weight(self):
|
||||
rows=[dict(case='a',component_id='x',seed0=metric(1),seed1=metric(2)),dict(case='b',component_id='x',seed0=metric(1),seed1=metric(4)),dict(case='c',component_id='y',seed0=metric(1),seed1=metric(9))]
|
||||
s=comparisons(rows)['relative_l2'];self.assertEqual(s['seed1_minus_seed0']['value'],5.)
|
||||
self.assertEqual(s['means'],dict(seed1=6.,seed0=1.));self.assertEqual(s['seed1_minus_seed0']['replicates'],2000)
|
||||
self.assertEqual(comparisons(rows),comparisons(rows[::-1]))
|
||||
|
||||
def test_undefined_component_not_silently_reweighted(self):
|
||||
rows=[dict(case='a',component_id='x',seed0=metric(1),seed1=metric(None)),dict(case='b',component_id='x',seed0=metric(1),seed1=metric(2)),dict(case='c',component_id='y',seed0=metric(1),seed1=metric(3))]
|
||||
s=comparisons(rows)['kendall_tau_b'];self.assertEqual(s['components'],dict(seed1=1,seed0=2));self.assertEqual(s['seed1_minus_seed0']['components'],1)
|
||||
self.assertEqual(s['undefined_parts']['seed1'],['a']);self.assertEqual(s['seed1_minus_seed0']['value'],2.)
|
||||
|
||||
def test_original_query_metric_ties_and_undefined(self):
|
||||
m=part_metrics(np.zeros(3),np.zeros(3),np.ones(3),np.ones(3,bool),np.array([3,1,2]),np.array([[0.,0,0],[1.,0,0],[2.,0,0]]),6)
|
||||
self.assertIsNone(m['relative_l2']['value']);self.assertIsNone(m['kendall_tau_b']['value']);self.assertEqual(m['last_location']['value'],0.)
|
||||
|
||||
def test_no_partial_cohort_summary(self):
|
||||
with self.assertRaises(ValueError):summary([dict(case='a',component_id='x',fold=0,seed0=metric(1),seed1=metric(2))])
|
||||
with self.assertRaises(ValueError):prediction_identity(Path('/synthetic'),{},dict(jobs=[]),[])
|
||||
|
||||
def test_manifest_requires_exact_complete_roles_and_hashes(self):
|
||||
root=Path('/synthetic');rows=[];entries=[];seed0={};jobs=[dict(id=f,test=[]) for f in range(5)]
|
||||
for i in range(552):
|
||||
case=str(i);f=i%5;group='g'+str(i%204);jobs[f]['test'].append(case)
|
||||
rows.append(dict(case=case,nodes=3,component_id=group,v5_cache=dict(output_sha256={'raw_ids.npy':'raw'})))
|
||||
seed0[case]=dict(path='/old/'+case,sha256='old',query_nodes=2)
|
||||
entries.append(dict(case=case,fold=f,path=str(root/'jobs'/str(f)/'prediction-records'/(case+'.npz')),sha256='new',nodes=3,query_nodes=2,component_id=group,seed0_sha256='old',raw_ids_sha256='raw',negative_full_predictions=0,negative_query_predictions=0))
|
||||
c=dict(rows=rows,seed0_predictions=seed0);tc=dict(jobs=jobs)
|
||||
with patch('phase3_seed1_postfit_common.checked'):
|
||||
self.assertEqual(len(prediction_identity(root,c,tc,entries)),552)
|
||||
for change in [dict(fold=4),dict(component_id='wrong'),dict(raw_ids_sha256='foreign'),dict(seed0_sha256='other'),dict(query_nodes=3)]:
|
||||
altered=copy.deepcopy(entries);altered[0].update(change)
|
||||
with self.assertRaises(ValueError):prediction_identity(root,c,tc,altered)
|
||||
with self.assertRaises(ValueError):prediction_identity(root,c,tc,entries[:-1]+[entries[0]])
|
||||
with patch('phase3_seed1_postfit_common.checked',side_effect=ValueError('Changed bytes')):
|
||||
with self.assertRaises(ValueError):prediction_identity(root,c,tc,entries)
|
||||
|
||||
def test_complete_summary_has_no_performance_gate(self):
|
||||
rows=[dict(case=str(i),component_id='g'+str(i%204),fold=(i%204)%5,seed0=metric(1.),seed1=metric(2.)) for i in range(552)]
|
||||
r=summary(rows);self.assertIsNone(r['performance_gate']);self.assertTrue(r['descriptive_only'])
|
||||
self.assertEqual(set(r['per_fold']),set(map(str,range(5))))
|
||||
self.assertEqual(r['pooled']['relative_l2']['seed1_minus_seed0']['value'],1.)
|
||||
|
||||
def test_failed_worker_recorded_peaks_and_monitor_type(self):
|
||||
class Peaks:
|
||||
witnesses={'peak_allocated_bytes':1,'peak_reserved_bytes':1}
|
||||
maxima={'peak_allocated_bytes':1,'peak_reserved_bytes':16*2**30}
|
||||
r=dict(status='PASS',returncode=0,monitor_errors=0);result=dict(status='COMPLETE',parts=2)
|
||||
self.assertIn('RECORDED_VRAM_LIMIT',final_reasons([],r,result,Peaks(),100,100,300,2))
|
||||
self.assertIn('WORKER_MONITOR_ERRORS',final_reasons([],dict(r,monitor_errors=[]),result,Peaks(),100,100,300,2))
|
||||
self.assertIn('INCOMPLETE_FOLD',final_reasons([],r,None,Peaks(),100,100,300,2))
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user