58 lines
7.0 KiB
Python
58 lines
7.0 KiB
Python
"""Prepare the one approved review during training; no target arrays or launches."""
|
|
import json,shutil,time,re
|
|
from pathlib import Path
|
|
from phase2_h1_worker_v2 import sha,checked,save
|
|
from phase2_h2_prepare import closure
|
|
from phase2_home_freeze_io import verify,freeze
|
|
from phase3_item1_postfit_common import read,config,MODEL_ARMS,expected_entries
|
|
B=Path('/home/frank14f/optane/MeltMind')
|
|
R=Path('/home/frank14f/MeltMind-runs/phase3-item1-postfit-2026-11-18')
|
|
T=R.parent/'phase3-data-sufficiency-v2-2026-11-18'
|
|
def main():
|
|
import psutil
|
|
start=time.process_time()
|
|
if R.exists():raise FileExistsError('Existing postfit preparation: inspect, never duplicate')
|
|
live=[]
|
|
for p in psutil.process_iter(['pid','cmdline']):
|
|
args=p.info['cmdline'] or []
|
|
if any(Path(x).name in {'phase3_item1_postfit.py','phase3_item1_postfit_worker.py'} for x in args):raise RuntimeError('Existing postfit execution')
|
|
if any(Path(x).name in {'phase3_data_sufficiency_v2_queue.py','phase3_data_sufficiency_v2_guard.py','phase3_data_sufficiency_v2_worker.py'} for x in args):live.append(dict(pid=p.pid,command=args))
|
|
tp=verify(T,'implementation');tc=read(T/'contract.json')
|
|
if tp['commit']!='73f94bbc77e5ed3156ec85ff4a32ec2c834fdb07' or sha(T/'contract.json')!='1bdf2331c5127e03bec160835eda9d04ff333c3f70d5fe12a0629ffa82ec5826':raise ValueError('Pinned training identity')
|
|
sources=dict(tc['source_sha256'])
|
|
for p,h in {**sources,**tc['code_sha256']}.items():checked(p,h)
|
|
pilot=read(tc['pilot_contract']);hpath=B/'artifacts/phase2-h1-execution-2026-11-17/contract.json';rows=read(hpath)['grouped_rows'];s1train=B/'artifacts/phase2-h2-s1-2026-11-17/contract.json'
|
|
s1=Path(tc['S1_reuse']['test_manifest']).parent;verify(s1,'predictions');verify(s1,'terminal')
|
|
for p in [T/'contract.json',T/'implementation-git-freeze.json',T/'implementation-authorization.json',Path(tc['pilot_contract']),hpath,s1train,Path(tc['S1_reuse']['test_manifest'])]:sources[str(p)]=sha(p)
|
|
old=read(tc['S1_reuse']['test_manifest'])['entries']
|
|
if len(old)!=121 or {e['case'] for e in old}!=set(tc['pools']['100']['test']):raise ValueError('S1 test manifest coverage')
|
|
for e in old:sources[e['path']]=e['sha256'];checked(e['path'],e['sha256'])
|
|
jobs=[]
|
|
for j,arm in zip(tc['jobs'],MODEL_ARMS):
|
|
tag='epoch200' if j['mode']=='epoch' else ('U' if j['target_steps']==32000 else '2U')
|
|
jobs.append(dict(id=j['id'],arm=arm,pool=j['pool'],mode=j['mode'],target_steps=j['target_steps'],tag=tag,expected_parts=sum(len(tc['pools'][str(j['pool'])][r]) for r in ['learning','validation','test'])))
|
|
jobs.append(dict(id=6,arm=MODEL_ARMS[6],pool=100,mode='epoch',target_steps=64000,tag='last_epoch_only',expected_parts=552))
|
|
universe=set().union(*(set(tc['pools']['100'][r]) for r in ['learning','validation','test']))
|
|
if len(universe)!=552 or not universe<=set(rows):raise ValueError('Original evaluation roster')
|
|
for pool in ['25','50','100']:
|
|
f=tc['pools'][pool]
|
|
if len({rows[x]['component_id'] for x in f['learning']})!=f['learning_components']:raise ValueError('Learning component counts')
|
|
log=Path('/tmp/phase3-item1-postfit-tests3.log').read_text()
|
|
if 'Ran 27 tests' not in log or not log.rstrip().endswith('OK'):raise ValueError('Actual27 tests PASS required')
|
|
comparison=Path(tc['comparison_contract']);checked(comparison,'332f2515911c99aadf2486086ddff82bd195d889faa8391296443e7aeacdfee9')
|
|
codes={str(B/p):h for p,h in closure(['phase3_item1_postfit','phase3_item1_postfit_common','phase3_item1_postfit_worker','phase3_item1_postfit_guard','phase3_item1_postfit_data','phase3_item1_postfit_statistics','prepare_phase3_item1_postfit','test_phase3_item1_postfit']).items()}
|
|
doc=B/'docs/phase3-item1-postfit-2026-11-18.md';codes[str(doc)]=sha(doc)
|
|
c=dict(version='phase3-item1-postfit-v1',approval='1794950613.195499',postfit_root=str(R),training_root=str(T),training_contract=str(T/'contract.json'),training_implementation=tp['commit'],S1_training_contract_sha256=sha(s1train),jobs=jobs,rows={x:rows[x] for x in sorted(universe)},source_sha256=sources,code_sha256=codes,comparison_contract=str(R/'comparison-contract.json'),comparison_contract_sha256=sha(comparison),postfit_gpu_cap=7200,postfit_cpu_cap=3600,item_gpu_cap=120000,gpu_uuid=tc['jobs'][0]['gpu_uuid'],worker_cpus=[0,1],coordinator_cpus=[12,13],new_fits=0,new_decodes=0,no_retry=True,no_auto_launch=True,review_stages=1,actual_checkpoint_hashes='Runtime only after six frozen terminals and process exit',all_predictions_frozen_before_target_read=True,expected_entries=4582,expected_unique_target_reads=552,expected_test_reads=121)
|
|
expected=expected_entries(c,tc)
|
|
if len(expected)!=4582:raise ValueError('Complete prediction cardinality')
|
|
R.mkdir();shutil.copyfile(comparison,R/'comparison-contract.json');save(R/'contract.json',c)
|
|
for attempt in [1,2,3]:shutil.copyfile('/tmp/phase3-item1-postfit-tests'+str(attempt)+'.log',R/('synthetic-attempt'+str(attempt)+'.log'))
|
|
save(R/'acceptance.json',dict(status='PASS',tests=27,elapsed_seconds=float(re.search(r"Ran 27 tests in ([0-9.]+)s",log).group(1)),log_sha256=sha(R/'synthetic-attempt3.log'),test_source_sha256=sha(B/'tools/test_phase3_item1_postfit.py'),scope='CPU synthetic and isolated Git/file validator; no real targets, prediction, fits or CUDA workload',original18_pass_preserved=True))
|
|
save(R/'training-live-snapshot.json',read(T/'live-progress.json'))
|
|
save(R/'preparation-audit.json',dict(status='PREPARED_NOT_LAUNCHED',unix=time.time(),metadata_cpu_seconds=time.process_time()-start,training_implementation=tp['commit'],training_code_pins_verified=len(tc['code_sha256']),training_processes=live,training_complete_exists=(T/'complete.json').exists(),future_new_checkpoint_hashes_not_adopted=True,comparison_byte_identical=True,S1_prediction_hashes_checked=121,expected_prediction_entries=len(expected),model_entries=3265,new_model_entries=3144,reused_model_entries=121,clock_entries=1317,unique_parts=552,test_parts=121,new_fits=0,new_predictions=0,new_scores=0,test_target_values_read=False,old_score_values_read=False,training_untouched=True,new_review_pending=True,no_waiter_started=True))
|
|
freeze(R,'implementation',[R/'contract.json',R/'comparison-contract.json',R/'acceptance.json',R/'preparation-audit.json',R/'training-live-snapshot.json',R/'synthetic-attempt1.log',R/'synthetic-attempt2.log',R/'synthetic-attempt3.log',*[Path(p) for p in codes]])
|
|
proof=verify(R,'implementation');binding=dict(reviewed_commit=proof['commit'],contract_sha256=sha(R/'contract.json'),implementation_proof_sha256=sha(R/'implementation-git-freeze.json'),comparison_contract_sha256=c['comparison_contract_sha256'],code_sha256=codes);save(R/'expected-review-binding.json',binding)
|
|
freeze(R,'review-binding',[R/'contract.json',R/'expected-review-binding.json',R/'implementation-git-freeze.json',R/'implementation-authorization.json']);config(R)
|
|
print(json.dumps(dict(root=str(R),implementation=proof['commit'],contract_sha256=sha(R/'contract.json'),proof_sha256=sha(R/'implementation-git-freeze.json'),binding_sha256=sha(R/'expected-review-binding.json'),code_files=len(codes),review_pending=True,launched=False)))
|
|
if __name__=='__main__':main()
|