"""Final-policy replacement for the retired idle waiter; original fit/postfit sources unchanged.""" import argparse import fcntl import json import os from pathlib import Path import shutil import subprocess import sys import time import psutil from freeze_g1_components import sha from s2_custody_common import read_json,checked from s2_recovery_supervisor import save from s3a_recovery_common import state from s3a_postfit import validate_terminal from s3a_final_process_policy import guard_final_policy REPO=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8') BRANCH='refs/heads/codex/s2-execution-diff-20261114' def freeze_files(paths,message): # This checkout is the existing publication checkout, never the unborn root .git. status=subprocess.check_output(['git','status','--porcelain','--untracked-files=no'],cwd=REPO,text=True) if status.strip():raise RuntimeError('Publication checkout has unrelated tracked changes') for relative in paths: p=Path(relative);dest=REPO/p if p.is_dir():shutil.copytree(p,dest,dirs_exist_ok=True) else:dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(p,dest) subprocess.run(['git','add','-f','--',*[str(p) for p in paths]],cwd=REPO,check=True) subprocess.run(['git','diff','--cached','--check','--','.',':(exclude)*.log'],cwd=REPO,check=True) subprocess.run(['git','-c','gc.auto=0','commit','-q','-m',message],cwd=REPO,check=True) return subprocess.check_output(['git','rev-parse','HEAD'],cwd=REPO,text=True).strip() def run(root,chain): chain.mkdir(exist_ok=True) with (chain/'lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (chain/'started.json').exists():raise FileExistsError('Postfit chain already attempted') c,scope,old,passed=state(root) policy=read_json(chain/'policy-code-freeze.json') if not policy.get('git_commit'):raise ValueError('Final policy must be Git fixed') for p,h in policy['source_sha256'].items():checked(p,h) retired=read_json(Path('artifacts/s3a-postfit-chain-2026-11-15/retirement-for-final-policy.json')) if not retired['exit_verified'] or psutil.pid_exists(retired['retired_pid']):raise RuntimeError('Old waiter still active') code=read_json(root/'postfit-code-freeze.json') for p,h in code['source_sha256'].items():checked(p,h) save(chain/'started.json',dict(unix=time.time(),pid=os.getpid(),root=str(root),postfit_code_freeze=code,policy_code_freeze=policy, no_fits_launched_by_this_chain=True,existing_recovery_only=True)) def mark(stage,**extra):save(chain/'progress.json',dict(stage=stage,unix=time.time(),**extra)) stage='WAIT_EXISTING_RECOVERY' try: mark(stage) while not (root/'batch-complete.json').exists(): guard_final_policy(root,chain,c) time.sleep(5) guard_final_policy(root,chain,c) batch=read_json(root/'batch-complete.json') if batch['status']!='ALL_S3A_FITS_TERMINAL':raise RuntimeError('Recovery stopped incomplete; no retry or prediction') frozen=read_json(root/'terminal-freeze.json');validate_terminal(batch,frozen) start=read_json(root/'batch-started.json') # A finished receipt can precede the coordinator's last exit by a moment. for _ in range(24): if not psutil.pid_exists(start['pid']):break p=psutil.Process(start['pid']) if str(root) not in p.cmdline():raise RuntimeError('Coordinator PID reused') time.sleep(5) else:raise RuntimeError('Coordinator still alive after terminal receipt') for p in psutil.process_iter(['pid','cmdline']): cmd=p.info['cmdline'] or [] if str(root) in cmd and any(Path(v).name=='run_s3a_fit.py' for v in cmd): raise RuntimeError('Fit worker remains alive') guard_final_policy(root,chain,c) stage='FREEZE_TERMINAL_MODELS';mark(stage) terminal_commit=freeze_files([str(root),scope['original_root']], 'Freeze complete two-source S3-A fits before test predictions') save(root/'terminal-git-freeze.json',dict(git_commit=terminal_commit, terminal_freeze_sha256=sha(root/'terminal-freeze.json'),before_predictions=True, remote_publication_separate=True)) guard_final_policy(root,chain,c) stage='PREDICT_ONCE';mark(stage,terminal_commit=terminal_commit) env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') with (chain/'predict-launch.log').open('x') as log: subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'predict'],env=env, stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920) if read_json(root/'postfit-predict-guard.json')['worker']['status']!='PASS': raise RuntimeError('Prediction guard failed; no retry/scoring') written=read_json(root/'predictions-written.json') if written['status']!='ALL_S3A_PREDICTIONS_WRITTEN':raise RuntimeError('Predictions not complete') stage='FREEZE_PREDICTIONS';mark(stage) prediction_commit=freeze_files([str(root)],'Freeze S3-A predictions and exact manifest before scoring') save(root/'predictions-complete.json',dict(status='ALL_S3A_PREDICTIONS_FROZEN',git_commit=prediction_commit, contract_sha256=sha(root/'contract.json'),terminal_freeze_sha256=sha(root/'terminal-freeze.json'), prediction_index_sha256=sha(root/'prediction-index.json'),prediction_files=written['prediction_files'], no_new_label_decode=True,remote_publication_separate=True)) guard_final_policy(root,chain,c) stage='SCORE_ONCE';mark(stage,prediction_commit=prediction_commit) with (chain/'score-launch.log').open('x') as log: subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'score'],env=env, stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920) guard=read_json(root/'postfit-score-guard.json') if guard['worker']['status']!='PASS':raise RuntimeError('Score guard failed; no retry') result=read_json(root/'scores/result.json') if result['status']!='S3A_SCORED':raise RuntimeError('Score result missing') for name,digest in result['output_sha256'].items():checked(root/'scores'/name,digest) save(chain/'scientific-complete.json',dict(status='S3A_SCORED',root=str(root), terminal_commit=terminal_commit,prediction_commit=prediction_commit, result_sha256=sha(root/'scores/result.json'),guard=guard)) stage='FREEZE_RESULTS';mark(stage) result_commit=freeze_files([str(root),str(chain)],'Preserve complete S3-A adaptive follow-up results and cumulative accounting') stage='WAIT_EXISTING_PUBLICATION';mark(stage,result_commit=result_commit) publication=Path('artifacts/publication-after-naive-2026-11-15/status.json') deadline=time.monotonic()+10800 while True: pub=read_json(publication) if pub['status']=='PUBLICATION_COMPLETE':break if pub['status']=='PUBLICATION_STOPPED':raise RuntimeError('Earlier publication stopped; preserve results locally') if time.monotonic()>deadline:raise TimeoutError('Earlier publication still pending') time.sleep(5) remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0] if remote!=policy['remote_base_for_results']:raise RuntimeError('Concurrent remote branch change; do not overwrite') stage='PUBLISH_RESULTS';mark(stage,result_commit=result_commit) with (chain/'publication.log').open('x') as log: subprocess.run(['git','-c','pack.window=0','-c','pack.threads=4','-c','core.compression=1', 'push','origin',result_commit+':'+BRANCH],cwd=REPO,stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10800) remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0] if remote!=result_commit:raise RuntimeError('Publication readback differs') save(chain/'complete.json',dict(status='S3A_SCIENCE_AND_PUBLICATION_COMPLETE',result_commit=result_commit, result_sha256=sha(root/'scores/result.json'),cumulative_worker_CPU_s=guard['cumulative_worker_CPU_s'])) mark('COMPLETE',result_commit=result_commit) return 0 except Exception as exc: save(chain/'stopped.json',dict(stage=stage,error=repr(exc),unix=time.time(),no_automatic_retry=True)) mark('STOPPED',failed_stage=stage,error=repr(exc)) raise if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('chain',type=Path);a=p.parse_args() raise SystemExit(run(a.root,a.chain))