"""Wait for the sole approved recovery; freeze, predict and score once within its budget.""" import argparse import fcntl import json import os from pathlib import Path import shutil import subprocess import sys import time import psutil from freeze_g1_components import sha from s2_custody_common import read_json,checked from s2_recovery_supervisor import save from s3a_recovery_common import state from s3a_postfit import validate_terminal REPO=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8') BRANCH='refs/heads/codex/s2-execution-diff-20261114' def freeze_files(paths,message): # This checkout is the existing publication checkout, never the unborn root .git. status=subprocess.check_output(['git','status','--porcelain','--untracked-files=no'],cwd=REPO,text=True) if status.strip():raise RuntimeError('Publication checkout has unrelated tracked changes') for relative in paths: p=Path(relative);dest=REPO/p if p.is_dir():shutil.copytree(p,dest,dirs_exist_ok=True) else:dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(p,dest) subprocess.run(['git','add','-f','--',*[str(p) for p in paths]],cwd=REPO,check=True) subprocess.run(['git','diff','--cached','--check','--','.',':(exclude)*.log'],cwd=REPO,check=True) subprocess.run(['git','-c','gc.auto=0','commit','-q','-m',message],cwd=REPO,check=True) return subprocess.check_output(['git','rev-parse','HEAD'],cwd=REPO,text=True).strip() def run(root,chain): chain.mkdir(exist_ok=True) with (chain/'lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (chain/'started.json').exists():raise FileExistsError('Postfit chain already attempted') c,scope,old,passed=state(root) code=read_json(root/'postfit-code-freeze.json') for p,h in code['source_sha256'].items():checked(p,h) save(chain/'started.json',dict(unix=time.time(),pid=os.getpid(),root=str(root),postfit_code_freeze=code, no_fits_launched_by_this_chain=True,existing_recovery_only=True)) def mark(stage,**extra):save(chain/'progress.json',dict(stage=stage,unix=time.time(),**extra)) stage='WAIT_EXISTING_RECOVERY' try: mark(stage) while not (root/'batch-complete.json').exists():time.sleep(5) batch=read_json(root/'batch-complete.json') if batch['status']!='ALL_S3A_FITS_TERMINAL':raise RuntimeError('Recovery stopped incomplete; no retry or prediction') frozen=read_json(root/'terminal-freeze.json');validate_terminal(batch,frozen) start=read_json(root/'batch-started.json') # A finished receipt can precede the coordinator's last exit by a moment. for _ in range(24): if not psutil.pid_exists(start['pid']):break p=psutil.Process(start['pid']) if str(root) not in p.cmdline():raise RuntimeError('Coordinator PID reused') time.sleep(5) else:raise RuntimeError('Coordinator still alive after terminal receipt') for p in psutil.process_iter(['pid','cmdline']): cmd=p.info['cmdline'] or [] if str(root) in cmd and any(Path(v).name=='run_s3a_fit.py' for v in cmd): raise RuntimeError('Fit worker remains alive') stage='FREEZE_TERMINAL_MODELS';mark(stage) terminal_commit=freeze_files([str(root),scope['original_root']], 'Freeze complete two-source S3-A fits before test predictions') save(root/'terminal-git-freeze.json',dict(git_commit=terminal_commit, terminal_freeze_sha256=sha(root/'terminal-freeze.json'),before_predictions=True, remote_publication_separate=True)) stage='PREDICT_ONCE';mark(stage,terminal_commit=terminal_commit) env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') with (chain/'predict-launch.log').open('x') as log: subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'predict'],env=env, stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920) if read_json(root/'postfit-predict-guard.json')['worker']['status']!='PASS': raise RuntimeError('Prediction guard failed; no retry/scoring') written=read_json(root/'predictions-written.json') if written['status']!='ALL_S3A_PREDICTIONS_WRITTEN':raise RuntimeError('Predictions not complete') stage='FREEZE_PREDICTIONS';mark(stage) prediction_commit=freeze_files([str(root)],'Freeze S3-A predictions and exact manifest before scoring') save(root/'predictions-complete.json',dict(status='ALL_S3A_PREDICTIONS_FROZEN',git_commit=prediction_commit, contract_sha256=sha(root/'contract.json'),terminal_freeze_sha256=sha(root/'terminal-freeze.json'), prediction_index_sha256=sha(root/'prediction-index.json'),prediction_files=written['prediction_files'], no_new_label_decode=True,remote_publication_separate=True)) stage='SCORE_ONCE';mark(stage,prediction_commit=prediction_commit) with (chain/'score-launch.log').open('x') as log: subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'score'],env=env, stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920) guard=read_json(root/'postfit-score-guard.json') if guard['worker']['status']!='PASS':raise RuntimeError('Score guard failed; no retry') result=read_json(root/'scores/result.json') if result['status']!='S3A_SCORED':raise RuntimeError('Score result missing') for name,digest in result['output_sha256'].items():checked(root/'scores'/name,digest) save(chain/'scientific-complete.json',dict(status='S3A_SCORED',root=str(root), terminal_commit=terminal_commit,prediction_commit=prediction_commit, result_sha256=sha(root/'scores/result.json'),guard=guard)) stage='FREEZE_RESULTS';mark(stage) result_commit=freeze_files([str(root),str(chain)],'Preserve complete S3-A adaptive follow-up results and cumulative accounting') stage='WAIT_EXISTING_PUBLICATION';mark(stage,result_commit=result_commit) publication=Path('artifacts/publication-after-naive-2026-11-15/status.json') deadline=time.monotonic()+10800 while True: pub=read_json(publication) if pub['status']=='PUBLICATION_COMPLETE':break if pub['status']=='PUBLICATION_STOPPED':raise RuntimeError('Earlier publication stopped; preserve results locally') if time.monotonic()>deadline:raise TimeoutError('Earlier publication still pending') time.sleep(5) remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0] if remote!=pub['target']:raise RuntimeError('Concurrent remote branch change; do not overwrite') stage='PUBLISH_RESULTS';mark(stage,result_commit=result_commit) with (chain/'publication.log').open('x') as log: subprocess.run(['git','-c','pack.window=0','-c','pack.threads=4','-c','core.compression=1', 'push','origin',result_commit+':'+BRANCH],cwd=REPO,stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10800) remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0] if remote!=result_commit:raise RuntimeError('Publication readback differs') save(chain/'complete.json',dict(status='S3A_SCIENCE_AND_PUBLICATION_COMPLETE',result_commit=result_commit, result_sha256=sha(root/'scores/result.json'),cumulative_worker_CPU_s=guard['cumulative_worker_CPU_s'])) mark('COMPLETE',result_commit=result_commit) return 0 except Exception as exc: save(chain/'stopped.json',dict(stage=stage,error=repr(exc),unix=time.time(),no_automatic_retry=True)) mark('STOPPED',failed_stage=stage,error=repr(exc)) raise if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('chain',type=Path);a=p.parse_args() raise SystemExit(run(a.root,a.chain))