Files
MeltMind/tools/continue_s3a_postfit.py
T

133 lines
8.4 KiB
Python

"""Wait for the sole approved recovery; freeze, predict and score once within its budget."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
import psutil
from freeze_g1_components import sha
from s2_custody_common import read_json,checked
from s2_recovery_supervisor import save
from s3a_recovery_common import state
from s3a_postfit import validate_terminal
REPO=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
BRANCH='refs/heads/codex/s2-execution-diff-20261114'
def freeze_files(paths,message):
# This checkout is the existing publication checkout, never the unborn root .git.
status=subprocess.check_output(['git','status','--porcelain','--untracked-files=no'],cwd=REPO,text=True)
if status.strip():raise RuntimeError('Publication checkout has unrelated tracked changes')
for relative in paths:
p=Path(relative);dest=REPO/p
if p.is_dir():shutil.copytree(p,dest,dirs_exist_ok=True)
else:dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(p,dest)
subprocess.run(['git','add','-f','--',*[str(p) for p in paths]],cwd=REPO,check=True)
subprocess.run(['git','diff','--cached','--check','--','.',':(exclude)*.log'],cwd=REPO,check=True)
subprocess.run(['git','-c','gc.auto=0','commit','-q','-m',message],cwd=REPO,check=True)
return subprocess.check_output(['git','rev-parse','HEAD'],cwd=REPO,text=True).strip()
def run(root,chain):
chain.mkdir(exist_ok=True)
with (chain/'lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (chain/'started.json').exists():raise FileExistsError('Postfit chain already attempted')
c,scope,old,passed=state(root)
code=read_json(root/'postfit-code-freeze.json')
for p,h in code['source_sha256'].items():checked(p,h)
save(chain/'started.json',dict(unix=time.time(),pid=os.getpid(),root=str(root),postfit_code_freeze=code,
no_fits_launched_by_this_chain=True,existing_recovery_only=True))
def mark(stage,**extra):save(chain/'progress.json',dict(stage=stage,unix=time.time(),**extra))
stage='WAIT_EXISTING_RECOVERY'
try:
mark(stage)
while not (root/'batch-complete.json').exists():time.sleep(5)
batch=read_json(root/'batch-complete.json')
if batch['status']!='ALL_S3A_FITS_TERMINAL':raise RuntimeError('Recovery stopped incomplete; no retry or prediction')
frozen=read_json(root/'terminal-freeze.json');validate_terminal(batch,frozen)
start=read_json(root/'batch-started.json')
# A finished receipt can precede the coordinator's last exit by a moment.
for _ in range(24):
if not psutil.pid_exists(start['pid']):break
p=psutil.Process(start['pid'])
if str(root) not in p.cmdline():raise RuntimeError('Coordinator PID reused')
time.sleep(5)
else:raise RuntimeError('Coordinator still alive after terminal receipt')
for p in psutil.process_iter(['pid','cmdline']):
cmd=p.info['cmdline'] or []
if str(root) in cmd and any(Path(v).name=='run_s3a_fit.py' for v in cmd):
raise RuntimeError('Fit worker remains alive')
stage='FREEZE_TERMINAL_MODELS';mark(stage)
terminal_commit=freeze_files([str(root),scope['original_root']],
'Freeze complete two-source S3-A fits before test predictions')
save(root/'terminal-git-freeze.json',dict(git_commit=terminal_commit,
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),before_predictions=True,
remote_publication_separate=True))
stage='PREDICT_ONCE';mark(stage,terminal_commit=terminal_commit)
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',
NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
with (chain/'predict-launch.log').open('x') as log:
subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'predict'],env=env,
stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920)
if read_json(root/'postfit-predict-guard.json')['worker']['status']!='PASS':
raise RuntimeError('Prediction guard failed; no retry/scoring')
written=read_json(root/'predictions-written.json')
if written['status']!='ALL_S3A_PREDICTIONS_WRITTEN':raise RuntimeError('Predictions not complete')
stage='FREEZE_PREDICTIONS';mark(stage)
prediction_commit=freeze_files([str(root)],'Freeze S3-A predictions and exact manifest before scoring')
save(root/'predictions-complete.json',dict(status='ALL_S3A_PREDICTIONS_FROZEN',git_commit=prediction_commit,
contract_sha256=sha(root/'contract.json'),terminal_freeze_sha256=sha(root/'terminal-freeze.json'),
prediction_index_sha256=sha(root/'prediction-index.json'),prediction_files=written['prediction_files'],
no_new_label_decode=True,remote_publication_separate=True))
stage='SCORE_ONCE';mark(stage,prediction_commit=prediction_commit)
with (chain/'score-launch.log').open('x') as log:
subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'score'],env=env,
stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920)
guard=read_json(root/'postfit-score-guard.json')
if guard['worker']['status']!='PASS':raise RuntimeError('Score guard failed; no retry')
result=read_json(root/'scores/result.json')
if result['status']!='S3A_SCORED':raise RuntimeError('Score result missing')
for name,digest in result['output_sha256'].items():checked(root/'scores'/name,digest)
save(chain/'scientific-complete.json',dict(status='S3A_SCORED',root=str(root),
terminal_commit=terminal_commit,prediction_commit=prediction_commit,
result_sha256=sha(root/'scores/result.json'),guard=guard))
stage='FREEZE_RESULTS';mark(stage)
result_commit=freeze_files([str(root),str(chain)],'Preserve complete S3-A adaptive follow-up results and cumulative accounting')
stage='WAIT_EXISTING_PUBLICATION';mark(stage,result_commit=result_commit)
publication=Path('artifacts/publication-after-naive-2026-11-15/status.json')
deadline=time.monotonic()+10800
while True:
pub=read_json(publication)
if pub['status']=='PUBLICATION_COMPLETE':break
if pub['status']=='PUBLICATION_STOPPED':raise RuntimeError('Earlier publication stopped; preserve results locally')
if time.monotonic()>deadline:raise TimeoutError('Earlier publication still pending')
time.sleep(5)
remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0]
if remote!=pub['target']:raise RuntimeError('Concurrent remote branch change; do not overwrite')
stage='PUBLISH_RESULTS';mark(stage,result_commit=result_commit)
with (chain/'publication.log').open('x') as log:
subprocess.run(['git','-c','pack.window=0','-c','pack.threads=4','-c','core.compression=1',
'push','origin',result_commit+':'+BRANCH],cwd=REPO,stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10800)
remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0]
if remote!=result_commit:raise RuntimeError('Publication readback differs')
save(chain/'complete.json',dict(status='S3A_SCIENCE_AND_PUBLICATION_COMPLETE',result_commit=result_commit,
result_sha256=sha(root/'scores/result.json'),cumulative_worker_CPU_s=guard['cumulative_worker_CPU_s']))
mark('COMPLETE',result_commit=result_commit)
return 0
except Exception as exc:
save(chain/'stopped.json',dict(stage=stage,error=repr(exc),unix=time.time(),no_automatic_retry=True))
mark('STOPPED',failed_stage=stage,error=repr(exc))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('chain',type=Path);a=p.parse_args()
raise SystemExit(run(a.root,a.chain))