Files
MeltMind/tools/continue_s3a_final_policy_postfit.py
T

145 lines
9.1 KiB
Python

"""Final-policy replacement for the retired idle waiter; original fit/postfit sources unchanged."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import time
import psutil
from freeze_g1_components import sha
from s2_custody_common import read_json,checked
from s2_recovery_supervisor import save
from s3a_recovery_common import state
from s3a_postfit import validate_terminal
from s3a_final_process_policy import guard_final_policy
REPO=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
BRANCH='refs/heads/codex/s2-execution-diff-20261114'
def freeze_files(paths,message):
# This checkout is the existing publication checkout, never the unborn root .git.
status=subprocess.check_output(['git','status','--porcelain','--untracked-files=no'],cwd=REPO,text=True)
if status.strip():raise RuntimeError('Publication checkout has unrelated tracked changes')
for relative in paths:
p=Path(relative);dest=REPO/p
if p.is_dir():shutil.copytree(p,dest,dirs_exist_ok=True)
else:dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(p,dest)
subprocess.run(['git','add','-f','--',*[str(p) for p in paths]],cwd=REPO,check=True)
subprocess.run(['git','diff','--cached','--check','--','.',':(exclude)*.log'],cwd=REPO,check=True)
subprocess.run(['git','-c','gc.auto=0','commit','-q','-m',message],cwd=REPO,check=True)
return subprocess.check_output(['git','rev-parse','HEAD'],cwd=REPO,text=True).strip()
def run(root,chain):
chain.mkdir(exist_ok=True)
with (chain/'lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (chain/'started.json').exists():raise FileExistsError('Postfit chain already attempted')
c,scope,old,passed=state(root)
policy=read_json(chain/'policy-code-freeze.json')
if not policy.get('git_commit'):raise ValueError('Final policy must be Git fixed')
for p,h in policy['source_sha256'].items():checked(p,h)
retired=read_json(Path('artifacts/s3a-postfit-chain-2026-11-15/retirement-for-final-policy.json'))
if not retired['exit_verified'] or psutil.pid_exists(retired['retired_pid']):raise RuntimeError('Old waiter still active')
code=read_json(root/'postfit-code-freeze.json')
for p,h in code['source_sha256'].items():checked(p,h)
save(chain/'started.json',dict(unix=time.time(),pid=os.getpid(),root=str(root),postfit_code_freeze=code,policy_code_freeze=policy,
no_fits_launched_by_this_chain=True,existing_recovery_only=True))
def mark(stage,**extra):save(chain/'progress.json',dict(stage=stage,unix=time.time(),**extra))
stage='WAIT_EXISTING_RECOVERY'
try:
mark(stage)
while not (root/'batch-complete.json').exists():
guard_final_policy(root,chain,c)
time.sleep(5)
guard_final_policy(root,chain,c)
batch=read_json(root/'batch-complete.json')
if batch['status']!='ALL_S3A_FITS_TERMINAL':raise RuntimeError('Recovery stopped incomplete; no retry or prediction')
frozen=read_json(root/'terminal-freeze.json');validate_terminal(batch,frozen)
start=read_json(root/'batch-started.json')
# A finished receipt can precede the coordinator's last exit by a moment.
for _ in range(24):
if not psutil.pid_exists(start['pid']):break
p=psutil.Process(start['pid'])
if str(root) not in p.cmdline():raise RuntimeError('Coordinator PID reused')
time.sleep(5)
else:raise RuntimeError('Coordinator still alive after terminal receipt')
for p in psutil.process_iter(['pid','cmdline']):
cmd=p.info['cmdline'] or []
if str(root) in cmd and any(Path(v).name=='run_s3a_fit.py' for v in cmd):
raise RuntimeError('Fit worker remains alive')
guard_final_policy(root,chain,c)
stage='FREEZE_TERMINAL_MODELS';mark(stage)
terminal_commit=freeze_files([str(root),scope['original_root']],
'Freeze complete two-source S3-A fits before test predictions')
save(root/'terminal-git-freeze.json',dict(git_commit=terminal_commit,
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),before_predictions=True,
remote_publication_separate=True))
guard_final_policy(root,chain,c)
stage='PREDICT_ONCE';mark(stage,terminal_commit=terminal_commit)
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',
NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
with (chain/'predict-launch.log').open('x') as log:
subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'predict'],env=env,
stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920)
if read_json(root/'postfit-predict-guard.json')['worker']['status']!='PASS':
raise RuntimeError('Prediction guard failed; no retry/scoring')
written=read_json(root/'predictions-written.json')
if written['status']!='ALL_S3A_PREDICTIONS_WRITTEN':raise RuntimeError('Predictions not complete')
stage='FREEZE_PREDICTIONS';mark(stage)
prediction_commit=freeze_files([str(root)],'Freeze S3-A predictions and exact manifest before scoring')
save(root/'predictions-complete.json',dict(status='ALL_S3A_PREDICTIONS_FROZEN',git_commit=prediction_commit,
contract_sha256=sha(root/'contract.json'),terminal_freeze_sha256=sha(root/'terminal-freeze.json'),
prediction_index_sha256=sha(root/'prediction-index.json'),prediction_files=written['prediction_files'],
no_new_label_decode=True,remote_publication_separate=True))
guard_final_policy(root,chain,c)
stage='SCORE_ONCE';mark(stage,prediction_commit=prediction_commit)
with (chain/'score-launch.log').open('x') as log:
subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'score'],env=env,
stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920)
guard=read_json(root/'postfit-score-guard.json')
if guard['worker']['status']!='PASS':raise RuntimeError('Score guard failed; no retry')
result=read_json(root/'scores/result.json')
if result['status']!='S3A_SCORED':raise RuntimeError('Score result missing')
for name,digest in result['output_sha256'].items():checked(root/'scores'/name,digest)
save(chain/'scientific-complete.json',dict(status='S3A_SCORED',root=str(root),
terminal_commit=terminal_commit,prediction_commit=prediction_commit,
result_sha256=sha(root/'scores/result.json'),guard=guard))
stage='FREEZE_RESULTS';mark(stage)
result_commit=freeze_files([str(root),str(chain)],'Preserve complete S3-A adaptive follow-up results and cumulative accounting')
stage='WAIT_EXISTING_PUBLICATION';mark(stage,result_commit=result_commit)
publication=Path('artifacts/publication-after-naive-2026-11-15/status.json')
deadline=time.monotonic()+10800
while True:
pub=read_json(publication)
if pub['status']=='PUBLICATION_COMPLETE':break
if pub['status']=='PUBLICATION_STOPPED':raise RuntimeError('Earlier publication stopped; preserve results locally')
if time.monotonic()>deadline:raise TimeoutError('Earlier publication still pending')
time.sleep(5)
remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0]
if remote!=policy['remote_base_for_results']:raise RuntimeError('Concurrent remote branch change; do not overwrite')
stage='PUBLISH_RESULTS';mark(stage,result_commit=result_commit)
with (chain/'publication.log').open('x') as log:
subprocess.run(['git','-c','pack.window=0','-c','pack.threads=4','-c','core.compression=1',
'push','origin',result_commit+':'+BRANCH],cwd=REPO,stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10800)
remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0]
if remote!=result_commit:raise RuntimeError('Publication readback differs')
save(chain/'complete.json',dict(status='S3A_SCIENCE_AND_PUBLICATION_COMPLETE',result_commit=result_commit,
result_sha256=sha(root/'scores/result.json'),cumulative_worker_CPU_s=guard['cumulative_worker_CPU_s']))
mark('COMPLETE',result_commit=result_commit)
return 0
except Exception as exc:
save(chain/'stopped.json',dict(stage=stage,error=repr(exc),unix=time.time(),no_automatic_retry=True))
mark('STOPPED',failed_stage=stage,error=repr(exc))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('chain',type=Path);a=p.parse_args()
raise SystemExit(run(a.root,a.chain))