133 lines
8.4 KiB
Python
133 lines
8.4 KiB
Python
"""Wait for the sole approved recovery; freeze, predict and score once within its budget."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import psutil
|
|
from freeze_g1_components import sha
|
|
from s2_custody_common import read_json,checked
|
|
from s2_recovery_supervisor import save
|
|
from s3a_recovery_common import state
|
|
from s3a_postfit import validate_terminal
|
|
|
|
|
|
REPO=Path('/tmp/meltmind-git-review-20261114-v1-29ebglg8')
|
|
BRANCH='refs/heads/codex/s2-execution-diff-20261114'
|
|
|
|
|
|
def freeze_files(paths,message):
|
|
# This checkout is the existing publication checkout, never the unborn root .git.
|
|
status=subprocess.check_output(['git','status','--porcelain','--untracked-files=no'],cwd=REPO,text=True)
|
|
if status.strip():raise RuntimeError('Publication checkout has unrelated tracked changes')
|
|
for relative in paths:
|
|
p=Path(relative);dest=REPO/p
|
|
if p.is_dir():shutil.copytree(p,dest,dirs_exist_ok=True)
|
|
else:dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(p,dest)
|
|
subprocess.run(['git','add','-f','--',*[str(p) for p in paths]],cwd=REPO,check=True)
|
|
subprocess.run(['git','diff','--cached','--check','--','.',':(exclude)*.log'],cwd=REPO,check=True)
|
|
subprocess.run(['git','-c','gc.auto=0','commit','-q','-m',message],cwd=REPO,check=True)
|
|
return subprocess.check_output(['git','rev-parse','HEAD'],cwd=REPO,text=True).strip()
|
|
|
|
|
|
def run(root,chain):
|
|
chain.mkdir(exist_ok=True)
|
|
with (chain/'lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (chain/'started.json').exists():raise FileExistsError('Postfit chain already attempted')
|
|
c,scope,old,passed=state(root)
|
|
code=read_json(root/'postfit-code-freeze.json')
|
|
for p,h in code['source_sha256'].items():checked(p,h)
|
|
save(chain/'started.json',dict(unix=time.time(),pid=os.getpid(),root=str(root),postfit_code_freeze=code,
|
|
no_fits_launched_by_this_chain=True,existing_recovery_only=True))
|
|
def mark(stage,**extra):save(chain/'progress.json',dict(stage=stage,unix=time.time(),**extra))
|
|
stage='WAIT_EXISTING_RECOVERY'
|
|
try:
|
|
mark(stage)
|
|
while not (root/'batch-complete.json').exists():time.sleep(5)
|
|
batch=read_json(root/'batch-complete.json')
|
|
if batch['status']!='ALL_S3A_FITS_TERMINAL':raise RuntimeError('Recovery stopped incomplete; no retry or prediction')
|
|
frozen=read_json(root/'terminal-freeze.json');validate_terminal(batch,frozen)
|
|
start=read_json(root/'batch-started.json')
|
|
# A finished receipt can precede the coordinator's last exit by a moment.
|
|
for _ in range(24):
|
|
if not psutil.pid_exists(start['pid']):break
|
|
p=psutil.Process(start['pid'])
|
|
if str(root) not in p.cmdline():raise RuntimeError('Coordinator PID reused')
|
|
time.sleep(5)
|
|
else:raise RuntimeError('Coordinator still alive after terminal receipt')
|
|
for p in psutil.process_iter(['pid','cmdline']):
|
|
cmd=p.info['cmdline'] or []
|
|
if str(root) in cmd and any(Path(v).name=='run_s3a_fit.py' for v in cmd):
|
|
raise RuntimeError('Fit worker remains alive')
|
|
stage='FREEZE_TERMINAL_MODELS';mark(stage)
|
|
terminal_commit=freeze_files([str(root),scope['original_root']],
|
|
'Freeze complete two-source S3-A fits before test predictions')
|
|
save(root/'terminal-git-freeze.json',dict(git_commit=terminal_commit,
|
|
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),before_predictions=True,
|
|
remote_publication_separate=True))
|
|
stage='PREDICT_ONCE';mark(stage,terminal_commit=terminal_commit)
|
|
env=dict(os.environ,OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',
|
|
NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
|
with (chain/'predict-launch.log').open('x') as log:
|
|
subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'predict'],env=env,
|
|
stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920)
|
|
if read_json(root/'postfit-predict-guard.json')['worker']['status']!='PASS':
|
|
raise RuntimeError('Prediction guard failed; no retry/scoring')
|
|
written=read_json(root/'predictions-written.json')
|
|
if written['status']!='ALL_S3A_PREDICTIONS_WRITTEN':raise RuntimeError('Predictions not complete')
|
|
stage='FREEZE_PREDICTIONS';mark(stage)
|
|
prediction_commit=freeze_files([str(root)],'Freeze S3-A predictions and exact manifest before scoring')
|
|
save(root/'predictions-complete.json',dict(status='ALL_S3A_PREDICTIONS_FROZEN',git_commit=prediction_commit,
|
|
contract_sha256=sha(root/'contract.json'),terminal_freeze_sha256=sha(root/'terminal-freeze.json'),
|
|
prediction_index_sha256=sha(root/'prediction-index.json'),prediction_files=written['prediction_files'],
|
|
no_new_label_decode=True,remote_publication_separate=True))
|
|
stage='SCORE_ONCE';mark(stage,prediction_commit=prediction_commit)
|
|
with (chain/'score-launch.log').open('x') as log:
|
|
subprocess.run([sys.executable,'tools/run_s3a_postfit.py',str(root),'score'],env=env,
|
|
stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10920)
|
|
guard=read_json(root/'postfit-score-guard.json')
|
|
if guard['worker']['status']!='PASS':raise RuntimeError('Score guard failed; no retry')
|
|
result=read_json(root/'scores/result.json')
|
|
if result['status']!='S3A_SCORED':raise RuntimeError('Score result missing')
|
|
for name,digest in result['output_sha256'].items():checked(root/'scores'/name,digest)
|
|
save(chain/'scientific-complete.json',dict(status='S3A_SCORED',root=str(root),
|
|
terminal_commit=terminal_commit,prediction_commit=prediction_commit,
|
|
result_sha256=sha(root/'scores/result.json'),guard=guard))
|
|
stage='FREEZE_RESULTS';mark(stage)
|
|
result_commit=freeze_files([str(root),str(chain)],'Preserve complete S3-A adaptive follow-up results and cumulative accounting')
|
|
stage='WAIT_EXISTING_PUBLICATION';mark(stage,result_commit=result_commit)
|
|
publication=Path('artifacts/publication-after-naive-2026-11-15/status.json')
|
|
deadline=time.monotonic()+10800
|
|
while True:
|
|
pub=read_json(publication)
|
|
if pub['status']=='PUBLICATION_COMPLETE':break
|
|
if pub['status']=='PUBLICATION_STOPPED':raise RuntimeError('Earlier publication stopped; preserve results locally')
|
|
if time.monotonic()>deadline:raise TimeoutError('Earlier publication still pending')
|
|
time.sleep(5)
|
|
remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0]
|
|
if remote!=pub['target']:raise RuntimeError('Concurrent remote branch change; do not overwrite')
|
|
stage='PUBLISH_RESULTS';mark(stage,result_commit=result_commit)
|
|
with (chain/'publication.log').open('x') as log:
|
|
subprocess.run(['git','-c','pack.window=0','-c','pack.threads=4','-c','core.compression=1',
|
|
'push','origin',result_commit+':'+BRANCH],cwd=REPO,stdout=log,stderr=subprocess.STDOUT,check=True,timeout=10800)
|
|
remote=subprocess.check_output(['git','ls-remote','origin',BRANCH],cwd=REPO,text=True,timeout=60).split()[0]
|
|
if remote!=result_commit:raise RuntimeError('Publication readback differs')
|
|
save(chain/'complete.json',dict(status='S3A_SCIENCE_AND_PUBLICATION_COMPLETE',result_commit=result_commit,
|
|
result_sha256=sha(root/'scores/result.json'),cumulative_worker_CPU_s=guard['cumulative_worker_CPU_s']))
|
|
mark('COMPLETE',result_commit=result_commit)
|
|
return 0
|
|
except Exception as exc:
|
|
save(chain/'stopped.json',dict(stage=stage,error=repr(exc),unix=time.time(),no_automatic_retry=True))
|
|
mark('STOPPED',failed_stage=stage,error=repr(exc))
|
|
raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('chain',type=Path);a=p.parse_args()
|
|
raise SystemExit(run(a.root,a.chain))
|