Files
MeltMind/tools/run_s3a_postfit.py

104 lines
5.5 KiB
Python

"""Bounded prediction then existing frozen scorer; no target IO during prediction."""
import argparse
import os
from pathlib import Path
import resource
import sys
import threading
import time
import joblib
import numpy as np
import psutil
from threadpoolctl import threadpool_limits
from freeze_g1_components import sha
from s2_custody_common import checked,read_json,prediction_inputs
from s2_recovery_supervisor import save,supervise,stop_group
from s2_recovery_monitoring import disk_usage
from run_s3a_score import expected_predictions,run as score
from s3a_postfit import gate,budget,predict_seconds
def predict(root):
c,scope,batch,frozen=gate(root)
if (root/'predictions-started.json').exists() or (root/'predictions').exists():
raise FileExistsError('No repeat prediction phase')
start=time.process_time()
save(root/'predictions-started.json',dict(unix=time.time(),pid=os.getpid(),
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),postfit_code_freeze=read_json(root/'postfit-code-freeze.json')))
roster=read_json(checked(c['roster'],c['roster_sha256']))['parts']
expected=expected_predictions(root,c,frozen['final_fits'],roster)
models={t['cell']:joblib.load(checked(t['model_path'],t['model_sha256']))
for t in frozen['final_fits'] if t['status']=='PASS'}
by_case={}
for path,(cell,row) in expected.items():by_case.setdefault(row['case'],[]).append((path,cell,row))
index={}
with threadpool_limits(limits=1):
for n,(case,items) in enumerate(sorted(by_case.items()),1):
d=prediction_inputs(items[0][2])
for path,cell,row in items:
values=predict_seconds(models[cell['index']],d,cell['arm'])
p=Path(path);p.parent.mkdir(parents=True,exist_ok=True)
if p.exists():raise FileExistsError('Duplicate prediction')
np.savez_compressed(p,seconds=values,raw_ids=d['raw_ids'],counts=d['counts'])
index[str(p)]=sha(p)
save(root/'prediction-progress.json',dict(parts=n,predictions=len(index),worker_CPU_s=time.process_time()-start))
if set(index)!=set(expected):raise ValueError('Missing/extra predicted keys')
save(root/'prediction-index.json',index)
save(root/'predictions-written.json',dict(status='ALL_S3A_PREDICTIONS_WRITTEN',contract_sha256=sha(root/'contract.json'),
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),prediction_index_sha256=sha(root/'prediction-index.json'),
prediction_files=len(index),test_parts=len(by_case),CPU_s=time.process_time()-start,
new_label_decodes=0,test_labels_read=False,new_fits=0,finite_negative_predictions_retained=True))
return 0
def guarded(root,phase):
c,scope,batch,frozen=gate(root)
if phase=='predict':
if (root/'postfit-started.json').exists():raise FileExistsError('No repeated postfit start')
save(root/'postfit-started.json',dict(unix=time.time(),includes_interphase_idle_wall=True))
begun=read_json(root/'postfit-started.json')['unix']
remaining_wall=c['limits']['prediction_scoring_wall_seconds']-(time.time()-begun)
if remaining_wall<=0:raise ValueError('Combined prediction/scoring wall cap exhausted')
previous=0.
if phase=='score':
old=read_json(root/'postfit-predict-guard.json')
if old['worker']['status']!='PASS':raise ValueError('Prediction guard failed')
previous=old['worker']['cpu_seconds']
cap=budget(batch['worker_cpu_seconds'],previous,c['limits'])
os.sched_setaffinity(0,[24])
command=[sys.executable,__file__,str(root),phase,'--worker']
done=threading.Event();reasons=[]
def watch():
while not done.is_set():
try:
if disk_usage(root)+disk_usage(Path(scope['original_root']))>c['limits']['disk_bytes']:
reasons.append('CUMULATIVE_DISK_CAP')
except Exception as exc:reasons.append('DISK_MONITOR_ERROR:'+repr(exc))
if reasons:
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:stop_group(p.pid)
except psutil.NoSuchProcess:pass
done.wait(1)
thread=threading.Thread(target=watch,daemon=True);thread.start()
try:
w=supervise(command,root/'postfit-guards'/phase,'worker',dict(total_cpu_seconds=cap,
job_cpu_seconds=cap,job_wall_seconds=remaining_wall,
tree_rss_bytes=c['limits']['per_fit_tree_rss_bytes'],disk_bytes=c['limits']['disk_bytes'],max_processes=1),poll=.2)
finally:done.set();thread.join(timeout=3)
if reasons:w.update(status='STOPPED_OR_FAILED',stop_reason=reasons[0])
save(root/('postfit-'+phase+'-guard.json'),dict(worker=w,prior_fit_worker_CPU_s=batch['worker_cpu_seconds'],
prior_postfit_worker_CPU_s=previous,postfit_worker_CPU_s=previous+w['cpu_seconds'],
cumulative_worker_CPU_s=batch['worker_cpu_seconds']+previous+w['cpu_seconds'],
total_CPU_cap=c['limits']['worker_cpu_seconds'],postfit_CPU_cap=c['limits']['postfit_reserved_cpu_seconds'],
guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
return 0 if w['status']=='PASS' else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score']);p.add_argument('--worker',action='store_true');a=p.parse_args()
if a.worker:
if a.phase=='predict':raise SystemExit(predict(a.root))
gate(a.root);score(a.root,a.root)
else:raise SystemExit(guarded(a.root,a.phase))