104 lines
5.5 KiB
Python
104 lines
5.5 KiB
Python
"""Bounded prediction then existing frozen scorer; no target IO during prediction."""
|
|
import argparse
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import sys
|
|
import threading
|
|
import time
|
|
import joblib
|
|
import numpy as np
|
|
import psutil
|
|
from threadpoolctl import threadpool_limits
|
|
from freeze_g1_components import sha
|
|
from s2_custody_common import checked,read_json,prediction_inputs
|
|
from s2_recovery_supervisor import save,supervise,stop_group
|
|
from s2_recovery_monitoring import disk_usage
|
|
from run_s3a_score import expected_predictions,run as score
|
|
from s3a_postfit import gate,budget,predict_seconds
|
|
|
|
|
|
def predict(root):
|
|
c,scope,batch,frozen=gate(root)
|
|
if (root/'predictions-started.json').exists() or (root/'predictions').exists():
|
|
raise FileExistsError('No repeat prediction phase')
|
|
start=time.process_time()
|
|
save(root/'predictions-started.json',dict(unix=time.time(),pid=os.getpid(),
|
|
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),postfit_code_freeze=read_json(root/'postfit-code-freeze.json')))
|
|
roster=read_json(checked(c['roster'],c['roster_sha256']))['parts']
|
|
expected=expected_predictions(root,c,frozen['final_fits'],roster)
|
|
models={t['cell']:joblib.load(checked(t['model_path'],t['model_sha256']))
|
|
for t in frozen['final_fits'] if t['status']=='PASS'}
|
|
by_case={}
|
|
for path,(cell,row) in expected.items():by_case.setdefault(row['case'],[]).append((path,cell,row))
|
|
index={}
|
|
with threadpool_limits(limits=1):
|
|
for n,(case,items) in enumerate(sorted(by_case.items()),1):
|
|
d=prediction_inputs(items[0][2])
|
|
for path,cell,row in items:
|
|
values=predict_seconds(models[cell['index']],d,cell['arm'])
|
|
p=Path(path);p.parent.mkdir(parents=True,exist_ok=True)
|
|
if p.exists():raise FileExistsError('Duplicate prediction')
|
|
np.savez_compressed(p,seconds=values,raw_ids=d['raw_ids'],counts=d['counts'])
|
|
index[str(p)]=sha(p)
|
|
save(root/'prediction-progress.json',dict(parts=n,predictions=len(index),worker_CPU_s=time.process_time()-start))
|
|
if set(index)!=set(expected):raise ValueError('Missing/extra predicted keys')
|
|
save(root/'prediction-index.json',index)
|
|
save(root/'predictions-written.json',dict(status='ALL_S3A_PREDICTIONS_WRITTEN',contract_sha256=sha(root/'contract.json'),
|
|
terminal_freeze_sha256=sha(root/'terminal-freeze.json'),prediction_index_sha256=sha(root/'prediction-index.json'),
|
|
prediction_files=len(index),test_parts=len(by_case),CPU_s=time.process_time()-start,
|
|
new_label_decodes=0,test_labels_read=False,new_fits=0,finite_negative_predictions_retained=True))
|
|
return 0
|
|
|
|
|
|
def guarded(root,phase):
|
|
c,scope,batch,frozen=gate(root)
|
|
if phase=='predict':
|
|
if (root/'postfit-started.json').exists():raise FileExistsError('No repeated postfit start')
|
|
save(root/'postfit-started.json',dict(unix=time.time(),includes_interphase_idle_wall=True))
|
|
begun=read_json(root/'postfit-started.json')['unix']
|
|
remaining_wall=c['limits']['prediction_scoring_wall_seconds']-(time.time()-begun)
|
|
if remaining_wall<=0:raise ValueError('Combined prediction/scoring wall cap exhausted')
|
|
previous=0.
|
|
if phase=='score':
|
|
old=read_json(root/'postfit-predict-guard.json')
|
|
if old['worker']['status']!='PASS':raise ValueError('Prediction guard failed')
|
|
previous=old['worker']['cpu_seconds']
|
|
cap=budget(batch['worker_cpu_seconds'],previous,c['limits'])
|
|
os.sched_setaffinity(0,[24])
|
|
command=[sys.executable,__file__,str(root),phase,'--worker']
|
|
done=threading.Event();reasons=[]
|
|
def watch():
|
|
while not done.is_set():
|
|
try:
|
|
if disk_usage(root)+disk_usage(Path(scope['original_root']))>c['limits']['disk_bytes']:
|
|
reasons.append('CUMULATIVE_DISK_CAP')
|
|
except Exception as exc:reasons.append('DISK_MONITOR_ERROR:'+repr(exc))
|
|
if reasons:
|
|
for p in psutil.Process().children(recursive=True):
|
|
try:
|
|
if p.cmdline()==command:stop_group(p.pid)
|
|
except psutil.NoSuchProcess:pass
|
|
done.wait(1)
|
|
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
|
try:
|
|
w=supervise(command,root/'postfit-guards'/phase,'worker',dict(total_cpu_seconds=cap,
|
|
job_cpu_seconds=cap,job_wall_seconds=remaining_wall,
|
|
tree_rss_bytes=c['limits']['per_fit_tree_rss_bytes'],disk_bytes=c['limits']['disk_bytes'],max_processes=1),poll=.2)
|
|
finally:done.set();thread.join(timeout=3)
|
|
if reasons:w.update(status='STOPPED_OR_FAILED',stop_reason=reasons[0])
|
|
save(root/('postfit-'+phase+'-guard.json'),dict(worker=w,prior_fit_worker_CPU_s=batch['worker_cpu_seconds'],
|
|
prior_postfit_worker_CPU_s=previous,postfit_worker_CPU_s=previous+w['cpu_seconds'],
|
|
cumulative_worker_CPU_s=batch['worker_cpu_seconds']+previous+w['cpu_seconds'],
|
|
total_CPU_cap=c['limits']['worker_cpu_seconds'],postfit_CPU_cap=c['limits']['postfit_reserved_cpu_seconds'],
|
|
guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
|
|
return 0 if w['status']=='PASS' else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score']);p.add_argument('--worker',action='store_true');a=p.parse_args()
|
|
if a.worker:
|
|
if a.phase=='predict':raise SystemExit(predict(a.root))
|
|
gate(a.root);score(a.root,a.root)
|
|
else:raise SystemExit(guarded(a.root,a.phase))
|