"""Bounded prediction then existing frozen scorer; no target IO during prediction.""" import argparse import os from pathlib import Path import resource import sys import threading import time import joblib import numpy as np import psutil from threadpoolctl import threadpool_limits from freeze_g1_components import sha from s2_custody_common import checked,read_json,prediction_inputs from s2_recovery_supervisor import save,supervise,stop_group from s2_recovery_monitoring import disk_usage from run_s3a_score import expected_predictions,run as score from s3a_postfit import gate,budget,predict_seconds def predict(root): c,scope,batch,frozen=gate(root) if (root/'predictions-started.json').exists() or (root/'predictions').exists(): raise FileExistsError('No repeat prediction phase') start=time.process_time() save(root/'predictions-started.json',dict(unix=time.time(),pid=os.getpid(), terminal_freeze_sha256=sha(root/'terminal-freeze.json'),postfit_code_freeze=read_json(root/'postfit-code-freeze.json'))) roster=read_json(checked(c['roster'],c['roster_sha256']))['parts'] expected=expected_predictions(root,c,frozen['final_fits'],roster) models={t['cell']:joblib.load(checked(t['model_path'],t['model_sha256'])) for t in frozen['final_fits'] if t['status']=='PASS'} by_case={} for path,(cell,row) in expected.items():by_case.setdefault(row['case'],[]).append((path,cell,row)) index={} with threadpool_limits(limits=1): for n,(case,items) in enumerate(sorted(by_case.items()),1): d=prediction_inputs(items[0][2]) for path,cell,row in items: values=predict_seconds(models[cell['index']],d,cell['arm']) p=Path(path);p.parent.mkdir(parents=True,exist_ok=True) if p.exists():raise FileExistsError('Duplicate prediction') np.savez_compressed(p,seconds=values,raw_ids=d['raw_ids'],counts=d['counts']) index[str(p)]=sha(p) save(root/'prediction-progress.json',dict(parts=n,predictions=len(index),worker_CPU_s=time.process_time()-start)) if set(index)!=set(expected):raise ValueError('Missing/extra predicted keys') save(root/'prediction-index.json',index) save(root/'predictions-written.json',dict(status='ALL_S3A_PREDICTIONS_WRITTEN',contract_sha256=sha(root/'contract.json'), terminal_freeze_sha256=sha(root/'terminal-freeze.json'),prediction_index_sha256=sha(root/'prediction-index.json'), prediction_files=len(index),test_parts=len(by_case),CPU_s=time.process_time()-start, new_label_decodes=0,test_labels_read=False,new_fits=0,finite_negative_predictions_retained=True)) return 0 def guarded(root,phase): c,scope,batch,frozen=gate(root) if phase=='predict': if (root/'postfit-started.json').exists():raise FileExistsError('No repeated postfit start') save(root/'postfit-started.json',dict(unix=time.time(),includes_interphase_idle_wall=True)) begun=read_json(root/'postfit-started.json')['unix'] remaining_wall=c['limits']['prediction_scoring_wall_seconds']-(time.time()-begun) if remaining_wall<=0:raise ValueError('Combined prediction/scoring wall cap exhausted') previous=0. if phase=='score': old=read_json(root/'postfit-predict-guard.json') if old['worker']['status']!='PASS':raise ValueError('Prediction guard failed') previous=old['worker']['cpu_seconds'] cap=budget(batch['worker_cpu_seconds'],previous,c['limits']) os.sched_setaffinity(0,[24]) command=[sys.executable,__file__,str(root),phase,'--worker'] done=threading.Event();reasons=[] def watch(): while not done.is_set(): try: if disk_usage(root)+disk_usage(Path(scope['original_root']))>c['limits']['disk_bytes']: reasons.append('CUMULATIVE_DISK_CAP') except Exception as exc:reasons.append('DISK_MONITOR_ERROR:'+repr(exc)) if reasons: for p in psutil.Process().children(recursive=True): try: if p.cmdline()==command:stop_group(p.pid) except psutil.NoSuchProcess:pass done.wait(1) thread=threading.Thread(target=watch,daemon=True);thread.start() try: w=supervise(command,root/'postfit-guards'/phase,'worker',dict(total_cpu_seconds=cap, job_cpu_seconds=cap,job_wall_seconds=remaining_wall, tree_rss_bytes=c['limits']['per_fit_tree_rss_bytes'],disk_bytes=c['limits']['disk_bytes'],max_processes=1),poll=.2) finally:done.set();thread.join(timeout=3) if reasons:w.update(status='STOPPED_OR_FAILED',stop_reason=reasons[0]) save(root/('postfit-'+phase+'-guard.json'),dict(worker=w,prior_fit_worker_CPU_s=batch['worker_cpu_seconds'], prior_postfit_worker_CPU_s=previous,postfit_worker_CPU_s=previous+w['cpu_seconds'], cumulative_worker_CPU_s=batch['worker_cpu_seconds']+previous+w['cpu_seconds'], total_CPU_cap=c['limits']['worker_cpu_seconds'],postfit_CPU_cap=c['limits']['postfit_reserved_cpu_seconds'], guard_CPU_s=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]))) return 0 if w['status']=='PASS' else 1 if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score']);p.add_argument('--worker',action='store_true');a=p.parse_args() if a.worker: if a.phase=='predict':raise SystemExit(predict(a.root)) gate(a.root);score(a.root,a.root) else:raise SystemExit(guarded(a.root,a.phase))