Files
MeltMind/tools/run_s3a_fit.py

69 lines
4.5 KiB
Python

"""One single-thread HPO trial or fresh final refit; launch only through the CPU guard."""
import argparse
import json
import os
from pathlib import Path
import time
import traceback
import joblib
import sklearn
from sklearn.ensemble import HistGradientBoostingRegressor
from threadpoolctl import threadpool_limits,threadpool_info
from freeze_g1_components import sha
from s2_recovery_supervisor import save
from s2_models.features import histgbrt_grid
from s3a_data import load_cell,learning_arrays,validation_score,columns
def run(root,index):
from s3a_runtime import contract
start=time.monotonic();cpu_start=time.process_time();c=contract(root);job=c['jobs'][index]
dest=root/'jobs'/f'job-{index:03d}'/'output';dest.mkdir(exist_ok=False)
save(dest/'started.json',dict(unix=time.time(),pid=os.getpid(),job=job,contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0))))
try:
assert job['arm'] in ('R2_E4','R2_G2')
assert sklearn.__version__=='1.7.2'
config_index=job.get('config_index');selection=None
if job['kind']=='FINAL':
path=root/'selections'/f"cell-{job['cell']:02d}.json";selection=json.loads(path.read_text())
assert selection['status']=='SELECTED' and selection['cell']==job['cell']
config_index=selection['config_index']
for item in selection['trials']:
assert sha(item['receipt_path'])==item['receipt_sha256']
save(dest/'selection.json',dict(path=str(path),sha256=sha(path),**selection))
params=histgbrt_grid()[config_index]
with threadpool_limits(limits=1):
pools=threadpool_info();assert all(p['num_threads']==1 for p in pools)
learning,validation,roles=load_cell(root,c,job)
save(dest/'access.json',dict(fold=job['fold'],pool=job['pool'],kind=job['kind'],
cases=[dict(case=p['case'],role=p['role'],label_mode=p['label_mode'],label_sha256=p['label_sha256']) for p in learning+validation],
outer_fold_test_labels_read=False,learning_only_final_refit=job['kind']=='FINAL'))
x,y,w=learning_arrays(learning);load_cpu=time.process_time()-cpu_start
save(dest/'progress.json',dict(stage='fit',training_queries=len(y),training_parts=len(learning),cpu_seconds=time.process_time()-cpu_start))
model=HistGradientBoostingRegressor(**params)
fit_start=time.process_time();model.fit(x,y,sample_weight=w);fit_cpu=time.process_time()-fit_start
assert model.n_iter_==params['max_iter']
value=errors=None
if job['kind']=='HPO':
save(dest/'progress.json',dict(stage='validation',fit_cpu_seconds=fit_cpu))
value,errors=validation_score(model,validation);save(dest/'validation-parts.json',errors)
joblib.dump(model,dest/'model.joblib',compress=3)
result=dict(status='COMPLETE',job=job,config_index=config_index,params=params,sklearn_version=sklearn.__version__,
columns=columns(job['arm']),target='T/t_set',learning_queries=len(y),learning_parts=len(learning),
learning_components=len({p['component_id'] for p in learning}),validation_parts=len(validation),
sample_weight_sum=float(w.sum()),validation_component_equal_relative_l2=value,fit_cpu_seconds=fit_cpu,
load_cpu_seconds=load_cpu,worker_elapsed_cpu_seconds=time.process_time()-cpu_start,wall_seconds=time.monotonic()-start,
threadpools=pools,cpu_affinity=sorted(os.sched_getaffinity(0)),model_sha256=sha(dest/'model.joblib'),
roster_sha256=c['roster_sha256'],test_predictions=False,test_labels_read=False,random_state=0,retry=False,
final_refit_learning_only=job['kind']=='FINAL',hpo_finished_configurations=selection['finished_configurations'] if selection else None)
save(dest/'result.json',result);save(dest/'progress.json',dict(stage='complete',cpu_seconds=time.process_time()-cpu_start))
print(json.dumps(result),flush=True);return 0
except Exception as exc:
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
cpu_seconds=time.process_time()-cpu_start,wall_seconds=time.monotonic()-start))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))