69 lines
4.5 KiB
Python
69 lines
4.5 KiB
Python
"""One single-thread HPO trial or fresh final refit; launch only through the CPU guard."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import time
|
|
import traceback
|
|
import joblib
|
|
import sklearn
|
|
from sklearn.ensemble import HistGradientBoostingRegressor
|
|
from threadpoolctl import threadpool_limits,threadpool_info
|
|
from freeze_g1_components import sha
|
|
from s2_recovery_supervisor import save
|
|
from s2_models.features import histgbrt_grid
|
|
from s3a_data import load_cell,learning_arrays,validation_score,columns
|
|
|
|
|
|
def run(root,index):
|
|
from s3a_runtime import contract
|
|
start=time.monotonic();cpu_start=time.process_time();c=contract(root);job=c['jobs'][index]
|
|
dest=root/'jobs'/f'job-{index:03d}'/'output';dest.mkdir(exist_ok=False)
|
|
save(dest/'started.json',dict(unix=time.time(),pid=os.getpid(),job=job,contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0))))
|
|
try:
|
|
assert job['arm'] in ('R2_E4','R2_G2')
|
|
assert sklearn.__version__=='1.7.2'
|
|
config_index=job.get('config_index');selection=None
|
|
if job['kind']=='FINAL':
|
|
path=root/'selections'/f"cell-{job['cell']:02d}.json";selection=json.loads(path.read_text())
|
|
assert selection['status']=='SELECTED' and selection['cell']==job['cell']
|
|
config_index=selection['config_index']
|
|
for item in selection['trials']:
|
|
assert sha(item['receipt_path'])==item['receipt_sha256']
|
|
save(dest/'selection.json',dict(path=str(path),sha256=sha(path),**selection))
|
|
params=histgbrt_grid()[config_index]
|
|
with threadpool_limits(limits=1):
|
|
pools=threadpool_info();assert all(p['num_threads']==1 for p in pools)
|
|
learning,validation,roles=load_cell(root,c,job)
|
|
save(dest/'access.json',dict(fold=job['fold'],pool=job['pool'],kind=job['kind'],
|
|
cases=[dict(case=p['case'],role=p['role'],label_mode=p['label_mode'],label_sha256=p['label_sha256']) for p in learning+validation],
|
|
outer_fold_test_labels_read=False,learning_only_final_refit=job['kind']=='FINAL'))
|
|
x,y,w=learning_arrays(learning);load_cpu=time.process_time()-cpu_start
|
|
save(dest/'progress.json',dict(stage='fit',training_queries=len(y),training_parts=len(learning),cpu_seconds=time.process_time()-cpu_start))
|
|
model=HistGradientBoostingRegressor(**params)
|
|
fit_start=time.process_time();model.fit(x,y,sample_weight=w);fit_cpu=time.process_time()-fit_start
|
|
assert model.n_iter_==params['max_iter']
|
|
value=errors=None
|
|
if job['kind']=='HPO':
|
|
save(dest/'progress.json',dict(stage='validation',fit_cpu_seconds=fit_cpu))
|
|
value,errors=validation_score(model,validation);save(dest/'validation-parts.json',errors)
|
|
joblib.dump(model,dest/'model.joblib',compress=3)
|
|
result=dict(status='COMPLETE',job=job,config_index=config_index,params=params,sklearn_version=sklearn.__version__,
|
|
columns=columns(job['arm']),target='T/t_set',learning_queries=len(y),learning_parts=len(learning),
|
|
learning_components=len({p['component_id'] for p in learning}),validation_parts=len(validation),
|
|
sample_weight_sum=float(w.sum()),validation_component_equal_relative_l2=value,fit_cpu_seconds=fit_cpu,
|
|
load_cpu_seconds=load_cpu,worker_elapsed_cpu_seconds=time.process_time()-cpu_start,wall_seconds=time.monotonic()-start,
|
|
threadpools=pools,cpu_affinity=sorted(os.sched_getaffinity(0)),model_sha256=sha(dest/'model.joblib'),
|
|
roster_sha256=c['roster_sha256'],test_predictions=False,test_labels_read=False,random_state=0,retry=False,
|
|
final_refit_learning_only=job['kind']=='FINAL',hpo_finished_configurations=selection['finished_configurations'] if selection else None)
|
|
save(dest/'result.json',result);save(dest/'progress.json',dict(stage='complete',cpu_seconds=time.process_time()-cpu_start))
|
|
print(json.dumps(result),flush=True);return 0
|
|
except Exception as exc:
|
|
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
|
|
cpu_seconds=time.process_time()-cpu_start,wall_seconds=time.monotonic()-start))
|
|
raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))
|