Implement independent R2 HPO and learning-only refits with first-fit cost gate

This commit is contained in:
Frank14f committed 2026-11-15 09:53:18 +08:00
1 parent 5034de7377
commit f560a3d5e1
10 files changed
+9624

No files matched your search

@@ -0,0 +1,26 @@
{
"status": "PASS_NOT_STARTED",
"tests_passed": 10,
"synthetic_fit": true,
"real_corpus_fit": false,
"contract_sha256": "edec86197769f44bf98f234795950f36c9fa42084ccb3c68448b63b440aa126e",
"roster_sha256": "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e",
"test_log_sha256": "3ebf40b5c8644fa7ad86944e9872e5c0e491b46ad04c77e8c27f2be6f7445f6b",
"sklearn_version": "1.7.2",
"single_thread_libraries_tested": [
"OpenBLAS",
"MKL",
"OpenMP"
],
"final_refit": "same_learning_pool_only",
"failed_HPO": "select_best_finished_configuration",
"ruling": "1794707190.141609",
"neural_code_pins_still_match": true,
"first_fit_only_before_cost_check": true,
"worker_cpus": [
8,
10,
12,
14
]
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,19 @@
{
"status": "FROZEN_NOT_STARTED",
"contract_sha256": "edec86197769f44bf98f234795950f36c9fa42084ccb3c68448b63b440aa126e",
"roster_sha256": "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e",
"HPO_trials": 180,
"final_fits": 15,
"worker_cpus": [
8,
10,
12,
14
],
"coordinator_cpus": [
30,
31
],
"labels_decoded": false,
"fits_started": false
}
@@ -0,0 +1,16 @@
test_actual_synthetic_histgbrt_fit_single_thread_and_fixed_params (__main__.R2Checks) ... {"status": "COMPLETE", "job": {"index": 0, "cell": 0, "fold": 0, "pool": 25, "kind": "HPO", "config_index": 0, "cost_units": 1, "cost_queries": 1}, "config_index": 0, "params": {"learning_rate": 0.05, "max_leaf_nodes": 31, "max_iter": 200, "min_samples_leaf": 20, "random_state": 0, "early_stopping": false}, "sklearn_version": "1.7.2", "columns": ["Vf", "grad_Vf_x", "grad_Vf_y", "grad_Vf_z", "Di_star", "grad_Di_x", "grad_Di_y", "grad_Di_z", "Dw", "Dw_star", "grad_Dw_x", "grad_Dw_y", "grad_Dw_z", "Sxx", "Sxy", "Sxz", "Syy", "Syz", "Szz", "t_set"], "target": "T/t_set", "learning_queries": 240, "learning_parts": 1, "learning_components": 1, "validation_parts": 1, "sample_weight_sum": 0.9999999999999999, "validation_component_equal_relative_l2": 0.014707062211418275, "fit_cpu_seconds": 0.25818808699999996, "load_cpu_seconds": 0.045540662999999926, "worker_elapsed_cpu_seconds": 0.38005410100000003, "wall_seconds": 0.3883439999772236, "threadpools": [{"user_api": "blas", "internal_api": "openblas", "num_threads": 1, "prefix": "libopenblas", "filepath": "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/site-packages/numpy.libs/libopenblas64_p-r0-0cf96a72.3.23.dev.so", "version": "0.3.23.dev", "threading_layer": "pthreads", "architecture": "Zen"}, {"user_api": "blas", "internal_api": "mkl", "num_threads": 1, "prefix": "libmkl_rt", "filepath": "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/libmkl_rt.so.2", "version": "2023.1-Product", "threading_layer": "gnu"}, {"user_api": "openmp", "internal_api": "openmp", "num_threads": 1, "prefix": "libgomp", "filepath": "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/libgomp.so.1.0.0", "version": null}], "cpu_affinity": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "model_sha256": "7536654c21366696414ec6f7c2b36828fe3967043f7d5e63b3bad3b19b408f6d", "roster_sha256": "fixed", "test_predictions": false, "test_labels_read": false, "random_state": 0, "retry": false, "final_refit_learning_only": false, "hpo_finished_configurations": null}
ok
test_final_loader_never_reads_validation (__main__.R2Checks) ... ok
test_first_fit_projection_and_failed_attempt_cost_bound (__main__.R2Checks) ... ok
test_grid_plan_has_180_unique_trials_15_final_and_real_roster (__main__.R2Checks) ... ok
test_guard_uses_cpu_cap_and_disjoint_affinity (__main__.R2Checks) ... ok
test_queue_first_fit_barrier_and_unique_195_attempts (__main__.R2Checks) ... ok
test_select_finished_configures_failure_and_tie_policy (__main__.R2Checks) ... ok
test_test_case_rejected_before_feature_or_target_io (__main__.R2Checks) ... ok
test_validation_is_component_equal_and_seconds_scale_correct (__main__.R2Checks) ... ok
test_weights_match_frozen_hierarchy_without_rescaling (__main__.R2Checks) ... ok
----------------------------------------------------------------------
Ran 10 tests in 1.707s
OK
+13
View File
@@ -0,0 +1,13 @@
# S2 R2-G24 独立执行合同
沿Claude1794699027.366469预算及1794707190.141609的最终裁量实现,不改运行中的神经源码。共5折×3组件比例池,每格12个固定HPO配置及一个final refit,即最多180 HPO+15 final。绑定617件统一roster,原七件准备失败在所有折一致排除。该限制必须在最终报告列明:共708645/18661224节点(约3.8%),四件属fold0 test;测试集缺少这些准备昂贵的大件。
固定v0.5几何映射与v0.6目标:复用已完成的20列R2-G24特征及相同合格query IDs/重复次数,不重做几何;直接预测T/t_set,秒值乘回t_set,不clip、不log变换。sklearn1.7.2 HistGradientBoostingRegressor;网格来自原固定histgbrt_grid:lr .05/.1 × max_leaf_nodes31/63 × max_iter200/500/1000,min_samples_leaf20、seed0、early_stopping=False。完整学习池权重为count/该件count总和/该组件件数/组件数,总和1,不再缩放。
每折当前test在任何特征/标签IO前拒绝。HPO仅在learning拟合,用inner validation的组件等权relative L2选择;同分按固定grid顺序。Claude1794707190明确:失败配置单独NA,记录原因,从成功完成配置中选最优,不能因一个trial失败就把整格废弃。只有12全部失败或final refit失败才整格NA。final以新估计器在相同learning pool重新拟合,不合并validation,不复用HPO模型权重,不读取validation标签。每格记录成功配置数量及全部失败配置。
独立根artifacts/s2-r2-main-2026-11-15;四单线程拟合工作者分别亲和CPU8/10/12/14,协调器30–31;神经feeders的CPU0–7不占用。环境OMP/MKL/OpenBLAS/NUMEXPR=1、CUDA不可见,fit内部threadpool_limits(1)并保存实际线程池快照。单fit完整工作者CPU≤1800s,全部worker累计≤360000s,活动监督器CPU保守纳入实时检查,终账分列。工程保护另限每fit墙钟7200s、RSS16GiB、输出5GiB、独立根100GiB;资源监测保留采样误差及退出复核,失败不重试。
先单独执行job0(fold0/pool25/lr.05/leaves31/200轮),取得真实成本回执再派下一项。用学习合格查询数×轮数×叶数缩放fit CPU、以查询数量缩放其余工作CPU;对剩余HPO及按最贵配置计价的15final加25%余量。投影≤100CPUh才继续,否则停止报告。这是工程估计,不是统计保证。若首trial失败,记录无有效实测投影;195次×30CPUmin=97.5CPUh仅作受限尝试成本上界,不能声称195次都会完成。其他配置依已批准失败政策仍可执行。
180 HPO终态后固定每格selection及12回执SHA,再启动final。预算中止的未派发项目明确NOT_RUN/NA,不重启不补跑。所有90神经+15R2最终格终态且统一固定检查点之前,禁止测试预测/评分;本R2执行器没有这些入口。B0与托管评分是后续独立步骤,不能把R2拟合完成称作整体研究完成。
+60
View File
@@ -0,0 +1,60 @@
"""Freeze the independent 180-HPO +15-final R2 plan, bound to the617 roster."""
import json
from pathlib import Path
from freeze_g1_components import sha
from s2_models.features import histgbrt_grid
from s2_main_data import select_cell
from s2_recovery_supervisor import save
ROOT=Path('artifacts/s2-r2-main-2026-11-15')
def build_jobs(roster):
grid=histgbrt_grid();jobs=[];cells=[]
for fold in range(5):
for pool in [25,50,100]:
rows,roles=select_cell(roster,fold,pool);index=len(cells)
learn=sum(rows[k]['preparation']['eligible_unique'] for k in roles['learning_parts'])
val=sum(rows[k]['preparation']['eligible_unique'] for k in roles['validation_parts'])
cell=dict(index=index,fold=fold,pool=pool,learning_queries=learn,validation_queries=val,
learning_parts=roles['learning_parts'],validation_parts=roles['validation_parts'],hpo_indices=[])
for cfg,params in enumerate(grid):
cell['hpo_indices'].append(len(jobs));jobs.append(dict(index=len(jobs),cell=index,fold=fold,pool=pool,kind='HPO',
config_index=cfg,params=params,cost_units=learn*params['max_iter']*params['max_leaf_nodes'],cost_queries=learn+val))
cells.append(cell)
for cell in cells:
jobs.append(dict(index=len(jobs),cell=cell['index'],fold=cell['fold'],pool=cell['pool'],kind='FINAL',
config_index=None,cost_units=cell['learning_queries']*1000*63,cost_queries=cell['learning_queries']))
return jobs,cells
def main():
assert not ROOT.exists(),'Never replace an R2 attempt root'
matrix_path=Path('artifacts/s2-main-matrix-2026-11-15/training-contract.json');matrix=json.loads(matrix_path.read_text())
roster=json.loads(Path(matrix['roster']).read_text());assert sha(matrix['roster'])==matrix['roster_sha256']
for p,h in matrix['code_sha256'].items():assert sha(p)==h,p
jobs,cells=build_jobs(roster)
added=['tools/s2_r2_data.py','tools/run_s2_r2_fit.py','tools/run_s2_r2.py','tools/prepare_s2_r2.py','tools/test_s2_r2.py']
paths=set(matrix['code_sha256'])|set(added)
sources=[str(matrix_path),matrix['roster'],matrix['old_labels_manifest'],
'artifacts/s2-e1-2026-11-14/preflight/claude-gre24-mapping-1b55e166.md',
'artifacts/s2-e1-2026-11-14/preflight/claude-target-9b71d565.md','docs/s2-r2-execution-2026-11-15.md']
c={k:matrix[k] for k in ['roster','roster_sha256','old_labels_manifest','old_labels_manifest_sha256','archive','archive_stat']}
c.update(version='S2-R2-G24-independent-v1',jobs=jobs,cells=cells,grid=histgbrt_grid(),sklearn_version='1.7.2',
authorizing_rulings=['1794699027.366469','1794707190.141609'],final_refit='same_learning_pool_only',
selection_policy='best_finished_config_fixed_grid_tie_order',
worker_cpus=[8,10,12,14],coordinator_cpus=[30,31],neural_reserved_cpus=list(range(8)),
limits=dict(workers=4,worker_cpu_seconds=100*3600,per_fit_cpu_seconds=1800,per_fit_wall_seconds=7200,
per_fit_tree_rss_bytes=16*2**30,per_fit_disk_bytes=5*2**30,disk_bytes=100*2**30,retries=0),
first_fit='Job0 alone, empirical full-grid query/iteration/leaf cost forecast with25percent reserve; continue if<=100CPUh. Failed firsttrial uses capped195*30min cost bound only, without a completion forecast.',
weighting='Eligible multiplicity / part count-sum / parts-in-component / learning component count; sum1, no rescaling.',
test_policy='Current fold test rejected before IO; final fit reads learning only; no test predictions or scoring before all90+15 terminal and unified custody.',
failure_policy='Failed configuration NA, no retry; select best finished by component-equal validation relativeL2; all12fail or finalfailure=>cellNA. Report finished/failed config count.',
code_sha256={p:sha(p) for p in sorted(paths)},source_sha256={p:sha(p) for p in sources})
ROOT.mkdir();save(ROOT/'contract.json',c)
save(ROOT/'prelaunch.json',dict(status='FROZEN_NOT_STARTED',contract_sha256=sha(ROOT/'contract.json'),roster_sha256=c['roster_sha256'],
HPO_trials=180,final_fits=15,worker_cpus=c['worker_cpus'],coordinator_cpus=c['coordinator_cpus'],labels_decoded=False,fits_started=False))
print(json.dumps(json.loads((ROOT/'prelaunch.json').read_text())))
if __name__=='__main__':main()
+166
View File
@@ -0,0 +1,166 @@
"""Independent four-worker CPU HPO/final queue; first-fit barrier, no neural mutations."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
from freeze_g1_components import sha
from s2_recovery_supervisor import save,supervise,tree_usage
from s2_recovery_monitoring import Monitor,disk_usage,budget_reason
from run_s2_pilot_preparation import terminate
def contract(root):
c=json.loads((root/'contract.json').read_text())
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():assert sha(p)==h,p
assert len(c['jobs'])==195 and len(c['cells'])==15 and c['limits']['worker_cpu_seconds']==360000
assert c['limits']['per_fit_cpu_seconds']==1800 and c['limits']['workers']==4
assert set(c['worker_cpus']).isdisjoint(range(8))
assert c['final_refit']=='same_learning_pool_only' and c['selection_policy']=='best_finished_config_fixed_grid_tie_order'
return c
def select_finished(cell,trials):
assert len(trials)==12 and sorted(t['config_index'] for t in trials)==list(range(12))
finished=[t for t in trials if t['status']=='PASS' and t.get('score') is not None and math.isfinite(t['score'])]
if not finished:return dict(status='NA_ALL_HPO_FAILED',cell=cell,finished_configurations=0,trials=trials,config_index=None)
chosen=min(finished,key=lambda t:(t['score'],t['config_index']))
return dict(status='SELECTED',cell=cell,config_index=chosen['config_index'],validation_score=chosen['score'],
finished_configurations=len(finished),failed_configurations=12-len(finished),trials=trials,
final_refit_set='same_learning_pool_only',ruling='1794707190.141609')
def cost_check(c,first):
cap=c['limits']['per_fit_cpu_seconds'];hard=len(c['jobs'])*cap
if first['worker']['status']!='PASS':
return dict(continue_within_cap=hard<=c['limits']['worker_cpu_seconds'],method='No successful empirical fit; capped195attempts upper-bound only, not a completion forecast',
projected_worker_cpu_seconds=None,capped_attempt_upper_bound_seconds=hard,first_status=first['worker']['status'])
path=Path(first['result_path']);r=json.loads(path.read_text());initial=c['jobs'][0]['cost_units']
remaining=sum(j['cost_units'] for j in c['jobs'][1:]);rate=r['fit_cpu_seconds']/initial
# Non-fit work scales by query rows, final config conservatively at63leaves/1000iterations.
overhead=max(0.,first['worker']['cpu_seconds']-r['fit_cpu_seconds'])
initial_rows=c['jobs'][0]['cost_queries']
forecast=first['worker']['cpu_seconds']+1.25*(rate*remaining+overhead*sum(j['cost_queries']/initial_rows for j in c['jobs'][1:]))
return dict(continue_within_cap=forecast<=c['limits']['worker_cpu_seconds'],projected_worker_cpu_seconds=forecast,
capped_attempt_upper_bound_seconds=hard,first_fit_cpu_seconds=r['fit_cpu_seconds'],first_worker_cpu_seconds=first['worker']['cpu_seconds'],
method='Observed fit CPU/(learning eligible queries*iterations*leaves), query-scaled nonfit CPU, final worst grid config,25percent reserve; heuristic not accuracy guarantee')
def guard(root,index,slot):
c=contract(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]])
dest=root/'jobs'/f'job-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
w=supervise([sys.executable,'tools/run_s2_r2_fit.py',str(root),str(index)],dest,'worker',
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=7200,tree_rss_bytes=16*2**30,
disk_bytes=5*2**30,max_processes=1),poll=.2)
path=dest/'output/result.json';result=None
if path.exists():result=json.loads(path.read_text())
if w['status']=='PASS' and (not result or result['status']!='COMPLETE'):
w.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT')
save(dest/'guard-result.json',dict(worker=w,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
cpu_affinity=sorted(os.sched_getaffinity(0)),result_path=str(path) if path.exists() else None,
result_sha256=sha(path) if path.exists() else None,job=c['jobs'][index]))
return 0 if w['status']=='PASS' else 1
def finalize_selections(root,c,done):
by_index={r['index']:r for r in done};directory=root/'selections';directory.mkdir(exist_ok=False)
selected={}
for cell in c['cells']:
trials=[]
for i in cell['hpo_indices']:
row=by_index[i];path=root/'jobs'/f'job-{i:03d}'/'guard-result.json'
result=json.loads(Path(row['result_path']).read_text()) if row.get('result_path') else None
trials.append(dict(index=i,config_index=c['jobs'][i]['config_index'],status=row['worker']['status'],
reason=row['worker']['stop_reason'] or (None if row['worker']['status']=='PASS' else 'WORKER_EXIT_'+str(row['worker']['returncode'])),
score=result.get('validation_component_equal_relative_l2') if result else None,
receipt_path=str(path),receipt_sha256=sha(path)))
decision=select_finished(cell['index'],trials);save(directory/f"cell-{cell['index']:02d}.json",decision);selected[cell['index']]=decision
return selected
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
def run_locked(root):
c=contract(root);assert not (root/'batch-started.json').exists(),'No repeat R2 queue'
os.sched_setaffinity(0,c['coordinator_cpus'])
start=time.monotonic();save(root/'batch-started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0))))
active={};done=[];next_index=0;reason=None;first_checked=False;selections=None;live_cpu=live_rss=disk=0
monitor=Monitor(root/'monitor-errors.jsonl')
env=dict(os.environ,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
def interrupted(*_):
nonlocal reason
reason='R2_COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
def terminate_active():terminate({k:(v['process'],v['log']) for k,v in active.items()})
try:
while active or (next_index<195 and reason is None):
for slot,item in list(active.items()):
code=item['process'].poll()
if code is None:continue
path=root/'jobs'/f"job-{item['index']:03d}"/'guard-result.json'
if not path.exists():
save(root/'infrastructure-stop.json',dict(reason='MISSING_GUARD_RECEIPT',index=item['index'],returncode=code,accounting_complete=False));raise RuntimeError('Missing R2 guard accounting')
row=json.loads(path.read_text());assert math.isfinite(row['worker']['cpu_seconds']) and row['worker']['cpu_seconds']>=0
done.append(dict(index=item['index'],**row));item['log'].close();del active[slot]
print(json.dumps(dict(event='R2_TERMINAL',index=item['index'],kind=c['jobs'][item['index']]['kind'],status=row['worker']['status'],completed=len(done))),flush=True)
def usage():
values=[tree_usage(v['process'].pid) for v in active.values()]
return sum(x[0] for x in values),sum(x[1] for x in values)
(live_cpu,live_rss),fresh=monitor.sample('live_usage',usage,(live_cpu,live_rss))
disk,ok=monitor.sample('disk',lambda:disk_usage(root),disk);fresh &= ok
spent=sum(x['worker']['cpu_seconds'] for x in done)
over=budget_reason(0,spent,live_cpu,disk,cpu_limit=c['limits']['worker_cpu_seconds'],disk_limit=c['limits']['disk_bytes'])
if over:reason=over
if not first_checked and len(done)==1 and not active and reason is None:
check=cost_check(c,done[0]);save(root/'first-fit-cost-check.json',check);first_checked=True
if not check['continue_within_cap']:reason='FIRST_FIT_COST_PROJECTION_OVER_CAP'
if next_index==180 and not active and len(done)==180 and selections is None and reason is None:
selections=finalize_selections(root,c,done)
if reason:terminate_active()
ceiling=1 if not first_checked else (180 if selections is None else 195)
while fresh and reason is None and len(active)<4 and next_index<ceiling:
i=next_index;job=c['jobs'][i]
if job['kind']=='FINAL' and selections[job['cell']]['status']!='SELECTED':
done.append(dict(index=i,job=job,worker=dict(status='NA',cpu_seconds=0.,returncode=0,stop_reason='ALL12_HPO_FAILED_NO_FINAL_FIT'),
supervisor_cpu_seconds=0.,attempted=False,result_path=None,result_sha256=None));next_index+=1;continue
slot=next(k for k in range(4) if k not in active);dest=root/'jobs'/f'job-{i:03d}';dest.mkdir(parents=True,exist_ok=True)
log=(dest/'guard.log').open('x')
proc=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i),'--slot',str(slot)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[slot]=dict(process=proc,log=log,index=i);next_index+=1
save(root/'progress.json',dict(completed=len(done),active={str(k):v['index'] for k,v in active.items()},next_index=next_index,
worker_cpu_seconds=spent,live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=disk,
first_fit_checked=first_checked,selections_frozen=selections is not None,stop_reason=reason,fresh_accounting=fresh,
monitor_errors=monitor.errors,elapsed_seconds=time.monotonic()-start))
if active or not fresh:time.sleep(.5)
finally:
terminate_active()
for item in active.values():item['process'].wait();item['log'].close()
attempted={x['index']:x for x in done};finals=[]
for job in c['jobs'][180:]:
row=attempted.get(job['index']);passed=row is not None and row['worker']['status']=='PASS'
result=json.loads(Path(row['result_path']).read_text()) if passed else None
finals.append(dict(cell=job['cell'],fold=job['fold'],pool=job['pool'],status='PASS' if passed else 'NA',
reason=None if passed else (row['worker']['stop_reason'] if row else 'NOT_RUN_'+str(reason)),
finished_configurations=selections[job['cell']]['finished_configurations'] if selections else None,
model_path=str(Path(row['result_path']).parent/'model.joblib') if passed else None,
model_sha256=result['model_sha256'] if result else None))
save(root/'batch-complete.json',dict(status='ALL_R2_TERMINAL',stop_reason=reason,final_fits=finals,jobs=done,
not_run_indices=[i for i in range(195) if i not in attempted],worker_cpu_seconds=sum(x['worker']['cpu_seconds'] for x in done),
guard_cpu_seconds=sum(x['supervisor_cpu_seconds'] for x in done),coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
disk_bytes=disk_usage(root),wall_seconds=time.monotonic()-start,monitor_errors=monitor.errors,roster_sha256=c['roster_sha256'],
test_predictions=False,test_scoring=False,retry=False))
return 0 if reason is None else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int);a=p.parse_args()
raise SystemExit(guard(a.root,a.guard,a.slot) if a.guard is not None else run(a.root))
+67
View File
@@ -0,0 +1,67 @@
"""One single-thread HPO trial or fresh final refit; launch only through the CPU guard."""
import argparse
import json
import os
from pathlib import Path
import time
import traceback
import joblib
import sklearn
from sklearn.ensemble import HistGradientBoostingRegressor
from threadpoolctl import threadpool_limits,threadpool_info
from freeze_g1_components import sha
from s2_recovery_supervisor import save
from s2_models.features import histgbrt_grid,R2_COLUMNS
from s2_r2_data import load_cell,learning_arrays,validation_score
def run(root,index):
from run_s2_r2 import contract
start=time.monotonic();cpu_start=time.process_time();c=contract(root);job=c['jobs'][index]
dest=root/'jobs'/f'job-{index:03d}'/'output';dest.mkdir(exist_ok=False)
save(dest/'started.json',dict(unix=time.time(),pid=os.getpid(),job=job,contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0))))
try:
assert sklearn.__version__=='1.7.2'
config_index=job.get('config_index');selection=None
if job['kind']=='FINAL':
path=root/'selections'/f"cell-{job['cell']:02d}.json";selection=json.loads(path.read_text())
assert selection['status']=='SELECTED' and selection['cell']==job['cell']
config_index=selection['config_index']
for item in selection['trials']:
assert sha(item['receipt_path'])==item['receipt_sha256']
save(dest/'selection.json',dict(path=str(path),sha256=sha(path),**selection))
params=histgbrt_grid()[config_index]
with threadpool_limits(limits=1):
pools=threadpool_info();assert all(p['num_threads']==1 for p in pools)
learning,validation,roles=load_cell(root,c,job)
save(dest/'access.json',dict(fold=job['fold'],pool=job['pool'],kind=job['kind'],
cases=[dict(case=p['case'],role=p['role'],label_mode=p['label_mode'],label_sha256=p['label_sha256']) for p in learning+validation],
outer_fold_test_labels_read=False,learning_only_final_refit=job['kind']=='FINAL'))
x,y,w=learning_arrays(learning);load_cpu=time.process_time()-cpu_start
save(dest/'progress.json',dict(stage='fit',training_queries=len(y),training_parts=len(learning),cpu_seconds=time.process_time()-cpu_start))
model=HistGradientBoostingRegressor(**params)
fit_start=time.process_time();model.fit(x,y,sample_weight=w);fit_cpu=time.process_time()-fit_start
assert model.n_iter_==params['max_iter']
value=errors=None
if job['kind']=='HPO':
save(dest/'progress.json',dict(stage='validation',fit_cpu_seconds=fit_cpu))
value,errors=validation_score(model,validation);save(dest/'validation-parts.json',errors)
joblib.dump(model,dest/'model.joblib',compress=3)
result=dict(status='COMPLETE',job=job,config_index=config_index,params=params,sklearn_version=sklearn.__version__,
columns=list(R2_COLUMNS),target='T/t_set',learning_queries=len(y),learning_parts=len(learning),
learning_components=len({p['component_id'] for p in learning}),validation_parts=len(validation),
sample_weight_sum=float(w.sum()),validation_component_equal_relative_l2=value,fit_cpu_seconds=fit_cpu,
load_cpu_seconds=load_cpu,worker_elapsed_cpu_seconds=time.process_time()-cpu_start,wall_seconds=time.monotonic()-start,
threadpools=pools,cpu_affinity=sorted(os.sched_getaffinity(0)),model_sha256=sha(dest/'model.joblib'),
roster_sha256=c['roster_sha256'],test_predictions=False,test_labels_read=False,random_state=0,retry=False,
final_refit_learning_only=job['kind']=='FINAL',hpo_finished_configurations=selection['finished_configurations'] if selection else None)
save(dest/'result.json',result);save(dest/'progress.json',dict(stage='complete',cpu_seconds=time.process_time()-cpu_start))
print(json.dumps(result),flush=True);return 0
except Exception as exc:
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
cpu_seconds=time.process_time()-cpu_start,wall_seconds=time.monotonic()-start))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))
+65
View File
@@ -0,0 +1,65 @@
"""R2 uses frozen query rows, direct targets and the same component/part weights."""
from collections import Counter
import json
from pathlib import Path
import numpy as np
from freeze_g1_components import sha
from s2_main_data import select_cell,ensure_targets
from s2_pilot_data import permitted_role
from s2_models.features import R2_COLUMNS
from s2_models.objectives import relative_l2
def read_part(root,c,row,roles,old_labels):
role=permitted_role(roles,row['case'])
prep=row['preparation'];source=Path(prep['preparation_root'])
label,label_sha,mode=ensure_targets(root,c,row,roles,old_labels)
for name in ['r2.npz','queries.npz']:assert sha(source/name)==prep['output_sha256'][name]
with np.load(source/'queries.npz',allow_pickle=False) as q:
mask=q['eligible_s2'];ids=q['anchor_ids'][mask];counts=q['multiplicity'][mask]
with np.load(source/'r2.npz',allow_pickle=False) as z:
x=z['features'];assert x.shape==(row['nodes'],len(R2_COLUMNS));x=x[ids].copy()
assert sha(label)==label_sha
with np.load(label,allow_pickle=False) as z:
assert np.array_equal(z['anchor_ids'],ids) and np.array_equal(z['multiplicity'],counts)
y=z['target_t_star'].copy();seconds=z['seconds'].copy()
assert len(y)>0 and np.isfinite(x).all() and np.isfinite(y).all() and (y>=0).all()
assert np.all(x[:,-1]==prep['t_set'])
assert np.array_equal(y,seconds/prep['t_set'])
return dict(case=row['case'],component_id=row['component_id'],role=role,x=x,y=y,seconds=seconds,
counts=counts,t_set=prep['t_set'],query_ids=ids,label_mode=mode,label_sha256=label_sha)
def learning_arrays(parts):
assert parts and all(p['role']=='learning' for p in parts)
groups=Counter(p['component_id'] for p in parts);ng=len(groups)
x=[];y=[];weights=[]
for p in parts:
counts=np.asarray(p['counts'],dtype=np.float64)
assert len(counts)==len(p['y'])==len(p['x']) and np.all(counts>0)
x.append(p['x']);y.append(p['y'])
weights.append(counts/counts.sum()/groups[p['component_id']]/ng)
w=np.concatenate(weights);assert np.isclose(w.sum(),1,rtol=0,atol=1e-12)
return np.concatenate(x),np.concatenate(y),w
def validation_score(model,parts):
assert parts and all(p['role']=='validation' for p in parts)
groups={};errors={}
for p in parts:
predicted=model.predict(p['x'])
value=relative_l2(predicted*p['t_set'],p['seconds'],p['counts'],np.ones(len(p['counts']),bool))['value']
if value is None or not np.isfinite(value):raise ValueError('Invalid validation relative L2')
errors[p['case']]=value;groups.setdefault(p['component_id'],[]).append(value)
return float(np.mean([np.mean(v) for v in groups.values()])),errors
def load_cell(root,c,job):
assert sha(c['roster'])==c['roster_sha256']
roster=json.loads(Path(c['roster']).read_text());rows,roles=select_cell(roster,job['fold'],job['pool'])
assert sha(c['old_labels_manifest'])==c['old_labels_manifest_sha256']
old={p['case']:p for p in json.loads(Path(c['old_labels_manifest']).read_text())['ready_parts']}
# Final refit reads learning only; validation was already used for HPO selection.
cases=roles['learning_parts']+(roles['validation_parts'] if job['kind']=='HPO' else [])
parts=[read_part(root,c,rows[k],roles,old) for k in sorted(cases)]
return [p for p in parts if p['role']=='learning'],[p for p in parts if p['role']=='validation'],roles
+136
View File
@@ -0,0 +1,136 @@
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import numpy as np
import run_s2_r2 as queue
import run_s2_r2_fit as fit
import s2_r2_data as data
from s2_models.objectives import hierarchy_weights
from prepare_s2_r2 import build_jobs
def synthetic_part(case,component,role,counts):
n=len(counts);x=np.zeros((n,20));x[:,-1]=2.
return dict(case=case,component_id=component,role=role,x=x,y=np.ones(n),seconds=np.ones(n)*2,counts=np.array(counts),t_set=2.,label_mode='synthetic',label_sha256='synthetic')
def fake_contract():
jobs=[];cells=[]
for cell in range(15):
ids=[]
for cfg in range(12):
ids.append(len(jobs));jobs.append(dict(index=len(jobs),cell=cell,fold=cell//3,pool=[25,50,100][cell%3],kind='HPO',config_index=cfg,cost_units=1,cost_queries=1))
cells.append(dict(index=cell,hpo_indices=ids))
for cell in range(15):jobs.append(dict(index=len(jobs),cell=cell,fold=cell//3,pool=[25,50,100][cell%3],kind='FINAL',cost_units=1,cost_queries=1))
return dict(jobs=jobs,cells=cells,worker_cpus=[8,10,12,14],coordinator_cpus=[30,31],roster_sha256='fixed',
limits=dict(worker_cpu_seconds=360000,per_fit_cpu_seconds=1800,workers=4,disk_bytes=100*2**30))
class R2Checks(unittest.TestCase):
def test_weights_match_frozen_hierarchy_without_rescaling(self):
parts=[synthetic_part('a','A','learning',[3,1]),synthetic_part('b','A','learning',[2]),synthetic_part('c','B','learning',[1,1])]
_,_,weights=data.learning_arrays(parts)
expected=hierarchy_weights([3,1,2,1,1],['a','a','b','c','c'],['A','A','A','B','B'])
np.testing.assert_array_equal(weights,expected);self.assertEqual(weights.sum(),1.)
def test_validation_is_component_equal_and_seconds_scale_correct(self):
parts=[synthetic_part('a','A','validation',[1]),synthetic_part('b','A','validation',[1]),synthetic_part('c','B','validation',[1])]
class Model:
def __init__(self):self.i=0
def predict(self,x):self.i+=1;return np.array([1.,3.,4.])[self.i-1:self.i]
value,errors=data.validation_score(Model(),parts)
self.assertEqual(errors,dict(a=0.,b=2.,c=3.));self.assertEqual(value,2.)
def test_test_case_rejected_before_feature_or_target_io(self):
roles=dict(learning_parts=['l'],validation_parts=['v'],sealed_test_parts=['t'])
with patch.object(data,'ensure_targets',side_effect=AssertionError('target IO')),patch.object(data,'sha',side_effect=AssertionError('feature IO')):
with self.assertRaises(PermissionError):data.read_part(Path('/none'),{},dict(case='t'),roles,{})
def test_final_loader_never_reads_validation(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);roster=root/'roster.json';old=root/'old.json'
roster.write_text('{}');old.write_text(json.dumps(dict(ready_parts=[])))
c=dict(roster=str(roster),roster_sha256=data.sha(roster),old_labels_manifest=str(old),old_labels_manifest_sha256=data.sha(old))
roles=dict(learning_parts=['l'],validation_parts=['v'],sealed_test_parts=['t']);rows={k:dict(case=k) for k in ['l','v']};seen=[]
def read(*args):
case=args[2]['case'];seen.append(case);return synthetic_part(case,case,'learning' if case=='l' else 'validation',[1])
with patch.object(data,'select_cell',return_value=(rows,roles)),patch.object(data,'read_part',read):
learning,validation,_=data.load_cell(root,c,dict(fold=0,pool=25,kind='FINAL'))
self.assertEqual(seen,['l']);self.assertEqual(validation,[])
def test_select_finished_configures_failure_and_tie_policy(self):
trials=[dict(config_index=i,status='STOPPED_OR_FAILED',score=None,reason='wall_limit') for i in range(12)]
self.assertEqual(queue.select_finished(0,trials)['status'],'NA_ALL_HPO_FAILED')
trials[3].update(status='PASS',score=.2);trials[1].update(status='PASS',score=.2)
r=queue.select_finished(0,trials);self.assertEqual(r['config_index'],1);self.assertEqual(r['finished_configurations'],2)
self.assertEqual(r['failed_configurations'],10)
def test_grid_plan_has_180_unique_trials_15_final_and_real_roster(self):
roster=json.loads(Path('artifacts/s2-main-preparation-2026-11-15/final-roster.json').read_text());jobs,cells=build_jobs(roster)
self.assertEqual((len(jobs),len(cells)),(195,15))
self.assertEqual(sum(j['kind']=='HPO' for j in jobs),180)
for cell in cells:self.assertEqual([jobs[i]['config_index'] for i in cell['hpo_indices']],list(range(12)))
self.assertTrue(all(j['kind']=='FINAL' and j['config_index'] is None for j in jobs[180:]))
def test_first_fit_projection_and_failed_attempt_cost_bound(self):
with tempfile.TemporaryDirectory() as t:
p=Path(t)/'result.json';p.write_text(json.dumps(dict(fit_cpu_seconds=1700)))
c=fake_contract();first=dict(worker=dict(status='PASS',cpu_seconds=1800),result_path=str(p))
self.assertFalse(queue.cost_check(c,first)['continue_within_cap'])
first['worker']['status']='STOPPED_OR_FAILED';check=queue.cost_check(c,first)
self.assertTrue(check['continue_within_cap']);self.assertEqual(check['capped_attempt_upper_bound_seconds'],351000)
self.assertIsNone(check['projected_worker_cpu_seconds'])
def test_guard_uses_cpu_cap_and_disjoint_affinity(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);seen=[]
def supervise(command,dest,name,limits,**kw):seen.append(limits);return dict(status='STOPPED_OR_FAILED',stop_reason='synthetic')
with patch.object(queue,'contract',return_value=fake_contract()),patch.object(queue.os,'sched_setaffinity') as affinity,patch.object(queue,'supervise',supervise):
self.assertEqual(queue.guard(root,0,0),1)
affinity.assert_called_once_with(0,[8])
self.assertEqual(seen[0]['job_cpu_seconds'],1800);self.assertEqual(seen[0]['max_processes'],1)
def test_actual_synthetic_histgbrt_fit_single_thread_and_fixed_params(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'contract.json').write_text('{}');(root/'jobs/job-000').mkdir(parents=True)
c=fake_contract();train=synthetic_part('learn','A','learning',[1]*240);val=synthetic_part('val','V','validation',[1]*40)
rng=np.random.default_rng(17)
for p in [train,val]:
p['x'][:,:19]=rng.normal(size=(len(p['y']),19));p['y']=1+.1*p['x'][:,0];p['seconds']=p['y']*2
with patch.object(queue,'contract',return_value=c),patch.object(fit,'load_cell',return_value=([train],[val],{})):
self.assertEqual(fit.run(root,0),0)
r=json.loads((root/'jobs/job-000/output/result.json').read_text())
self.assertEqual(r['params']['max_iter'],200);self.assertFalse(r['params']['early_stopping'])
self.assertTrue(all(p['num_threads']==1 for p in r['threadpools']));self.assertFalse(r['test_predictions'])
def test_queue_first_fit_barrier_and_unique_195_attempts(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'contract.json').write_text('{}');c=fake_contract();spawned=[];peak=[0]
class Process:
def __init__(self,index):self.index=index;self.pid=1000000+index;self.polls=0
def poll(self):
self.polls+=1
if self.polls==1:return None
job=c['jobs'][self.index];dest=root/'jobs'/f'job-{self.index:03d}';out=dest/'output';out.mkdir(exist_ok=True)
result=out/'result.json';result.write_text(json.dumps(dict(status='COMPLETE',fit_cpu_seconds=.001,validation_component_equal_relative_l2=job.get('config_index',0)/100,model_sha256='synthetic')))
(dest/'guard-result.json').write_text(json.dumps(dict(worker=dict(status='PASS',cpu_seconds=.01,returncode=0,stop_reason=None),
supervisor_cpu_seconds=.001,result_path=str(result),result_sha256=data.sha(result),job=job)))
return 0
def wait(self):return 0
def spawn(command,**kw):
i=int(command[-3])
if i>0:self.assertTrue((root/'first-fit-cost-check.json').exists())
if i>=180:self.assertTrue((root/'selections'/f"cell-{c['jobs'][i]['cell']:02d}.json").exists())
p=Process(i);spawned.append(p);peak[0]=max(peak[0],sum(p.polls<2 for p in spawned));return p
with patch.object(queue,'contract',return_value=c),patch.object(queue.os,'sched_setaffinity'),patch.object(queue.subprocess,'Popen',spawn),patch.object(queue,'tree_usage',return_value=(0.,1,1)),patch.object(queue.time,'sleep'),patch('builtins.print'):
self.assertEqual(queue.run(root),0)
r=json.loads((root/'batch-complete.json').read_text());self.assertEqual(len(spawned),195);self.assertEqual(len({p.index for p in spawned}),195)
self.assertLessEqual(peak[0],4);self.assertEqual(len(r['final_fits']),15);self.assertTrue(all(x['status']=='PASS' for x in r['final_fits']))
with patch.object(queue,'contract',return_value=c),patch.object(queue.os,'sched_setaffinity'),patch.object(queue.subprocess,'Popen') as spawn:
with self.assertRaises(AssertionError):queue.run(root)
spawn.assert_not_called()
if __name__=='__main__':unittest.main(verbosity=2)