Implement independent R2 HPO and learning-only refits with first-fit cost gate
This commit is contained in:
1 parent
5034de7377
commit
f560a3d5e1
10 files changed
+9624
No files matched your search
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"status": "PASS_NOT_STARTED",
|
||||
"tests_passed": 10,
|
||||
"synthetic_fit": true,
|
||||
"real_corpus_fit": false,
|
||||
"contract_sha256": "edec86197769f44bf98f234795950f36c9fa42084ccb3c68448b63b440aa126e",
|
||||
"roster_sha256": "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e",
|
||||
"test_log_sha256": "3ebf40b5c8644fa7ad86944e9872e5c0e491b46ad04c77e8c27f2be6f7445f6b",
|
||||
"sklearn_version": "1.7.2",
|
||||
"single_thread_libraries_tested": [
|
||||
"OpenBLAS",
|
||||
"MKL",
|
||||
"OpenMP"
|
||||
],
|
||||
"final_refit": "same_learning_pool_only",
|
||||
"failed_HPO": "select_best_finished_configuration",
|
||||
"ruling": "1794707190.141609",
|
||||
"neural_code_pins_still_match": true,
|
||||
"first_fit_only_before_cost_check": true,
|
||||
"worker_cpus": [
|
||||
8,
|
||||
10,
|
||||
12,
|
||||
14
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"status": "FROZEN_NOT_STARTED",
|
||||
"contract_sha256": "edec86197769f44bf98f234795950f36c9fa42084ccb3c68448b63b440aa126e",
|
||||
"roster_sha256": "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e",
|
||||
"HPO_trials": 180,
|
||||
"final_fits": 15,
|
||||
"worker_cpus": [
|
||||
8,
|
||||
10,
|
||||
12,
|
||||
14
|
||||
],
|
||||
"coordinator_cpus": [
|
||||
30,
|
||||
31
|
||||
],
|
||||
"labels_decoded": false,
|
||||
"fits_started": false
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
test_actual_synthetic_histgbrt_fit_single_thread_and_fixed_params (__main__.R2Checks) ... {"status": "COMPLETE", "job": {"index": 0, "cell": 0, "fold": 0, "pool": 25, "kind": "HPO", "config_index": 0, "cost_units": 1, "cost_queries": 1}, "config_index": 0, "params": {"learning_rate": 0.05, "max_leaf_nodes": 31, "max_iter": 200, "min_samples_leaf": 20, "random_state": 0, "early_stopping": false}, "sklearn_version": "1.7.2", "columns": ["Vf", "grad_Vf_x", "grad_Vf_y", "grad_Vf_z", "Di_star", "grad_Di_x", "grad_Di_y", "grad_Di_z", "Dw", "Dw_star", "grad_Dw_x", "grad_Dw_y", "grad_Dw_z", "Sxx", "Sxy", "Sxz", "Syy", "Syz", "Szz", "t_set"], "target": "T/t_set", "learning_queries": 240, "learning_parts": 1, "learning_components": 1, "validation_parts": 1, "sample_weight_sum": 0.9999999999999999, "validation_component_equal_relative_l2": 0.014707062211418275, "fit_cpu_seconds": 0.25818808699999996, "load_cpu_seconds": 0.045540662999999926, "worker_elapsed_cpu_seconds": 0.38005410100000003, "wall_seconds": 0.3883439999772236, "threadpools": [{"user_api": "blas", "internal_api": "openblas", "num_threads": 1, "prefix": "libopenblas", "filepath": "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/site-packages/numpy.libs/libopenblas64_p-r0-0cf96a72.3.23.dev.so", "version": "0.3.23.dev", "threading_layer": "pthreads", "architecture": "Zen"}, {"user_api": "blas", "internal_api": "mkl", "num_threads": 1, "prefix": "libmkl_rt", "filepath": "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/libmkl_rt.so.2", "version": "2023.1-Product", "threading_layer": "gnu"}, {"user_api": "openmp", "internal_api": "openmp", "num_threads": 1, "prefix": "libgomp", "filepath": "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/libgomp.so.1.0.0", "version": null}], "cpu_affinity": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31], "model_sha256": "7536654c21366696414ec6f7c2b36828fe3967043f7d5e63b3bad3b19b408f6d", "roster_sha256": "fixed", "test_predictions": false, "test_labels_read": false, "random_state": 0, "retry": false, "final_refit_learning_only": false, "hpo_finished_configurations": null}
|
||||
ok
|
||||
test_final_loader_never_reads_validation (__main__.R2Checks) ... ok
|
||||
test_first_fit_projection_and_failed_attempt_cost_bound (__main__.R2Checks) ... ok
|
||||
test_grid_plan_has_180_unique_trials_15_final_and_real_roster (__main__.R2Checks) ... ok
|
||||
test_guard_uses_cpu_cap_and_disjoint_affinity (__main__.R2Checks) ... ok
|
||||
test_queue_first_fit_barrier_and_unique_195_attempts (__main__.R2Checks) ... ok
|
||||
test_select_finished_configures_failure_and_tie_policy (__main__.R2Checks) ... ok
|
||||
test_test_case_rejected_before_feature_or_target_io (__main__.R2Checks) ... ok
|
||||
test_validation_is_component_equal_and_seconds_scale_correct (__main__.R2Checks) ... ok
|
||||
test_weights_match_frozen_hierarchy_without_rescaling (__main__.R2Checks) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 10 tests in 1.707s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,13 @@
|
||||
# S2 R2-G24 独立执行合同
|
||||
|
||||
沿Claude1794699027.366469预算及1794707190.141609的最终裁量实现,不改运行中的神经源码。共5折×3组件比例池,每格12个固定HPO配置及一个final refit,即最多180 HPO+15 final。绑定617件统一roster,原七件准备失败在所有折一致排除。该限制必须在最终报告列明:共708645/18661224节点(约3.8%),四件属fold0 test;测试集缺少这些准备昂贵的大件。
|
||||
|
||||
固定v0.5几何映射与v0.6目标:复用已完成的20列R2-G24特征及相同合格query IDs/重复次数,不重做几何;直接预测T/t_set,秒值乘回t_set,不clip、不log变换。sklearn1.7.2 HistGradientBoostingRegressor;网格来自原固定histgbrt_grid:lr .05/.1 × max_leaf_nodes31/63 × max_iter200/500/1000,min_samples_leaf20、seed0、early_stopping=False。完整学习池权重为count/该件count总和/该组件件数/组件数,总和1,不再缩放。
|
||||
|
||||
每折当前test在任何特征/标签IO前拒绝。HPO仅在learning拟合,用inner validation的组件等权relative L2选择;同分按固定grid顺序。Claude1794707190明确:失败配置单独NA,记录原因,从成功完成配置中选最优,不能因一个trial失败就把整格废弃。只有12全部失败或final refit失败才整格NA。final以新估计器在相同learning pool重新拟合,不合并validation,不复用HPO模型权重,不读取validation标签。每格记录成功配置数量及全部失败配置。
|
||||
|
||||
独立根artifacts/s2-r2-main-2026-11-15;四单线程拟合工作者分别亲和CPU8/10/12/14,协调器30–31;神经feeders的CPU0–7不占用。环境OMP/MKL/OpenBLAS/NUMEXPR=1、CUDA不可见,fit内部threadpool_limits(1)并保存实际线程池快照。单fit完整工作者CPU≤1800s,全部worker累计≤360000s,活动监督器CPU保守纳入实时检查,终账分列。工程保护另限每fit墙钟7200s、RSS16GiB、输出5GiB、独立根100GiB;资源监测保留采样误差及退出复核,失败不重试。
|
||||
|
||||
先单独执行job0(fold0/pool25/lr.05/leaves31/200轮),取得真实成本回执再派下一项。用学习合格查询数×轮数×叶数缩放fit CPU、以查询数量缩放其余工作CPU;对剩余HPO及按最贵配置计价的15final加25%余量。投影≤100CPUh才继续,否则停止报告。这是工程估计,不是统计保证。若首trial失败,记录无有效实测投影;195次×30CPUmin=97.5CPUh仅作受限尝试成本上界,不能声称195次都会完成。其他配置依已批准失败政策仍可执行。
|
||||
|
||||
180 HPO终态后固定每格selection及12回执SHA,再启动final。预算中止的未派发项目明确NOT_RUN/NA,不重启不补跑。所有90神经+15R2最终格终态且统一固定检查点之前,禁止测试预测/评分;本R2执行器没有这些入口。B0与托管评分是后续独立步骤,不能把R2拟合完成称作整体研究完成。
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Freeze the independent 180-HPO +15-final R2 plan, bound to the617 roster."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
from freeze_g1_components import sha
|
||||
from s2_models.features import histgbrt_grid
|
||||
from s2_main_data import select_cell
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
ROOT=Path('artifacts/s2-r2-main-2026-11-15')
|
||||
|
||||
|
||||
def build_jobs(roster):
|
||||
grid=histgbrt_grid();jobs=[];cells=[]
|
||||
for fold in range(5):
|
||||
for pool in [25,50,100]:
|
||||
rows,roles=select_cell(roster,fold,pool);index=len(cells)
|
||||
learn=sum(rows[k]['preparation']['eligible_unique'] for k in roles['learning_parts'])
|
||||
val=sum(rows[k]['preparation']['eligible_unique'] for k in roles['validation_parts'])
|
||||
cell=dict(index=index,fold=fold,pool=pool,learning_queries=learn,validation_queries=val,
|
||||
learning_parts=roles['learning_parts'],validation_parts=roles['validation_parts'],hpo_indices=[])
|
||||
for cfg,params in enumerate(grid):
|
||||
cell['hpo_indices'].append(len(jobs));jobs.append(dict(index=len(jobs),cell=index,fold=fold,pool=pool,kind='HPO',
|
||||
config_index=cfg,params=params,cost_units=learn*params['max_iter']*params['max_leaf_nodes'],cost_queries=learn+val))
|
||||
cells.append(cell)
|
||||
for cell in cells:
|
||||
jobs.append(dict(index=len(jobs),cell=cell['index'],fold=cell['fold'],pool=cell['pool'],kind='FINAL',
|
||||
config_index=None,cost_units=cell['learning_queries']*1000*63,cost_queries=cell['learning_queries']))
|
||||
return jobs,cells
|
||||
|
||||
|
||||
def main():
|
||||
assert not ROOT.exists(),'Never replace an R2 attempt root'
|
||||
matrix_path=Path('artifacts/s2-main-matrix-2026-11-15/training-contract.json');matrix=json.loads(matrix_path.read_text())
|
||||
roster=json.loads(Path(matrix['roster']).read_text());assert sha(matrix['roster'])==matrix['roster_sha256']
|
||||
for p,h in matrix['code_sha256'].items():assert sha(p)==h,p
|
||||
jobs,cells=build_jobs(roster)
|
||||
added=['tools/s2_r2_data.py','tools/run_s2_r2_fit.py','tools/run_s2_r2.py','tools/prepare_s2_r2.py','tools/test_s2_r2.py']
|
||||
paths=set(matrix['code_sha256'])|set(added)
|
||||
sources=[str(matrix_path),matrix['roster'],matrix['old_labels_manifest'],
|
||||
'artifacts/s2-e1-2026-11-14/preflight/claude-gre24-mapping-1b55e166.md',
|
||||
'artifacts/s2-e1-2026-11-14/preflight/claude-target-9b71d565.md','docs/s2-r2-execution-2026-11-15.md']
|
||||
c={k:matrix[k] for k in ['roster','roster_sha256','old_labels_manifest','old_labels_manifest_sha256','archive','archive_stat']}
|
||||
c.update(version='S2-R2-G24-independent-v1',jobs=jobs,cells=cells,grid=histgbrt_grid(),sklearn_version='1.7.2',
|
||||
authorizing_rulings=['1794699027.366469','1794707190.141609'],final_refit='same_learning_pool_only',
|
||||
selection_policy='best_finished_config_fixed_grid_tie_order',
|
||||
worker_cpus=[8,10,12,14],coordinator_cpus=[30,31],neural_reserved_cpus=list(range(8)),
|
||||
limits=dict(workers=4,worker_cpu_seconds=100*3600,per_fit_cpu_seconds=1800,per_fit_wall_seconds=7200,
|
||||
per_fit_tree_rss_bytes=16*2**30,per_fit_disk_bytes=5*2**30,disk_bytes=100*2**30,retries=0),
|
||||
first_fit='Job0 alone, empirical full-grid query/iteration/leaf cost forecast with25percent reserve; continue if<=100CPUh. Failed firsttrial uses capped195*30min cost bound only, without a completion forecast.',
|
||||
weighting='Eligible multiplicity / part count-sum / parts-in-component / learning component count; sum1, no rescaling.',
|
||||
test_policy='Current fold test rejected before IO; final fit reads learning only; no test predictions or scoring before all90+15 terminal and unified custody.',
|
||||
failure_policy='Failed configuration NA, no retry; select best finished by component-equal validation relativeL2; all12fail or finalfailure=>cellNA. Report finished/failed config count.',
|
||||
code_sha256={p:sha(p) for p in sorted(paths)},source_sha256={p:sha(p) for p in sources})
|
||||
ROOT.mkdir();save(ROOT/'contract.json',c)
|
||||
save(ROOT/'prelaunch.json',dict(status='FROZEN_NOT_STARTED',contract_sha256=sha(ROOT/'contract.json'),roster_sha256=c['roster_sha256'],
|
||||
HPO_trials=180,final_fits=15,worker_cpus=c['worker_cpus'],coordinator_cpus=c['coordinator_cpus'],labels_decoded=False,fits_started=False))
|
||||
print(json.dumps(json.loads((ROOT/'prelaunch.json').read_text())))
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Independent four-worker CPU HPO/final queue; first-fit barrier, no neural mutations."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_supervisor import save,supervise,tree_usage
|
||||
from s2_recovery_monitoring import Monitor,disk_usage,budget_reason
|
||||
from run_s2_pilot_preparation import terminate
|
||||
|
||||
|
||||
def contract(root):
|
||||
c=json.loads((root/'contract.json').read_text())
|
||||
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():assert sha(p)==h,p
|
||||
assert len(c['jobs'])==195 and len(c['cells'])==15 and c['limits']['worker_cpu_seconds']==360000
|
||||
assert c['limits']['per_fit_cpu_seconds']==1800 and c['limits']['workers']==4
|
||||
assert set(c['worker_cpus']).isdisjoint(range(8))
|
||||
assert c['final_refit']=='same_learning_pool_only' and c['selection_policy']=='best_finished_config_fixed_grid_tie_order'
|
||||
return c
|
||||
|
||||
|
||||
def select_finished(cell,trials):
|
||||
assert len(trials)==12 and sorted(t['config_index'] for t in trials)==list(range(12))
|
||||
finished=[t for t in trials if t['status']=='PASS' and t.get('score') is not None and math.isfinite(t['score'])]
|
||||
if not finished:return dict(status='NA_ALL_HPO_FAILED',cell=cell,finished_configurations=0,trials=trials,config_index=None)
|
||||
chosen=min(finished,key=lambda t:(t['score'],t['config_index']))
|
||||
return dict(status='SELECTED',cell=cell,config_index=chosen['config_index'],validation_score=chosen['score'],
|
||||
finished_configurations=len(finished),failed_configurations=12-len(finished),trials=trials,
|
||||
final_refit_set='same_learning_pool_only',ruling='1794707190.141609')
|
||||
|
||||
|
||||
def cost_check(c,first):
|
||||
cap=c['limits']['per_fit_cpu_seconds'];hard=len(c['jobs'])*cap
|
||||
if first['worker']['status']!='PASS':
|
||||
return dict(continue_within_cap=hard<=c['limits']['worker_cpu_seconds'],method='No successful empirical fit; capped195attempts upper-bound only, not a completion forecast',
|
||||
projected_worker_cpu_seconds=None,capped_attempt_upper_bound_seconds=hard,first_status=first['worker']['status'])
|
||||
path=Path(first['result_path']);r=json.loads(path.read_text());initial=c['jobs'][0]['cost_units']
|
||||
remaining=sum(j['cost_units'] for j in c['jobs'][1:]);rate=r['fit_cpu_seconds']/initial
|
||||
# Non-fit work scales by query rows, final config conservatively at63leaves/1000iterations.
|
||||
overhead=max(0.,first['worker']['cpu_seconds']-r['fit_cpu_seconds'])
|
||||
initial_rows=c['jobs'][0]['cost_queries']
|
||||
forecast=first['worker']['cpu_seconds']+1.25*(rate*remaining+overhead*sum(j['cost_queries']/initial_rows for j in c['jobs'][1:]))
|
||||
return dict(continue_within_cap=forecast<=c['limits']['worker_cpu_seconds'],projected_worker_cpu_seconds=forecast,
|
||||
capped_attempt_upper_bound_seconds=hard,first_fit_cpu_seconds=r['fit_cpu_seconds'],first_worker_cpu_seconds=first['worker']['cpu_seconds'],
|
||||
method='Observed fit CPU/(learning eligible queries*iterations*leaves), query-scaled nonfit CPU, final worst grid config,25percent reserve; heuristic not accuracy guarantee')
|
||||
|
||||
|
||||
def guard(root,index,slot):
|
||||
c=contract(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]])
|
||||
dest=root/'jobs'/f'job-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
|
||||
w=supervise([sys.executable,'tools/run_s2_r2_fit.py',str(root),str(index)],dest,'worker',
|
||||
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=7200,tree_rss_bytes=16*2**30,
|
||||
disk_bytes=5*2**30,max_processes=1),poll=.2)
|
||||
path=dest/'output/result.json';result=None
|
||||
if path.exists():result=json.loads(path.read_text())
|
||||
if w['status']=='PASS' and (not result or result['status']!='COMPLETE'):
|
||||
w.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_COMPLETE_RESULT')
|
||||
save(dest/'guard-result.json',dict(worker=w,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
cpu_affinity=sorted(os.sched_getaffinity(0)),result_path=str(path) if path.exists() else None,
|
||||
result_sha256=sha(path) if path.exists() else None,job=c['jobs'][index]))
|
||||
return 0 if w['status']=='PASS' else 1
|
||||
|
||||
|
||||
def finalize_selections(root,c,done):
|
||||
by_index={r['index']:r for r in done};directory=root/'selections';directory.mkdir(exist_ok=False)
|
||||
selected={}
|
||||
for cell in c['cells']:
|
||||
trials=[]
|
||||
for i in cell['hpo_indices']:
|
||||
row=by_index[i];path=root/'jobs'/f'job-{i:03d}'/'guard-result.json'
|
||||
result=json.loads(Path(row['result_path']).read_text()) if row.get('result_path') else None
|
||||
trials.append(dict(index=i,config_index=c['jobs'][i]['config_index'],status=row['worker']['status'],
|
||||
reason=row['worker']['stop_reason'] or (None if row['worker']['status']=='PASS' else 'WORKER_EXIT_'+str(row['worker']['returncode'])),
|
||||
score=result.get('validation_component_equal_relative_l2') if result else None,
|
||||
receipt_path=str(path),receipt_sha256=sha(path)))
|
||||
decision=select_finished(cell['index'],trials);save(directory/f"cell-{cell['index']:02d}.json",decision);selected[cell['index']]=decision
|
||||
return selected
|
||||
|
||||
|
||||
def run(root):
|
||||
with (root/'batch.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
|
||||
|
||||
|
||||
def run_locked(root):
|
||||
c=contract(root);assert not (root/'batch-started.json').exists(),'No repeat R2 queue'
|
||||
os.sched_setaffinity(0,c['coordinator_cpus'])
|
||||
start=time.monotonic();save(root/'batch-started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0))))
|
||||
active={};done=[];next_index=0;reason=None;first_checked=False;selections=None;live_cpu=live_rss=disk=0
|
||||
monitor=Monitor(root/'monitor-errors.jsonl')
|
||||
env=dict(os.environ,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
||||
def interrupted(*_):
|
||||
nonlocal reason
|
||||
reason='R2_COORDINATOR_INTERRUPTED'
|
||||
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
||||
def terminate_active():terminate({k:(v['process'],v['log']) for k,v in active.items()})
|
||||
try:
|
||||
while active or (next_index<195 and reason is None):
|
||||
for slot,item in list(active.items()):
|
||||
code=item['process'].poll()
|
||||
if code is None:continue
|
||||
path=root/'jobs'/f"job-{item['index']:03d}"/'guard-result.json'
|
||||
if not path.exists():
|
||||
save(root/'infrastructure-stop.json',dict(reason='MISSING_GUARD_RECEIPT',index=item['index'],returncode=code,accounting_complete=False));raise RuntimeError('Missing R2 guard accounting')
|
||||
row=json.loads(path.read_text());assert math.isfinite(row['worker']['cpu_seconds']) and row['worker']['cpu_seconds']>=0
|
||||
done.append(dict(index=item['index'],**row));item['log'].close();del active[slot]
|
||||
print(json.dumps(dict(event='R2_TERMINAL',index=item['index'],kind=c['jobs'][item['index']]['kind'],status=row['worker']['status'],completed=len(done))),flush=True)
|
||||
def usage():
|
||||
values=[tree_usage(v['process'].pid) for v in active.values()]
|
||||
return sum(x[0] for x in values),sum(x[1] for x in values)
|
||||
(live_cpu,live_rss),fresh=monitor.sample('live_usage',usage,(live_cpu,live_rss))
|
||||
disk,ok=monitor.sample('disk',lambda:disk_usage(root),disk);fresh &= ok
|
||||
spent=sum(x['worker']['cpu_seconds'] for x in done)
|
||||
over=budget_reason(0,spent,live_cpu,disk,cpu_limit=c['limits']['worker_cpu_seconds'],disk_limit=c['limits']['disk_bytes'])
|
||||
if over:reason=over
|
||||
if not first_checked and len(done)==1 and not active and reason is None:
|
||||
check=cost_check(c,done[0]);save(root/'first-fit-cost-check.json',check);first_checked=True
|
||||
if not check['continue_within_cap']:reason='FIRST_FIT_COST_PROJECTION_OVER_CAP'
|
||||
if next_index==180 and not active and len(done)==180 and selections is None and reason is None:
|
||||
selections=finalize_selections(root,c,done)
|
||||
if reason:terminate_active()
|
||||
ceiling=1 if not first_checked else (180 if selections is None else 195)
|
||||
while fresh and reason is None and len(active)<4 and next_index<ceiling:
|
||||
i=next_index;job=c['jobs'][i]
|
||||
if job['kind']=='FINAL' and selections[job['cell']]['status']!='SELECTED':
|
||||
done.append(dict(index=i,job=job,worker=dict(status='NA',cpu_seconds=0.,returncode=0,stop_reason='ALL12_HPO_FAILED_NO_FINAL_FIT'),
|
||||
supervisor_cpu_seconds=0.,attempted=False,result_path=None,result_sha256=None));next_index+=1;continue
|
||||
slot=next(k for k in range(4) if k not in active);dest=root/'jobs'/f'job-{i:03d}';dest.mkdir(parents=True,exist_ok=True)
|
||||
log=(dest/'guard.log').open('x')
|
||||
proc=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i),'--slot',str(slot)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
active[slot]=dict(process=proc,log=log,index=i);next_index+=1
|
||||
save(root/'progress.json',dict(completed=len(done),active={str(k):v['index'] for k,v in active.items()},next_index=next_index,
|
||||
worker_cpu_seconds=spent,live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=disk,
|
||||
first_fit_checked=first_checked,selections_frozen=selections is not None,stop_reason=reason,fresh_accounting=fresh,
|
||||
monitor_errors=monitor.errors,elapsed_seconds=time.monotonic()-start))
|
||||
if active or not fresh:time.sleep(.5)
|
||||
finally:
|
||||
terminate_active()
|
||||
for item in active.values():item['process'].wait();item['log'].close()
|
||||
attempted={x['index']:x for x in done};finals=[]
|
||||
for job in c['jobs'][180:]:
|
||||
row=attempted.get(job['index']);passed=row is not None and row['worker']['status']=='PASS'
|
||||
result=json.loads(Path(row['result_path']).read_text()) if passed else None
|
||||
finals.append(dict(cell=job['cell'],fold=job['fold'],pool=job['pool'],status='PASS' if passed else 'NA',
|
||||
reason=None if passed else (row['worker']['stop_reason'] if row else 'NOT_RUN_'+str(reason)),
|
||||
finished_configurations=selections[job['cell']]['finished_configurations'] if selections else None,
|
||||
model_path=str(Path(row['result_path']).parent/'model.joblib') if passed else None,
|
||||
model_sha256=result['model_sha256'] if result else None))
|
||||
save(root/'batch-complete.json',dict(status='ALL_R2_TERMINAL',stop_reason=reason,final_fits=finals,jobs=done,
|
||||
not_run_indices=[i for i in range(195) if i not in attempted],worker_cpu_seconds=sum(x['worker']['cpu_seconds'] for x in done),
|
||||
guard_cpu_seconds=sum(x['supervisor_cpu_seconds'] for x in done),coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
disk_bytes=disk_usage(root),wall_seconds=time.monotonic()-start,monitor_errors=monitor.errors,roster_sha256=c['roster_sha256'],
|
||||
test_predictions=False,test_scoring=False,retry=False))
|
||||
return 0 if reason is None else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int);a=p.parse_args()
|
||||
raise SystemExit(guard(a.root,a.guard,a.slot) if a.guard is not None else run(a.root))
|
||||
@@ -0,0 +1,67 @@
|
||||
"""One single-thread HPO trial or fresh final refit; launch only through the CPU guard."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
import traceback
|
||||
import joblib
|
||||
import sklearn
|
||||
from sklearn.ensemble import HistGradientBoostingRegressor
|
||||
from threadpoolctl import threadpool_limits,threadpool_info
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_supervisor import save
|
||||
from s2_models.features import histgbrt_grid,R2_COLUMNS
|
||||
from s2_r2_data import load_cell,learning_arrays,validation_score
|
||||
|
||||
|
||||
def run(root,index):
|
||||
from run_s2_r2 import contract
|
||||
start=time.monotonic();cpu_start=time.process_time();c=contract(root);job=c['jobs'][index]
|
||||
dest=root/'jobs'/f'job-{index:03d}'/'output';dest.mkdir(exist_ok=False)
|
||||
save(dest/'started.json',dict(unix=time.time(),pid=os.getpid(),job=job,contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0))))
|
||||
try:
|
||||
assert sklearn.__version__=='1.7.2'
|
||||
config_index=job.get('config_index');selection=None
|
||||
if job['kind']=='FINAL':
|
||||
path=root/'selections'/f"cell-{job['cell']:02d}.json";selection=json.loads(path.read_text())
|
||||
assert selection['status']=='SELECTED' and selection['cell']==job['cell']
|
||||
config_index=selection['config_index']
|
||||
for item in selection['trials']:
|
||||
assert sha(item['receipt_path'])==item['receipt_sha256']
|
||||
save(dest/'selection.json',dict(path=str(path),sha256=sha(path),**selection))
|
||||
params=histgbrt_grid()[config_index]
|
||||
with threadpool_limits(limits=1):
|
||||
pools=threadpool_info();assert all(p['num_threads']==1 for p in pools)
|
||||
learning,validation,roles=load_cell(root,c,job)
|
||||
save(dest/'access.json',dict(fold=job['fold'],pool=job['pool'],kind=job['kind'],
|
||||
cases=[dict(case=p['case'],role=p['role'],label_mode=p['label_mode'],label_sha256=p['label_sha256']) for p in learning+validation],
|
||||
outer_fold_test_labels_read=False,learning_only_final_refit=job['kind']=='FINAL'))
|
||||
x,y,w=learning_arrays(learning);load_cpu=time.process_time()-cpu_start
|
||||
save(dest/'progress.json',dict(stage='fit',training_queries=len(y),training_parts=len(learning),cpu_seconds=time.process_time()-cpu_start))
|
||||
model=HistGradientBoostingRegressor(**params)
|
||||
fit_start=time.process_time();model.fit(x,y,sample_weight=w);fit_cpu=time.process_time()-fit_start
|
||||
assert model.n_iter_==params['max_iter']
|
||||
value=errors=None
|
||||
if job['kind']=='HPO':
|
||||
save(dest/'progress.json',dict(stage='validation',fit_cpu_seconds=fit_cpu))
|
||||
value,errors=validation_score(model,validation);save(dest/'validation-parts.json',errors)
|
||||
joblib.dump(model,dest/'model.joblib',compress=3)
|
||||
result=dict(status='COMPLETE',job=job,config_index=config_index,params=params,sklearn_version=sklearn.__version__,
|
||||
columns=list(R2_COLUMNS),target='T/t_set',learning_queries=len(y),learning_parts=len(learning),
|
||||
learning_components=len({p['component_id'] for p in learning}),validation_parts=len(validation),
|
||||
sample_weight_sum=float(w.sum()),validation_component_equal_relative_l2=value,fit_cpu_seconds=fit_cpu,
|
||||
load_cpu_seconds=load_cpu,worker_elapsed_cpu_seconds=time.process_time()-cpu_start,wall_seconds=time.monotonic()-start,
|
||||
threadpools=pools,cpu_affinity=sorted(os.sched_getaffinity(0)),model_sha256=sha(dest/'model.joblib'),
|
||||
roster_sha256=c['roster_sha256'],test_predictions=False,test_labels_read=False,random_state=0,retry=False,
|
||||
final_refit_learning_only=job['kind']=='FINAL',hpo_finished_configurations=selection['finished_configurations'] if selection else None)
|
||||
save(dest/'result.json',result);save(dest/'progress.json',dict(stage='complete',cpu_seconds=time.process_time()-cpu_start))
|
||||
print(json.dumps(result),flush=True);return 0
|
||||
except Exception as exc:
|
||||
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
|
||||
cpu_seconds=time.process_time()-cpu_start,wall_seconds=time.monotonic()-start))
|
||||
raise
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))
|
||||
@@ -0,0 +1,65 @@
|
||||
"""R2 uses frozen query rows, direct targets and the same component/part weights."""
|
||||
from collections import Counter
|
||||
import json
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
from freeze_g1_components import sha
|
||||
from s2_main_data import select_cell,ensure_targets
|
||||
from s2_pilot_data import permitted_role
|
||||
from s2_models.features import R2_COLUMNS
|
||||
from s2_models.objectives import relative_l2
|
||||
|
||||
|
||||
def read_part(root,c,row,roles,old_labels):
|
||||
role=permitted_role(roles,row['case'])
|
||||
prep=row['preparation'];source=Path(prep['preparation_root'])
|
||||
label,label_sha,mode=ensure_targets(root,c,row,roles,old_labels)
|
||||
for name in ['r2.npz','queries.npz']:assert sha(source/name)==prep['output_sha256'][name]
|
||||
with np.load(source/'queries.npz',allow_pickle=False) as q:
|
||||
mask=q['eligible_s2'];ids=q['anchor_ids'][mask];counts=q['multiplicity'][mask]
|
||||
with np.load(source/'r2.npz',allow_pickle=False) as z:
|
||||
x=z['features'];assert x.shape==(row['nodes'],len(R2_COLUMNS));x=x[ids].copy()
|
||||
assert sha(label)==label_sha
|
||||
with np.load(label,allow_pickle=False) as z:
|
||||
assert np.array_equal(z['anchor_ids'],ids) and np.array_equal(z['multiplicity'],counts)
|
||||
y=z['target_t_star'].copy();seconds=z['seconds'].copy()
|
||||
assert len(y)>0 and np.isfinite(x).all() and np.isfinite(y).all() and (y>=0).all()
|
||||
assert np.all(x[:,-1]==prep['t_set'])
|
||||
assert np.array_equal(y,seconds/prep['t_set'])
|
||||
return dict(case=row['case'],component_id=row['component_id'],role=role,x=x,y=y,seconds=seconds,
|
||||
counts=counts,t_set=prep['t_set'],query_ids=ids,label_mode=mode,label_sha256=label_sha)
|
||||
|
||||
|
||||
def learning_arrays(parts):
|
||||
assert parts and all(p['role']=='learning' for p in parts)
|
||||
groups=Counter(p['component_id'] for p in parts);ng=len(groups)
|
||||
x=[];y=[];weights=[]
|
||||
for p in parts:
|
||||
counts=np.asarray(p['counts'],dtype=np.float64)
|
||||
assert len(counts)==len(p['y'])==len(p['x']) and np.all(counts>0)
|
||||
x.append(p['x']);y.append(p['y'])
|
||||
weights.append(counts/counts.sum()/groups[p['component_id']]/ng)
|
||||
w=np.concatenate(weights);assert np.isclose(w.sum(),1,rtol=0,atol=1e-12)
|
||||
return np.concatenate(x),np.concatenate(y),w
|
||||
|
||||
|
||||
def validation_score(model,parts):
|
||||
assert parts and all(p['role']=='validation' for p in parts)
|
||||
groups={};errors={}
|
||||
for p in parts:
|
||||
predicted=model.predict(p['x'])
|
||||
value=relative_l2(predicted*p['t_set'],p['seconds'],p['counts'],np.ones(len(p['counts']),bool))['value']
|
||||
if value is None or not np.isfinite(value):raise ValueError('Invalid validation relative L2')
|
||||
errors[p['case']]=value;groups.setdefault(p['component_id'],[]).append(value)
|
||||
return float(np.mean([np.mean(v) for v in groups.values()])),errors
|
||||
|
||||
|
||||
def load_cell(root,c,job):
|
||||
assert sha(c['roster'])==c['roster_sha256']
|
||||
roster=json.loads(Path(c['roster']).read_text());rows,roles=select_cell(roster,job['fold'],job['pool'])
|
||||
assert sha(c['old_labels_manifest'])==c['old_labels_manifest_sha256']
|
||||
old={p['case']:p for p in json.loads(Path(c['old_labels_manifest']).read_text())['ready_parts']}
|
||||
# Final refit reads learning only; validation was already used for HPO selection.
|
||||
cases=roles['learning_parts']+(roles['validation_parts'] if job['kind']=='HPO' else [])
|
||||
parts=[read_part(root,c,rows[k],roles,old) for k in sorted(cases)]
|
||||
return [p for p in parts if p['role']=='learning'],[p for p in parts if p['role']=='validation'],roles
|
||||
@@ -0,0 +1,136 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import numpy as np
|
||||
import run_s2_r2 as queue
|
||||
import run_s2_r2_fit as fit
|
||||
import s2_r2_data as data
|
||||
from s2_models.objectives import hierarchy_weights
|
||||
from prepare_s2_r2 import build_jobs
|
||||
|
||||
|
||||
def synthetic_part(case,component,role,counts):
|
||||
n=len(counts);x=np.zeros((n,20));x[:,-1]=2.
|
||||
return dict(case=case,component_id=component,role=role,x=x,y=np.ones(n),seconds=np.ones(n)*2,counts=np.array(counts),t_set=2.,label_mode='synthetic',label_sha256='synthetic')
|
||||
|
||||
|
||||
def fake_contract():
|
||||
jobs=[];cells=[]
|
||||
for cell in range(15):
|
||||
ids=[]
|
||||
for cfg in range(12):
|
||||
ids.append(len(jobs));jobs.append(dict(index=len(jobs),cell=cell,fold=cell//3,pool=[25,50,100][cell%3],kind='HPO',config_index=cfg,cost_units=1,cost_queries=1))
|
||||
cells.append(dict(index=cell,hpo_indices=ids))
|
||||
for cell in range(15):jobs.append(dict(index=len(jobs),cell=cell,fold=cell//3,pool=[25,50,100][cell%3],kind='FINAL',cost_units=1,cost_queries=1))
|
||||
return dict(jobs=jobs,cells=cells,worker_cpus=[8,10,12,14],coordinator_cpus=[30,31],roster_sha256='fixed',
|
||||
limits=dict(worker_cpu_seconds=360000,per_fit_cpu_seconds=1800,workers=4,disk_bytes=100*2**30))
|
||||
|
||||
|
||||
class R2Checks(unittest.TestCase):
|
||||
def test_weights_match_frozen_hierarchy_without_rescaling(self):
|
||||
parts=[synthetic_part('a','A','learning',[3,1]),synthetic_part('b','A','learning',[2]),synthetic_part('c','B','learning',[1,1])]
|
||||
_,_,weights=data.learning_arrays(parts)
|
||||
expected=hierarchy_weights([3,1,2,1,1],['a','a','b','c','c'],['A','A','A','B','B'])
|
||||
np.testing.assert_array_equal(weights,expected);self.assertEqual(weights.sum(),1.)
|
||||
|
||||
def test_validation_is_component_equal_and_seconds_scale_correct(self):
|
||||
parts=[synthetic_part('a','A','validation',[1]),synthetic_part('b','A','validation',[1]),synthetic_part('c','B','validation',[1])]
|
||||
class Model:
|
||||
def __init__(self):self.i=0
|
||||
def predict(self,x):self.i+=1;return np.array([1.,3.,4.])[self.i-1:self.i]
|
||||
value,errors=data.validation_score(Model(),parts)
|
||||
self.assertEqual(errors,dict(a=0.,b=2.,c=3.));self.assertEqual(value,2.)
|
||||
|
||||
def test_test_case_rejected_before_feature_or_target_io(self):
|
||||
roles=dict(learning_parts=['l'],validation_parts=['v'],sealed_test_parts=['t'])
|
||||
with patch.object(data,'ensure_targets',side_effect=AssertionError('target IO')),patch.object(data,'sha',side_effect=AssertionError('feature IO')):
|
||||
with self.assertRaises(PermissionError):data.read_part(Path('/none'),{},dict(case='t'),roles,{})
|
||||
|
||||
def test_final_loader_never_reads_validation(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);roster=root/'roster.json';old=root/'old.json'
|
||||
roster.write_text('{}');old.write_text(json.dumps(dict(ready_parts=[])))
|
||||
c=dict(roster=str(roster),roster_sha256=data.sha(roster),old_labels_manifest=str(old),old_labels_manifest_sha256=data.sha(old))
|
||||
roles=dict(learning_parts=['l'],validation_parts=['v'],sealed_test_parts=['t']);rows={k:dict(case=k) for k in ['l','v']};seen=[]
|
||||
def read(*args):
|
||||
case=args[2]['case'];seen.append(case);return synthetic_part(case,case,'learning' if case=='l' else 'validation',[1])
|
||||
with patch.object(data,'select_cell',return_value=(rows,roles)),patch.object(data,'read_part',read):
|
||||
learning,validation,_=data.load_cell(root,c,dict(fold=0,pool=25,kind='FINAL'))
|
||||
self.assertEqual(seen,['l']);self.assertEqual(validation,[])
|
||||
|
||||
def test_select_finished_configures_failure_and_tie_policy(self):
|
||||
trials=[dict(config_index=i,status='STOPPED_OR_FAILED',score=None,reason='wall_limit') for i in range(12)]
|
||||
self.assertEqual(queue.select_finished(0,trials)['status'],'NA_ALL_HPO_FAILED')
|
||||
trials[3].update(status='PASS',score=.2);trials[1].update(status='PASS',score=.2)
|
||||
r=queue.select_finished(0,trials);self.assertEqual(r['config_index'],1);self.assertEqual(r['finished_configurations'],2)
|
||||
self.assertEqual(r['failed_configurations'],10)
|
||||
|
||||
def test_grid_plan_has_180_unique_trials_15_final_and_real_roster(self):
|
||||
roster=json.loads(Path('artifacts/s2-main-preparation-2026-11-15/final-roster.json').read_text());jobs,cells=build_jobs(roster)
|
||||
self.assertEqual((len(jobs),len(cells)),(195,15))
|
||||
self.assertEqual(sum(j['kind']=='HPO' for j in jobs),180)
|
||||
for cell in cells:self.assertEqual([jobs[i]['config_index'] for i in cell['hpo_indices']],list(range(12)))
|
||||
self.assertTrue(all(j['kind']=='FINAL' and j['config_index'] is None for j in jobs[180:]))
|
||||
|
||||
def test_first_fit_projection_and_failed_attempt_cost_bound(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
p=Path(t)/'result.json';p.write_text(json.dumps(dict(fit_cpu_seconds=1700)))
|
||||
c=fake_contract();first=dict(worker=dict(status='PASS',cpu_seconds=1800),result_path=str(p))
|
||||
self.assertFalse(queue.cost_check(c,first)['continue_within_cap'])
|
||||
first['worker']['status']='STOPPED_OR_FAILED';check=queue.cost_check(c,first)
|
||||
self.assertTrue(check['continue_within_cap']);self.assertEqual(check['capped_attempt_upper_bound_seconds'],351000)
|
||||
self.assertIsNone(check['projected_worker_cpu_seconds'])
|
||||
|
||||
def test_guard_uses_cpu_cap_and_disjoint_affinity(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);seen=[]
|
||||
def supervise(command,dest,name,limits,**kw):seen.append(limits);return dict(status='STOPPED_OR_FAILED',stop_reason='synthetic')
|
||||
with patch.object(queue,'contract',return_value=fake_contract()),patch.object(queue.os,'sched_setaffinity') as affinity,patch.object(queue,'supervise',supervise):
|
||||
self.assertEqual(queue.guard(root,0,0),1)
|
||||
affinity.assert_called_once_with(0,[8])
|
||||
self.assertEqual(seen[0]['job_cpu_seconds'],1800);self.assertEqual(seen[0]['max_processes'],1)
|
||||
|
||||
def test_actual_synthetic_histgbrt_fit_single_thread_and_fixed_params(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);(root/'contract.json').write_text('{}');(root/'jobs/job-000').mkdir(parents=True)
|
||||
c=fake_contract();train=synthetic_part('learn','A','learning',[1]*240);val=synthetic_part('val','V','validation',[1]*40)
|
||||
rng=np.random.default_rng(17)
|
||||
for p in [train,val]:
|
||||
p['x'][:,:19]=rng.normal(size=(len(p['y']),19));p['y']=1+.1*p['x'][:,0];p['seconds']=p['y']*2
|
||||
with patch.object(queue,'contract',return_value=c),patch.object(fit,'load_cell',return_value=([train],[val],{})):
|
||||
self.assertEqual(fit.run(root,0),0)
|
||||
r=json.loads((root/'jobs/job-000/output/result.json').read_text())
|
||||
self.assertEqual(r['params']['max_iter'],200);self.assertFalse(r['params']['early_stopping'])
|
||||
self.assertTrue(all(p['num_threads']==1 for p in r['threadpools']));self.assertFalse(r['test_predictions'])
|
||||
|
||||
def test_queue_first_fit_barrier_and_unique_195_attempts(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);(root/'contract.json').write_text('{}');c=fake_contract();spawned=[];peak=[0]
|
||||
class Process:
|
||||
def __init__(self,index):self.index=index;self.pid=1000000+index;self.polls=0
|
||||
def poll(self):
|
||||
self.polls+=1
|
||||
if self.polls==1:return None
|
||||
job=c['jobs'][self.index];dest=root/'jobs'/f'job-{self.index:03d}';out=dest/'output';out.mkdir(exist_ok=True)
|
||||
result=out/'result.json';result.write_text(json.dumps(dict(status='COMPLETE',fit_cpu_seconds=.001,validation_component_equal_relative_l2=job.get('config_index',0)/100,model_sha256='synthetic')))
|
||||
(dest/'guard-result.json').write_text(json.dumps(dict(worker=dict(status='PASS',cpu_seconds=.01,returncode=0,stop_reason=None),
|
||||
supervisor_cpu_seconds=.001,result_path=str(result),result_sha256=data.sha(result),job=job)))
|
||||
return 0
|
||||
def wait(self):return 0
|
||||
def spawn(command,**kw):
|
||||
i=int(command[-3])
|
||||
if i>0:self.assertTrue((root/'first-fit-cost-check.json').exists())
|
||||
if i>=180:self.assertTrue((root/'selections'/f"cell-{c['jobs'][i]['cell']:02d}.json").exists())
|
||||
p=Process(i);spawned.append(p);peak[0]=max(peak[0],sum(p.polls<2 for p in spawned));return p
|
||||
with patch.object(queue,'contract',return_value=c),patch.object(queue.os,'sched_setaffinity'),patch.object(queue.subprocess,'Popen',spawn),patch.object(queue,'tree_usage',return_value=(0.,1,1)),patch.object(queue.time,'sleep'),patch('builtins.print'):
|
||||
self.assertEqual(queue.run(root),0)
|
||||
r=json.loads((root/'batch-complete.json').read_text());self.assertEqual(len(spawned),195);self.assertEqual(len({p.index for p in spawned}),195)
|
||||
self.assertLessEqual(peak[0],4);self.assertEqual(len(r['final_fits']),15);self.assertTrue(all(x['status']=='PASS' for x in r['final_fits']))
|
||||
with patch.object(queue,'contract',return_value=c),patch.object(queue.os,'sched_setaffinity'),patch.object(queue.subprocess,'Popen') as spawn:
|
||||
with self.assertRaises(AssertionError):queue.run(root)
|
||||
spawn.assert_not_called()
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user