Implement approved bounded S3-A queue with first-trial cost checkpoint
This commit is contained in:
1 parent
b0390231f0
commit
6142ad69d0
12 files changed
+19110
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"version": "S3-A-queue-v1",
|
||||
"contract_sha256": "263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8",
|
||||
"review_sha256": "1092dd9926d6a1a486da21cf2276901e01a67c2f8110a0fe2fbf26a808046bea",
|
||||
"approved_plan_commit": "b0390231f0161fa55ae72494a32a38863e1b96c0",
|
||||
"approval_slack": "1794732829.567629",
|
||||
"code_sha256": {
|
||||
"tools/s3a_runtime.py": "919c6198971505ea0b2cfbb8e2b09be7d322eab2261f4ca820c05bbab8dcf98a",
|
||||
"tools/run_s3a_fit.py": "a77f6166ba69e93a74d7b5aca7233d5730baf896ebfc25fc923856d97ba3f78c",
|
||||
"tools/run_s3a_queue.py": "cb98694d4860b7e004f5d20636900f4ffa12439f07fdd8cfbd77789e53e92ae1",
|
||||
"tools/test_s3a_queue.py": "a7fabe301eeb4968cb31a984a6bdbe64cb43a01adf9bc8c539c5c926fb564a59",
|
||||
"tools/prepare_s3a_execution.py": "f183c98f825b5a6efe042dcbdc567ae0eeb4d356f9d1a0d43b917b70d2992c0c"
|
||||
},
|
||||
"scope": "Fit queue only; stops at immutable terminal freeze, never automatically evaluates test labels",
|
||||
"first_trials": [
|
||||
0,
|
||||
12
|
||||
],
|
||||
"dispatch": "0,12 then all remaining indices in ascending order",
|
||||
"maximum_new_fits": 390,
|
||||
"GPU": false,
|
||||
"retries": 0
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"approved": true,
|
||||
"slack_ts": "1794732829.567629",
|
||||
"slack_url": "https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794732829567629",
|
||||
"git_commit": "b0390231f0161fa55ae72494a32a38863e1b96c0",
|
||||
"contract_sha256": "263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8",
|
||||
"ruling": "Go ahead and implement and launch the queue under the contract. No further review from me before the result.",
|
||||
"review_scope": "Claude spot-checked s3a_statistics.py and contract; not a full executor review",
|
||||
"result_headline_required": "Follow-up exploration with S2 results known, not a fresh confirmatory test"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"status": "IMPLEMENTED_NOT_STARTED",
|
||||
"contract_sha256": "263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8",
|
||||
"implementation_sha256": "a2bd051b83502129473038b0e98d5d0a0726e1f0d5c6f0e718ccfad1c77f1526",
|
||||
"fits_started": false,
|
||||
"baseline_refits": 0,
|
||||
"labels_decoded": false,
|
||||
"approved": true
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests": 8,
|
||||
"failures": 0,
|
||||
"errors": 0,
|
||||
"worker_CPU_s": 13.534670686,
|
||||
"wall_s": 13.561030525015667,
|
||||
"real_dataset_fits": 0,
|
||||
"synthetic_model_fits": 1,
|
||||
"output": "test_390_unique_attempts_two_sequential_witnesses_and_four_workers (test_s3a_queue.QueueChecks) ... ok\ntest_actual_synthetic_HistGBRT_has24columns_fixedgrid_and_singlethread (test_s3a_queue.QueueChecks) ... ok\ntest_failed_final_kept_as_cell_NA_in_terminal_freeze (test_s3a_queue.QueueChecks) ... ok\ntest_first_failure_stops_without_second_trial_or_retry (test_s3a_queue.QueueChecks) ... ok\ntest_guard_uses_approved_limits_and_one_process (test_s3a_queue.QueueChecks) ... ok\ntest_missing_guard_is_infrastructure_stop_not_completed_batch (test_s3a_queue.QueueChecks) ... ok\ntest_nonfirst_hpo_failure_is_NA_config_and_final_fits_remaining (test_s3a_queue.QueueChecks) ... ok\ntest_reservation_stop_never_starts_next_task_or_freezes_complete (test_s3a_queue.QueueChecks) ... ok\n\n----------------------------------------------------------------------\nRan 8 tests in 13.561s\n\nOK\n",
|
||||
"code_sha256": {
|
||||
"tools/run_s3a_queue.py": "cb98694d4860b7e004f5d20636900f4ffa12439f07fdd8cfbd77789e53e92ae1",
|
||||
"tools/run_s3a_fit.py": "a77f6166ba69e93a74d7b5aca7233d5730baf896ebfc25fc923856d97ba3f78c",
|
||||
"tools/s3a_runtime.py": "919c6198971505ea0b2cfbb8e2b09be7d322eab2261f4ca820c05bbab8dcf98a",
|
||||
"tools/test_s3a_queue.py": "a7fabe301eeb4968cb31a984a6bdbe64cb43a01adf9bc8c539c5c926fb564a59",
|
||||
"tools/prepare_s3a_execution.py": "f183c98f825b5a6efe042dcbdc567ae0eeb4d356f9d1a0d43b917b70d2992c0c"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"approved": true,
|
||||
"slack_ts": "1794732829.567629",
|
||||
"slack_url": "https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794732829567629",
|
||||
"git_commit": "b0390231f0161fa55ae72494a32a38863e1b96c0",
|
||||
"contract_sha256": "263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8",
|
||||
"ruling": "Go ahead and implement and launch the queue under the contract. No further review from me before the result.",
|
||||
"review_scope": "Claude spot-checked s3a_statistics.py and contract; not a full executor review",
|
||||
"result_headline_required": "Follow-up exploration with S2 results known, not a fresh confirmatory test"
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
# S3-A 队列实现与启动前验收
|
||||
|
||||
Claude1794732829.567629已批准b0390231合同,明确允许实现并启动,结果前不再主审。其实际审查范围为合同和s3a_statistics.py定点核读;不把该主审写成独立逐行执行器审查。保持“已知S2结果后的探索,非新确认性测试”结果标题。
|
||||
|
||||
实现新增run_s3a_queue.py/run_s3a_fit.py/s3a_runtime.py,不改原S2或b0390231冻结代码。新root为artifacts/s3a-execution-2026-11-15,科学合同SHA263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8完全相同,implementation.json单独固定新执行代码。主审凭据也保存于原规划root。
|
||||
|
||||
按固定job0(E4)、job12(G2)依次单独首试;两项都PASS且双臂外推含余量≤18h,继续最多4单线程CPU worker。随后未尝试HPO按原ID顺序;360HPO全终态后冻结30格选择,再30个learning-only final。选优/平手/失败直接复用原select_finished/finalize_selections,未重新拟合R2。原fit仅替换新数据入口和列名。
|
||||
|
||||
在途任务按完整剩余单job cap保守预留,并额外计入活跃树CPU,不会为了跑完而放宽预算。所有fit真实资源由原CPU guard跟踪,失败仍计费;缺guard回执标INFRA/accounting incomplete,不制造终态。全390终态且无全局停止才写terminal-freeze;只完成拟合队列,不自动跨过预测/评分冻结次序。后续预测/评分入口在已有2h预留内执行,并在运行前单独记录代码版本。
|
||||
|
||||
8项新增验收PASS:完整390唯一调度、两个首试顺序和4并发;首试失败停止;缺guard为未完成;普通HPO失败按11个成功配置选优;final失败保留NA;预算预留拒绝继续;guard的30min CPU/单进程约束;真实合成24列HistGBRT固定200迭代/单线程fit。合成验收不读真实标签或真实拟合。验收raw日志与源码hash见queue-acceptance.json。原17项合同/统计验收保持有效,不重复已完成S2/D3。
|
||||
|
||||
本文件为实现验收,实际启动时间、PID、首试开销与门结果以batch-started/progress/guard-result/first-fit-cost-check和后续actual-start-audit为准。
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Bind approved S3-A plan to new queue sources; does not launch workers."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from freeze_g1_components import sha
|
||||
from run_s3a_score import review_gate
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
PLAN = Path('artifacts/s3a-planning-2026-11-15')
|
||||
ROOT = Path('artifacts/s3a-execution-2026-11-15')
|
||||
SOURCES = ['tools/s3a_runtime.py', 'tools/run_s3a_fit.py', 'tools/run_s3a_queue.py',
|
||||
'tools/test_s3a_queue.py', 'tools/prepare_s3a_execution.py']
|
||||
|
||||
|
||||
def main():
|
||||
c = review_gate(PLAN)
|
||||
ROOT.mkdir(exist_ok=False)
|
||||
shutil.copy2(PLAN/'contract.json', ROOT/'contract.json')
|
||||
shutil.copy2(PLAN/'main-review.json', ROOT/'main-review.json')
|
||||
save(ROOT/'implementation.json', dict(version='S3-A-queue-v1',
|
||||
contract_sha256=sha(ROOT/'contract.json'), review_sha256=sha(ROOT/'main-review.json'),
|
||||
approved_plan_commit='b0390231f0161fa55ae72494a32a38863e1b96c0',
|
||||
approval_slack='1794732829.567629',
|
||||
code_sha256={p: sha(p) for p in SOURCES},
|
||||
scope='Fit queue only; stops at immutable terminal freeze, never automatically evaluates test labels',
|
||||
first_trials=[0, 12], dispatch='0,12 then all remaining indices in ascending order',
|
||||
maximum_new_fits=len(c['jobs']), GPU=False, retries=0))
|
||||
save(ROOT/'prelaunch.json', dict(status='IMPLEMENTED_NOT_STARTED',
|
||||
contract_sha256=sha(ROOT/'contract.json'), implementation_sha256=sha(ROOT/'implementation.json'),
|
||||
fits_started=False, baseline_refits=0, labels_decoded=False, approved=True))
|
||||
print(json.dumps(dict(root=str(ROOT), implementation_sha256=sha(ROOT/'implementation.json'))))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,68 @@
|
||||
"""One single-thread HPO trial or fresh final refit; launch only through the CPU guard."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
import traceback
|
||||
import joblib
|
||||
import sklearn
|
||||
from sklearn.ensemble import HistGradientBoostingRegressor
|
||||
from threadpoolctl import threadpool_limits,threadpool_info
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_supervisor import save
|
||||
from s2_models.features import histgbrt_grid
|
||||
from s3a_data import load_cell,learning_arrays,validation_score,columns
|
||||
|
||||
|
||||
def run(root,index):
|
||||
from s3a_runtime import contract
|
||||
start=time.monotonic();cpu_start=time.process_time();c=contract(root);job=c['jobs'][index]
|
||||
dest=root/'jobs'/f'job-{index:03d}'/'output';dest.mkdir(exist_ok=False)
|
||||
save(dest/'started.json',dict(unix=time.time(),pid=os.getpid(),job=job,contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0))))
|
||||
try:
|
||||
assert job['arm'] in ('R2_E4','R2_G2')
|
||||
assert sklearn.__version__=='1.7.2'
|
||||
config_index=job.get('config_index');selection=None
|
||||
if job['kind']=='FINAL':
|
||||
path=root/'selections'/f"cell-{job['cell']:02d}.json";selection=json.loads(path.read_text())
|
||||
assert selection['status']=='SELECTED' and selection['cell']==job['cell']
|
||||
config_index=selection['config_index']
|
||||
for item in selection['trials']:
|
||||
assert sha(item['receipt_path'])==item['receipt_sha256']
|
||||
save(dest/'selection.json',dict(path=str(path),sha256=sha(path),**selection))
|
||||
params=histgbrt_grid()[config_index]
|
||||
with threadpool_limits(limits=1):
|
||||
pools=threadpool_info();assert all(p['num_threads']==1 for p in pools)
|
||||
learning,validation,roles=load_cell(root,c,job)
|
||||
save(dest/'access.json',dict(fold=job['fold'],pool=job['pool'],kind=job['kind'],
|
||||
cases=[dict(case=p['case'],role=p['role'],label_mode=p['label_mode'],label_sha256=p['label_sha256']) for p in learning+validation],
|
||||
outer_fold_test_labels_read=False,learning_only_final_refit=job['kind']=='FINAL'))
|
||||
x,y,w=learning_arrays(learning);load_cpu=time.process_time()-cpu_start
|
||||
save(dest/'progress.json',dict(stage='fit',training_queries=len(y),training_parts=len(learning),cpu_seconds=time.process_time()-cpu_start))
|
||||
model=HistGradientBoostingRegressor(**params)
|
||||
fit_start=time.process_time();model.fit(x,y,sample_weight=w);fit_cpu=time.process_time()-fit_start
|
||||
assert model.n_iter_==params['max_iter']
|
||||
value=errors=None
|
||||
if job['kind']=='HPO':
|
||||
save(dest/'progress.json',dict(stage='validation',fit_cpu_seconds=fit_cpu))
|
||||
value,errors=validation_score(model,validation);save(dest/'validation-parts.json',errors)
|
||||
joblib.dump(model,dest/'model.joblib',compress=3)
|
||||
result=dict(status='COMPLETE',job=job,config_index=config_index,params=params,sklearn_version=sklearn.__version__,
|
||||
columns=columns(job['arm']),target='T/t_set',learning_queries=len(y),learning_parts=len(learning),
|
||||
learning_components=len({p['component_id'] for p in learning}),validation_parts=len(validation),
|
||||
sample_weight_sum=float(w.sum()),validation_component_equal_relative_l2=value,fit_cpu_seconds=fit_cpu,
|
||||
load_cpu_seconds=load_cpu,worker_elapsed_cpu_seconds=time.process_time()-cpu_start,wall_seconds=time.monotonic()-start,
|
||||
threadpools=pools,cpu_affinity=sorted(os.sched_getaffinity(0)),model_sha256=sha(dest/'model.joblib'),
|
||||
roster_sha256=c['roster_sha256'],test_predictions=False,test_labels_read=False,random_state=0,retry=False,
|
||||
final_refit_learning_only=job['kind']=='FINAL',hpo_finished_configurations=selection['finished_configurations'] if selection else None)
|
||||
save(dest/'result.json',result);save(dest/'progress.json',dict(stage='complete',cpu_seconds=time.process_time()-cpu_start))
|
||||
print(json.dumps(result),flush=True);return 0
|
||||
except Exception as exc:
|
||||
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
|
||||
cpu_seconds=time.process_time()-cpu_start,wall_seconds=time.monotonic()-start))
|
||||
raise
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))
|
||||
@@ -0,0 +1,234 @@
|
||||
"""S3-A single-attempt CPU queue: two sequential witnesses then bounded four-way HPO."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from freeze_g1_components import sha
|
||||
from s2_custody_common import checked, read_json
|
||||
from s2_recovery_supervisor import save, supervise, tree_usage
|
||||
from s2_recovery_monitoring import Monitor, disk_usage, budget_reason
|
||||
from run_s2_pilot_preparation import terminate
|
||||
from run_s2_r2 import finalize_selections
|
||||
from s3a_budget import first_fit_projection, can_dispatch
|
||||
from s3a_runtime import contract
|
||||
|
||||
FIRST = [0, 12]
|
||||
|
||||
|
||||
def dispatch_order():
|
||||
return FIRST + [i for i in range(390) if i not in FIRST]
|
||||
|
||||
|
||||
def guard(root, index, slot):
|
||||
c = contract(root)
|
||||
os.sched_setaffinity(0, [c['worker_cpus'][slot]])
|
||||
limits = c['limits']
|
||||
dest = root/'jobs'/f'job-{index:03d}'
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
w = supervise([sys.executable, 'tools/run_s3a_fit.py', str(root), str(index)], dest, 'worker',
|
||||
dict(total_cpu_seconds=limits['per_fit_cpu_seconds']+1,
|
||||
job_cpu_seconds=limits['per_fit_cpu_seconds'], job_wall_seconds=limits['per_fit_wall_seconds'],
|
||||
tree_rss_bytes=limits['per_fit_tree_rss_bytes'], disk_bytes=limits['per_fit_disk_bytes'], max_processes=1), poll=.2)
|
||||
path = dest/'output/result.json'
|
||||
result = read_json(path) if path.exists() else None
|
||||
if w['status'] == 'PASS' and (not result or result['status'] != 'COMPLETE'):
|
||||
w.update(status='STOPPED_OR_FAILED', stop_reason='MISSING_COMPLETE_RESULT')
|
||||
if w['status'] == 'PASS':
|
||||
checked(dest/'output/model.joblib', result['model_sha256'])
|
||||
save(dest/'guard-result.json', dict(worker=w, supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
cpu_affinity=sorted(os.sched_getaffinity(0)), result_path=str(path) if path.exists() else None,
|
||||
result_sha256=sha(path) if path.exists() else None, job=c['jobs'][index]))
|
||||
return 0 if w['status'] == 'PASS' else 1
|
||||
|
||||
|
||||
def first_check(c, done):
|
||||
samples = {}
|
||||
for i in FIRST:
|
||||
item = next(r for r in done if r['index'] == i)
|
||||
result = read_json(checked(item['result_path'], item['result_sha256'])) if item.get('result_path') else {}
|
||||
samples[c['jobs'][i]['arm']] = dict(status=item['worker']['status'],
|
||||
worker_cpu_seconds=item['worker']['cpu_seconds'], fit_cpu_seconds=result.get('fit_cpu_seconds', 0.))
|
||||
return first_fit_projection(c, samples)
|
||||
|
||||
|
||||
def freeze_terminal(root, c, done, selections, reason, started, monitor, accounting_complete):
|
||||
attempts = {r['index']: r for r in done}
|
||||
finals = []
|
||||
for job in c['jobs'][360:]:
|
||||
row = attempts.get(job['index'])
|
||||
passed = row is not None and row['worker']['status'] == 'PASS'
|
||||
result = read_json(checked(row['result_path'], row['result_sha256'])) if passed else None
|
||||
model_path = str(Path(row['result_path']).parent/'model.joblib') if passed else None
|
||||
if passed:
|
||||
checked(model_path, result['model_sha256'])
|
||||
finals.append(dict(cell=job['cell'], arm=job['arm'], fold=job['fold'], pool=job['pool'],
|
||||
status='PASS' if passed else 'NA',
|
||||
reason=None if passed else ((row['worker']['stop_reason'] or 'WORKER_EXIT_'+str(row['worker']['returncode'])) if row else 'NOT_RUN_'+str(reason)),
|
||||
finished_configurations=selections[job['cell']]['finished_configurations'] if selections else None,
|
||||
model_path=model_path, model_sha256=result['model_sha256'] if result else None))
|
||||
complete = reason is None and len(attempts) == 390 and accounting_complete
|
||||
result = dict(status='ALL_S3A_FITS_TERMINAL' if complete else 'S3A_STOPPED_INCOMPLETE',
|
||||
contract_sha256=sha(root/'contract.json'), stop_reason=reason, accounting_complete=accounting_complete,
|
||||
final_fits=finals, jobs=done, not_run_indices=[i for i in range(390) if i not in attempts],
|
||||
worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
|
||||
guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),
|
||||
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
disk_bytes=disk_usage(root), wall_seconds=time.monotonic()-started,
|
||||
monitor_errors=monitor.errors, roster_sha256=c['roster_sha256'],
|
||||
test_predictions=False, test_scoring=False, retry=False, baseline_refits=0)
|
||||
save(root/'batch-complete.json', result)
|
||||
if complete:
|
||||
save(root/'terminal-freeze.json', dict(status='ALL_S3A_FITS_TERMINAL', contract_sha256=sha(root/'contract.json'),
|
||||
hpo_terminal_indices=list(range(360)), final_fits=finals,
|
||||
batch_receipt=str(root/'batch-complete.json'), batch_receipt_sha256=sha(root/'batch-complete.json'),
|
||||
test_predictions_started=False))
|
||||
return 0 if complete else 1
|
||||
|
||||
|
||||
def run(root):
|
||||
with (root/'batch.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
return run_locked(root)
|
||||
|
||||
|
||||
def run_locked(root):
|
||||
c = contract(root)
|
||||
if (root/'batch-started.json').exists():
|
||||
raise FileExistsError('No restart/retry of an attempted S3-A queue')
|
||||
os.sched_setaffinity(0, c['coordinator_cpus'])
|
||||
start = time.monotonic()
|
||||
save(root/'batch-started.json', dict(pid=os.getpid(), unix=time.time(), contract_sha256=sha(root/'contract.json'),
|
||||
implementation_freeze=read_json(root/'implementation-freeze.json'), cpu_affinity=sorted(os.sched_getaffinity(0))))
|
||||
active, done = {}, []
|
||||
order = dispatch_order()
|
||||
cursor = 0
|
||||
reason = None
|
||||
accounting_complete = True
|
||||
first_checked = False
|
||||
selections = None
|
||||
live_cpu = live_rss = disk = 0
|
||||
monitor = Monitor(root/'monitor-errors.jsonl')
|
||||
env = dict(os.environ, OMP_NUM_THREADS='1', MKL_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
|
||||
|
||||
def interrupted(*_):
|
||||
nonlocal reason
|
||||
reason = 'S3A_COORDINATOR_INTERRUPTED'
|
||||
|
||||
signal.signal(signal.SIGTERM, interrupted)
|
||||
signal.signal(signal.SIGINT, interrupted)
|
||||
|
||||
def terminate_active():
|
||||
terminate({k: (v['process'], v['log']) for k, v in active.items()})
|
||||
|
||||
def collect():
|
||||
nonlocal reason, accounting_complete
|
||||
for slot, item in list(active.items()):
|
||||
code = item['process'].poll()
|
||||
if code is None:
|
||||
continue
|
||||
path = root/'jobs'/f"job-{item['index']:03d}"/'guard-result.json'
|
||||
if not path.exists():
|
||||
reason = 'MISSING_GUARD_RECEIPT'
|
||||
accounting_complete = False
|
||||
save(root/'infrastructure-stop.json', dict(reason=reason, index=item['index'], returncode=code,
|
||||
last_observed_guard_tree_CPU_s=item.get('cpu_peak', 0.), accounting_complete=False))
|
||||
else:
|
||||
row = read_json(path)
|
||||
if row['job'] != c['jobs'][item['index']] or not math.isfinite(row['worker']['cpu_seconds']) or row['worker']['cpu_seconds'] < 0:
|
||||
raise ValueError('Invalid job/accounting receipt')
|
||||
if row.get('result_path'):
|
||||
checked(row['result_path'], row['result_sha256'])
|
||||
done.append(dict(index=item['index'], **row))
|
||||
print(json.dumps(dict(event='S3A_TERMINAL', index=item['index'], arm=row['job']['arm'],
|
||||
status=row['worker']['status'], completed=len(done))), flush=True)
|
||||
item['log'].close()
|
||||
del active[slot]
|
||||
|
||||
try:
|
||||
while active or (cursor < 390 and reason is None):
|
||||
collect()
|
||||
|
||||
def usage():
|
||||
values = []
|
||||
for v in active.values():
|
||||
value = tree_usage(v['process'].pid)
|
||||
v['cpu_peak'] = max(v.get('cpu_peak', 0.), value[0])
|
||||
values.append((v['cpu_peak'], value[1]))
|
||||
return sum(v[0] for v in values), sum(v[1] for v in values)
|
||||
|
||||
(live_cpu, live_rss), fresh = monitor.sample('live_usage', usage, (live_cpu, live_rss))
|
||||
disk, ok = monitor.sample('disk', lambda: disk_usage(root), disk)
|
||||
fresh &= ok
|
||||
spent = sum(r['worker']['cpu_seconds'] for r in done)
|
||||
over = budget_reason(0, spent, live_cpu, disk, cpu_limit=c['limits']['fitting_worker_cpu_seconds'], disk_limit=c['limits']['disk_bytes'])
|
||||
if over:
|
||||
reason = over
|
||||
if not first_checked and not active and done and any(r['worker']['status'] != 'PASS' for r in done):
|
||||
reason = 'FIRST_TRIAL_FAILED_NO_RETRY'
|
||||
if not first_checked and len(done) == 2 and not active and reason is None:
|
||||
check = first_check(c, done)
|
||||
save(root/'first-fit-cost-check.json', check)
|
||||
first_checked = True
|
||||
if not check['continue_within_cap']:
|
||||
reason = 'FIRST_TRIAL_COST_PROJECTION_OVER_CAP'
|
||||
if cursor == 360 and not active and len(done) == 360 and selections is None and reason is None:
|
||||
selections = finalize_selections(root, c, done)
|
||||
if reason:
|
||||
terminate_active()
|
||||
ceiling = (len(done)+1 if len(done) < 2 else 2) if not first_checked else (360 if selections is None else 390)
|
||||
parallel = 1 if not first_checked else 4
|
||||
while fresh and reason is None and len(active) < parallel and cursor < ceiling:
|
||||
i = order[cursor]
|
||||
job = c['jobs'][i]
|
||||
if job['kind'] == 'FINAL' and selections[job['cell']]['status'] != 'SELECTED':
|
||||
done.append(dict(index=i, job=job, worker=dict(status='NA', cpu_seconds=0., returncode=0,
|
||||
stop_reason='ALL12_HPO_FAILED_NO_FINAL_FIT'), supervisor_cpu_seconds=0., attempted=False, result_path=None, result_sha256=None))
|
||||
cursor += 1
|
||||
continue
|
||||
# Conservatively reserve full caps of active workers plus the next task.
|
||||
cap = c['limits']['per_fit_cpu_seconds']
|
||||
if not can_dispatch(c, spent, live_cpu, [cap for _ in active], cap):
|
||||
if not active:
|
||||
reason = 'INSUFFICIENT_REMAINING_FIT_CAP_NO_LAUNCH'
|
||||
break
|
||||
slot = next(k for k in range(4) if k not in active)
|
||||
dest = root/'jobs'/f'job-{i:03d}'
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
log = (dest/'guard.log').open('x')
|
||||
proc = subprocess.Popen([sys.executable, __file__, str(root), '--guard', str(i), '--slot', str(slot)],
|
||||
env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
active[slot] = dict(process=proc, log=log, index=i, cpu_peak=0.)
|
||||
cursor += 1
|
||||
save(root/'progress.json', dict(completed=len(done), active={str(k): v['index'] for k,v in active.items()},
|
||||
dispatch_cursor=cursor, worker_cpu_seconds=spent, live_guard_and_worker_cpu_seconds=live_cpu,
|
||||
active_tree_rss_bytes=live_rss, disk_bytes=disk, first_fit_checked=first_checked,
|
||||
selections_frozen=selections is not None, stop_reason=reason, fresh_accounting=fresh,
|
||||
monitor_errors=monitor.errors, elapsed_seconds=time.monotonic()-start))
|
||||
if active or not fresh:
|
||||
time.sleep(.5)
|
||||
except Exception as exc:
|
||||
reason = 'COORDINATOR_EXCEPTION'
|
||||
save(root/'coordinator-failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc()))
|
||||
finally:
|
||||
terminate_active()
|
||||
for item in active.values():
|
||||
item['process'].wait()
|
||||
collect()
|
||||
return freeze_terminal(root, c, done, selections, reason, start, monitor, accounting_complete)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('root', type=Path)
|
||||
p.add_argument('--guard', type=int)
|
||||
p.add_argument('--slot', type=int)
|
||||
a = p.parse_args()
|
||||
raise SystemExit(guard(a.root, a.guard, a.slot) if a.guard is not None else run(a.root))
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Execution pins and one-main-review gate; original approved code stays unchanged."""
|
||||
from pathlib import Path
|
||||
from freeze_g1_components import sha
|
||||
from s2_custody_common import checked, read_json
|
||||
from run_s3a_score import review_gate
|
||||
|
||||
|
||||
def contract(root):
|
||||
root = Path(root)
|
||||
c = review_gate(root)
|
||||
implementation = read_json(root/'implementation.json')
|
||||
if implementation['contract_sha256'] != sha(root/'contract.json'):
|
||||
raise ValueError('Implementation bound to different scientific contract')
|
||||
checked(root/'main-review.json', implementation['review_sha256'])
|
||||
for path, digest in implementation['code_sha256'].items():
|
||||
checked(path, digest)
|
||||
frozen = read_json(root/'implementation-freeze.json')
|
||||
if not frozen.get('git_commit') or frozen['implementation_sha256'] != sha(root/'implementation.json'):
|
||||
raise ValueError('Implementation must be Git frozen before launch')
|
||||
if len(c['jobs']) != 390 or len(c['cells']) != 30 or c['limits']['worker_cpu_seconds'] != 72000:
|
||||
raise ValueError('Unexpected S3-A scope')
|
||||
return c
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Execution acceptance with synthetic data/processes; no real dataset fitting."""
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import numpy as np
|
||||
import run_s3a_queue as queue
|
||||
import run_s3a_fit as fit
|
||||
import s3a_runtime as runtime
|
||||
from freeze_g1_components import sha
|
||||
from s2_custody_common import read_json
|
||||
|
||||
|
||||
def example_contract():
|
||||
return copy.deepcopy(read_json('artifacts/s3a-planning-2026-11-15/contract.json'))
|
||||
|
||||
|
||||
class QueueChecks(unittest.TestCase):
|
||||
def simulated(self, root, c, fail=None, missing=None, budget_stop=False):
|
||||
spawned=[];peak=[0]
|
||||
(root/'contract.json').write_text('{}')
|
||||
(root/'implementation-freeze.json').write_text('{}')
|
||||
owner=self
|
||||
class Process:
|
||||
def __init__(self,index):self.index=index;self.pid=900000+index;self.polls=0
|
||||
def poll(self):
|
||||
self.polls+=1
|
||||
if self.polls<2:return None
|
||||
if self.index==missing:return 1
|
||||
job=c['jobs'][self.index];dest=root/'jobs'/f'job-{self.index:03d}';out=dest/'output';out.mkdir(exist_ok=True)
|
||||
model=out/'model.joblib';model.write_bytes(b'synthetic')
|
||||
result=out/'result.json'
|
||||
result.write_text(json.dumps(dict(status='COMPLETE',fit_cpu_seconds=.01,
|
||||
validation_component_equal_relative_l2=job.get('config_index',0)/100 if job['kind']=='HPO' else None,model_sha256=sha(model))))
|
||||
failed=self.index==fail
|
||||
receipt=dict(worker=dict(status='STOPPED_OR_FAILED' if failed else 'PASS',cpu_seconds=.02,returncode=1 if failed else 0,stop_reason='synthetic_failure' if failed else None),
|
||||
supervisor_cpu_seconds=.001,result_path=str(result),result_sha256=sha(result),job=job)
|
||||
(dest/'guard-result.json').write_text(json.dumps(receipt))
|
||||
return 1 if failed else 0
|
||||
def wait(self):self.polls=2;return self.poll()
|
||||
def spawn(command,**kwargs):
|
||||
i=int(command[-3])
|
||||
if len(spawned)==1:owner.assertEqual(i,12)
|
||||
if len(spawned)>1:owner.assertTrue((root/'first-fit-cost-check.json').exists())
|
||||
if i>=360:owner.assertTrue((root/'selections'/f"cell-{c['jobs'][i]['cell']:02d}.json").exists())
|
||||
proc=Process(i);spawned.append(proc);peak[0]=max(peak[0],sum(p.polls<2 for p in spawned));return proc
|
||||
def dispatch(*args):return len(spawned)<2 if budget_stop else True
|
||||
with patch.object(queue,'contract',return_value=c),patch.object(queue.os,'sched_setaffinity'),patch.object(queue.subprocess,'Popen',spawn),patch.object(queue,'tree_usage',return_value=(0.,1,1)),patch.object(queue.time,'sleep'),patch.object(queue,'terminate'),patch.object(queue,'can_dispatch',dispatch),patch('builtins.print'):
|
||||
code=queue.run(root)
|
||||
return code,spawned,peak[0]
|
||||
|
||||
def test_390_unique_attempts_two_sequential_witnesses_and_four_workers(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);c=example_contract();code,spawned,peak=self.simulated(root,c)
|
||||
self.assertEqual(code,0);self.assertEqual([p.index for p in spawned[:2]],[0,12])
|
||||
self.assertEqual(len(spawned),390);self.assertEqual(len({p.index for p in spawned}),390);self.assertEqual(peak,4)
|
||||
final=read_json(root/'terminal-freeze.json');self.assertEqual(len(final['final_fits']),30)
|
||||
self.assertEqual(final['hpo_terminal_indices'],list(range(360)))
|
||||
self.assertTrue(all(f['status']=='PASS' for f in final['final_fits']))
|
||||
with patch.object(queue,'contract',return_value=c),patch.object(queue.subprocess,'Popen') as spawn:
|
||||
with self.assertRaises(FileExistsError):queue.run(root)
|
||||
spawn.assert_not_called()
|
||||
|
||||
def test_first_failure_stops_without_second_trial_or_retry(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);code,spawned,_=self.simulated(root,example_contract(),fail=0)
|
||||
self.assertEqual(code,1);self.assertEqual(len(spawned),1)
|
||||
self.assertFalse((root/'terminal-freeze.json').exists())
|
||||
result=read_json(root/'batch-complete.json')
|
||||
self.assertEqual(result['stop_reason'],'FIRST_TRIAL_FAILED_NO_RETRY')
|
||||
self.assertEqual(result['worker_cpu_seconds'],.02)
|
||||
|
||||
def test_missing_guard_is_infrastructure_stop_not_completed_batch(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);code,spawned,_=self.simulated(root,example_contract(),missing=0)
|
||||
self.assertEqual(code,1);self.assertEqual(len(spawned),1)
|
||||
self.assertFalse(read_json(root/'batch-complete.json')['accounting_complete'])
|
||||
self.assertFalse((root/'terminal-freeze.json').exists())
|
||||
|
||||
def test_nonfirst_hpo_failure_is_NA_config_and_final_fits_remaining(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);code,spawned,_=self.simulated(root,example_contract(),fail=1)
|
||||
self.assertEqual(code,0);decision=read_json(root/'selections/cell-00.json')
|
||||
self.assertEqual(decision['finished_configurations'],11)
|
||||
self.assertEqual(decision['config_index'],0)
|
||||
self.assertEqual(read_json(root/'terminal-freeze.json')['final_fits'][0]['status'],'PASS')
|
||||
|
||||
def test_failed_final_kept_as_cell_NA_in_terminal_freeze(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);code,_,_=self.simulated(root,example_contract(),fail=360)
|
||||
self.assertEqual(code,0)
|
||||
rows=read_json(root/'terminal-freeze.json')['final_fits']
|
||||
self.assertEqual(rows[0]['status'],'NA');self.assertEqual(rows[0]['reason'],'synthetic_failure')
|
||||
self.assertEqual(sum(f['status']=='PASS' for f in rows),29)
|
||||
|
||||
def test_reservation_stop_never_starts_next_task_or_freezes_complete(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);code,spawned,_=self.simulated(root,example_contract(),budget_stop=True)
|
||||
self.assertEqual(code,1);self.assertEqual(len(spawned),2)
|
||||
self.assertEqual(read_json(root/'batch-complete.json')['stop_reason'],'INSUFFICIENT_REMAINING_FIT_CAP_NO_LAUNCH')
|
||||
self.assertFalse((root/'terminal-freeze.json').exists())
|
||||
|
||||
def test_guard_uses_approved_limits_and_one_process(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);seen=[]
|
||||
def supervise(command,dest,name,limits,**kwargs):
|
||||
seen.append((command,limits));return dict(status='STOPPED_OR_FAILED',stop_reason='synthetic')
|
||||
with patch.object(queue,'contract',return_value=example_contract()),patch.object(queue.os,'sched_setaffinity'),patch.object(queue,'supervise',supervise):
|
||||
self.assertEqual(queue.guard(root,0,0),1)
|
||||
self.assertIn('tools/run_s3a_fit.py',seen[0][0])
|
||||
self.assertEqual(seen[0][1]['job_cpu_seconds'],1800)
|
||||
self.assertEqual(seen[0][1]['max_processes'],1)
|
||||
|
||||
def test_actual_synthetic_HistGBRT_has24columns_fixedgrid_and_singlethread(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);(root/'contract.json').write_text('{}');(root/'jobs/job-000').mkdir(parents=True)
|
||||
rng=np.random.default_rng(88)
|
||||
def part(case,role,n):
|
||||
x=rng.normal(size=(n,24));y=1+.1*x[:,0]
|
||||
return dict(case=case,component_id=case,role=role,x=x,y=y,seconds=y*2,counts=np.ones(n),t_set=2.,label_mode='synthetic',label_sha256='synthetic')
|
||||
with patch.object(runtime,'contract',return_value=example_contract()),patch.object(fit,'load_cell',return_value=([part('a','learning',240)],[part('b','validation',40)],{})),patch('builtins.print'):
|
||||
self.assertEqual(fit.run(root,0),0)
|
||||
r=read_json(root/'jobs/job-000/output/result.json')
|
||||
self.assertEqual(len(r['columns']),24);self.assertEqual(r['params']['max_iter'],200)
|
||||
self.assertFalse(r['params']['early_stopping']);self.assertTrue(all(p['num_threads']==1 for p in r['threadpools']))
|
||||
self.assertFalse(r['test_labels_read'])
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user