Implement approved bounded S3-A queue with first-trial cost checkpoint

This commit is contained in:
Frank14f committed 2026-11-15 17:00:10 +08:00
1 parent b0390231f0
commit 6142ad69d0
12 files changed
+19110

No files matched your search

File diff suppressed because it is too large. Load diff
@@ -0,0 +1,23 @@
{
"version": "S3-A-queue-v1",
"contract_sha256": "263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8",
"review_sha256": "1092dd9926d6a1a486da21cf2276901e01a67c2f8110a0fe2fbf26a808046bea",
"approved_plan_commit": "b0390231f0161fa55ae72494a32a38863e1b96c0",
"approval_slack": "1794732829.567629",
"code_sha256": {
"tools/s3a_runtime.py": "919c6198971505ea0b2cfbb8e2b09be7d322eab2261f4ca820c05bbab8dcf98a",
"tools/run_s3a_fit.py": "a77f6166ba69e93a74d7b5aca7233d5730baf896ebfc25fc923856d97ba3f78c",
"tools/run_s3a_queue.py": "cb98694d4860b7e004f5d20636900f4ffa12439f07fdd8cfbd77789e53e92ae1",
"tools/test_s3a_queue.py": "a7fabe301eeb4968cb31a984a6bdbe64cb43a01adf9bc8c539c5c926fb564a59",
"tools/prepare_s3a_execution.py": "f183c98f825b5a6efe042dcbdc567ae0eeb4d356f9d1a0d43b917b70d2992c0c"
},
"scope": "Fit queue only; stops at immutable terminal freeze, never automatically evaluates test labels",
"first_trials": [
0,
12
],
"dispatch": "0,12 then all remaining indices in ascending order",
"maximum_new_fits": 390,
"GPU": false,
"retries": 0
}
@@ -0,0 +1,10 @@
{
"approved": true,
"slack_ts": "1794732829.567629",
"slack_url": "https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794732829567629",
"git_commit": "b0390231f0161fa55ae72494a32a38863e1b96c0",
"contract_sha256": "263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8",
"ruling": "Go ahead and implement and launch the queue under the contract. No further review from me before the result.",
"review_scope": "Claude spot-checked s3a_statistics.py and contract; not a full executor review",
"result_headline_required": "Follow-up exploration with S2 results known, not a fresh confirmatory test"
}
@@ -0,0 +1,9 @@
{
"status": "IMPLEMENTED_NOT_STARTED",
"contract_sha256": "263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8",
"implementation_sha256": "a2bd051b83502129473038b0e98d5d0a0726e1f0d5c6f0e718ccfad1c77f1526",
"fits_started": false,
"baseline_refits": 0,
"labels_decoded": false,
"approved": true
}
@@ -0,0 +1,18 @@
{
"status": "PASS",
"tests": 8,
"failures": 0,
"errors": 0,
"worker_CPU_s": 13.534670686,
"wall_s": 13.561030525015667,
"real_dataset_fits": 0,
"synthetic_model_fits": 1,
"output": "test_390_unique_attempts_two_sequential_witnesses_and_four_workers (test_s3a_queue.QueueChecks) ... ok\ntest_actual_synthetic_HistGBRT_has24columns_fixedgrid_and_singlethread (test_s3a_queue.QueueChecks) ... ok\ntest_failed_final_kept_as_cell_NA_in_terminal_freeze (test_s3a_queue.QueueChecks) ... ok\ntest_first_failure_stops_without_second_trial_or_retry (test_s3a_queue.QueueChecks) ... ok\ntest_guard_uses_approved_limits_and_one_process (test_s3a_queue.QueueChecks) ... ok\ntest_missing_guard_is_infrastructure_stop_not_completed_batch (test_s3a_queue.QueueChecks) ... ok\ntest_nonfirst_hpo_failure_is_NA_config_and_final_fits_remaining (test_s3a_queue.QueueChecks) ... ok\ntest_reservation_stop_never_starts_next_task_or_freezes_complete (test_s3a_queue.QueueChecks) ... ok\n\n----------------------------------------------------------------------\nRan 8 tests in 13.561s\n\nOK\n",
"code_sha256": {
"tools/run_s3a_queue.py": "cb98694d4860b7e004f5d20636900f4ffa12439f07fdd8cfbd77789e53e92ae1",
"tools/run_s3a_fit.py": "a77f6166ba69e93a74d7b5aca7233d5730baf896ebfc25fc923856d97ba3f78c",
"tools/s3a_runtime.py": "919c6198971505ea0b2cfbb8e2b09be7d322eab2261f4ca820c05bbab8dcf98a",
"tools/test_s3a_queue.py": "a7fabe301eeb4968cb31a984a6bdbe64cb43a01adf9bc8c539c5c926fb564a59",
"tools/prepare_s3a_execution.py": "f183c98f825b5a6efe042dcbdc567ae0eeb4d356f9d1a0d43b917b70d2992c0c"
}
}
@@ -0,0 +1,10 @@
{
"approved": true,
"slack_ts": "1794732829.567629",
"slack_url": "https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794732829567629",
"git_commit": "b0390231f0161fa55ae72494a32a38863e1b96c0",
"contract_sha256": "263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8",
"ruling": "Go ahead and implement and launch the queue under the contract. No further review from me before the result.",
"review_scope": "Claude spot-checked s3a_statistics.py and contract; not a full executor review",
"result_headline_required": "Follow-up exploration with S2 results known, not a fresh confirmatory test"
}
@@ -0,0 +1,13 @@
# S3-A 队列实现与启动前验收
Claude1794732829.567629已批准b0390231合同,明确允许实现并启动,结果前不再主审。其实际审查范围为合同和s3a_statistics.py定点核读;不把该主审写成独立逐行执行器审查。保持“已知S2结果后的探索,非新确认性测试”结果标题。
实现新增run_s3a_queue.py/run_s3a_fit.py/s3a_runtime.py,不改原S2或b0390231冻结代码。新root为artifacts/s3a-execution-2026-11-15,科学合同SHA263b0b42b832d958ccf5843215403d629cf9d037084951d10f2af40bb89348d8完全相同,implementation.json单独固定新执行代码。主审凭据也保存于原规划root。
按固定job0(E4)、job12(G2)依次单独首试;两项都PASS且双臂外推含余量≤18h,继续最多4单线程CPU worker。随后未尝试HPO按原ID顺序;360HPO全终态后冻结30格选择,再30个learning-only final。选优/平手/失败直接复用原select_finished/finalize_selections,未重新拟合R2。原fit仅替换新数据入口和列名。
在途任务按完整剩余单job cap保守预留,并额外计入活跃树CPU,不会为了跑完而放宽预算。所有fit真实资源由原CPU guard跟踪,失败仍计费;缺guard回执标INFRA/accounting incomplete,不制造终态。全390终态且无全局停止才写terminal-freeze;只完成拟合队列,不自动跨过预测/评分冻结次序。后续预测/评分入口在已有2h预留内执行,并在运行前单独记录代码版本。
8项新增验收PASS:完整390唯一调度、两个首试顺序和4并发;首试失败停止;缺guard为未完成;普通HPO失败按11个成功配置选优;final失败保留NA;预算预留拒绝继续;guard的30min CPU/单进程约束;真实合成24列HistGBRT固定200迭代/单线程fit。合成验收不读真实标签或真实拟合。验收raw日志与源码hash见queue-acceptance.json。原17项合同/统计验收保持有效,不重复已完成S2/D3。
本文件为实现验收,实际启动时间、PID、首试开销与门结果以batch-started/progress/guard-result/first-fit-cost-check和后续actual-start-audit为准。
+35
View File
@@ -0,0 +1,35 @@
"""Bind approved S3-A plan to new queue sources; does not launch workers."""
import json
from pathlib import Path
import shutil
from freeze_g1_components import sha
from run_s3a_score import review_gate
from s2_recovery_supervisor import save
PLAN = Path('artifacts/s3a-planning-2026-11-15')
ROOT = Path('artifacts/s3a-execution-2026-11-15')
SOURCES = ['tools/s3a_runtime.py', 'tools/run_s3a_fit.py', 'tools/run_s3a_queue.py',
'tools/test_s3a_queue.py', 'tools/prepare_s3a_execution.py']
def main():
c = review_gate(PLAN)
ROOT.mkdir(exist_ok=False)
shutil.copy2(PLAN/'contract.json', ROOT/'contract.json')
shutil.copy2(PLAN/'main-review.json', ROOT/'main-review.json')
save(ROOT/'implementation.json', dict(version='S3-A-queue-v1',
contract_sha256=sha(ROOT/'contract.json'), review_sha256=sha(ROOT/'main-review.json'),
approved_plan_commit='b0390231f0161fa55ae72494a32a38863e1b96c0',
approval_slack='1794732829.567629',
code_sha256={p: sha(p) for p in SOURCES},
scope='Fit queue only; stops at immutable terminal freeze, never automatically evaluates test labels',
first_trials=[0, 12], dispatch='0,12 then all remaining indices in ascending order',
maximum_new_fits=len(c['jobs']), GPU=False, retries=0))
save(ROOT/'prelaunch.json', dict(status='IMPLEMENTED_NOT_STARTED',
contract_sha256=sha(ROOT/'contract.json'), implementation_sha256=sha(ROOT/'implementation.json'),
fits_started=False, baseline_refits=0, labels_decoded=False, approved=True))
print(json.dumps(dict(root=str(ROOT), implementation_sha256=sha(ROOT/'implementation.json'))))
if __name__ == '__main__':
main()
+68
View File
@@ -0,0 +1,68 @@
"""One single-thread HPO trial or fresh final refit; launch only through the CPU guard."""
import argparse
import json
import os
from pathlib import Path
import time
import traceback
import joblib
import sklearn
from sklearn.ensemble import HistGradientBoostingRegressor
from threadpoolctl import threadpool_limits,threadpool_info
from freeze_g1_components import sha
from s2_recovery_supervisor import save
from s2_models.features import histgbrt_grid
from s3a_data import load_cell,learning_arrays,validation_score,columns
def run(root,index):
from s3a_runtime import contract
start=time.monotonic();cpu_start=time.process_time();c=contract(root);job=c['jobs'][index]
dest=root/'jobs'/f'job-{index:03d}'/'output';dest.mkdir(exist_ok=False)
save(dest/'started.json',dict(unix=time.time(),pid=os.getpid(),job=job,contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0))))
try:
assert job['arm'] in ('R2_E4','R2_G2')
assert sklearn.__version__=='1.7.2'
config_index=job.get('config_index');selection=None
if job['kind']=='FINAL':
path=root/'selections'/f"cell-{job['cell']:02d}.json";selection=json.loads(path.read_text())
assert selection['status']=='SELECTED' and selection['cell']==job['cell']
config_index=selection['config_index']
for item in selection['trials']:
assert sha(item['receipt_path'])==item['receipt_sha256']
save(dest/'selection.json',dict(path=str(path),sha256=sha(path),**selection))
params=histgbrt_grid()[config_index]
with threadpool_limits(limits=1):
pools=threadpool_info();assert all(p['num_threads']==1 for p in pools)
learning,validation,roles=load_cell(root,c,job)
save(dest/'access.json',dict(fold=job['fold'],pool=job['pool'],kind=job['kind'],
cases=[dict(case=p['case'],role=p['role'],label_mode=p['label_mode'],label_sha256=p['label_sha256']) for p in learning+validation],
outer_fold_test_labels_read=False,learning_only_final_refit=job['kind']=='FINAL'))
x,y,w=learning_arrays(learning);load_cpu=time.process_time()-cpu_start
save(dest/'progress.json',dict(stage='fit',training_queries=len(y),training_parts=len(learning),cpu_seconds=time.process_time()-cpu_start))
model=HistGradientBoostingRegressor(**params)
fit_start=time.process_time();model.fit(x,y,sample_weight=w);fit_cpu=time.process_time()-fit_start
assert model.n_iter_==params['max_iter']
value=errors=None
if job['kind']=='HPO':
save(dest/'progress.json',dict(stage='validation',fit_cpu_seconds=fit_cpu))
value,errors=validation_score(model,validation);save(dest/'validation-parts.json',errors)
joblib.dump(model,dest/'model.joblib',compress=3)
result=dict(status='COMPLETE',job=job,config_index=config_index,params=params,sklearn_version=sklearn.__version__,
columns=columns(job['arm']),target='T/t_set',learning_queries=len(y),learning_parts=len(learning),
learning_components=len({p['component_id'] for p in learning}),validation_parts=len(validation),
sample_weight_sum=float(w.sum()),validation_component_equal_relative_l2=value,fit_cpu_seconds=fit_cpu,
load_cpu_seconds=load_cpu,worker_elapsed_cpu_seconds=time.process_time()-cpu_start,wall_seconds=time.monotonic()-start,
threadpools=pools,cpu_affinity=sorted(os.sched_getaffinity(0)),model_sha256=sha(dest/'model.joblib'),
roster_sha256=c['roster_sha256'],test_predictions=False,test_labels_read=False,random_state=0,retry=False,
final_refit_learning_only=job['kind']=='FINAL',hpo_finished_configurations=selection['finished_configurations'] if selection else None)
save(dest/'result.json',result);save(dest/'progress.json',dict(stage='complete',cpu_seconds=time.process_time()-cpu_start))
print(json.dumps(result),flush=True);return 0
except Exception as exc:
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
cpu_seconds=time.process_time()-cpu_start,wall_seconds=time.monotonic()-start))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))
+234
View File
@@ -0,0 +1,234 @@
"""S3-A single-attempt CPU queue: two sequential witnesses then bounded four-way HPO."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
import traceback
from freeze_g1_components import sha
from s2_custody_common import checked, read_json
from s2_recovery_supervisor import save, supervise, tree_usage
from s2_recovery_monitoring import Monitor, disk_usage, budget_reason
from run_s2_pilot_preparation import terminate
from run_s2_r2 import finalize_selections
from s3a_budget import first_fit_projection, can_dispatch
from s3a_runtime import contract
FIRST = [0, 12]
def dispatch_order():
return FIRST + [i for i in range(390) if i not in FIRST]
def guard(root, index, slot):
c = contract(root)
os.sched_setaffinity(0, [c['worker_cpus'][slot]])
limits = c['limits']
dest = root/'jobs'/f'job-{index:03d}'
dest.mkdir(parents=True, exist_ok=True)
w = supervise([sys.executable, 'tools/run_s3a_fit.py', str(root), str(index)], dest, 'worker',
dict(total_cpu_seconds=limits['per_fit_cpu_seconds']+1,
job_cpu_seconds=limits['per_fit_cpu_seconds'], job_wall_seconds=limits['per_fit_wall_seconds'],
tree_rss_bytes=limits['per_fit_tree_rss_bytes'], disk_bytes=limits['per_fit_disk_bytes'], max_processes=1), poll=.2)
path = dest/'output/result.json'
result = read_json(path) if path.exists() else None
if w['status'] == 'PASS' and (not result or result['status'] != 'COMPLETE'):
w.update(status='STOPPED_OR_FAILED', stop_reason='MISSING_COMPLETE_RESULT')
if w['status'] == 'PASS':
checked(dest/'output/model.joblib', result['model_sha256'])
save(dest/'guard-result.json', dict(worker=w, supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
cpu_affinity=sorted(os.sched_getaffinity(0)), result_path=str(path) if path.exists() else None,
result_sha256=sha(path) if path.exists() else None, job=c['jobs'][index]))
return 0 if w['status'] == 'PASS' else 1
def first_check(c, done):
samples = {}
for i in FIRST:
item = next(r for r in done if r['index'] == i)
result = read_json(checked(item['result_path'], item['result_sha256'])) if item.get('result_path') else {}
samples[c['jobs'][i]['arm']] = dict(status=item['worker']['status'],
worker_cpu_seconds=item['worker']['cpu_seconds'], fit_cpu_seconds=result.get('fit_cpu_seconds', 0.))
return first_fit_projection(c, samples)
def freeze_terminal(root, c, done, selections, reason, started, monitor, accounting_complete):
attempts = {r['index']: r for r in done}
finals = []
for job in c['jobs'][360:]:
row = attempts.get(job['index'])
passed = row is not None and row['worker']['status'] == 'PASS'
result = read_json(checked(row['result_path'], row['result_sha256'])) if passed else None
model_path = str(Path(row['result_path']).parent/'model.joblib') if passed else None
if passed:
checked(model_path, result['model_sha256'])
finals.append(dict(cell=job['cell'], arm=job['arm'], fold=job['fold'], pool=job['pool'],
status='PASS' if passed else 'NA',
reason=None if passed else ((row['worker']['stop_reason'] or 'WORKER_EXIT_'+str(row['worker']['returncode'])) if row else 'NOT_RUN_'+str(reason)),
finished_configurations=selections[job['cell']]['finished_configurations'] if selections else None,
model_path=model_path, model_sha256=result['model_sha256'] if result else None))
complete = reason is None and len(attempts) == 390 and accounting_complete
result = dict(status='ALL_S3A_FITS_TERMINAL' if complete else 'S3A_STOPPED_INCOMPLETE',
contract_sha256=sha(root/'contract.json'), stop_reason=reason, accounting_complete=accounting_complete,
final_fits=finals, jobs=done, not_run_indices=[i for i in range(390) if i not in attempts],
worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
disk_bytes=disk_usage(root), wall_seconds=time.monotonic()-started,
monitor_errors=monitor.errors, roster_sha256=c['roster_sha256'],
test_predictions=False, test_scoring=False, retry=False, baseline_refits=0)
save(root/'batch-complete.json', result)
if complete:
save(root/'terminal-freeze.json', dict(status='ALL_S3A_FITS_TERMINAL', contract_sha256=sha(root/'contract.json'),
hpo_terminal_indices=list(range(360)), final_fits=finals,
batch_receipt=str(root/'batch-complete.json'), batch_receipt_sha256=sha(root/'batch-complete.json'),
test_predictions_started=False))
return 0 if complete else 1
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
return run_locked(root)
def run_locked(root):
c = contract(root)
if (root/'batch-started.json').exists():
raise FileExistsError('No restart/retry of an attempted S3-A queue')
os.sched_setaffinity(0, c['coordinator_cpus'])
start = time.monotonic()
save(root/'batch-started.json', dict(pid=os.getpid(), unix=time.time(), contract_sha256=sha(root/'contract.json'),
implementation_freeze=read_json(root/'implementation-freeze.json'), cpu_affinity=sorted(os.sched_getaffinity(0))))
active, done = {}, []
order = dispatch_order()
cursor = 0
reason = None
accounting_complete = True
first_checked = False
selections = None
live_cpu = live_rss = disk = 0
monitor = Monitor(root/'monitor-errors.jsonl')
env = dict(os.environ, OMP_NUM_THREADS='1', MKL_NUM_THREADS='1', OPENBLAS_NUM_THREADS='1', NUMEXPR_NUM_THREADS='1', CUDA_VISIBLE_DEVICES='')
def interrupted(*_):
nonlocal reason
reason = 'S3A_COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM, interrupted)
signal.signal(signal.SIGINT, interrupted)
def terminate_active():
terminate({k: (v['process'], v['log']) for k, v in active.items()})
def collect():
nonlocal reason, accounting_complete
for slot, item in list(active.items()):
code = item['process'].poll()
if code is None:
continue
path = root/'jobs'/f"job-{item['index']:03d}"/'guard-result.json'
if not path.exists():
reason = 'MISSING_GUARD_RECEIPT'
accounting_complete = False
save(root/'infrastructure-stop.json', dict(reason=reason, index=item['index'], returncode=code,
last_observed_guard_tree_CPU_s=item.get('cpu_peak', 0.), accounting_complete=False))
else:
row = read_json(path)
if row['job'] != c['jobs'][item['index']] or not math.isfinite(row['worker']['cpu_seconds']) or row['worker']['cpu_seconds'] < 0:
raise ValueError('Invalid job/accounting receipt')
if row.get('result_path'):
checked(row['result_path'], row['result_sha256'])
done.append(dict(index=item['index'], **row))
print(json.dumps(dict(event='S3A_TERMINAL', index=item['index'], arm=row['job']['arm'],
status=row['worker']['status'], completed=len(done))), flush=True)
item['log'].close()
del active[slot]
try:
while active or (cursor < 390 and reason is None):
collect()
def usage():
values = []
for v in active.values():
value = tree_usage(v['process'].pid)
v['cpu_peak'] = max(v.get('cpu_peak', 0.), value[0])
values.append((v['cpu_peak'], value[1]))
return sum(v[0] for v in values), sum(v[1] for v in values)
(live_cpu, live_rss), fresh = monitor.sample('live_usage', usage, (live_cpu, live_rss))
disk, ok = monitor.sample('disk', lambda: disk_usage(root), disk)
fresh &= ok
spent = sum(r['worker']['cpu_seconds'] for r in done)
over = budget_reason(0, spent, live_cpu, disk, cpu_limit=c['limits']['fitting_worker_cpu_seconds'], disk_limit=c['limits']['disk_bytes'])
if over:
reason = over
if not first_checked and not active and done and any(r['worker']['status'] != 'PASS' for r in done):
reason = 'FIRST_TRIAL_FAILED_NO_RETRY'
if not first_checked and len(done) == 2 and not active and reason is None:
check = first_check(c, done)
save(root/'first-fit-cost-check.json', check)
first_checked = True
if not check['continue_within_cap']:
reason = 'FIRST_TRIAL_COST_PROJECTION_OVER_CAP'
if cursor == 360 and not active and len(done) == 360 and selections is None and reason is None:
selections = finalize_selections(root, c, done)
if reason:
terminate_active()
ceiling = (len(done)+1 if len(done) < 2 else 2) if not first_checked else (360 if selections is None else 390)
parallel = 1 if not first_checked else 4
while fresh and reason is None and len(active) < parallel and cursor < ceiling:
i = order[cursor]
job = c['jobs'][i]
if job['kind'] == 'FINAL' and selections[job['cell']]['status'] != 'SELECTED':
done.append(dict(index=i, job=job, worker=dict(status='NA', cpu_seconds=0., returncode=0,
stop_reason='ALL12_HPO_FAILED_NO_FINAL_FIT'), supervisor_cpu_seconds=0., attempted=False, result_path=None, result_sha256=None))
cursor += 1
continue
# Conservatively reserve full caps of active workers plus the next task.
cap = c['limits']['per_fit_cpu_seconds']
if not can_dispatch(c, spent, live_cpu, [cap for _ in active], cap):
if not active:
reason = 'INSUFFICIENT_REMAINING_FIT_CAP_NO_LAUNCH'
break
slot = next(k for k in range(4) if k not in active)
dest = root/'jobs'/f'job-{i:03d}'
dest.mkdir(parents=True, exist_ok=True)
log = (dest/'guard.log').open('x')
proc = subprocess.Popen([sys.executable, __file__, str(root), '--guard', str(i), '--slot', str(slot)],
env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
active[slot] = dict(process=proc, log=log, index=i, cpu_peak=0.)
cursor += 1
save(root/'progress.json', dict(completed=len(done), active={str(k): v['index'] for k,v in active.items()},
dispatch_cursor=cursor, worker_cpu_seconds=spent, live_guard_and_worker_cpu_seconds=live_cpu,
active_tree_rss_bytes=live_rss, disk_bytes=disk, first_fit_checked=first_checked,
selections_frozen=selections is not None, stop_reason=reason, fresh_accounting=fresh,
monitor_errors=monitor.errors, elapsed_seconds=time.monotonic()-start))
if active or not fresh:
time.sleep(.5)
except Exception as exc:
reason = 'COORDINATOR_EXCEPTION'
save(root/'coordinator-failure.json', dict(type=type(exc).__name__, message=str(exc), traceback=traceback.format_exc()))
finally:
terminate_active()
for item in active.values():
item['process'].wait()
collect()
return freeze_terminal(root, c, done, selections, reason, start, monitor, accounting_complete)
if __name__ == '__main__':
p = argparse.ArgumentParser()
p.add_argument('root', type=Path)
p.add_argument('--guard', type=int)
p.add_argument('--slot', type=int)
a = p.parse_args()
raise SystemExit(guard(a.root, a.guard, a.slot) if a.guard is not None else run(a.root))
+22
View File
@@ -0,0 +1,22 @@
"""Execution pins and one-main-review gate; original approved code stays unchanged."""
from pathlib import Path
from freeze_g1_components import sha
from s2_custody_common import checked, read_json
from run_s3a_score import review_gate
def contract(root):
root = Path(root)
c = review_gate(root)
implementation = read_json(root/'implementation.json')
if implementation['contract_sha256'] != sha(root/'contract.json'):
raise ValueError('Implementation bound to different scientific contract')
checked(root/'main-review.json', implementation['review_sha256'])
for path, digest in implementation['code_sha256'].items():
checked(path, digest)
frozen = read_json(root/'implementation-freeze.json')
if not frozen.get('git_commit') or frozen['implementation_sha256'] != sha(root/'implementation.json'):
raise ValueError('Implementation must be Git frozen before launch')
if len(c['jobs']) != 390 or len(c['cells']) != 30 or c['limits']['worker_cpu_seconds'] != 72000:
raise ValueError('Unexpected S3-A scope')
return c
+132
View File
@@ -0,0 +1,132 @@
"""Execution acceptance with synthetic data/processes; no real dataset fitting."""
import copy
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import numpy as np
import run_s3a_queue as queue
import run_s3a_fit as fit
import s3a_runtime as runtime
from freeze_g1_components import sha
from s2_custody_common import read_json
def example_contract():
return copy.deepcopy(read_json('artifacts/s3a-planning-2026-11-15/contract.json'))
class QueueChecks(unittest.TestCase):
def simulated(self, root, c, fail=None, missing=None, budget_stop=False):
spawned=[];peak=[0]
(root/'contract.json').write_text('{}')
(root/'implementation-freeze.json').write_text('{}')
owner=self
class Process:
def __init__(self,index):self.index=index;self.pid=900000+index;self.polls=0
def poll(self):
self.polls+=1
if self.polls<2:return None
if self.index==missing:return 1
job=c['jobs'][self.index];dest=root/'jobs'/f'job-{self.index:03d}';out=dest/'output';out.mkdir(exist_ok=True)
model=out/'model.joblib';model.write_bytes(b'synthetic')
result=out/'result.json'
result.write_text(json.dumps(dict(status='COMPLETE',fit_cpu_seconds=.01,
validation_component_equal_relative_l2=job.get('config_index',0)/100 if job['kind']=='HPO' else None,model_sha256=sha(model))))
failed=self.index==fail
receipt=dict(worker=dict(status='STOPPED_OR_FAILED' if failed else 'PASS',cpu_seconds=.02,returncode=1 if failed else 0,stop_reason='synthetic_failure' if failed else None),
supervisor_cpu_seconds=.001,result_path=str(result),result_sha256=sha(result),job=job)
(dest/'guard-result.json').write_text(json.dumps(receipt))
return 1 if failed else 0
def wait(self):self.polls=2;return self.poll()
def spawn(command,**kwargs):
i=int(command[-3])
if len(spawned)==1:owner.assertEqual(i,12)
if len(spawned)>1:owner.assertTrue((root/'first-fit-cost-check.json').exists())
if i>=360:owner.assertTrue((root/'selections'/f"cell-{c['jobs'][i]['cell']:02d}.json").exists())
proc=Process(i);spawned.append(proc);peak[0]=max(peak[0],sum(p.polls<2 for p in spawned));return proc
def dispatch(*args):return len(spawned)<2 if budget_stop else True
with patch.object(queue,'contract',return_value=c),patch.object(queue.os,'sched_setaffinity'),patch.object(queue.subprocess,'Popen',spawn),patch.object(queue,'tree_usage',return_value=(0.,1,1)),patch.object(queue.time,'sleep'),patch.object(queue,'terminate'),patch.object(queue,'can_dispatch',dispatch),patch('builtins.print'):
code=queue.run(root)
return code,spawned,peak[0]
def test_390_unique_attempts_two_sequential_witnesses_and_four_workers(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);c=example_contract();code,spawned,peak=self.simulated(root,c)
self.assertEqual(code,0);self.assertEqual([p.index for p in spawned[:2]],[0,12])
self.assertEqual(len(spawned),390);self.assertEqual(len({p.index for p in spawned}),390);self.assertEqual(peak,4)
final=read_json(root/'terminal-freeze.json');self.assertEqual(len(final['final_fits']),30)
self.assertEqual(final['hpo_terminal_indices'],list(range(360)))
self.assertTrue(all(f['status']=='PASS' for f in final['final_fits']))
with patch.object(queue,'contract',return_value=c),patch.object(queue.subprocess,'Popen') as spawn:
with self.assertRaises(FileExistsError):queue.run(root)
spawn.assert_not_called()
def test_first_failure_stops_without_second_trial_or_retry(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);code,spawned,_=self.simulated(root,example_contract(),fail=0)
self.assertEqual(code,1);self.assertEqual(len(spawned),1)
self.assertFalse((root/'terminal-freeze.json').exists())
result=read_json(root/'batch-complete.json')
self.assertEqual(result['stop_reason'],'FIRST_TRIAL_FAILED_NO_RETRY')
self.assertEqual(result['worker_cpu_seconds'],.02)
def test_missing_guard_is_infrastructure_stop_not_completed_batch(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);code,spawned,_=self.simulated(root,example_contract(),missing=0)
self.assertEqual(code,1);self.assertEqual(len(spawned),1)
self.assertFalse(read_json(root/'batch-complete.json')['accounting_complete'])
self.assertFalse((root/'terminal-freeze.json').exists())
def test_nonfirst_hpo_failure_is_NA_config_and_final_fits_remaining(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);code,spawned,_=self.simulated(root,example_contract(),fail=1)
self.assertEqual(code,0);decision=read_json(root/'selections/cell-00.json')
self.assertEqual(decision['finished_configurations'],11)
self.assertEqual(decision['config_index'],0)
self.assertEqual(read_json(root/'terminal-freeze.json')['final_fits'][0]['status'],'PASS')
def test_failed_final_kept_as_cell_NA_in_terminal_freeze(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);code,_,_=self.simulated(root,example_contract(),fail=360)
self.assertEqual(code,0)
rows=read_json(root/'terminal-freeze.json')['final_fits']
self.assertEqual(rows[0]['status'],'NA');self.assertEqual(rows[0]['reason'],'synthetic_failure')
self.assertEqual(sum(f['status']=='PASS' for f in rows),29)
def test_reservation_stop_never_starts_next_task_or_freezes_complete(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);code,spawned,_=self.simulated(root,example_contract(),budget_stop=True)
self.assertEqual(code,1);self.assertEqual(len(spawned),2)
self.assertEqual(read_json(root/'batch-complete.json')['stop_reason'],'INSUFFICIENT_REMAINING_FIT_CAP_NO_LAUNCH')
self.assertFalse((root/'terminal-freeze.json').exists())
def test_guard_uses_approved_limits_and_one_process(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);seen=[]
def supervise(command,dest,name,limits,**kwargs):
seen.append((command,limits));return dict(status='STOPPED_OR_FAILED',stop_reason='synthetic')
with patch.object(queue,'contract',return_value=example_contract()),patch.object(queue.os,'sched_setaffinity'),patch.object(queue,'supervise',supervise):
self.assertEqual(queue.guard(root,0,0),1)
self.assertIn('tools/run_s3a_fit.py',seen[0][0])
self.assertEqual(seen[0][1]['job_cpu_seconds'],1800)
self.assertEqual(seen[0][1]['max_processes'],1)
def test_actual_synthetic_HistGBRT_has24columns_fixedgrid_and_singlethread(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'contract.json').write_text('{}');(root/'jobs/job-000').mkdir(parents=True)
rng=np.random.default_rng(88)
def part(case,role,n):
x=rng.normal(size=(n,24));y=1+.1*x[:,0]
return dict(case=case,component_id=case,role=role,x=x,y=y,seconds=y*2,counts=np.ones(n),t_set=2.,label_mode='synthetic',label_sha256='synthetic')
with patch.object(runtime,'contract',return_value=example_contract()),patch.object(fit,'load_cell',return_value=([part('a','learning',240)],[part('b','validation',40)],{})),patch('builtins.print'):
self.assertEqual(fit.run(root,0),0)
r=read_json(root/'jobs/job-000/output/result.json')
self.assertEqual(len(r['columns']),24);self.assertEqual(r['params']['max_iter'],200)
self.assertFalse(r['params']['early_stopping']);self.assertTrue(all(p['num_threads']==1 for p in r['threadpools']))
self.assertFalse(r['test_labels_read'])
if __name__=='__main__':
unittest.main(verbosity=2)