Implement role-scoped 90 fresh fits with paired four-GPU scheduling and budget gate

This commit is contained in:
Frank14f committed 2026-11-15 09:35:39 +08:00
1 parent 31012ae4d3
commit e3fa0af79a
11 files changed
+9462

No files matched your search

@@ -0,0 +1,102 @@
{
"status": "PASS_NOT_STARTED",
"matrix_checks": 10,
"memory_guard_checks": 7,
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56",
"roster_sha256": "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e",
"code_sha256": {
"tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"tools/prepare_s2_incident_recovery.py": "4f772573dedc2b7921986e25eaedb14f91d98b235421829c284f3b40b85710aa",
"tools/prepare_s2_main_matrix.py": "43b48d82a813175be22be41cb63aaa5d04499000e80197c7405538eff3052ede",
"tools/prepare_s2_precision_pilot.py": "c3a71b0ff58bcea4433ff93e18ef14d3a00521438a76baaa69830a5ee8ac098a",
"tools/prepare_s2_recovered_pilot_training.py": "ab5a4274363598ff933ff873df2b00497f069b1fb45756a1d1dba8d723294f68",
"tools/run_s2_encoder_diagnostic.py": "8f31d312fb53bfdb9b1804af064e463d3989f1b33c4aa7279a967ca82dd2c0a4",
"tools/run_s2_main_matrix.py": "cebd3ccdea80c2914f6bbc1c3de293870535689f224c547c656a860b64af14da",
"tools/run_s2_main_training.py": "7fb18f015ab2eaa4968ff50c8b140285f9db6eb822398b2789d6de40f1aea2e7",
"tools/run_s2_pilot_probe.py": "2360ab7b44b179144a5ecd2d691e33cd36a6ec4bfeac71452bf062e7f42f81eb",
"tools/run_s2_pilot_probe_v2.py": "61c08499346d50405d0c254de54deea24b3522256bf1c8b0aec3e9f4ddab6390",
"tools/run_s2_pilot_training.py": "d5e543478543243fe1db3475a4d4cd135fbdcf631d3bce48c4b5af8b2edafb22",
"tools/run_s2_pilot_training_v2.py": "f461cb43bc602a32677d7dcfb8a322e206531ad152fa33bb4e36ea8546a1c0f2",
"tools/run_s2_precision_continuation.py": "9fb5a22007788f0c18c77a82b6a8b76fbb53a98b49cc9251bc49a7a5b8312f2f",
"tools/run_s2_precision_pilot.py": "ac8347be1ca11f48cb67abbf5e5c78b39af46ed3a7c6f89b49098fd5e77a6904",
"tools/run_s2_recovered_pilot.py": "56962009bf5eaaee599ffea6a3d454372e0aea2842fa79eac609fd4a24ecbb39",
"tools/run_s2_recovery_chain.py": "76d0808bdb2d624cfbaadf5a3c5e7e9a0c2be5d5ac41d4ac3fd9ccd48743933c",
"tools/s2_e1_supervisor.py": "6753c9ceb0b932047ba505bd15681f109e9407bdb58fd96c55e1431a3f1b4a02",
"tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
"tools/s2_incident_recovery_manifest.py": "8cbb2e1067c2cc22762d8ae3cf192aa2c5fa7c7a199a22e7a39a25570ee1c3b1",
"tools/s2_main_data.py": "bda31f5e9335b927315292f57e6f689787eacc5b32f7fae3fa23eb13b00343f0",
"tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
"tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
"tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
"tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
"tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
"tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
"tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
"tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
"tools/s2_models_v2/__init__.py": "0302d333bcd8b33e85aaa6a4e7822e7e1ccfc66343136107fea6f4ca283ba1ec",
"tools/s2_models_v2/attention.py": "3829e34ca30942c863699c4d43fb6fa104452c6c04c4d606c11019d3dde16ffa",
"tools/s2_models_v2/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
"tools/s2_models_v2/network.py": "a443470caeecdf529504bffab3ca8cc0b70ce44b505bad60f9c63db9709dbb83",
"tools/s2_pilot_data.py": "82c1fd18ec51757479c8fd7dda2db4d5f38c366d77fedcca3216a08998214c97",
"tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
"tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
"tools/s2_precision_pilot_gpu_guard.py": "5cf0f41de34b28394dbee4672b5bd34eb9c4fe2344822aa3d648f20561d96328",
"tools/s2_recovered_pilot_gpu_guard.py": "9a27f47918c865ce37e46b0be54144dc3f9ab1058e4eddc9f6fbc771f4959eb0",
"tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
"tools/test_s2_main_matrix.py": "d9b05506dd6e5a8a94cccaee28c6d8239bb3c6cdbe31567be1fdd04559154feb"
},
"numerical_loop": "Byte-identical training/validation epoch loop to pinned run_s2_pilot_training_v2.py; bootstrap adapts fold/pool/seed and role-checked targets only.",
"source_pin_validation": true,
"labels_decoded": false,
"gpu_work_started": false,
"cpu_affinities": [
[
0,
1
],
[
2,
3
],
[
4,
5
],
[
6,
7
]
],
"r2_reserved_cpus": [
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31
],
"test_log_sha256": {
"synthetic-tests.txt": "3d9cfaf01bdb3370741f9f38c4801856f430985c03921a08b575732a5fdd6ba5",
"memory-guard-tests.txt": "33ec082a255158f45f02b5f9cec0ed74eeaa19a68982e74df328992e14d3b1b7"
}
}
@@ -0,0 +1,12 @@
test_allocator_is_required_before_device_or_worker_call (__main__.RecordedPeakTests) ... ok
test_allocator_reaches_supervised_cpu_child_without_cuda_work (__main__.RecordedPeakTests) ... ok
test_failed_worker_final_scan_catches_breach_without_result (__main__.RecordedPeakTests) ... ok
test_invalid_recorded_peak_rejected (__main__.RecordedPeakTests) ... ok
test_maximum_survives_lower_replacement_and_deletion (__main__.RecordedPeakTests) ... ok
test_nested_history_and_result_both_contribute (__main__.RecordedPeakTests) ... ok
test_only_progress_reserved_breach_stops_live_worker (__main__.RecordedPeakTests) ... ok
----------------------------------------------------------------------
Ran 7 tests in 0.122s
OK
@@ -0,0 +1,27 @@
{
"status": "FROZEN_NOT_STARTED",
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56",
"roster_sha256": "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e",
"runs": 90,
"blocks": 45,
"labels_decoded": false,
"gpu_work_started": false,
"gpu_cpu_affinities": [
[
0,
1
],
[
2,
3
],
[
4,
5
],
[
6,
7
]
]
}
@@ -0,0 +1,15 @@
test_budget_reservation_and_first_eight_dispatch_gate (__main__.MatrixChecks) ... ok
test_current_fold_not_other_fold_controls_access (__main__.MatrixChecks) ... ok
test_current_test_rejected_before_any_target_io_even_old_cache (__main__.MatrixChecks) ... ok
test_existing_start_marker_rejects_duplicate_matrix (__main__.MatrixChecks) ... ok
test_failed_N_does_not_retry_or_skip_paired_E (__main__.MatrixChecks) ... ok
test_first_eight_projection_and_failed_cost_guard (__main__.MatrixChecks) ... ok
test_full_fake_scheduler_barrier_unique_blocks_and_90_terminal (__main__.MatrixChecks) ... ok
test_lazy_targets_created_once_and_reused_only_after_role_check (__main__.MatrixChecks) ... ok
test_numerical_loop_unchanged_and_no_checkpoint_loading (__main__.MatrixChecks) ... ok
test_real_roster_all_15_selections_match_frozen_counts (__main__.MatrixChecks) ... ok
----------------------------------------------------------------------
Ran 10 tests in 0.232s
OK
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,15 @@
# S2 统一名单后的 fresh 矩阵执行
准备终态已固定为 `31012ae4d38b4b088eb2f8d3a5851ef2ab55e710`:131 尝试中124通过、7超时,493旧成功仅复用。最终617件的统一名单 SHA256 为 `fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e`。排除7件共708645节点,原全集18661224节点;各折/池实际计数已固定在准备根 `final-fold-pool-counts.csv`。本次不再启动准备,也不复用pilot权重。
新矩阵根 `artifacts/s2-main-matrix-2026-11-15`,合同 SHA256 `198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56`。执行授权沿Claude1794699027.366469。固定5折×3组件比例池×3种子×N/E共90 fresh runs,45个相邻N/E块;宽128按可行性选择,宽度效应未检验。E_max/cosine42、AdamW及AMP跳步规则不变。新工作者的完整训练/验证epoch循环与已固定v2 pilot逐字一致;只适配fold/pool/seed、名单和输入访问,没有加载旧checkpoint。
每个fold/pool只枚举该折成功learning/validation,标准化统计只用该学习池。读取缓存或压缩包之前拒绝本折test。可验证兼容的旧非测试目标复用;缺少的目标在受监督run内按当前折允许角色惰性提取并留回执,按折加锁缓存,失败/中断不静默重试。本折test始终不进入该折拟合或验证;其他折的角色不能绕过当前折检查。所有90+15终态及检查点冻结后才统一托管测试预测、评分;本执行器没有测试预测或评分入口。
四张V100每卡一块,块内N后E,每个run都有三次空闲确认和已验收的累计显存峰值监督。各卡CPU亲和分别[0,1]、[2,3]、[4,5]、[6,7],记录在合同及回执;CPU8–31留给其他工作和后续R2。环境显式设置expandable_segments:True;每run≤1800s、15GiB显存、16GiB进程树RSS、5GiB输出。每次空闲等待最多600s;它与设置、失败耗时均计入46聚合GPU小时。监督仍是采样与退出复核,不把采样上限描述为连续硬件配额。
协调器按实际子进程存活时长累计每块GPU时间,并在派发时为每个活动块预留5000s;不按并行日历时长低报GPU小时。首四块共八个run全部终态前不派第五块;使用成功观察到的各臂最大wall/node、剩余实际节点数及25%余量重估,超过46h或两臂缺可用成本时停止报告。预算不足时尚未派发的run明确NA。任一臂失败仍完成其配对臂的既定尝试;失败不重试、不替补,配对有效性单独记录。
验收:10项矩阵CPU合成检查及7项既有累计显存监督检查全部PASS。覆盖15个真实名单选择与固定计数一致、当前折test在I/O前拒绝(即使有旧缓存)、折角色转换、惰性目标只创建一次及缓存后再次角色检查、训练循环未改变/无权重加载、预算预留、首八屏障和投影停止、N失败仍执行E一次、45块/90终态唯一派发、禁止重复矩阵启动,以及历史失败路径显存峰值监督。没有在这些验收中读取真实新增标签或启动GPU计算。
执行状态以 `matrix-started.json`、`matrix-progress.json`、各block/cell回执及 `first-eight-checkpoint.json` 为准。`prelaunch.json`、`acceptance.json` 是启动前固定证据,不可据此覆盖之后的真实状态。不得因聊天闲置重启。后续R2尚需独立执行实现与验收,预算为单fit30CPU-min、总100workerCPU-h、首fit成本检查;R2未全部终态时禁止托管测试解封。
+70
View File
@@ -0,0 +1,70 @@
"""Freeze 90 fresh cells against the one final roster; no label decoding or GPU work."""
import json
import os
from pathlib import Path
from freeze_g1_components import sha
from s2_recovery_supervisor import save
from s2_main_data import select_cell
ROOT=Path('artifacts/s2-main-matrix-2026-11-15')
PREP=Path('artifacts/s2-main-preparation-2026-11-15')
GPUS=['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a']
def build_runs(roster):
runs=[];blocks=[];cells=[]
for fold in range(5):
for pool in [25,50,100]:
records,roles=select_cell(roster,fold,pool)
cell=dict(fold=fold,pool=pool,roles=roles,learning_nodes=sum(records[k]['nodes'] for k in roles['learning_parts']),
validation_nodes=sum(records[k]['nodes'] for k in roles['validation_parts']))
cells.append(cell)
for seed in [0,1,2]:
indices=[]
for arm in ['N','E']:
indices.append(len(runs));runs.append(dict(name=f'fold{fold}-pool{pool}-{arm}128-seed{seed}',
fold=fold,pool=pool,seed=seed,arm=arm,width=128,initialization='fresh_seeded',
cost_nodes=cell['learning_nodes']+cell['validation_nodes']))
blocks.append(dict(fold=fold,pool=pool,seed=seed,run_indices=indices))
return runs,blocks,cells
def main():
assert not ROOT.exists(),'Do not overwrite matrix plan or execution'
roster_path=PREP/'final-roster.json';roster=json.loads(roster_path.read_text())
rr=json.loads((PREP/'roster-receipt.json').read_text());assert rr['status']=='FROZEN' and sha(roster_path)==rr['sha256']
batch=json.loads((PREP/'batch-complete.json').read_text());assert batch['status']=='ATTEMPTS_COMPLETE' and batch['stop_reason'] is None
prior=json.loads((PREP/'contract.json').read_text());runs,blocks,cells=build_runs(roster)
assert len(runs)==90 and len(blocks)==45 and len(cells)==15 and sum(p['included'] for p in roster['parts'])==617
pilot=json.loads(Path('artifacts/s2-pilot-v2-continuation-2026-11-15/training-contract.json').read_text())
for p,h in pilot['code_sha256'].items():assert sha(p)==h,p
added=['tools/s2_main_data.py','tools/run_s2_main_training.py','tools/run_s2_main_matrix.py','tools/prepare_s2_main_matrix.py',
'tools/test_s2_main_matrix.py','tools/s2_main_memory_guard.py','tools/s2_gpu_idle_gate.py',
'tools/s2_recovery_monitoring.py','tools/s2_recovery_supervisor.py','tools/s2_recovery_gpu_supervisor.py']
paths=set(pilot['code_sha256'])|set(added)
available=sorted(os.sched_getaffinity(0));assert len(available)>=12
affinities=[available[2*i:2*i+2] for i in range(4)]
old_labels='artifacts/s2-pilot-training-2026-11-15/success-manifest.json'
sources=[str(roster_path),str(PREP/'roster-receipt.json'),str(PREP/'batch-complete.json'),old_labels]
c=dict(version='S2-fresh-main-matrix-v1',runs=runs,blocks=blocks,cells=cells,E_max=42,cosine_T_max=42,
roster=str(roster_path),roster_sha256=sha(roster_path),old_labels_manifest=old_labels,old_labels_manifest_sha256=sha(old_labels),
archive=prior['archive'],archive_stat=prior['archive_stat'],gpu_uuids=GPUS,gpu_cpu_affinities=affinities,
cpu_affinity_available=available,cpu_affinity_reserved_for_other_work=available[8:],
code_sha256={p:sha(p) for p in sorted(paths)},source_sha256={p:sha(p) for p in sources},
limits=dict(aggregate_gpu_seconds=46*3600,run_wall_seconds=1800,idle_seconds_per_run=600,block_wall_seconds=5000,
max_concurrent_gpus=4,run_vram_bytes=15*2**30,run_host_rss_bytes=16*2**30,run_disk_bytes=5*2**30,retries=0),
allocator_conf='expandable_segments:True',authorizing_rulings=['1794698179.161599','1794698734.967459','1794699027.366469'],
capacity='128 chosen by feasibility; width effect not tested',pilot_weights_reused=False,
target_access='Current fold learning/validation only; reject current fold test before cache/archive IO. Other folds roles do not grant access.',
test_predictions='No predictions/scoring until all90 neural +15 R2 final fits terminal and checkpoints frozen; unified custody only.',
supervision='N/E adjacent per fold/pool/seed block, one block perGPU, reserve5000s per active block; charge coordinator-observed block lifetime including idle/setup and failures.',
first_eight='All four initial blocks terminal before any fifth block; project remaining work with per-arm max successful wall/node and25percent reserve; stop if over46h or no valid rate.',
failure_policy='No retry; retain checkpoint/failure evidence, failed run NA; pair invalid if either arm NA or standardizer mismatch. Budget unstarted cells terminal NA with explicit reason.')
ROOT.mkdir();save(ROOT/'training-contract.json',c)
save(ROOT/'prelaunch.json',dict(status='FROZEN_NOT_STARTED',contract_sha256=sha(ROOT/'training-contract.json'),
roster_sha256=c['roster_sha256'],runs=90,blocks=45,labels_decoded=False,gpu_work_started=False,gpu_cpu_affinities=affinities))
print(json.dumps(json.loads((ROOT/'prelaunch.json').read_text())))
if __name__=='__main__':main()
+182
View File
@@ -0,0 +1,182 @@
"""Four isolated GPU block workers; adjacent N/E, first-eight barrier, aggregate wall ledger."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
import traceback
import psutil
from freeze_g1_components import sha
from s2_recovery_supervisor import save
from s2_gpu_idle_gate import recorded_wait
from s2_main_memory_guard import guarded_worker,ALLOCATOR_CONF
BLOCK_WALL_CAP=5000.
def contract(root):
c=json.loads((root/'training-contract.json').read_text())
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():assert sha(p)==h,p
assert c['roster_sha256']==sha(c['roster']) and len(c['runs'])==90 and len(c['blocks'])==45
assert c['limits']['aggregate_gpu_seconds']==46*3600 and c['limits']['run_wall_seconds']==1800
assert c['E_max']==c['cosine_T_max']==42
assert len(set(c['gpu_uuids']))==len(c['gpu_uuids'])==4
for b in c['blocks']:
n,e=[c['runs'][i] for i in b['run_indices']]
assert (n['arm'],e['arm'])==('N','E')
assert (n['fold'],n['pool'],n['seed'])==(e['fold'],e['pool'],e['seed'])
return c
def first_eight_projection(c,blocks,charged):
assert len(blocks)==4 and sum(len(b['runs']) for b in blocks)==8
rates={arm:[] for arm in ['N','E']}
for b in blocks:
for row in b['runs']:
run=c['runs'][row['index']]
if row['status']=='PASS':rates[run['arm']].append(row['cell_wall_seconds']/run['cost_nodes'])
if any(not v for v in rates.values()):
return dict(continue_within_cap=False,reason='FIRST8_NO_VALID_COST_FOR_BOTH_ARMS',observed_charged_gpu_seconds=charged)
rate={a:max(v) for a,v in rates.items()}
completed={row['index'] for b in blocks for row in b['runs']}
projected=charged+1.25*sum(rate[r['arm']]*r['cost_nodes'] for i,r in enumerate(c['runs']) if i not in completed)
return dict(continue_within_cap=projected<=c['limits']['aggregate_gpu_seconds'],projected_gpu_seconds=projected,
observed_charged_gpu_seconds=charged,seconds_per_node_by_arm=rate,reserve_multiplier=1.25,
method='Maximum successful observed per-arm cell wall/node (idle/setup included), remaining cells plus25percent; all actual block wall already charged.')
def can_dispatch(c,next_block,active_count,charged,checkpoint):
ceiling=len(c['blocks']) if checkpoint else 4
return next_block<ceiling and active_count<4 and charged+(active_count+1)*BLOCK_WALL_CAP<=c['limits']['aggregate_gpu_seconds']
def block(root,index,gpu_slot):
c=contract(root);spec=c['blocks'][index];uuid=c['gpu_uuids'][gpu_slot]
dest=root/'blocks'/f'block-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
with (dest/'block.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
assert not (dest/'started.json').exists(),'No repeated block'
with Path('/tmp/meltmind-s2-'+uuid+'.lock').open('a') as gpu_lock:
fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return block_locked(root,c,spec,dest,index,gpu_slot)
def block_locked(root,c,spec,dest,index,gpu_slot):
begin=time.monotonic();uuid=c['gpu_uuids'][gpu_slot];rows=[]
os.sched_setaffinity(0,c['gpu_cpu_affinities'][gpu_slot])
os.environ['PYTORCH_CUDA_ALLOC_CONF']=ALLOCATOR_CONF
save(dest/'started.json',dict(pid=os.getpid(),started_unix=time.time(),gpu_uuid=uuid,block=index,
run_indices=spec['run_indices'],cpu_affinity=sorted(os.sched_getaffinity(0)),contract_sha256=sha(root/'training-contract.json')))
try:
for i in spec['run_indices']:
run=c['runs'][i];cell=root/'cells'/run['name'];cell.mkdir(parents=True,exist_ok=False)
cell_begin=time.monotonic();row=dict(index=i,name=run['name'],arm=run['arm'],status='NA',reason=None)
try:
if (root/'cancel.json').exists():raise RuntimeError('MATRIX_CANCELLED_BEFORE_RUN')
if time.monotonic()-begin>BLOCK_WALL_CAP-2405:raise RuntimeError('BLOCK_REMAINING_BUDGET_INSUFFICIENT')
idle=recorded_wait(cell,'before-run',uuid,600)
if idle['status']!='IDLE_CONFIRMED':raise RuntimeError('GPU_IDLE_TIMEOUT_NO_RETRY')
if (root/'cancel.json').exists():raise RuntimeError('MATRIX_CANCELLED_AFTER_IDLE')
receipt,result=guarded_worker(cell,run['name'],uuid,1800,
[sys.executable,'tools/run_s2_main_training.py',str(root),str(i)],cumulative_worker_cpu_limit_seconds=1801)
row.update(status='PASS' if receipt['status']=='PASS' else 'NA',reason=(receipt.get('stop_reason') or ('WORKER_EXIT_'+str(receipt['returncode']) if receipt['status']!='PASS' else None)),
guard_receipt=str(cell/(run['name']+'-guard-receipt.json')),guard_sha256=sha(cell/(run['name']+'-guard-receipt.json')),
worker_status=receipt['status'],worker_returncode=receipt['returncode'],worker_cpu_seconds=receipt['cpu_seconds'],
checkpoint_sha256=result.get('checkpoint_sha256') if result else None,
standardizer_fingerprint=result.get('standardizer_fingerprint') if result else None)
except Exception as exc:
row.update(status='NA',reason=type(exc).__name__+': '+str(exc),traceback=traceback.format_exc())
row['cell_wall_seconds']=time.monotonic()-cell_begin;rows.append(row)
save(cell/'terminal.json',row);save(dest/'progress.json',dict(runs=rows,elapsed_seconds=time.monotonic()-begin))
finally:
# A failed arm never causes its paired arm to be silently replaced or retried.
present={r['index'] for r in rows}
for i in spec['run_indices']:
if i not in present:rows.append(dict(index=i,name=c['runs'][i]['name'],status='NA',reason='BLOCK_INTERRUPTED_BEFORE_RUN',cell_wall_seconds=0))
valid=all(r['status']=='PASS' for r in rows)
if valid and rows[0]['standardizer_fingerprint']!=rows[1]['standardizer_fingerprint']:valid=False
save(dest/'terminal.json',dict(status='TERMINAL',block=index,gpu_uuid=uuid,runs=rows,pair_valid=valid,
pair_invalid_reason=None if valid else 'AT_LEAST_ONE_ARM_NA_OR_STANDARDIZER_MISMATCH',
block_wall_seconds=time.monotonic()-begin,cpu_affinity=sorted(os.sched_getaffinity(0)),retry=False,test_predictions=False))
def stop_owned_workers(active):
for item in active.values():
try:children=psutil.Process(item['process'].pid).children(recursive=True)
except psutil.NoSuchProcess:continue
for p in children:
try:
if 'tools/run_s2_main_training.py' in p.cmdline():os.killpg(p.pid,signal.SIGKILL)
except (ProcessLookupError,psutil.NoSuchProcess):pass
def run(root):
with (root/'matrix.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
def run_locked(root):
c=contract(root);assert not (root/'matrix-started.json').exists(),'No repeated matrix launch'
save(root/'matrix-started.json',dict(pid=os.getpid(),started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
start=time.monotonic();active={};done=[];next_block=0;reason=None;checkpoint=False;charged=0.
def interrupt(*_):
nonlocal reason
reason='MATRIX_COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM,interrupt);signal.signal(signal.SIGINT,interrupt)
env=dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=ALLOCATOR_CONF,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
try:
while active or (next_block<len(c['blocks']) and reason is None):
for slot,item in list(active.items()):
code=item['process'].poll();elapsed=time.monotonic()-item['begin']
if elapsed>BLOCK_WALL_CAP and code is None:reason='BLOCK_WALL_LIMIT'
if code is None:continue
path=root/'blocks'/f"block-{item['index']:03d}"/'terminal.json'
if not path.exists():
reason='BLOCK_EXIT_WITHOUT_TERMINAL_RECEIPT'
result=dict(status='INFRASTRUCTURE_FAILED',block=item['index'],runs=[dict(index=i,name=c['runs'][i]['name'],status='NA',reason=reason,cell_wall_seconds=0) for i in c['blocks'][item['index']]['run_indices']],pair_valid=False)
else:result=json.loads(path.read_text())
result.update(charged_gpu_seconds=elapsed,coordinator_observed_returncode=code)
save(root/'blocks'/f"block-{item['index']:03d}"/'accounted.json',result)
done.append(result);charged+=elapsed;item['log'].close();del active[slot]
print(json.dumps(dict(event='BLOCK_TERMINAL',block=result['block'],run_statuses=[r['status'] for r in result['runs']],charged_gpu_hours=charged/3600)),flush=True)
live=sum(time.monotonic()-x['begin'] for x in active.values())
if charged+live>=c['limits']['aggregate_gpu_seconds']:reason='AGGREGATE_GPU_WALL_LIMIT'
if not checkpoint and len(done)==4 and not active and reason is None:
projection=first_eight_projection(c,done,charged);save(root/'first-eight-checkpoint.json',projection)
if projection['continue_within_cap']:checkpoint=True
else:reason='FIRST_EIGHT_PROJECTION_STOP'
if reason:
if not (root/'cancel.json').exists():save(root/'cancel.json',dict(reason=reason,unix=time.time()))
stop_owned_workers(active)
while reason is None and can_dispatch(c,next_block,len(active),charged,checkpoint):
slot=next(s for s in range(4) if s not in active);idx=next_block
dest=root/'blocks'/f'block-{idx:03d}';dest.mkdir(parents=True,exist_ok=True)
log=(dest/'block.log').open('x');begin=time.monotonic()
p=subprocess.Popen([sys.executable,__file__,str(root),'--block',str(idx),'--gpu-slot',str(slot)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[slot]=dict(process=p,log=log,begin=begin,index=idx);next_block+=1
if not active and next_block<len(c['blocks']) and reason is None and checkpoint:reason='AGGREGATE_BUDGET_RESERVATION_STOP'
save(root/'matrix-progress.json',dict(completed_blocks=len(done),terminal_runs=2*len(done),active_blocks={str(k):v['index'] for k,v in active.items()},
next_block=next_block,settled_gpu_seconds=charged,active_gpu_seconds=live,first_eight_passed=checkpoint,stop_reason=reason,elapsed_seconds=time.monotonic()-start))
if active:time.sleep(.5)
finally:
if active:
save(root/'cancel.json',dict(reason=reason or 'COORDINATOR_EXIT',unix=time.time()));stop_owned_workers(active)
for item in active.values():item['process'].wait();item['log'].close()
rows=[row for b in done for row in b['runs']];seen={r['index'] for r in rows}
for i,r in enumerate(c['runs']):
if i not in seen:rows.append(dict(index=i,name=r['name'],status='NA',reason='NOT_RUN_'+str(reason)))
result=dict(status='ALL_NEURAL_TERMINAL',stop_reason=reason,runs=sorted(rows,key=lambda r:r['index']),blocks=done,
charged_gpu_seconds=charged,passed=sum(r['status']=='PASS' for r in rows),NA=sum(r['status']!='PASS' for r in rows),
roster_sha256=c['roster_sha256'],contract_sha256=sha(root/'training-contract.json'),retry=False,
test_predictions=False,test_scoring=False,r2_required_before_custody=True)
save(root/'matrix-complete.json',result)
return 0 if reason is None else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--block',type=int);p.add_argument('--gpu-slot',type=int);a=p.parse_args()
raise SystemExit(block(a.root,a.block,a.gpu_slot) if a.block is not None else run(a.root))
+148
View File
@@ -0,0 +1,148 @@
"""One fresh frozen matrix cell; unchanged v2 numerical training loop inside the GPU guard."""
import argparse
from collections import Counter
from collections.abc import Mapping
import json
from pathlib import Path
import random
import time
import traceback
import numpy as np
import torch
from freeze_g1_components import sha
from s2_e1_supervisor import save
from s2_models_v2.network import QueryModel,TRAINING_CONTRACT
from s2_models.normalization import BaseStandardizer,fit_base_standardizer
from s2_models.objectives import relative_l2
from s2_pilot_data import permitted_role
from s2_main_data import cell_records
from s2_pilot_train import part_weights,train_part,component_score,SkipPolicy
class LearningBases(Mapping):
def __init__(self,records,ids,roles,cache=None):self.records=records;self.ids=tuple(ids);self.roles=roles;self.cache=cache
def __iter__(self):return iter(self.ids)
def __len__(self):return len(self.ids)
def __getitem__(self,case):
if case not in self.ids or permitted_role(self.roles,case)!='learning':raise PermissionError('Learning-only normalization')
if self.cache is not None:return self.cache[case]['base']
assert sha(Path(self.records[case]['preparation_root'])/'base.npy')==self.records[case]['input_sha256']['base.npy']
return np.load(Path(self.records[case]['preparation_root'])/'base.npy',allow_pickle=False,mmap_mode='r')
def read_part_arrays(record,roles,*,synthetic_targets=False):
case=record['case'];role=permitted_role(roles,case)
if synthetic_targets and role!='learning':raise PermissionError('Timing probe accepts only learning parts')
root=Path(record['preparation_root']);label=Path(record['label_path'])
assert sha(root/'base.npy')==record['input_sha256']['base.npy']
assert sha(root/'queries.npz')==record['input_sha256']['queries.npz']
with np.load(root/'queries.npz',allow_pickle=False) as q:
mask=q['eligible_s2'];ids=q['anchor_ids'][mask];g=q['g'][mask];edge=q['edge_summary'][mask];counts=q['multiplicity'][mask]
if synthetic_targets:
target=np.zeros(len(ids));seconds=None
else:
assert sha(label)==record['input_sha256']['labels.npz']
with np.load(label,allow_pickle=False) as z:
assert np.array_equal(ids,z['anchor_ids']) and np.array_equal(counts,z['multiplicity'])
target=z['target_t_star'];seconds=z['seconds']
base=np.load(root/'base.npy',allow_pickle=False)
if len(base)!=record['nodes']:raise ValueError('Full-node base size mismatch')
return dict(base=base,ids=ids,gaps=g,edge=edge,target=target,counts=counts,truth_seconds=seconds)
def to_device(data,device):
result={k:v for k,v in data.items() if k not in ['base','ids','gaps','edge','target']}
for key in ['base','ids','gaps','edge','target']:
result[key]=torch.as_tensor(np.array(data[key],copy=True),dtype=torch.long if key=='ids' else torch.float32,device=device)
return result
def load_part(record,roles,device):
return to_device(read_part_arrays(record,roles),device)
def run(root,index):
started=time.monotonic();c=json.loads((root/'training-contract.json').read_text());run=c['runs'][index]
dest=root/'cells'/run['name']/run['name'];dest.mkdir(exist_ok=False);save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
try:
for p,h in c['code_sha256'].items():assert sha(p)==h,p
records,roles,manifest,target_audit=cell_records(root,c,run)
save(dest/'target-access.json',dict(fold=run['fold'],pool=run['pool'],roster_sha256=c['roster_sha256'],access=target_audit,outer_fold_test_labels_read=False))
learning=sorted(p for p in records if permitted_role(roles,p)=='learning')
validation=sorted(p for p in records if permitted_role(roles,p)=='validation')
if not learning or not validation:raise ValueError('Empty successful learning/validation roster')
if any(p in roles['sealed_test_parts'] for p in records):raise PermissionError('Test case present in ready manifest')
assert run['seed'] in [0,1,2] and run['fold'] in range(5) and run['pool'] in [25,50,100]
assert run['width']==128 and c['E_max']==42 and run['initialization']=='fresh_seeded'
torch.set_num_threads(1);random.seed(run['seed']);np.random.seed(run['seed']);torch.manual_seed(run['seed'])
if not torch.cuda.is_available() or torch.cuda.device_count()!=1:raise RuntimeError('Exactly one visible CUDA device required')
device=torch.device('cuda:0');torch.cuda.manual_seed_all(run['seed']);torch.cuda.reset_peak_memory_stats()
torch.backends.cuda.matmul.allow_tf32=False
if 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Frozen pilot requires one V100')
# Check hashes once, cache raw FP64 arrays, and fit only the learning subset.
cache={case:read_part_arrays(records[case],roles) for case in sorted(records)}
stats=fit_base_standardizer(LearningBases(records,learning,roles,cache),learning_part_ids=learning,fold=str(run['fold']),pool=str(run['pool']))
save(dest/'standardizer.json',dict(**stats.as_dict(),fingerprint=stats.fingerprint))
model=QueryModel(run['arm'],run['width'],standardizer=stats).to(device)
optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5)
epoch_limit=int(c['E_max']);assert 1<=epoch_limit<=200 and c['cosine_T_max']==epoch_limit
scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=epoch_limit)
scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(run['seed']))
group_counts=Counter(records[p]['component_id'] for p in learning);group_number=len(group_counts)
val_groups={p:records[p]['component_id'] for p in validation}
best=float('inf');best_epoch=None;stale=0;history=[];updates=0;attempted_updates=0;skip_policy=SkipPolicy()
for epoch in range(epoch_limit):
model.train();loss_sum=0.;skip_policy.start_epoch()
order=[learning[int(j)] for j in rng.permutation(len(learning))]
for j,case in enumerate(order):
record=records[case];data=to_device(cache[case],device)
weights=torch.tensor(part_weights(data['counts'],group_counts[record['component_id']],group_number,len(learning)),dtype=torch.float32,device=device)
step_info={}
loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=step_info)
loss_sum+=loss/len(learning);attempted_updates+=1;updates+=int(not step_info['step_skipped'])
skip_policy.observe(step_info['step_skipped'])
del data,weights
if j%16==0:
torch.cuda.synchronize();save(dest/'progress.json',dict(stage='training',epoch=epoch,parts_done=j+1,parts_total=len(learning),updates=updates,elapsed_seconds=time.monotonic()-started,
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()))
amp_summary=skip_policy.summary();save(dest/f'epoch-{epoch:03d}-amp.json',amp_summary)
skip_policy.finish_epoch(epoch)
model.eval();errors={}
with torch.no_grad():
for case in validation:
data=to_device(cache[case],device)
with torch.autocast(device_type='cuda',enabled=False):
pred=model(data['base'],data['ids'],data['gaps'],data['edge'] if run['arm']=='E' else None)
predicted_seconds=pred.float().cpu().numpy().astype(np.float64)*records[case]['t_set']
score=relative_l2(predicted_seconds,data['truth_seconds'],data['counts'],np.ones(len(data['counts']),bool))
errors[case]=score['value'];del data,pred
value=component_score(errors,val_groups);torch.cuda.synchronize()
row=dict(epoch=epoch,amp=amp_summary,attempted_updates=attempted_updates,training_global_weighted_mse=loss_sum,validation_component_equal_relative_l2=value,updates=updates,elapsed_seconds=time.monotonic()-started,
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())
history.append(row);save(dest/'history.json',history)
if value<best:
best=value;best_epoch=epoch;stale=0
torch.save(dict(state_dict={k:v.detach().cpu() for k,v in model.state_dict().items()},arm=run['arm'],width=run['width'],seed=run['seed'],epoch=epoch,
standardizer=stats.as_dict(),standardizer_fingerprint=stats.fingerprint),dest/'best.pt')
save(dest/'best-validation-parts.json',errors)
else:stale+=1
scheduler.step()
save(dest/'progress.json',dict(stage='epoch_complete',**row,patience_count=stale))
if stale>=20:break
torch.cuda.synchronize()
result=dict(status='COMPLETE',run=run,E_max=epoch_limit,cosine_T_max=epoch_limit,epochs=len(history),attempted_updates=attempted_updates,updates=updates,best_epoch=best_epoch,best_validation_score=best,
learning_parts=len(learning),validation_parts=len(validation),learning_components=group_number,validation_components=len(set(val_groups.values())),
roster_sha256=c['roster_sha256'],fold=run['fold'],pool=run['pool'],fresh_initialization=True,failed_preparation_parts_dropped=manifest['failed_parts'],standardizer_fingerprint=stats.fingerprint,
wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(),
test_labels_read=False,test_predictions=False,capacity_selected=False,checkpoint_sha256=sha(dest/'best.pt'))
save(dest/'result.json',result);print(json.dumps(result),flush=True);return 0
except Exception as exc:
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-started,
peak_allocated_bytes=torch.cuda.max_memory_allocated() if torch.cuda.is_initialized() else 0,
peak_reserved_bytes=torch.cuda.max_memory_reserved() if torch.cuda.is_initialized() else 0,
epoch=locals().get('epoch'),amp=skip_policy.summary() if 'skip_policy' in locals() else None))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))
+84
View File
@@ -0,0 +1,84 @@
"""Frozen fold/pool selection and lazy targets, rejecting that fold's test before I/O."""
import fcntl
import json
from pathlib import Path
import numpy as np
from freeze_g1_components import sha
from s2_recovery_supervisor import save
from s2_pilot_data import permitted_role,read_non_test_fill_time,validate_query_targets
def select_cell(roster,fold,pool):
assert fold in range(5) and pool in [25,50,100]
selected={};roles=dict(fold=fold,pool=pool,learning_parts=[],validation_parts=[],sealed_test_parts=[])
for row in roster['parts']:
a=row['assignments'][fold];assert a['fold']==fold
if a['role']=='test':roles['sealed_test_parts'].append(row['case']);continue
if not row['included']:continue
if a['role']=='learning' and str(pool) not in a['learning_pools']:continue
roles[a['role']+'_parts'].append(row['case']);selected[row['case']]=row
for key in ['learning_parts','validation_parts','sealed_test_parts']:roles[key].sort()
assert roles['learning_parts'] and roles['validation_parts']
assert not set(selected)&set(roles['sealed_test_parts'])
return selected,roles
def ensure_targets(root,c,row,roles,old_labels):
case=row['case'];role=permitted_role(roles,case) # before any file/cache/archive access
prep=row['preparation'];source=Path(prep['preparation_root'])
if case in old_labels:
old=old_labels[case]
assert old['nodes']==row['nodes'] and old['t_set']==prep['t_set']
assert old['input_sha256']['queries.npz']==prep['output_sha256']['queries.npz']
path=Path(old['label_path']);digest=old['input_sha256']['labels.npz']
assert sha(path)==digest
return path,digest,'previous_non_test_target_reused_after_current_fold_role_check'
cache=Path(root)/'fold-targets'/str(roles['fold'])/case;cache.mkdir(parents=True,exist_ok=True)
with (cache/'target.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX)
if (cache/'failure.json').exists():raise RuntimeError('Prior target preparation failure; no silent retry')
if (cache/'receipt.json').exists():
receipt=json.loads((cache/'receipt.json').read_text())
assert receipt['case']==case and receipt['fold']==roles['fold']
assert receipt['queries_sha256']==prep['output_sha256']['queries.npz']
assert receipt['roster_sha256']==c['roster_sha256']
assert sha(cache/'labels.npz')==receipt['sha256']
return cache/'labels.npz',receipt['sha256'],'current_fold_non_test_target_cache'
if (cache/'started.json').exists():raise RuntimeError('Interrupted target preparation; no silent retry')
save(cache/'started.json',dict(case=case,fold=roles['fold'],role_at_creation=role,roster_sha256=c['roster_sha256']))
try:
st=Path(c['archive']).stat();ver=c['archive_stat']
assert (st.st_size,st.st_mtime_ns,st.st_ino)==(ver['bytes'],ver['mtime_ns'],ver['inode'])
for name in ['geometry.npz','queries.npz']:assert sha(source/name)==prep['output_sha256'][name]
with np.load(source/'geometry.npz',allow_pickle=False) as z:raw_ids=z['raw_node_ids']
with np.load(source/'queries.npz',allow_pickle=False) as q:
ids=q['anchor_ids'];eligible=q['eligible_s2'];counts=q['multiplicity']
values,provenance=read_non_test_fill_time(c['archive'],case,raw_ids,roles)
target=validate_query_targets(values,ids,eligible,prep['t_set'])
np.savez_compressed(cache/'labels.npz',anchor_ids=ids[eligible],raw_node_ids=raw_ids[ids[eligible]],
multiplicity=counts[eligible],seconds=values[ids[eligible]],target_t_star=target)
digest=sha(cache/'labels.npz')
save(cache/'receipt.json',dict(case=case,fold=roles['fold'],role_at_creation=role,sha256=digest,
queries_sha256=prep['output_sha256']['queries.npz'],roster_sha256=c['roster_sha256'],provenance=provenance,
outer_fold_test_labels_read=False))
return cache/'labels.npz',digest,'new_current_fold_non_test_target'
except Exception as exc:
save(cache/'failure.json',dict(error_type=type(exc).__name__,message=str(exc),case=case,fold=roles['fold']))
raise
def cell_records(root,c,run):
assert sha(c['roster'])==c['roster_sha256']
roster=json.loads(Path(c['roster']).read_text())
rows,roles=select_cell(roster,run['fold'],run['pool'])
assert sha(c['old_labels_manifest'])==c['old_labels_manifest_sha256']
old={r['case']:r for r in json.loads(Path(c['old_labels_manifest']).read_text())['ready_parts']}
records={};audit=[]
for case,row in sorted(rows.items()):
label,digest,mode=ensure_targets(root,c,row,roles,old);prep=row['preparation']
records[case]=dict(case=case,nodes=row['nodes'],component_id=row['component_id'],t_set=prep['t_set'],
preparation_root=prep['preparation_root'],label_path=str(label),
input_sha256={'base.npy':prep['output_sha256']['base.npy'],'queries.npz':prep['output_sha256']['queries.npz'],'labels.npz':digest})
audit.append(dict(case=case,role=permitted_role(roles,case),mode=mode,label_sha256=digest))
manifest=dict(failed_parts=[dict(case=r['case'],reason=r['exclusion_reason']) for r in roster['parts'] if not r['included']])
return records,roles,manifest,audit
+133
View File
@@ -0,0 +1,133 @@
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import numpy as np
import s2_main_data as data
import run_s2_main_matrix as matrix
from prepare_s2_main_matrix import build_runs
def fake_plan():
runs=[];blocks=[]
for i in range(45):
indices=[]
for arm in ['N','E']:
indices.append(len(runs));runs.append(dict(name=f'{i}-{arm}',arm=arm,cost_nodes=100))
blocks.append(dict(run_indices=indices))
return dict(runs=runs,blocks=blocks,limits=dict(aggregate_gpu_seconds=46*3600),gpu_uuids=['g0','g1','g2','g3'],gpu_cpu_affinities=[[0],[1],[2],[3]],roster_sha256='frozen')
class MatrixChecks(unittest.TestCase):
def test_real_roster_all_15_selections_match_frozen_counts(self):
r=json.loads(Path('artifacts/s2-main-preparation-2026-11-15/final-roster.json').read_text())
runs,blocks,cells=build_runs(r)
self.assertEqual((len(runs),len(blocks),len(cells)),(90,45,15))
for cell,count in zip(cells,r['fold_pool_counts']):
for role in ['learning','validation']:
self.assertEqual(len(cell['roles'][role+'_parts']),count[role]['parts'])
self.assertEqual(cell[role+'_nodes'],count[role]['nodes'])
self.assertFalse(set(cell['roles']['learning_parts'])&set(cell['roles']['sealed_test_parts']))
for block in blocks:
n,e=[runs[i] for i in block['run_indices']]
self.assertEqual((n['arm'],e['arm']),('N','E'))
self.assertEqual((n['fold'],n['pool'],n['seed']),(e['fold'],e['pool'],e['seed']))
def test_current_test_rejected_before_any_target_io_even_old_cache(self):
roles=dict(learning_parts=['learn'],validation_parts=['val'],sealed_test_parts=['test'])
with patch.object(data,'sha',side_effect=AssertionError('IO')),patch.object(data.Path,'mkdir',side_effect=AssertionError('IO')),patch.object(data,'read_non_test_fill_time',side_effect=AssertionError('decode')):
with self.assertRaises(PermissionError):data.ensure_targets(Path('/nonexistent'),{},dict(case='test'),roles,{'test':{}})
def test_current_fold_not_other_fold_controls_access(self):
r=dict(parts=[dict(case='a',included=True,assignments=[dict(fold=0,role='test',learning_pools=[]),dict(fold=1,role='learning',learning_pools=['25','50','100'])]),
dict(case='b',included=True,assignments=[dict(fold=0,role='learning',learning_pools=['25']),dict(fold=1,role='validation',learning_pools=[])]),
dict(case='c',included=True,assignments=[dict(fold=0,role='validation',learning_pools=[]),dict(fold=1,role='test',learning_pools=[])])])
a,_=data.select_cell(r,0,25);b,_=data.select_cell(r,1,25)
self.assertNotIn('a',a);self.assertIn('a',b);self.assertNotIn('c',b)
def test_lazy_targets_created_once_and_reused_only_after_role_check(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);prep=root/'geom';prep.mkdir();archive=root/'archive';archive.write_bytes(b'synthetic')
np.savez(prep/'geometry.npz',raw_node_ids=np.array([0,1,2]))
np.savez(prep/'queries.npz',anchor_ids=np.array([1,2]),eligible_s2=np.array([True,False]),multiplicity=np.array([1500,500]))
hashes={n:data.sha(prep/n) for n in ['geometry.npz','queries.npz']};st=archive.stat()
c=dict(archive=str(archive),archive_stat=dict(bytes=st.st_size,mtime_ns=st.st_mtime_ns,inode=st.st_ino),roster_sha256='fixed')
row=dict(case='a',preparation=dict(preparation_root=str(prep),output_sha256=hashes,t_set=2.))
roles=dict(fold=1,learning_parts=['a'],validation_parts=[],sealed_test_parts=[])
with patch.object(data,'read_non_test_fill_time',return_value=(np.array([0.,4.,8.]),dict(decoded_label_arrays=['Fill_time']))) as decode:
path,digest,mode=data.ensure_targets(root,c,row,roles,{})
path2,digest2,mode2=data.ensure_targets(root,c,row,roles,{})
self.assertEqual(decode.call_count,1);self.assertEqual((path,digest),(path2,digest2))
with np.load(path,allow_pickle=False) as z:
np.testing.assert_array_equal(z['target_t_star'],[2.]);np.testing.assert_array_equal(z['anchor_ids'],[1])
roles.update(learning_parts=[],sealed_test_parts=['a'])
with self.assertRaises(PermissionError):data.ensure_targets(root,c,row,roles,{})
self.assertEqual(decode.call_count,1)
def test_numerical_loop_unchanged_and_no_checkpoint_loading(self):
old=Path('tools/run_s2_pilot_training_v2.py').read_text();new=Path('tools/run_s2_main_training.py').read_text()
extract=lambda s:s[s.index(' for epoch in range(epoch_limit):'):s.index(' result=dict')]
self.assertEqual(extract(old),extract(new));self.assertNotIn('torch.load(',new);self.assertNotIn('load_state_dict(',new)
def test_budget_reservation_and_first_eight_dispatch_gate(self):
c=fake_plan();cap=c['limits']['aggregate_gpu_seconds']
self.assertFalse(matrix.can_dispatch(c,4,0,0,False))
self.assertTrue(matrix.can_dispatch(c,4,0,0,True))
self.assertFalse(matrix.can_dispatch(c,5,3,cap-4*matrix.BLOCK_WALL_CAP+1,True))
self.assertTrue(matrix.can_dispatch(c,5,3,cap-4*matrix.BLOCK_WALL_CAP,True))
def test_first_eight_projection_and_failed_cost_guard(self):
c=fake_plan();blocks=[dict(runs=[dict(index=i,status='PASS',cell_wall_seconds=100) for i in b['run_indices']]) for b in c['blocks'][:4]]
self.assertTrue(matrix.first_eight_projection(c,blocks,1000)['continue_within_cap'])
for b in blocks:
for r in b['runs']:r['cell_wall_seconds']=2000
self.assertFalse(matrix.first_eight_projection(c,blocks,16000)['continue_within_cap'])
for b in blocks:
for r in b['runs']:r['status']='NA'
self.assertFalse(matrix.first_eight_projection(c,blocks,1000)['continue_within_cap'])
def test_failed_N_does_not_retry_or_skip_paired_E(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'training-contract.json').write_text('{}');dest=root/'blocks'/'block-000';dest.mkdir(parents=True)
c=fake_plan();calls=[]
def guard(cell,name,uuid,wall,command,**kw):
calls.append(name);r=dict(status='STOPPED_OR_FAILED' if len(calls)==1 else 'PASS',stop_reason='synthetic_failure' if len(calls)==1 else None,returncode=1 if len(calls)==1 else 0,cpu_seconds=1)
(cell/(name+'-guard-receipt.json')).write_text(json.dumps(r))
return r,dict(checkpoint_sha256='fresh',standardizer_fingerprint='shared')
with patch.object(matrix.os,'sched_setaffinity'),patch.object(matrix.os,'sched_getaffinity',return_value={0}),patch.object(matrix,'recorded_wait',return_value=dict(status='IDLE_CONFIRMED')),patch.object(matrix,'guarded_worker',guard):
matrix.block_locked(root,c,c['blocks'][0],dest,0,0)
r=json.loads((dest/'terminal.json').read_text());self.assertEqual(calls,['0-N','0-E'])
self.assertEqual([x['status'] for x in r['runs']],['NA','PASS']);self.assertFalse(r['pair_valid'])
def test_full_fake_scheduler_barrier_unique_blocks_and_90_terminal(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'training-contract.json').write_text('{}');c=fake_plan();clock=[0.];spawned=[];peak=[0]
class Process:
def __init__(self,i):self.i=i;self.pid=1000000+i;self.polls=0
def poll(self):
self.polls+=1
if self.polls==1:return None
dest=root/'blocks'/f'block-{self.i:03d}'/'terminal.json'
dest.write_text(json.dumps(dict(block=self.i,runs=[dict(index=j,status='PASS',cell_wall_seconds=1) for j in c['blocks'][self.i]['run_indices']],pair_valid=True)))
return 0
def wait(self):return 0
def spawn(command,**kw):
i=int(command[-3])
if i>=4:self.assertTrue((root/'first-eight-checkpoint.json').exists())
p=Process(i);spawned.append(p);peak[0]=max(peak[0],sum(x.polls<2 for x in spawned));return p
with patch.object(matrix,'contract',return_value=c),patch.object(matrix.subprocess,'Popen',spawn),patch.object(matrix.time,'monotonic',side_effect=lambda:clock[0]),patch.object(matrix.time,'sleep',side_effect=lambda n:clock.__setitem__(0,clock[0]+1)),patch('builtins.print'):
self.assertEqual(matrix.run(root),0)
r=json.loads((root/'matrix-complete.json').read_text())
self.assertEqual(len(spawned),45);self.assertEqual(len({p.i for p in spawned}),45);self.assertLessEqual(peak[0],4)
self.assertEqual((r['passed'],r['NA']),(90,0));self.assertFalse(r['test_predictions']);self.assertTrue(r['r2_required_before_custody'])
def test_existing_start_marker_rejects_duplicate_matrix(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'matrix-started.json').write_text('{}')
with patch.object(matrix,'contract',return_value=fake_plan()),patch.object(matrix.subprocess,'Popen') as spawn:
with self.assertRaises(AssertionError):matrix.run(root)
spawn.assert_not_called()
if __name__=='__main__':unittest.main(verbosity=2)