Implement role-scoped 90 fresh fits with paired four-GPU scheduling and budget gate
This commit is contained in:
1 parent
31012ae4d3
commit
e3fa0af79a
11 files changed
+9462
No files matched your search
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"status": "PASS_NOT_STARTED",
|
||||
"matrix_checks": 10,
|
||||
"memory_guard_checks": 7,
|
||||
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56",
|
||||
"roster_sha256": "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e",
|
||||
"code_sha256": {
|
||||
"tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"tools/prepare_s2_incident_recovery.py": "4f772573dedc2b7921986e25eaedb14f91d98b235421829c284f3b40b85710aa",
|
||||
"tools/prepare_s2_main_matrix.py": "43b48d82a813175be22be41cb63aaa5d04499000e80197c7405538eff3052ede",
|
||||
"tools/prepare_s2_precision_pilot.py": "c3a71b0ff58bcea4433ff93e18ef14d3a00521438a76baaa69830a5ee8ac098a",
|
||||
"tools/prepare_s2_recovered_pilot_training.py": "ab5a4274363598ff933ff873df2b00497f069b1fb45756a1d1dba8d723294f68",
|
||||
"tools/run_s2_encoder_diagnostic.py": "8f31d312fb53bfdb9b1804af064e463d3989f1b33c4aa7279a967ca82dd2c0a4",
|
||||
"tools/run_s2_main_matrix.py": "cebd3ccdea80c2914f6bbc1c3de293870535689f224c547c656a860b64af14da",
|
||||
"tools/run_s2_main_training.py": "7fb18f015ab2eaa4968ff50c8b140285f9db6eb822398b2789d6de40f1aea2e7",
|
||||
"tools/run_s2_pilot_probe.py": "2360ab7b44b179144a5ecd2d691e33cd36a6ec4bfeac71452bf062e7f42f81eb",
|
||||
"tools/run_s2_pilot_probe_v2.py": "61c08499346d50405d0c254de54deea24b3522256bf1c8b0aec3e9f4ddab6390",
|
||||
"tools/run_s2_pilot_training.py": "d5e543478543243fe1db3475a4d4cd135fbdcf631d3bce48c4b5af8b2edafb22",
|
||||
"tools/run_s2_pilot_training_v2.py": "f461cb43bc602a32677d7dcfb8a322e206531ad152fa33bb4e36ea8546a1c0f2",
|
||||
"tools/run_s2_precision_continuation.py": "9fb5a22007788f0c18c77a82b6a8b76fbb53a98b49cc9251bc49a7a5b8312f2f",
|
||||
"tools/run_s2_precision_pilot.py": "ac8347be1ca11f48cb67abbf5e5c78b39af46ed3a7c6f89b49098fd5e77a6904",
|
||||
"tools/run_s2_recovered_pilot.py": "56962009bf5eaaee599ffea6a3d454372e0aea2842fa79eac609fd4a24ecbb39",
|
||||
"tools/run_s2_recovery_chain.py": "76d0808bdb2d624cfbaadf5a3c5e7e9a0c2be5d5ac41d4ac3fd9ccd48743933c",
|
||||
"tools/s2_e1_supervisor.py": "6753c9ceb0b932047ba505bd15681f109e9407bdb58fd96c55e1431a3f1b4a02",
|
||||
"tools/s2_gpu_idle_gate.py": "e2b1613e7e6ea398a8d555220a9365a1ad998ab4ef4b2bf8ec7cc4e31f8b4b9f",
|
||||
"tools/s2_incident_recovery_manifest.py": "8cbb2e1067c2cc22762d8ae3cf192aa2c5fa7c7a199a22e7a39a25570ee1c3b1",
|
||||
"tools/s2_main_data.py": "bda31f5e9335b927315292f57e6f689787eacc5b32f7fae3fa23eb13b00343f0",
|
||||
"tools/s2_main_memory_guard.py": "e094f5443e11518efe9018215c11718a8e18aa462a977bf43e974b2d0b2aebb1",
|
||||
"tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609",
|
||||
"tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61",
|
||||
"tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
|
||||
"tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071",
|
||||
"tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0",
|
||||
"tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1",
|
||||
"tools/s2_models_v2/__init__.py": "0302d333bcd8b33e85aaa6a4e7822e7e1ccfc66343136107fea6f4ca283ba1ec",
|
||||
"tools/s2_models_v2/attention.py": "3829e34ca30942c863699c4d43fb6fa104452c6c04c4d606c11019d3dde16ffa",
|
||||
"tools/s2_models_v2/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75",
|
||||
"tools/s2_models_v2/network.py": "a443470caeecdf529504bffab3ca8cc0b70ce44b505bad60f9c63db9709dbb83",
|
||||
"tools/s2_pilot_data.py": "82c1fd18ec51757479c8fd7dda2db4d5f38c366d77fedcca3216a08998214c97",
|
||||
"tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948",
|
||||
"tools/s2_precision_continuation_guard.py": "01235d897c7c03e5ffa109c58b14f5d24d128e8f1ba873e1f76cfb14e02f7a7f",
|
||||
"tools/s2_precision_pilot_gpu_guard.py": "5cf0f41de34b28394dbee4672b5bd34eb9c4fe2344822aa3d648f20561d96328",
|
||||
"tools/s2_recovered_pilot_gpu_guard.py": "9a27f47918c865ce37e46b0be54144dc3f9ab1058e4eddc9f6fbc771f4959eb0",
|
||||
"tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15",
|
||||
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"tools/test_s2_main_matrix.py": "d9b05506dd6e5a8a94cccaee28c6d8239bb3c6cdbe31567be1fdd04559154feb"
|
||||
},
|
||||
"numerical_loop": "Byte-identical training/validation epoch loop to pinned run_s2_pilot_training_v2.py; bootstrap adapts fold/pool/seed and role-checked targets only.",
|
||||
"source_pin_validation": true,
|
||||
"labels_decoded": false,
|
||||
"gpu_work_started": false,
|
||||
"cpu_affinities": [
|
||||
[
|
||||
0,
|
||||
1
|
||||
],
|
||||
[
|
||||
2,
|
||||
3
|
||||
],
|
||||
[
|
||||
4,
|
||||
5
|
||||
],
|
||||
[
|
||||
6,
|
||||
7
|
||||
]
|
||||
],
|
||||
"r2_reserved_cpus": [
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
11,
|
||||
12,
|
||||
13,
|
||||
14,
|
||||
15,
|
||||
16,
|
||||
17,
|
||||
18,
|
||||
19,
|
||||
20,
|
||||
21,
|
||||
22,
|
||||
23,
|
||||
24,
|
||||
25,
|
||||
26,
|
||||
27,
|
||||
28,
|
||||
29,
|
||||
30,
|
||||
31
|
||||
],
|
||||
"test_log_sha256": {
|
||||
"synthetic-tests.txt": "3d9cfaf01bdb3370741f9f38c4801856f430985c03921a08b575732a5fdd6ba5",
|
||||
"memory-guard-tests.txt": "33ec082a255158f45f02b5f9cec0ed74eeaa19a68982e74df328992e14d3b1b7"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
test_allocator_is_required_before_device_or_worker_call (__main__.RecordedPeakTests) ... ok
|
||||
test_allocator_reaches_supervised_cpu_child_without_cuda_work (__main__.RecordedPeakTests) ... ok
|
||||
test_failed_worker_final_scan_catches_breach_without_result (__main__.RecordedPeakTests) ... ok
|
||||
test_invalid_recorded_peak_rejected (__main__.RecordedPeakTests) ... ok
|
||||
test_maximum_survives_lower_replacement_and_deletion (__main__.RecordedPeakTests) ... ok
|
||||
test_nested_history_and_result_both_contribute (__main__.RecordedPeakTests) ... ok
|
||||
test_only_progress_reserved_breach_stops_live_worker (__main__.RecordedPeakTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 7 tests in 0.122s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"status": "FROZEN_NOT_STARTED",
|
||||
"contract_sha256": "198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56",
|
||||
"roster_sha256": "fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e",
|
||||
"runs": 90,
|
||||
"blocks": 45,
|
||||
"labels_decoded": false,
|
||||
"gpu_work_started": false,
|
||||
"gpu_cpu_affinities": [
|
||||
[
|
||||
0,
|
||||
1
|
||||
],
|
||||
[
|
||||
2,
|
||||
3
|
||||
],
|
||||
[
|
||||
4,
|
||||
5
|
||||
],
|
||||
[
|
||||
6,
|
||||
7
|
||||
]
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
test_budget_reservation_and_first_eight_dispatch_gate (__main__.MatrixChecks) ... ok
|
||||
test_current_fold_not_other_fold_controls_access (__main__.MatrixChecks) ... ok
|
||||
test_current_test_rejected_before_any_target_io_even_old_cache (__main__.MatrixChecks) ... ok
|
||||
test_existing_start_marker_rejects_duplicate_matrix (__main__.MatrixChecks) ... ok
|
||||
test_failed_N_does_not_retry_or_skip_paired_E (__main__.MatrixChecks) ... ok
|
||||
test_first_eight_projection_and_failed_cost_guard (__main__.MatrixChecks) ... ok
|
||||
test_full_fake_scheduler_barrier_unique_blocks_and_90_terminal (__main__.MatrixChecks) ... ok
|
||||
test_lazy_targets_created_once_and_reused_only_after_role_check (__main__.MatrixChecks) ... ok
|
||||
test_numerical_loop_unchanged_and_no_checkpoint_loading (__main__.MatrixChecks) ... ok
|
||||
test_real_roster_all_15_selections_match_frozen_counts (__main__.MatrixChecks) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 10 tests in 0.232s
|
||||
|
||||
OK
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,15 @@
|
||||
# S2 统一名单后的 fresh 矩阵执行
|
||||
|
||||
准备终态已固定为 `31012ae4d38b4b088eb2f8d3a5851ef2ab55e710`:131 尝试中124通过、7超时,493旧成功仅复用。最终617件的统一名单 SHA256 为 `fb0c984f2e553343a598ce2e3c7012d3017053dc0fef58b5aaf46dd3c67a207e`。排除7件共708645节点,原全集18661224节点;各折/池实际计数已固定在准备根 `final-fold-pool-counts.csv`。本次不再启动准备,也不复用pilot权重。
|
||||
|
||||
新矩阵根 `artifacts/s2-main-matrix-2026-11-15`,合同 SHA256 `198e5711ed2e7485167068769652c866867229e09ecf0d09b21612debc3b7a56`。执行授权沿Claude1794699027.366469。固定5折×3组件比例池×3种子×N/E共90 fresh runs,45个相邻N/E块;宽128按可行性选择,宽度效应未检验。E_max/cosine42、AdamW及AMP跳步规则不变。新工作者的完整训练/验证epoch循环与已固定v2 pilot逐字一致;只适配fold/pool/seed、名单和输入访问,没有加载旧checkpoint。
|
||||
|
||||
每个fold/pool只枚举该折成功learning/validation,标准化统计只用该学习池。读取缓存或压缩包之前拒绝本折test。可验证兼容的旧非测试目标复用;缺少的目标在受监督run内按当前折允许角色惰性提取并留回执,按折加锁缓存,失败/中断不静默重试。本折test始终不进入该折拟合或验证;其他折的角色不能绕过当前折检查。所有90+15终态及检查点冻结后才统一托管测试预测、评分;本执行器没有测试预测或评分入口。
|
||||
|
||||
四张V100每卡一块,块内N后E,每个run都有三次空闲确认和已验收的累计显存峰值监督。各卡CPU亲和分别[0,1]、[2,3]、[4,5]、[6,7],记录在合同及回执;CPU8–31留给其他工作和后续R2。环境显式设置expandable_segments:True;每run≤1800s、15GiB显存、16GiB进程树RSS、5GiB输出。每次空闲等待最多600s;它与设置、失败耗时均计入46聚合GPU小时。监督仍是采样与退出复核,不把采样上限描述为连续硬件配额。
|
||||
|
||||
协调器按实际子进程存活时长累计每块GPU时间,并在派发时为每个活动块预留5000s;不按并行日历时长低报GPU小时。首四块共八个run全部终态前不派第五块;使用成功观察到的各臂最大wall/node、剩余实际节点数及25%余量重估,超过46h或两臂缺可用成本时停止报告。预算不足时尚未派发的run明确NA。任一臂失败仍完成其配对臂的既定尝试;失败不重试、不替补,配对有效性单独记录。
|
||||
|
||||
验收:10项矩阵CPU合成检查及7项既有累计显存监督检查全部PASS。覆盖15个真实名单选择与固定计数一致、当前折test在I/O前拒绝(即使有旧缓存)、折角色转换、惰性目标只创建一次及缓存后再次角色检查、训练循环未改变/无权重加载、预算预留、首八屏障和投影停止、N失败仍执行E一次、45块/90终态唯一派发、禁止重复矩阵启动,以及历史失败路径显存峰值监督。没有在这些验收中读取真实新增标签或启动GPU计算。
|
||||
|
||||
执行状态以 `matrix-started.json`、`matrix-progress.json`、各block/cell回执及 `first-eight-checkpoint.json` 为准。`prelaunch.json`、`acceptance.json` 是启动前固定证据,不可据此覆盖之后的真实状态。不得因聊天闲置重启。后续R2尚需独立执行实现与验收,预算为单fit30CPU-min、总100workerCPU-h、首fit成本检查;R2未全部终态时禁止托管测试解封。
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Freeze 90 fresh cells against the one final roster; no label decoding or GPU work."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_supervisor import save
|
||||
from s2_main_data import select_cell
|
||||
|
||||
ROOT=Path('artifacts/s2-main-matrix-2026-11-15')
|
||||
PREP=Path('artifacts/s2-main-preparation-2026-11-15')
|
||||
GPUS=['GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c','GPU-f1c9fd5a-e917-56e0-21fb-24035ba44a36',
|
||||
'GPU-b8c11c01-c91f-aa0b-abf6-dad35085cc76','GPU-8b71b032-1b75-2891-935c-ce106a53034a']
|
||||
|
||||
|
||||
def build_runs(roster):
|
||||
runs=[];blocks=[];cells=[]
|
||||
for fold in range(5):
|
||||
for pool in [25,50,100]:
|
||||
records,roles=select_cell(roster,fold,pool)
|
||||
cell=dict(fold=fold,pool=pool,roles=roles,learning_nodes=sum(records[k]['nodes'] for k in roles['learning_parts']),
|
||||
validation_nodes=sum(records[k]['nodes'] for k in roles['validation_parts']))
|
||||
cells.append(cell)
|
||||
for seed in [0,1,2]:
|
||||
indices=[]
|
||||
for arm in ['N','E']:
|
||||
indices.append(len(runs));runs.append(dict(name=f'fold{fold}-pool{pool}-{arm}128-seed{seed}',
|
||||
fold=fold,pool=pool,seed=seed,arm=arm,width=128,initialization='fresh_seeded',
|
||||
cost_nodes=cell['learning_nodes']+cell['validation_nodes']))
|
||||
blocks.append(dict(fold=fold,pool=pool,seed=seed,run_indices=indices))
|
||||
return runs,blocks,cells
|
||||
|
||||
|
||||
def main():
|
||||
assert not ROOT.exists(),'Do not overwrite matrix plan or execution'
|
||||
roster_path=PREP/'final-roster.json';roster=json.loads(roster_path.read_text())
|
||||
rr=json.loads((PREP/'roster-receipt.json').read_text());assert rr['status']=='FROZEN' and sha(roster_path)==rr['sha256']
|
||||
batch=json.loads((PREP/'batch-complete.json').read_text());assert batch['status']=='ATTEMPTS_COMPLETE' and batch['stop_reason'] is None
|
||||
prior=json.loads((PREP/'contract.json').read_text());runs,blocks,cells=build_runs(roster)
|
||||
assert len(runs)==90 and len(blocks)==45 and len(cells)==15 and sum(p['included'] for p in roster['parts'])==617
|
||||
pilot=json.loads(Path('artifacts/s2-pilot-v2-continuation-2026-11-15/training-contract.json').read_text())
|
||||
for p,h in pilot['code_sha256'].items():assert sha(p)==h,p
|
||||
added=['tools/s2_main_data.py','tools/run_s2_main_training.py','tools/run_s2_main_matrix.py','tools/prepare_s2_main_matrix.py',
|
||||
'tools/test_s2_main_matrix.py','tools/s2_main_memory_guard.py','tools/s2_gpu_idle_gate.py',
|
||||
'tools/s2_recovery_monitoring.py','tools/s2_recovery_supervisor.py','tools/s2_recovery_gpu_supervisor.py']
|
||||
paths=set(pilot['code_sha256'])|set(added)
|
||||
available=sorted(os.sched_getaffinity(0));assert len(available)>=12
|
||||
affinities=[available[2*i:2*i+2] for i in range(4)]
|
||||
old_labels='artifacts/s2-pilot-training-2026-11-15/success-manifest.json'
|
||||
sources=[str(roster_path),str(PREP/'roster-receipt.json'),str(PREP/'batch-complete.json'),old_labels]
|
||||
c=dict(version='S2-fresh-main-matrix-v1',runs=runs,blocks=blocks,cells=cells,E_max=42,cosine_T_max=42,
|
||||
roster=str(roster_path),roster_sha256=sha(roster_path),old_labels_manifest=old_labels,old_labels_manifest_sha256=sha(old_labels),
|
||||
archive=prior['archive'],archive_stat=prior['archive_stat'],gpu_uuids=GPUS,gpu_cpu_affinities=affinities,
|
||||
cpu_affinity_available=available,cpu_affinity_reserved_for_other_work=available[8:],
|
||||
code_sha256={p:sha(p) for p in sorted(paths)},source_sha256={p:sha(p) for p in sources},
|
||||
limits=dict(aggregate_gpu_seconds=46*3600,run_wall_seconds=1800,idle_seconds_per_run=600,block_wall_seconds=5000,
|
||||
max_concurrent_gpus=4,run_vram_bytes=15*2**30,run_host_rss_bytes=16*2**30,run_disk_bytes=5*2**30,retries=0),
|
||||
allocator_conf='expandable_segments:True',authorizing_rulings=['1794698179.161599','1794698734.967459','1794699027.366469'],
|
||||
capacity='128 chosen by feasibility; width effect not tested',pilot_weights_reused=False,
|
||||
target_access='Current fold learning/validation only; reject current fold test before cache/archive IO. Other folds roles do not grant access.',
|
||||
test_predictions='No predictions/scoring until all90 neural +15 R2 final fits terminal and checkpoints frozen; unified custody only.',
|
||||
supervision='N/E adjacent per fold/pool/seed block, one block perGPU, reserve5000s per active block; charge coordinator-observed block lifetime including idle/setup and failures.',
|
||||
first_eight='All four initial blocks terminal before any fifth block; project remaining work with per-arm max successful wall/node and25percent reserve; stop if over46h or no valid rate.',
|
||||
failure_policy='No retry; retain checkpoint/failure evidence, failed run NA; pair invalid if either arm NA or standardizer mismatch. Budget unstarted cells terminal NA with explicit reason.')
|
||||
ROOT.mkdir();save(ROOT/'training-contract.json',c)
|
||||
save(ROOT/'prelaunch.json',dict(status='FROZEN_NOT_STARTED',contract_sha256=sha(ROOT/'training-contract.json'),
|
||||
roster_sha256=c['roster_sha256'],runs=90,blocks=45,labels_decoded=False,gpu_work_started=False,gpu_cpu_affinities=affinities))
|
||||
print(json.dumps(json.loads((ROOT/'prelaunch.json').read_text())))
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Four isolated GPU block workers; adjacent N/E, first-eight barrier, aggregate wall ledger."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import psutil
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_supervisor import save
|
||||
from s2_gpu_idle_gate import recorded_wait
|
||||
from s2_main_memory_guard import guarded_worker,ALLOCATOR_CONF
|
||||
|
||||
BLOCK_WALL_CAP=5000.
|
||||
|
||||
|
||||
def contract(root):
|
||||
c=json.loads((root/'training-contract.json').read_text())
|
||||
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():assert sha(p)==h,p
|
||||
assert c['roster_sha256']==sha(c['roster']) and len(c['runs'])==90 and len(c['blocks'])==45
|
||||
assert c['limits']['aggregate_gpu_seconds']==46*3600 and c['limits']['run_wall_seconds']==1800
|
||||
assert c['E_max']==c['cosine_T_max']==42
|
||||
assert len(set(c['gpu_uuids']))==len(c['gpu_uuids'])==4
|
||||
for b in c['blocks']:
|
||||
n,e=[c['runs'][i] for i in b['run_indices']]
|
||||
assert (n['arm'],e['arm'])==('N','E')
|
||||
assert (n['fold'],n['pool'],n['seed'])==(e['fold'],e['pool'],e['seed'])
|
||||
return c
|
||||
|
||||
|
||||
def first_eight_projection(c,blocks,charged):
|
||||
assert len(blocks)==4 and sum(len(b['runs']) for b in blocks)==8
|
||||
rates={arm:[] for arm in ['N','E']}
|
||||
for b in blocks:
|
||||
for row in b['runs']:
|
||||
run=c['runs'][row['index']]
|
||||
if row['status']=='PASS':rates[run['arm']].append(row['cell_wall_seconds']/run['cost_nodes'])
|
||||
if any(not v for v in rates.values()):
|
||||
return dict(continue_within_cap=False,reason='FIRST8_NO_VALID_COST_FOR_BOTH_ARMS',observed_charged_gpu_seconds=charged)
|
||||
rate={a:max(v) for a,v in rates.items()}
|
||||
completed={row['index'] for b in blocks for row in b['runs']}
|
||||
projected=charged+1.25*sum(rate[r['arm']]*r['cost_nodes'] for i,r in enumerate(c['runs']) if i not in completed)
|
||||
return dict(continue_within_cap=projected<=c['limits']['aggregate_gpu_seconds'],projected_gpu_seconds=projected,
|
||||
observed_charged_gpu_seconds=charged,seconds_per_node_by_arm=rate,reserve_multiplier=1.25,
|
||||
method='Maximum successful observed per-arm cell wall/node (idle/setup included), remaining cells plus25percent; all actual block wall already charged.')
|
||||
|
||||
|
||||
def can_dispatch(c,next_block,active_count,charged,checkpoint):
|
||||
ceiling=len(c['blocks']) if checkpoint else 4
|
||||
return next_block<ceiling and active_count<4 and charged+(active_count+1)*BLOCK_WALL_CAP<=c['limits']['aggregate_gpu_seconds']
|
||||
|
||||
|
||||
def block(root,index,gpu_slot):
|
||||
c=contract(root);spec=c['blocks'][index];uuid=c['gpu_uuids'][gpu_slot]
|
||||
dest=root/'blocks'/f'block-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
|
||||
with (dest/'block.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
assert not (dest/'started.json').exists(),'No repeated block'
|
||||
with Path('/tmp/meltmind-s2-'+uuid+'.lock').open('a') as gpu_lock:
|
||||
fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
return block_locked(root,c,spec,dest,index,gpu_slot)
|
||||
|
||||
|
||||
def block_locked(root,c,spec,dest,index,gpu_slot):
|
||||
begin=time.monotonic();uuid=c['gpu_uuids'][gpu_slot];rows=[]
|
||||
os.sched_setaffinity(0,c['gpu_cpu_affinities'][gpu_slot])
|
||||
os.environ['PYTORCH_CUDA_ALLOC_CONF']=ALLOCATOR_CONF
|
||||
save(dest/'started.json',dict(pid=os.getpid(),started_unix=time.time(),gpu_uuid=uuid,block=index,
|
||||
run_indices=spec['run_indices'],cpu_affinity=sorted(os.sched_getaffinity(0)),contract_sha256=sha(root/'training-contract.json')))
|
||||
try:
|
||||
for i in spec['run_indices']:
|
||||
run=c['runs'][i];cell=root/'cells'/run['name'];cell.mkdir(parents=True,exist_ok=False)
|
||||
cell_begin=time.monotonic();row=dict(index=i,name=run['name'],arm=run['arm'],status='NA',reason=None)
|
||||
try:
|
||||
if (root/'cancel.json').exists():raise RuntimeError('MATRIX_CANCELLED_BEFORE_RUN')
|
||||
if time.monotonic()-begin>BLOCK_WALL_CAP-2405:raise RuntimeError('BLOCK_REMAINING_BUDGET_INSUFFICIENT')
|
||||
idle=recorded_wait(cell,'before-run',uuid,600)
|
||||
if idle['status']!='IDLE_CONFIRMED':raise RuntimeError('GPU_IDLE_TIMEOUT_NO_RETRY')
|
||||
if (root/'cancel.json').exists():raise RuntimeError('MATRIX_CANCELLED_AFTER_IDLE')
|
||||
receipt,result=guarded_worker(cell,run['name'],uuid,1800,
|
||||
[sys.executable,'tools/run_s2_main_training.py',str(root),str(i)],cumulative_worker_cpu_limit_seconds=1801)
|
||||
row.update(status='PASS' if receipt['status']=='PASS' else 'NA',reason=(receipt.get('stop_reason') or ('WORKER_EXIT_'+str(receipt['returncode']) if receipt['status']!='PASS' else None)),
|
||||
guard_receipt=str(cell/(run['name']+'-guard-receipt.json')),guard_sha256=sha(cell/(run['name']+'-guard-receipt.json')),
|
||||
worker_status=receipt['status'],worker_returncode=receipt['returncode'],worker_cpu_seconds=receipt['cpu_seconds'],
|
||||
checkpoint_sha256=result.get('checkpoint_sha256') if result else None,
|
||||
standardizer_fingerprint=result.get('standardizer_fingerprint') if result else None)
|
||||
except Exception as exc:
|
||||
row.update(status='NA',reason=type(exc).__name__+': '+str(exc),traceback=traceback.format_exc())
|
||||
row['cell_wall_seconds']=time.monotonic()-cell_begin;rows.append(row)
|
||||
save(cell/'terminal.json',row);save(dest/'progress.json',dict(runs=rows,elapsed_seconds=time.monotonic()-begin))
|
||||
finally:
|
||||
# A failed arm never causes its paired arm to be silently replaced or retried.
|
||||
present={r['index'] for r in rows}
|
||||
for i in spec['run_indices']:
|
||||
if i not in present:rows.append(dict(index=i,name=c['runs'][i]['name'],status='NA',reason='BLOCK_INTERRUPTED_BEFORE_RUN',cell_wall_seconds=0))
|
||||
valid=all(r['status']=='PASS' for r in rows)
|
||||
if valid and rows[0]['standardizer_fingerprint']!=rows[1]['standardizer_fingerprint']:valid=False
|
||||
save(dest/'terminal.json',dict(status='TERMINAL',block=index,gpu_uuid=uuid,runs=rows,pair_valid=valid,
|
||||
pair_invalid_reason=None if valid else 'AT_LEAST_ONE_ARM_NA_OR_STANDARDIZER_MISMATCH',
|
||||
block_wall_seconds=time.monotonic()-begin,cpu_affinity=sorted(os.sched_getaffinity(0)),retry=False,test_predictions=False))
|
||||
|
||||
|
||||
def stop_owned_workers(active):
|
||||
for item in active.values():
|
||||
try:children=psutil.Process(item['process'].pid).children(recursive=True)
|
||||
except psutil.NoSuchProcess:continue
|
||||
for p in children:
|
||||
try:
|
||||
if 'tools/run_s2_main_training.py' in p.cmdline():os.killpg(p.pid,signal.SIGKILL)
|
||||
except (ProcessLookupError,psutil.NoSuchProcess):pass
|
||||
|
||||
|
||||
def run(root):
|
||||
with (root/'matrix.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
|
||||
|
||||
|
||||
def run_locked(root):
|
||||
c=contract(root);assert not (root/'matrix-started.json').exists(),'No repeated matrix launch'
|
||||
save(root/'matrix-started.json',dict(pid=os.getpid(),started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
|
||||
start=time.monotonic();active={};done=[];next_block=0;reason=None;checkpoint=False;charged=0.
|
||||
def interrupt(*_):
|
||||
nonlocal reason
|
||||
reason='MATRIX_COORDINATOR_INTERRUPTED'
|
||||
signal.signal(signal.SIGTERM,interrupt);signal.signal(signal.SIGINT,interrupt)
|
||||
env=dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=ALLOCATOR_CONF,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
||||
try:
|
||||
while active or (next_block<len(c['blocks']) and reason is None):
|
||||
for slot,item in list(active.items()):
|
||||
code=item['process'].poll();elapsed=time.monotonic()-item['begin']
|
||||
if elapsed>BLOCK_WALL_CAP and code is None:reason='BLOCK_WALL_LIMIT'
|
||||
if code is None:continue
|
||||
path=root/'blocks'/f"block-{item['index']:03d}"/'terminal.json'
|
||||
if not path.exists():
|
||||
reason='BLOCK_EXIT_WITHOUT_TERMINAL_RECEIPT'
|
||||
result=dict(status='INFRASTRUCTURE_FAILED',block=item['index'],runs=[dict(index=i,name=c['runs'][i]['name'],status='NA',reason=reason,cell_wall_seconds=0) for i in c['blocks'][item['index']]['run_indices']],pair_valid=False)
|
||||
else:result=json.loads(path.read_text())
|
||||
result.update(charged_gpu_seconds=elapsed,coordinator_observed_returncode=code)
|
||||
save(root/'blocks'/f"block-{item['index']:03d}"/'accounted.json',result)
|
||||
done.append(result);charged+=elapsed;item['log'].close();del active[slot]
|
||||
print(json.dumps(dict(event='BLOCK_TERMINAL',block=result['block'],run_statuses=[r['status'] for r in result['runs']],charged_gpu_hours=charged/3600)),flush=True)
|
||||
live=sum(time.monotonic()-x['begin'] for x in active.values())
|
||||
if charged+live>=c['limits']['aggregate_gpu_seconds']:reason='AGGREGATE_GPU_WALL_LIMIT'
|
||||
if not checkpoint and len(done)==4 and not active and reason is None:
|
||||
projection=first_eight_projection(c,done,charged);save(root/'first-eight-checkpoint.json',projection)
|
||||
if projection['continue_within_cap']:checkpoint=True
|
||||
else:reason='FIRST_EIGHT_PROJECTION_STOP'
|
||||
if reason:
|
||||
if not (root/'cancel.json').exists():save(root/'cancel.json',dict(reason=reason,unix=time.time()))
|
||||
stop_owned_workers(active)
|
||||
while reason is None and can_dispatch(c,next_block,len(active),charged,checkpoint):
|
||||
slot=next(s for s in range(4) if s not in active);idx=next_block
|
||||
dest=root/'blocks'/f'block-{idx:03d}';dest.mkdir(parents=True,exist_ok=True)
|
||||
log=(dest/'block.log').open('x');begin=time.monotonic()
|
||||
p=subprocess.Popen([sys.executable,__file__,str(root),'--block',str(idx),'--gpu-slot',str(slot)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
active[slot]=dict(process=p,log=log,begin=begin,index=idx);next_block+=1
|
||||
if not active and next_block<len(c['blocks']) and reason is None and checkpoint:reason='AGGREGATE_BUDGET_RESERVATION_STOP'
|
||||
save(root/'matrix-progress.json',dict(completed_blocks=len(done),terminal_runs=2*len(done),active_blocks={str(k):v['index'] for k,v in active.items()},
|
||||
next_block=next_block,settled_gpu_seconds=charged,active_gpu_seconds=live,first_eight_passed=checkpoint,stop_reason=reason,elapsed_seconds=time.monotonic()-start))
|
||||
if active:time.sleep(.5)
|
||||
finally:
|
||||
if active:
|
||||
save(root/'cancel.json',dict(reason=reason or 'COORDINATOR_EXIT',unix=time.time()));stop_owned_workers(active)
|
||||
for item in active.values():item['process'].wait();item['log'].close()
|
||||
rows=[row for b in done for row in b['runs']];seen={r['index'] for r in rows}
|
||||
for i,r in enumerate(c['runs']):
|
||||
if i not in seen:rows.append(dict(index=i,name=r['name'],status='NA',reason='NOT_RUN_'+str(reason)))
|
||||
result=dict(status='ALL_NEURAL_TERMINAL',stop_reason=reason,runs=sorted(rows,key=lambda r:r['index']),blocks=done,
|
||||
charged_gpu_seconds=charged,passed=sum(r['status']=='PASS' for r in rows),NA=sum(r['status']!='PASS' for r in rows),
|
||||
roster_sha256=c['roster_sha256'],contract_sha256=sha(root/'training-contract.json'),retry=False,
|
||||
test_predictions=False,test_scoring=False,r2_required_before_custody=True)
|
||||
save(root/'matrix-complete.json',result)
|
||||
return 0 if reason is None else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--block',type=int);p.add_argument('--gpu-slot',type=int);a=p.parse_args()
|
||||
raise SystemExit(block(a.root,a.block,a.gpu_slot) if a.block is not None else run(a.root))
|
||||
@@ -0,0 +1,148 @@
|
||||
"""One fresh frozen matrix cell; unchanged v2 numerical training loop inside the GPU guard."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from collections.abc import Mapping
|
||||
import json
|
||||
from pathlib import Path
|
||||
import random
|
||||
import time
|
||||
import traceback
|
||||
import numpy as np
|
||||
import torch
|
||||
from freeze_g1_components import sha
|
||||
from s2_e1_supervisor import save
|
||||
from s2_models_v2.network import QueryModel,TRAINING_CONTRACT
|
||||
from s2_models.normalization import BaseStandardizer,fit_base_standardizer
|
||||
from s2_models.objectives import relative_l2
|
||||
from s2_pilot_data import permitted_role
|
||||
from s2_main_data import cell_records
|
||||
from s2_pilot_train import part_weights,train_part,component_score,SkipPolicy
|
||||
|
||||
|
||||
class LearningBases(Mapping):
|
||||
def __init__(self,records,ids,roles,cache=None):self.records=records;self.ids=tuple(ids);self.roles=roles;self.cache=cache
|
||||
def __iter__(self):return iter(self.ids)
|
||||
def __len__(self):return len(self.ids)
|
||||
def __getitem__(self,case):
|
||||
if case not in self.ids or permitted_role(self.roles,case)!='learning':raise PermissionError('Learning-only normalization')
|
||||
if self.cache is not None:return self.cache[case]['base']
|
||||
assert sha(Path(self.records[case]['preparation_root'])/'base.npy')==self.records[case]['input_sha256']['base.npy']
|
||||
return np.load(Path(self.records[case]['preparation_root'])/'base.npy',allow_pickle=False,mmap_mode='r')
|
||||
|
||||
|
||||
def read_part_arrays(record,roles,*,synthetic_targets=False):
|
||||
case=record['case'];role=permitted_role(roles,case)
|
||||
if synthetic_targets and role!='learning':raise PermissionError('Timing probe accepts only learning parts')
|
||||
root=Path(record['preparation_root']);label=Path(record['label_path'])
|
||||
assert sha(root/'base.npy')==record['input_sha256']['base.npy']
|
||||
assert sha(root/'queries.npz')==record['input_sha256']['queries.npz']
|
||||
with np.load(root/'queries.npz',allow_pickle=False) as q:
|
||||
mask=q['eligible_s2'];ids=q['anchor_ids'][mask];g=q['g'][mask];edge=q['edge_summary'][mask];counts=q['multiplicity'][mask]
|
||||
if synthetic_targets:
|
||||
target=np.zeros(len(ids));seconds=None
|
||||
else:
|
||||
assert sha(label)==record['input_sha256']['labels.npz']
|
||||
with np.load(label,allow_pickle=False) as z:
|
||||
assert np.array_equal(ids,z['anchor_ids']) and np.array_equal(counts,z['multiplicity'])
|
||||
target=z['target_t_star'];seconds=z['seconds']
|
||||
base=np.load(root/'base.npy',allow_pickle=False)
|
||||
if len(base)!=record['nodes']:raise ValueError('Full-node base size mismatch')
|
||||
return dict(base=base,ids=ids,gaps=g,edge=edge,target=target,counts=counts,truth_seconds=seconds)
|
||||
|
||||
|
||||
def to_device(data,device):
|
||||
result={k:v for k,v in data.items() if k not in ['base','ids','gaps','edge','target']}
|
||||
for key in ['base','ids','gaps','edge','target']:
|
||||
result[key]=torch.as_tensor(np.array(data[key],copy=True),dtype=torch.long if key=='ids' else torch.float32,device=device)
|
||||
return result
|
||||
|
||||
|
||||
def load_part(record,roles,device):
|
||||
return to_device(read_part_arrays(record,roles),device)
|
||||
|
||||
|
||||
def run(root,index):
|
||||
started=time.monotonic();c=json.loads((root/'training-contract.json').read_text());run=c['runs'][index]
|
||||
dest=root/'cells'/run['name']/run['name'];dest.mkdir(exist_ok=False);save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
|
||||
try:
|
||||
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
||||
records,roles,manifest,target_audit=cell_records(root,c,run)
|
||||
save(dest/'target-access.json',dict(fold=run['fold'],pool=run['pool'],roster_sha256=c['roster_sha256'],access=target_audit,outer_fold_test_labels_read=False))
|
||||
learning=sorted(p for p in records if permitted_role(roles,p)=='learning')
|
||||
validation=sorted(p for p in records if permitted_role(roles,p)=='validation')
|
||||
if not learning or not validation:raise ValueError('Empty successful learning/validation roster')
|
||||
if any(p in roles['sealed_test_parts'] for p in records):raise PermissionError('Test case present in ready manifest')
|
||||
assert run['seed'] in [0,1,2] and run['fold'] in range(5) and run['pool'] in [25,50,100]
|
||||
assert run['width']==128 and c['E_max']==42 and run['initialization']=='fresh_seeded'
|
||||
torch.set_num_threads(1);random.seed(run['seed']);np.random.seed(run['seed']);torch.manual_seed(run['seed'])
|
||||
if not torch.cuda.is_available() or torch.cuda.device_count()!=1:raise RuntimeError('Exactly one visible CUDA device required')
|
||||
device=torch.device('cuda:0');torch.cuda.manual_seed_all(run['seed']);torch.cuda.reset_peak_memory_stats()
|
||||
torch.backends.cuda.matmul.allow_tf32=False
|
||||
if 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Frozen pilot requires one V100')
|
||||
# Check hashes once, cache raw FP64 arrays, and fit only the learning subset.
|
||||
cache={case:read_part_arrays(records[case],roles) for case in sorted(records)}
|
||||
stats=fit_base_standardizer(LearningBases(records,learning,roles,cache),learning_part_ids=learning,fold=str(run['fold']),pool=str(run['pool']))
|
||||
save(dest/'standardizer.json',dict(**stats.as_dict(),fingerprint=stats.fingerprint))
|
||||
model=QueryModel(run['arm'],run['width'],standardizer=stats).to(device)
|
||||
optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5)
|
||||
epoch_limit=int(c['E_max']);assert 1<=epoch_limit<=200 and c['cosine_T_max']==epoch_limit
|
||||
scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=epoch_limit)
|
||||
scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(run['seed']))
|
||||
group_counts=Counter(records[p]['component_id'] for p in learning);group_number=len(group_counts)
|
||||
val_groups={p:records[p]['component_id'] for p in validation}
|
||||
best=float('inf');best_epoch=None;stale=0;history=[];updates=0;attempted_updates=0;skip_policy=SkipPolicy()
|
||||
for epoch in range(epoch_limit):
|
||||
model.train();loss_sum=0.;skip_policy.start_epoch()
|
||||
order=[learning[int(j)] for j in rng.permutation(len(learning))]
|
||||
for j,case in enumerate(order):
|
||||
record=records[case];data=to_device(cache[case],device)
|
||||
weights=torch.tensor(part_weights(data['counts'],group_counts[record['component_id']],group_number,len(learning)),dtype=torch.float32,device=device)
|
||||
step_info={}
|
||||
loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=step_info)
|
||||
loss_sum+=loss/len(learning);attempted_updates+=1;updates+=int(not step_info['step_skipped'])
|
||||
skip_policy.observe(step_info['step_skipped'])
|
||||
del data,weights
|
||||
if j%16==0:
|
||||
torch.cuda.synchronize();save(dest/'progress.json',dict(stage='training',epoch=epoch,parts_done=j+1,parts_total=len(learning),updates=updates,elapsed_seconds=time.monotonic()-started,
|
||||
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()))
|
||||
amp_summary=skip_policy.summary();save(dest/f'epoch-{epoch:03d}-amp.json',amp_summary)
|
||||
skip_policy.finish_epoch(epoch)
|
||||
model.eval();errors={}
|
||||
with torch.no_grad():
|
||||
for case in validation:
|
||||
data=to_device(cache[case],device)
|
||||
with torch.autocast(device_type='cuda',enabled=False):
|
||||
pred=model(data['base'],data['ids'],data['gaps'],data['edge'] if run['arm']=='E' else None)
|
||||
predicted_seconds=pred.float().cpu().numpy().astype(np.float64)*records[case]['t_set']
|
||||
score=relative_l2(predicted_seconds,data['truth_seconds'],data['counts'],np.ones(len(data['counts']),bool))
|
||||
errors[case]=score['value'];del data,pred
|
||||
value=component_score(errors,val_groups);torch.cuda.synchronize()
|
||||
row=dict(epoch=epoch,amp=amp_summary,attempted_updates=attempted_updates,training_global_weighted_mse=loss_sum,validation_component_equal_relative_l2=value,updates=updates,elapsed_seconds=time.monotonic()-started,
|
||||
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())
|
||||
history.append(row);save(dest/'history.json',history)
|
||||
if value<best:
|
||||
best=value;best_epoch=epoch;stale=0
|
||||
torch.save(dict(state_dict={k:v.detach().cpu() for k,v in model.state_dict().items()},arm=run['arm'],width=run['width'],seed=run['seed'],epoch=epoch,
|
||||
standardizer=stats.as_dict(),standardizer_fingerprint=stats.fingerprint),dest/'best.pt')
|
||||
save(dest/'best-validation-parts.json',errors)
|
||||
else:stale+=1
|
||||
scheduler.step()
|
||||
save(dest/'progress.json',dict(stage='epoch_complete',**row,patience_count=stale))
|
||||
if stale>=20:break
|
||||
torch.cuda.synchronize()
|
||||
result=dict(status='COMPLETE',run=run,E_max=epoch_limit,cosine_T_max=epoch_limit,epochs=len(history),attempted_updates=attempted_updates,updates=updates,best_epoch=best_epoch,best_validation_score=best,
|
||||
learning_parts=len(learning),validation_parts=len(validation),learning_components=group_number,validation_components=len(set(val_groups.values())),
|
||||
roster_sha256=c['roster_sha256'],fold=run['fold'],pool=run['pool'],fresh_initialization=True,failed_preparation_parts_dropped=manifest['failed_parts'],standardizer_fingerprint=stats.fingerprint,
|
||||
wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(),
|
||||
test_labels_read=False,test_predictions=False,capacity_selected=False,checkpoint_sha256=sha(dest/'best.pt'))
|
||||
save(dest/'result.json',result);print(json.dumps(result),flush=True);return 0
|
||||
except Exception as exc:
|
||||
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-started,
|
||||
peak_allocated_bytes=torch.cuda.max_memory_allocated() if torch.cuda.is_initialized() else 0,
|
||||
peak_reserved_bytes=torch.cuda.max_memory_reserved() if torch.cuda.is_initialized() else 0,
|
||||
epoch=locals().get('epoch'),amp=skip_policy.summary() if 'skip_policy' in locals() else None))
|
||||
raise
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Frozen fold/pool selection and lazy targets, rejecting that fold's test before I/O."""
|
||||
import fcntl
|
||||
import json
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_supervisor import save
|
||||
from s2_pilot_data import permitted_role,read_non_test_fill_time,validate_query_targets
|
||||
|
||||
|
||||
def select_cell(roster,fold,pool):
|
||||
assert fold in range(5) and pool in [25,50,100]
|
||||
selected={};roles=dict(fold=fold,pool=pool,learning_parts=[],validation_parts=[],sealed_test_parts=[])
|
||||
for row in roster['parts']:
|
||||
a=row['assignments'][fold];assert a['fold']==fold
|
||||
if a['role']=='test':roles['sealed_test_parts'].append(row['case']);continue
|
||||
if not row['included']:continue
|
||||
if a['role']=='learning' and str(pool) not in a['learning_pools']:continue
|
||||
roles[a['role']+'_parts'].append(row['case']);selected[row['case']]=row
|
||||
for key in ['learning_parts','validation_parts','sealed_test_parts']:roles[key].sort()
|
||||
assert roles['learning_parts'] and roles['validation_parts']
|
||||
assert not set(selected)&set(roles['sealed_test_parts'])
|
||||
return selected,roles
|
||||
|
||||
|
||||
def ensure_targets(root,c,row,roles,old_labels):
|
||||
case=row['case'];role=permitted_role(roles,case) # before any file/cache/archive access
|
||||
prep=row['preparation'];source=Path(prep['preparation_root'])
|
||||
if case in old_labels:
|
||||
old=old_labels[case]
|
||||
assert old['nodes']==row['nodes'] and old['t_set']==prep['t_set']
|
||||
assert old['input_sha256']['queries.npz']==prep['output_sha256']['queries.npz']
|
||||
path=Path(old['label_path']);digest=old['input_sha256']['labels.npz']
|
||||
assert sha(path)==digest
|
||||
return path,digest,'previous_non_test_target_reused_after_current_fold_role_check'
|
||||
cache=Path(root)/'fold-targets'/str(roles['fold'])/case;cache.mkdir(parents=True,exist_ok=True)
|
||||
with (cache/'target.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX)
|
||||
if (cache/'failure.json').exists():raise RuntimeError('Prior target preparation failure; no silent retry')
|
||||
if (cache/'receipt.json').exists():
|
||||
receipt=json.loads((cache/'receipt.json').read_text())
|
||||
assert receipt['case']==case and receipt['fold']==roles['fold']
|
||||
assert receipt['queries_sha256']==prep['output_sha256']['queries.npz']
|
||||
assert receipt['roster_sha256']==c['roster_sha256']
|
||||
assert sha(cache/'labels.npz')==receipt['sha256']
|
||||
return cache/'labels.npz',receipt['sha256'],'current_fold_non_test_target_cache'
|
||||
if (cache/'started.json').exists():raise RuntimeError('Interrupted target preparation; no silent retry')
|
||||
save(cache/'started.json',dict(case=case,fold=roles['fold'],role_at_creation=role,roster_sha256=c['roster_sha256']))
|
||||
try:
|
||||
st=Path(c['archive']).stat();ver=c['archive_stat']
|
||||
assert (st.st_size,st.st_mtime_ns,st.st_ino)==(ver['bytes'],ver['mtime_ns'],ver['inode'])
|
||||
for name in ['geometry.npz','queries.npz']:assert sha(source/name)==prep['output_sha256'][name]
|
||||
with np.load(source/'geometry.npz',allow_pickle=False) as z:raw_ids=z['raw_node_ids']
|
||||
with np.load(source/'queries.npz',allow_pickle=False) as q:
|
||||
ids=q['anchor_ids'];eligible=q['eligible_s2'];counts=q['multiplicity']
|
||||
values,provenance=read_non_test_fill_time(c['archive'],case,raw_ids,roles)
|
||||
target=validate_query_targets(values,ids,eligible,prep['t_set'])
|
||||
np.savez_compressed(cache/'labels.npz',anchor_ids=ids[eligible],raw_node_ids=raw_ids[ids[eligible]],
|
||||
multiplicity=counts[eligible],seconds=values[ids[eligible]],target_t_star=target)
|
||||
digest=sha(cache/'labels.npz')
|
||||
save(cache/'receipt.json',dict(case=case,fold=roles['fold'],role_at_creation=role,sha256=digest,
|
||||
queries_sha256=prep['output_sha256']['queries.npz'],roster_sha256=c['roster_sha256'],provenance=provenance,
|
||||
outer_fold_test_labels_read=False))
|
||||
return cache/'labels.npz',digest,'new_current_fold_non_test_target'
|
||||
except Exception as exc:
|
||||
save(cache/'failure.json',dict(error_type=type(exc).__name__,message=str(exc),case=case,fold=roles['fold']))
|
||||
raise
|
||||
|
||||
|
||||
def cell_records(root,c,run):
|
||||
assert sha(c['roster'])==c['roster_sha256']
|
||||
roster=json.loads(Path(c['roster']).read_text())
|
||||
rows,roles=select_cell(roster,run['fold'],run['pool'])
|
||||
assert sha(c['old_labels_manifest'])==c['old_labels_manifest_sha256']
|
||||
old={r['case']:r for r in json.loads(Path(c['old_labels_manifest']).read_text())['ready_parts']}
|
||||
records={};audit=[]
|
||||
for case,row in sorted(rows.items()):
|
||||
label,digest,mode=ensure_targets(root,c,row,roles,old);prep=row['preparation']
|
||||
records[case]=dict(case=case,nodes=row['nodes'],component_id=row['component_id'],t_set=prep['t_set'],
|
||||
preparation_root=prep['preparation_root'],label_path=str(label),
|
||||
input_sha256={'base.npy':prep['output_sha256']['base.npy'],'queries.npz':prep['output_sha256']['queries.npz'],'labels.npz':digest})
|
||||
audit.append(dict(case=case,role=permitted_role(roles,case),mode=mode,label_sha256=digest))
|
||||
manifest=dict(failed_parts=[dict(case=r['case'],reason=r['exclusion_reason']) for r in roster['parts'] if not r['included']])
|
||||
return records,roles,manifest,audit
|
||||
@@ -0,0 +1,133 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import numpy as np
|
||||
import s2_main_data as data
|
||||
import run_s2_main_matrix as matrix
|
||||
from prepare_s2_main_matrix import build_runs
|
||||
|
||||
|
||||
def fake_plan():
|
||||
runs=[];blocks=[]
|
||||
for i in range(45):
|
||||
indices=[]
|
||||
for arm in ['N','E']:
|
||||
indices.append(len(runs));runs.append(dict(name=f'{i}-{arm}',arm=arm,cost_nodes=100))
|
||||
blocks.append(dict(run_indices=indices))
|
||||
return dict(runs=runs,blocks=blocks,limits=dict(aggregate_gpu_seconds=46*3600),gpu_uuids=['g0','g1','g2','g3'],gpu_cpu_affinities=[[0],[1],[2],[3]],roster_sha256='frozen')
|
||||
|
||||
|
||||
class MatrixChecks(unittest.TestCase):
|
||||
def test_real_roster_all_15_selections_match_frozen_counts(self):
|
||||
r=json.loads(Path('artifacts/s2-main-preparation-2026-11-15/final-roster.json').read_text())
|
||||
runs,blocks,cells=build_runs(r)
|
||||
self.assertEqual((len(runs),len(blocks),len(cells)),(90,45,15))
|
||||
for cell,count in zip(cells,r['fold_pool_counts']):
|
||||
for role in ['learning','validation']:
|
||||
self.assertEqual(len(cell['roles'][role+'_parts']),count[role]['parts'])
|
||||
self.assertEqual(cell[role+'_nodes'],count[role]['nodes'])
|
||||
self.assertFalse(set(cell['roles']['learning_parts'])&set(cell['roles']['sealed_test_parts']))
|
||||
for block in blocks:
|
||||
n,e=[runs[i] for i in block['run_indices']]
|
||||
self.assertEqual((n['arm'],e['arm']),('N','E'))
|
||||
self.assertEqual((n['fold'],n['pool'],n['seed']),(e['fold'],e['pool'],e['seed']))
|
||||
|
||||
def test_current_test_rejected_before_any_target_io_even_old_cache(self):
|
||||
roles=dict(learning_parts=['learn'],validation_parts=['val'],sealed_test_parts=['test'])
|
||||
with patch.object(data,'sha',side_effect=AssertionError('IO')),patch.object(data.Path,'mkdir',side_effect=AssertionError('IO')),patch.object(data,'read_non_test_fill_time',side_effect=AssertionError('decode')):
|
||||
with self.assertRaises(PermissionError):data.ensure_targets(Path('/nonexistent'),{},dict(case='test'),roles,{'test':{}})
|
||||
|
||||
def test_current_fold_not_other_fold_controls_access(self):
|
||||
r=dict(parts=[dict(case='a',included=True,assignments=[dict(fold=0,role='test',learning_pools=[]),dict(fold=1,role='learning',learning_pools=['25','50','100'])]),
|
||||
dict(case='b',included=True,assignments=[dict(fold=0,role='learning',learning_pools=['25']),dict(fold=1,role='validation',learning_pools=[])]),
|
||||
dict(case='c',included=True,assignments=[dict(fold=0,role='validation',learning_pools=[]),dict(fold=1,role='test',learning_pools=[])])])
|
||||
a,_=data.select_cell(r,0,25);b,_=data.select_cell(r,1,25)
|
||||
self.assertNotIn('a',a);self.assertIn('a',b);self.assertNotIn('c',b)
|
||||
|
||||
def test_lazy_targets_created_once_and_reused_only_after_role_check(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);prep=root/'geom';prep.mkdir();archive=root/'archive';archive.write_bytes(b'synthetic')
|
||||
np.savez(prep/'geometry.npz',raw_node_ids=np.array([0,1,2]))
|
||||
np.savez(prep/'queries.npz',anchor_ids=np.array([1,2]),eligible_s2=np.array([True,False]),multiplicity=np.array([1500,500]))
|
||||
hashes={n:data.sha(prep/n) for n in ['geometry.npz','queries.npz']};st=archive.stat()
|
||||
c=dict(archive=str(archive),archive_stat=dict(bytes=st.st_size,mtime_ns=st.st_mtime_ns,inode=st.st_ino),roster_sha256='fixed')
|
||||
row=dict(case='a',preparation=dict(preparation_root=str(prep),output_sha256=hashes,t_set=2.))
|
||||
roles=dict(fold=1,learning_parts=['a'],validation_parts=[],sealed_test_parts=[])
|
||||
with patch.object(data,'read_non_test_fill_time',return_value=(np.array([0.,4.,8.]),dict(decoded_label_arrays=['Fill_time']))) as decode:
|
||||
path,digest,mode=data.ensure_targets(root,c,row,roles,{})
|
||||
path2,digest2,mode2=data.ensure_targets(root,c,row,roles,{})
|
||||
self.assertEqual(decode.call_count,1);self.assertEqual((path,digest),(path2,digest2))
|
||||
with np.load(path,allow_pickle=False) as z:
|
||||
np.testing.assert_array_equal(z['target_t_star'],[2.]);np.testing.assert_array_equal(z['anchor_ids'],[1])
|
||||
roles.update(learning_parts=[],sealed_test_parts=['a'])
|
||||
with self.assertRaises(PermissionError):data.ensure_targets(root,c,row,roles,{})
|
||||
self.assertEqual(decode.call_count,1)
|
||||
|
||||
def test_numerical_loop_unchanged_and_no_checkpoint_loading(self):
|
||||
old=Path('tools/run_s2_pilot_training_v2.py').read_text();new=Path('tools/run_s2_main_training.py').read_text()
|
||||
extract=lambda s:s[s.index(' for epoch in range(epoch_limit):'):s.index(' result=dict')]
|
||||
self.assertEqual(extract(old),extract(new));self.assertNotIn('torch.load(',new);self.assertNotIn('load_state_dict(',new)
|
||||
|
||||
def test_budget_reservation_and_first_eight_dispatch_gate(self):
|
||||
c=fake_plan();cap=c['limits']['aggregate_gpu_seconds']
|
||||
self.assertFalse(matrix.can_dispatch(c,4,0,0,False))
|
||||
self.assertTrue(matrix.can_dispatch(c,4,0,0,True))
|
||||
self.assertFalse(matrix.can_dispatch(c,5,3,cap-4*matrix.BLOCK_WALL_CAP+1,True))
|
||||
self.assertTrue(matrix.can_dispatch(c,5,3,cap-4*matrix.BLOCK_WALL_CAP,True))
|
||||
|
||||
def test_first_eight_projection_and_failed_cost_guard(self):
|
||||
c=fake_plan();blocks=[dict(runs=[dict(index=i,status='PASS',cell_wall_seconds=100) for i in b['run_indices']]) for b in c['blocks'][:4]]
|
||||
self.assertTrue(matrix.first_eight_projection(c,blocks,1000)['continue_within_cap'])
|
||||
for b in blocks:
|
||||
for r in b['runs']:r['cell_wall_seconds']=2000
|
||||
self.assertFalse(matrix.first_eight_projection(c,blocks,16000)['continue_within_cap'])
|
||||
for b in blocks:
|
||||
for r in b['runs']:r['status']='NA'
|
||||
self.assertFalse(matrix.first_eight_projection(c,blocks,1000)['continue_within_cap'])
|
||||
|
||||
def test_failed_N_does_not_retry_or_skip_paired_E(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);(root/'training-contract.json').write_text('{}');dest=root/'blocks'/'block-000';dest.mkdir(parents=True)
|
||||
c=fake_plan();calls=[]
|
||||
def guard(cell,name,uuid,wall,command,**kw):
|
||||
calls.append(name);r=dict(status='STOPPED_OR_FAILED' if len(calls)==1 else 'PASS',stop_reason='synthetic_failure' if len(calls)==1 else None,returncode=1 if len(calls)==1 else 0,cpu_seconds=1)
|
||||
(cell/(name+'-guard-receipt.json')).write_text(json.dumps(r))
|
||||
return r,dict(checkpoint_sha256='fresh',standardizer_fingerprint='shared')
|
||||
with patch.object(matrix.os,'sched_setaffinity'),patch.object(matrix.os,'sched_getaffinity',return_value={0}),patch.object(matrix,'recorded_wait',return_value=dict(status='IDLE_CONFIRMED')),patch.object(matrix,'guarded_worker',guard):
|
||||
matrix.block_locked(root,c,c['blocks'][0],dest,0,0)
|
||||
r=json.loads((dest/'terminal.json').read_text());self.assertEqual(calls,['0-N','0-E'])
|
||||
self.assertEqual([x['status'] for x in r['runs']],['NA','PASS']);self.assertFalse(r['pair_valid'])
|
||||
|
||||
def test_full_fake_scheduler_barrier_unique_blocks_and_90_terminal(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);(root/'training-contract.json').write_text('{}');c=fake_plan();clock=[0.];spawned=[];peak=[0]
|
||||
class Process:
|
||||
def __init__(self,i):self.i=i;self.pid=1000000+i;self.polls=0
|
||||
def poll(self):
|
||||
self.polls+=1
|
||||
if self.polls==1:return None
|
||||
dest=root/'blocks'/f'block-{self.i:03d}'/'terminal.json'
|
||||
dest.write_text(json.dumps(dict(block=self.i,runs=[dict(index=j,status='PASS',cell_wall_seconds=1) for j in c['blocks'][self.i]['run_indices']],pair_valid=True)))
|
||||
return 0
|
||||
def wait(self):return 0
|
||||
def spawn(command,**kw):
|
||||
i=int(command[-3])
|
||||
if i>=4:self.assertTrue((root/'first-eight-checkpoint.json').exists())
|
||||
p=Process(i);spawned.append(p);peak[0]=max(peak[0],sum(x.polls<2 for x in spawned));return p
|
||||
with patch.object(matrix,'contract',return_value=c),patch.object(matrix.subprocess,'Popen',spawn),patch.object(matrix.time,'monotonic',side_effect=lambda:clock[0]),patch.object(matrix.time,'sleep',side_effect=lambda n:clock.__setitem__(0,clock[0]+1)),patch('builtins.print'):
|
||||
self.assertEqual(matrix.run(root),0)
|
||||
r=json.loads((root/'matrix-complete.json').read_text())
|
||||
self.assertEqual(len(spawned),45);self.assertEqual(len({p.i for p in spawned}),45);self.assertLessEqual(peak[0],4)
|
||||
self.assertEqual((r['passed'],r['NA']),(90,0));self.assertFalse(r['test_predictions']);self.assertTrue(r['r2_required_before_custody'])
|
||||
|
||||
def test_existing_start_marker_rejects_duplicate_matrix(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);(root/'matrix-started.json').write_text('{}')
|
||||
with patch.object(matrix,'contract',return_value=fake_plan()),patch.object(matrix.subprocess,'Popen') as spawn:
|
||||
with self.assertRaises(AssertionError):matrix.run(root)
|
||||
spawn.assert_not_called()
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user