Freeze approved 131 label-free preparation and guarded roster finalization
This commit is contained in:
1 parent
0af3dab8d1
commit
bf5203c2ae
10 files changed
+16899
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"status": "PASS_FROZEN_NOT_STARTED",
|
||||
"tests_passed": 10,
|
||||
"test_log_sha256": "bfccebc343951402d9ba1fb1d8d0574a8b475e3f265a09fe267f7a1a9ea92976",
|
||||
"contract_sha256": "0a3c628c81c008eab53e9337406b84ec2063edcd49529bcc7a82e3e4a3d719a3",
|
||||
"code_sha256": {
|
||||
"tools/analyze_support_capacity_conditions.py": "6bfc3ab91c48ff1c28a4b6d56092c7c8bd1f73e7f263c54bf857858cf84fd2c9",
|
||||
"tools/audit_samples.py": "a3daf70ec3bcb77bf46d9dc4887d552cf7c422117880a169969780012278cd70",
|
||||
"tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
|
||||
"tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
|
||||
"tools/freeze_s2_main_roster.py": "f2c8b844d9c1c1096d6bd395022b5f766f51c15bbb4659b495bee728e2af0f31",
|
||||
"tools/g1_candidate_fingerprints.py": "2d027bd95f8da2cbfc0381d051f6a37949a5abe6b27e6b37d747d43c5b6612a0",
|
||||
"tools/g1_registration_cost.py": "c3d722a8cfd067dfd726f8a8729e72c37dbc63e2c24ba1b0e3d767d47f7756ba",
|
||||
"tools/o2_rep.py": "8c784ea04f6c14fd47ab79e1f2160c19ed48344bf24e6ab4255ac23ab3eb2003",
|
||||
"tools/physical_support_probe.py": "4513745d0f2c00e5b5b545ef87382efb2cf8057b1166068b3c85b7c1854a91f7",
|
||||
"tools/prepare_s2_main_inputs.py": "15043e599c8564075084ccb5c10fed41262d9d23594c9c5e5dc19a0410e6d6fd",
|
||||
"tools/prepare_s2_pilot_inputs.py": "8a04afe82c0add692c7cd5182b3fd70919ed9979f417af7223fba64753ca3144",
|
||||
"tools/prepare_s2_ten_part_measurement.py": "a6c8ec22ebd541a2e68f72f4c3049c893adc9f55e4a706f3e9c2d4681f1a8a6f",
|
||||
"tools/projector_remesh_probe.py": "90411c11dde855d3a17d651c859a0a5c8d3241c47375a0487eea1c9631abc55a",
|
||||
"tools/route_feasibility.py": "00e6b67d9079fc9c5d371064d5bfeb1a9950ce8b677bbe2c89c9976f45fb23d2",
|
||||
"tools/run_e6_real_kernel.py": "45a3ade5d1cbc52828b83dcb4a1b884e83063c7eea3f68851e464f4d1e2ce68a",
|
||||
"tools/run_s2_main_preparation.py": "705b798d5c1b56831e5af114818dbbbd4f5419002d55dc1a3ac4d715e740f5e0",
|
||||
"tools/run_s2_pilot_preparation.py": "08f07593a5f5f434ff1bcfdf952b9079500e2378ea3afb2e4d7e152db1a58def",
|
||||
"tools/run_s2_ten_part_measurement.py": "4108352a44368bd5581b21df029a84db5f00afad04de3ff73716ad4fa214296c",
|
||||
"tools/run_volume_quadrature_trial.py": "eff0c71b4cb795b1b9992a1111b2f28c23b5f8f5e34b483f79512644008ac725",
|
||||
"tools/s2_e1_supervisor.py": "6753c9ceb0b932047ba505bd15681f109e9407bdb58fd96c55e1431a3f1b4a02",
|
||||
"tools/s2_main_prepare.py": "4a2083f7400091d2a344af378dfdb3c9a5b94a4e95d1897a5576ae535a94a875",
|
||||
"tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
|
||||
"tools/s2_pilot_data.py": "82c1fd18ec51757479c8fd7dda2db4d5f38c366d77fedcca3216a08998214c97",
|
||||
"tools/s2_pilot_prepare.py": "0987a5fe49816557d2cd8ccaeaf01de5effcb5d080221f2c93eb834719d413d2",
|
||||
"tools/s2_preprocess.py": "0bb59c6b5be166b379c924ca68aacfcaca6737386b71be61a43dc5433ca53419",
|
||||
"tools/s2_preprocess_pilot.py": "46cec13e13c0b94e5a51444b5d5ee78579dec9a3aa42ed24294e0a0deec55833",
|
||||
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"tools/test_s2_main_preparation.py": "075c752b337e65690ad0694eae7a3102d46620c7925ac7f69ff6d42358c12bc9",
|
||||
"tools/test_s2_pilot_data.py": "cd9a5f69a5207d0ea62e8c150df737992ab640a08d82b182d504c9559e729562",
|
||||
"tools/test_s2_pilot_preparation.py": "71667a38b352ad64e588653b7d3545e71a473fb151de775e515e174b5358403c",
|
||||
"tools/test_s2_preprocess.py": "6f70d4392d3800b8146fbdd92b3c06b08638e45167fefb7261bf7ecf30f3faf4"
|
||||
},
|
||||
"preparation_cases": 131,
|
||||
"reuse_only_cases": 493,
|
||||
"new_worker_cpu_hours": 20,
|
||||
"prior_worker_cpu_hours": 34.75965079444441,
|
||||
"current_cumulative_disk_bytes": 6513541120,
|
||||
"labels_decoded": false,
|
||||
"workers_started": false,
|
||||
"planner_validation": "All131 source geometry/cache hashes and all493 successful geometry feature output hashes verified. Initial metadata schema mismatch corrected before root creation; no scientific attempt occurred."
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,15 @@
|
||||
test_131_terminal_and_transient_monitor_failure_without_retry (__main__.MainPreparationChecks) ... ok
|
||||
test_actual_geometry_cache_whitelist_never_opens_label (__main__.MainPreparationChecks) ... ok
|
||||
test_cumulative_disk_cap_blocks_dispatch (__main__.MainPreparationChecks) ... ok
|
||||
test_existing_batch_marker_refuses_restart (__main__.MainPreparationChecks) ... ok
|
||||
test_failed_part_excluded_globally_original_component_and_nested_pools (__main__.MainPreparationChecks) ... ok
|
||||
test_geometry_prefix_rejects_field_before_geometry_without_decode (__main__.MainPreparationChecks) ... ok
|
||||
test_guards_apply_60_and_30_minutes_to_label_free_worker (__main__.MainPreparationChecks) ... ok
|
||||
test_label_free_adapter_accepts_test_case_without_label_io (__main__.MainPreparationChecks) ... ok
|
||||
test_new_20h_cap_stops_and_does_not_charge_prior_34h (__main__.MainPreparationChecks) ... ok
|
||||
test_zero_eligible_queries_fail_without_fabricating_ready (__main__.MainPreparationChecks) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 10 tests in 0.099s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,18 @@
|
||||
# S2 主矩阵前的 131 件无标签准备
|
||||
|
||||
Claude [1794699027.366469](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794699027366469) 已批准准备预算和后续执行顺序。本次入口完成实现及验收,合同冻结后实际启动;实时状态以新目录的启动、进度和终态回执为准。
|
||||
|
||||
- 新目录:`artifacts/s2-main-preparation-2026-11-15`。合同 SHA256:`0a3c628c81c008eab53e9337406b84ec2063edcd49529bcc7a82e3e4a3d719a3`。
|
||||
- 固定原计划 131 件及顺序:6 件已记录的超时实件各 60 分钟,125 件 fold0 test 几何各 30 分钟;每件进程树 RSS 1.25 GiB,最多 16 工作者,无重试。
|
||||
- 新增 20 worker CPU-h 单独计账;历史 34.7596507944 worker CPU-h 另列,不再次扣入新增额度。运行中保守计入活动监督器 CPU;结束后工作者与监督器分列。旧准备、事故隔离、恢复、十件测量和新目录累计不超过 100 GiB。
|
||||
- 直接调用原固定 `s2_preprocess_pilot.run` 数值入口,独立包装器只检查无标签和共同查询资格。没有修改旧源代码、旧失败记录或旧合同。所有 131 件都不解码标签、不拟合、不使用 GPU。
|
||||
- 原 493 件成功结果仅复用。已核验其几何/特征输出哈希;未打开标签数组。复用清单单独固定为 `reuse-493.json`。
|
||||
- 单批次锁与启动标记拒绝重复启动;每件监督锁拒绝重试。监测保留原恢复监督器的临时文件竞争容错。若监督器退出但无法取得计账回执,则停止并记录未知计账,不能把未知 CPU 当零。
|
||||
|
||||
10 项 CPU 合成检查通过(0.099 秒):测试角色的无标签入口、实际混合容器白名单、几何前出现场数据时拒绝解码、60/30 分钟限额、131 项有界调度及监测瞬态错误、独立新增 CPU 上限、累计磁盘上限、重复启动拒绝、失败跨折排除及组件身份/嵌套池保持、零资格查询失败。日志及预启动核验在新目录中。
|
||||
|
||||
首次组装时发现历史合并回执中旧批与恢复批顶层字段不同,已改为使用统一的 `worker.status`。此错误发生在创建新准备目录和启动任何工作者之前;未消耗任何科学尝试。
|
||||
|
||||
全部 131 项终态且没有全局停止后,监督入口自动合并原 493 成功结果,校验输出并生成唯一 `final-roster.json` 和 `roster-receipt.json`。失败实件跨五折统一排除,保留原组件、划分和原因;报告每折每个组件比例池的实际件数、节点数及组件数。若全局预算停止或计账未知,不生成可供训练的冻结名单。
|
||||
|
||||
后续 90 个神经模型均从头训练,不复用 pilot 权重;只有 roster 已冻结后方可执行。总 46 GPU-h 含等待/设置,每个 run 最多 30 分钟、最多四张 V100、N/E 相邻、首八 run 重估预算。R2 沿批准的单 fit 30 CPU-min、总 100 worker CPU-h,并记录留给 GPU 数据供给的 CPU 配置。全部 90+15 终态且检查点冻结后统一托管预测及评分;本准备入口不启动训练或解封。
|
||||
@@ -0,0 +1,91 @@
|
||||
"""One metadata-only roster across five folds, after the 131 terminal attempts."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
|
||||
def read(path):
|
||||
return json.loads(Path(path).read_text())
|
||||
|
||||
|
||||
def validate_preparation(path,expected_sha=None):
|
||||
path=Path(path)
|
||||
if expected_sha is not None:assert sha(path/'result.json')==expected_sha
|
||||
r=read(path/'result.json')
|
||||
assert r['case']==path.name
|
||||
assert r['status']=='COMPLETE' and r['label_arrays_decoded']==[]
|
||||
assert r['eligible_unique']>0 and r['eligible_slots']>0,'No common eligible query'
|
||||
for name,digest in r['output_sha256'].items():
|
||||
assert 'label' not in name and sha(path/name)==digest,(path,name)
|
||||
return dict(preparation_root=str(path),result_sha256=sha(path/'result.json'),output_sha256=r['output_sha256'],
|
||||
nodes=r['nodes'],t_set=r['scales']['t_set'],eligible_unique=r['eligible_unique'],eligible_slots=r['eligible_slots'])
|
||||
|
||||
|
||||
def assemble(parts,reused,new_jobs,new_sources):
|
||||
"""Preserve original component IDs and all assignments, including excluded parts."""
|
||||
assert len({p['case'] for p in parts})==len(parts)
|
||||
records={p['case']:dict(p,included=False,exclusion_reason='NOT_PREPARED') for p in parts}
|
||||
assert not (set(reused)&{j['case'] for j in new_jobs})
|
||||
for case,source in reused.items():
|
||||
assert source['nodes']==records[case]['nodes']
|
||||
records[case].update(included=True,exclusion_reason=None,preparation=source,attempt_kind='REUSED_PREVIOUS_PASS')
|
||||
for job in new_jobs:
|
||||
case=job['case'];r=records[case];w=job['worker']
|
||||
if w['status']=='PASS' and case in new_sources:
|
||||
assert new_sources[case]['nodes']==r['nodes']
|
||||
r.update(included=True,exclusion_reason=None,preparation=new_sources[case],attempt_kind='NEW_LABEL_FREE_ATTEMPT')
|
||||
else:
|
||||
r.update(exclusion_reason=w.get('stop_reason') or w['status'],attempt_kind='NEW_LABEL_FREE_ATTEMPT',terminal_worker=w)
|
||||
assert set(reused)|{j['case'] for j in new_jobs}==set(records)
|
||||
rows=[records[p['case']] for p in parts]
|
||||
counts=[]
|
||||
for fold in range(5):
|
||||
for pool in [25,50,100]:
|
||||
groups={role:[] for role in ['learning','validation','test']}
|
||||
for p in rows:
|
||||
a=p['assignments'][fold]
|
||||
assert a['fold']==fold
|
||||
if not p['included']:continue
|
||||
if a['role']=='learning' and str(pool) not in a['learning_pools']:continue
|
||||
groups[a['role']].append(p)
|
||||
counts.append(dict(fold=fold,pool=pool,**{role:dict(parts=len(items),nodes=sum(p['nodes'] for p in items),
|
||||
components=len({p['component_id'] for p in items})) for role,items in groups.items()}))
|
||||
return rows,counts
|
||||
|
||||
|
||||
def freeze(root):
|
||||
root=Path(root)
|
||||
assert not (root/'final-roster.json').exists(),'Never replace a frozen roster'
|
||||
from run_s2_main_preparation import contract
|
||||
c=contract(root);batch=read(root/'batch-complete.json')
|
||||
assert batch['status']=='ATTEMPTS_COMPLETE' and batch['stop_reason'] is None
|
||||
assert len(batch['jobs'])==131 and not batch['not_run_indices']
|
||||
assert sorted(j['index'] for j in batch['jobs'])==list(range(131))
|
||||
sources={}
|
||||
for j in batch['jobs']:
|
||||
assert j['case']==c['cases'][j['index']]['case']
|
||||
guard=root/'jobs'/f"job-{j['index']:04d}"/'guard-result.json'
|
||||
assert read(guard)['worker']==j['worker']
|
||||
if j['worker']['status']=='PASS':
|
||||
sources[j['case']]=validate_preparation(root/'parts'/j['case'])
|
||||
reuse=read(root/'reuse-493.json')['parts']
|
||||
old={r['case']:validate_preparation(r['preparation_root'],r['result_sha256']) for r in reuse}
|
||||
parts=read('artifacts/s2-g1-components-2026-11-15/custodian/part-splits.json')
|
||||
assert len(parts)==624 and len(old)==493
|
||||
rows,counts=assemble(parts,old,batch['jobs'],sources)
|
||||
roster=dict(status='FROZEN',version='S2-main-five-fold-roster-v1',parts=rows,fold_pool_counts=counts,
|
||||
included=sum(r['included'] for r in rows),excluded=[r['case'] for r in rows if not r['included']],
|
||||
contract_sha256=sha(root/'contract.json'),batch_sha256=sha(root/'batch-complete.json'),
|
||||
splits_sha256=sha('artifacts/s2-g1-components-2026-11-15/custodian/part-splits.json'),
|
||||
labels_decoded=False,models_reused=False,neural_runs_required=90,r2_final_fits_required=15,
|
||||
test_unsealing='Only unified custody after all90+15 terminal and checkpoints frozen')
|
||||
save(root/'final-roster.json',roster)
|
||||
save(root/'roster-receipt.json',dict(status='FROZEN',sha256=sha(root/'final-roster.json'),included=roster['included'],
|
||||
excluded=roster['excluded'],neural_started=False,r2_started=False,labels_decoded=False))
|
||||
print(json.dumps(read(root/'roster-receipt.json')),flush=True)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);freeze(p.parse_args().root)
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Freeze the approved 131 geometry-only attempts; preserve the old 493 successes."""
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
ROOT = Path('artifacts/s2-main-preparation-2026-11-15')
|
||||
PLAN = Path('artifacts/s2-main-engineering-2026-11-15/planned-preparation-131.json')
|
||||
SPLITS = Path('artifacts/s2-g1-components-2026-11-15/custodian/part-splits.json')
|
||||
OLD = Path('artifacts/s2-pilot-inputs-2026-11-15/contract.json')
|
||||
MERGED = Path('artifacts/s2-preparation-incident-2026-11-15/merged-recovery-terminal.json')
|
||||
SUCCESS = Path('artifacts/s2-pilot-training-2026-11-15/success-manifest.json')
|
||||
|
||||
|
||||
def read(path):
|
||||
return json.loads(Path(path).read_text())
|
||||
|
||||
|
||||
def main():
|
||||
assert not ROOT.exists(), 'Never overwrite an existing attempt'
|
||||
plan=read(PLAN);old=read(OLD);merged=read(MERGED);success=read(SUCCESS)
|
||||
parts={p['case']:p for p in read(SPLITS)}
|
||||
prior={p['case']:p for p in old['cases']}
|
||||
failed={p['case'] for p in merged['jobs'] if p['worker']['status']!='PASS'}
|
||||
passed={p['case'] for p in success['ready_parts']}
|
||||
expected=failed | {k for k,p in parts.items() if p['assignments'][0]['role']=='test'}
|
||||
assert len(parts)==624 and len(failed)==6 and len(passed)==493
|
||||
assert len(plan['jobs'])==131 and {p['case'] for p in plan['jobs']}==expected
|
||||
assert not expected & passed and expected | passed == set(parts)
|
||||
g1=read('artifacts/s2-g1-registration-2026-11-14/custodian/plan.json')['geometry_cache']
|
||||
settings_path=Path('artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json')
|
||||
settings={p['member'].removesuffix('.log'):p for p in read(settings_path)}
|
||||
counts={p['case']:p for p in csv.DictReader(open('artifacts/s2-e1-2026-11-14/preflight/geometry_counts.csv'))}
|
||||
cases=[]
|
||||
for job in plan['jobs']:
|
||||
case=job['case'];p=parts[case]
|
||||
assert job['decode_labels'] is False and job['retries']==0
|
||||
assert job['nodes']==p['nodes'] and job['component_id']==p['component_id']
|
||||
assert job['wall_limit_seconds']==(3600 if case in failed else 1800)
|
||||
if case in failed:
|
||||
spec=json.loads(json.dumps(prior[case]))
|
||||
assert not any(k in spec for k in ['reuse_preparation','prior_failed_preparation'])
|
||||
else:
|
||||
setting=settings[case]
|
||||
spec=dict(case=case,numeric_prefix=int(case.split('_')[0]),source_key=p['source_key'],
|
||||
component_id=p['component_id'],tet_nodes=p['nodes'],tet_cells=int(counts[case]['tet_cells']),
|
||||
t_set=setting['setting']['seconds'],t_set_provenance=setting,g1_geometry=g1[case]['path'],
|
||||
input_sha256={g1[case]['path']:g1[case]['sha256'],str(settings_path):sha(settings_path)})
|
||||
cache=Path('artifacts/s1-panel-precheck-2026-11-14')/case
|
||||
if (cache/'result.json').exists():
|
||||
r=read(cache/'result.json');assert r['status']=='COUNT_COMPLETE'
|
||||
spec.update(legacy_geometry=str(cache/'geometry.npz'),existing_distance=str(cache/'boundary_distance_raw.npy'))
|
||||
for name in ['geometry.npz','boundary_distance_raw.npy']:
|
||||
spec['input_sha256'][str(cache/name)]=r['output_sha256'][name]
|
||||
draw=Path('artifacts/s1-sampled-draw-2026-11-14')/case/'draw.npz'
|
||||
if draw.exists():
|
||||
r=read(draw.parent/'receipt.json');assert r['status']=='DRAW_COMPLETE'
|
||||
spec['existing_primary_draw']=str(draw);spec['input_sha256'][str(draw)]=r['output_sha256']['draw.npz']
|
||||
o2=Path('artifacts/s1-sampled-o2-2026-11-14/panel')/(case+'_original')
|
||||
if (o2/'result.json').exists():
|
||||
r=read(o2/'result.json');assert r['status']=='COMPLETE'
|
||||
spec.update(existing_sampled_o2_nodes=str(o2/'nodes.npz'),existing_sampled_o2_result=str(o2/'result.json'))
|
||||
spec['input_sha256'][str(o2/'nodes.npz')]=r['output_sha256']['nodes.npz']
|
||||
spec['input_sha256'][str(o2/'result.json')]=sha(o2/'result.json')
|
||||
spec.update(attempt=job)
|
||||
for path,digest in spec['input_sha256'].items():assert sha(path)==digest,path
|
||||
cases.append(spec)
|
||||
# Only hash geometry/features; neither open nor decode any labels.npz.
|
||||
reused=[]
|
||||
for row in success['ready_parts']:
|
||||
source=Path(row['preparation_root']);r=read(source/'result.json')
|
||||
assert r['status']=='COMPLETE' and r['label_arrays_decoded']==[]
|
||||
assert r['case']==row['case'] and r['nodes']==parts[row['case']]['nodes']
|
||||
for name,digest in r['output_sha256'].items():
|
||||
assert 'label' not in name and sha(source/name)==digest,(source,name)
|
||||
reused.append(dict(case=row['case'],preparation_root=str(source),
|
||||
result_sha256=sha(source/'result.json'),output_sha256=r['output_sha256']))
|
||||
base=read('artifacts/s2-ten-part-measurement-2026-11-15/contract.json')
|
||||
for path,digest in old['code_sha256'].items():assert sha(path)==digest,path
|
||||
paths=set(old['code_sha256']) | {'tools/prepare_s2_main_inputs.py','tools/run_s2_main_preparation.py',
|
||||
'tools/freeze_s2_main_roster.py','tools/test_s2_main_preparation.py','tools/s2_main_prepare.py',
|
||||
'tools/s2_recovery_monitoring.py','tools/s2_recovery_supervisor.py'}
|
||||
sources={str(p):sha(p) for p in [PLAN,SPLITS,OLD,MERGED,SUCCESS,settings_path]}
|
||||
ROOT.mkdir()
|
||||
save(ROOT/'reuse-493.json',dict(status='VERIFIED_GEOMETRY_FEATURES_ONLY',parts=reused,labels_decoded=False))
|
||||
sources[str(ROOT/'reuse-493.json')]=sha(ROOT/'reuse-493.json')
|
||||
c={k:base[k] for k in ['fixed_split_commit','archive','archive_stat','scales','draw','eligibility']}
|
||||
c.update(version='S2-main-131-label-free-v1',authorizing_rulings=['1794698734.967459','1794699027.366469'],
|
||||
cases=cases,code_sha256={p:sha(p) for p in sorted(paths)},source_sha256=sources,
|
||||
prior_worker_cpu_seconds=merged['worker_cpu_seconds'],
|
||||
limits=dict(worker_cpu_hours=20,workers=16,disk_bytes=100*2**30,per_part_tree_rss_bytes=1342177280,retries=0),
|
||||
budget_roots=['artifacts/s2-ten-part-measurement-2026-11-15','artifacts/s2-pilot-inputs-2026-11-15',
|
||||
'artifacts/s2-preparation-incident-2026-11-15','artifacts/s2-pilot-recovery-inputs-2026-11-15',str(ROOT)],
|
||||
input_policy='Geometry/features only for all 131; no labels decoded, no fit, no GPU. Original 493 reused without label reads.',
|
||||
accounting='20 additional worker CPU-h; historical CPU recorded separately; live guard CPU conservatively included in cap.',
|
||||
failure_policy='One labelled new attempt per planned case; no retry. Retain prior failures. Remaining failures excluded globally across all five folds with original component identity.',
|
||||
dispatch='Frozen plan order, at most16; no first50 gate; freeze roster only after all131 terminal within approved budget.')
|
||||
save(ROOT/'contract.json',c)
|
||||
print(json.dumps(dict(status='FROZEN_NOT_STARTED',root=str(ROOT),contract_sha256=sha(ROOT/'contract.json'),
|
||||
cases=len(cases),reused=len(reused),labels_decoded=False)))
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Approved 131 geometry-only attempts; independent stage CPU and cumulative disk."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_supervisor import supervise,save,tree_usage
|
||||
from s2_recovery_monitoring import disk_usage,Monitor,budget_reason
|
||||
from run_s2_pilot_preparation import terminate
|
||||
|
||||
|
||||
def contract(root):
|
||||
c=json.loads((root/'contract.json').read_text())
|
||||
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
||||
for p,h in c['source_sha256'].items():assert sha(p)==h,p
|
||||
assert len(c['cases'])==131 and len({s['case'] for s in c['cases']})==131
|
||||
for spec in c['cases']:
|
||||
a=spec['attempt']
|
||||
assert a['decode_labels'] is False and a['retries']==0
|
||||
expected=3600 if a['kind']=='labelled_timeout_repreparation' else 1800
|
||||
assert a['wall_limit_seconds']==expected and a['tree_rss_bytes']==1342177280
|
||||
roots=[Path(p).resolve() for p in c['budget_roots']]
|
||||
assert all(a!=b and a not in b.parents and b not in a.parents for i,a in enumerate(roots) for b in roots[i+1:])
|
||||
return c
|
||||
|
||||
|
||||
def guard(root,index):
|
||||
c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True)
|
||||
wall=c['cases'][index]['attempt']['wall_limit_seconds']
|
||||
r=supervise([sys.executable,'tools/s2_main_prepare.py',str(root),str(index)],job,'worker',
|
||||
dict(total_cpu_seconds=wall+1,job_cpu_seconds=wall,job_wall_seconds=wall,tree_rss_bytes=1342177280,
|
||||
disk_bytes=100*2**30,max_processes=1),poll=.25)
|
||||
monitor=Monitor(job/'accounting-errors.jsonl')
|
||||
monitor.retry('guard_receipt',lambda:save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
attempt=c['cases'][index]['attempt'],label_arrays_decoded=[])),.25)
|
||||
return 0 if r['status']=='PASS' else 1
|
||||
|
||||
|
||||
def run(root):
|
||||
with (root/'batch.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
|
||||
|
||||
|
||||
def run_locked(root):
|
||||
c=contract(root)
|
||||
assert not (root/'batch-started.json').exists(),'One main preparation batch only; never restart'
|
||||
assert c['limits']['workers']==16
|
||||
assert c['limits']['worker_cpu_hours']==20 and c['limits']['disk_bytes']==100*2**30
|
||||
monitor=Monitor(root/'monitor-errors.jsonl');start=time.monotonic()
|
||||
save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0)),
|
||||
limits=c['limits'],labels_decoded=False))
|
||||
active={};done=[];next_index=0;reason=None;live_cpu=live_rss=used=peak_rss=0;fresh=False
|
||||
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
||||
def interrupted(*_):
|
||||
nonlocal reason
|
||||
reason='MAIN_PREPARATION_COORDINATOR_INTERRUPTED'
|
||||
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
||||
def read_account(path):
|
||||
r=json.loads(path.read_text());w=r['worker']
|
||||
assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0
|
||||
assert math.isfinite(r['supervisor_cpu_seconds']) and r['supervisor_cpu_seconds']>=0
|
||||
assert isinstance(w['returncode'],int) and isinstance(w['status'],str)
|
||||
assert 'stop_reason' in w
|
||||
return r
|
||||
try:
|
||||
while active or (next_index<len(c['cases']) and reason is None):
|
||||
cycle_ok=True
|
||||
for i,(p,log) in list(active.items()):
|
||||
code,ok=monitor.sample(f'poll_{i}',p.poll,None);cycle_ok &= ok
|
||||
if ok and code is not None:
|
||||
path=root/'jobs'/f'job-{i:04d}'/'guard-result.json'
|
||||
r,ok=monitor.sample(f'account_{i}',lambda path=path:read_account(path),None);cycle_ok &= ok
|
||||
if not ok:
|
||||
reason='GUARD_ACCOUNTING_UNAVAILABLE'
|
||||
terminate(active)
|
||||
save(root/'infrastructure-stop.json',dict(status=reason,index=i,guard_returncode=code,
|
||||
active_indices=list(active),not_run_indices=list(range(next_index,len(c['cases']))),
|
||||
cpu_accounting_complete=False,prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],
|
||||
settled_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
|
||||
contract_sha256=sha(root/'contract.json'),retry=False))
|
||||
raise RuntimeError(reason)
|
||||
done.append(dict(index=i,case=c['cases'][i]['case'],returncode=code,**r));del active[i];log.close()
|
||||
if reason is None and r['worker']['returncode']<0 and r['worker']['stop_reason'] is None:reason='UNEXPLAINED_MAIN_PREPARATION_WORKER_SIGNAL'
|
||||
print(json.dumps(dict(event='FINISHED',index=i,completed=len(done),status=r['worker']['status'])),flush=True)
|
||||
def sample_trees():
|
||||
values=[tree_usage(p.pid) for p,_ in active.values()]
|
||||
return sum(v[0] for v in values),sum(v[1] for v in values)
|
||||
(live_cpu,live_rss),ok=monitor.sample('aggregate_live_usage',sample_trees,(live_cpu,live_rss));cycle_ok &= ok
|
||||
peak_rss=max(peak_rss,live_rss)
|
||||
used,ok=monitor.sample('aggregate_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),used);cycle_ok &= ok
|
||||
spent=sum(r['worker']['cpu_seconds'] for r in done)
|
||||
over=budget_reason(0,spent,live_cpu,used,cpu_limit=c['limits']['worker_cpu_hours']*3600,disk_limit=c['limits']['disk_bytes'])
|
||||
if over:reason=over
|
||||
if reason:terminate(active)
|
||||
progress=dict(completed=len(done),active=list(active),next_index=next_index,stop_reason=reason,
|
||||
prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],stage_worker_cpu_seconds=spent,
|
||||
cumulative_settled_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,
|
||||
live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=used,
|
||||
elapsed_seconds=time.monotonic()-start,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,
|
||||
fresh_accounting=cycle_ok,no_new_first50_barrier=True)
|
||||
_,ok=monitor.sample('progress_write',lambda:save(root/'progress.json',progress),None);cycle_ok &= ok
|
||||
# Do not dispatch new work on a stale accounting cycle. Existing
|
||||
# workers keep their independent wall/CPU/RSS guards while re-polling.
|
||||
while cycle_ok and reason is None and len(active)<c['limits']['workers'] and next_index<len(c['cases']):
|
||||
i=next_index;job=root/'jobs'/f'job-{i:04d}';job.mkdir(parents=True,exist_ok=True)
|
||||
log=(job/'guard.log').open('x')
|
||||
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
active[i]=(p,log);next_index+=1
|
||||
if active or (not cycle_ok and reason is None):time.sleep(.5)
|
||||
finally:
|
||||
terminate(active)
|
||||
for p,log in active.values():
|
||||
p.wait();log.close()
|
||||
used=monitor.retry('final_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),.5)
|
||||
spent=sum(r['worker']['cpu_seconds'] for r in done)
|
||||
if spent>c['limits']['worker_cpu_hours']*3600:reason='GLOBAL_CPU_LIMIT_AFTER_EXIT'
|
||||
if used>100*2**30:reason='GLOBAL_DISK_LIMIT_AFTER_EXIT'
|
||||
result=dict(status='ATTEMPTS_COMPLETE' if len(done)==len(c['cases']) and reason is None else 'STOPPED',stop_reason=reason,jobs=done,
|
||||
not_run_indices=list(range(next_index,len(c['cases']))),prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],worker_cpu_seconds=spent,
|
||||
cumulative_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),
|
||||
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
|
||||
peak_aggregate_rss_bytes=peak_rss,disk_bytes=used,failed=sum(r['worker']['status']!='PASS' for r in done),
|
||||
monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,retry=False,replacement=False,test_labels_read=False)
|
||||
monitor.retry('batch_receipt',lambda:save(root/'batch-complete.json',result),.5)
|
||||
print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True)
|
||||
if result['status']=='ATTEMPTS_COMPLETE':
|
||||
from freeze_s2_main_roster import freeze
|
||||
freeze(root)
|
||||
return 0 if result['status']=='ATTEMPTS_COMPLETE' else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args()
|
||||
sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root))
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Geometry-only main adapter; all numerical preparation remains pinned and unchanged."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import s2_preprocess_pilot
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
|
||||
def run(root,index):
|
||||
spec=json.loads((root/'contract.json').read_text())['cases'][index]
|
||||
assert spec['attempt']['decode_labels'] is False
|
||||
assert 'reuse_preparation' not in spec and 'prior_failed_preparation' not in spec
|
||||
code=s2_preprocess_pilot.run(root,index)
|
||||
if code:return code
|
||||
dest=root/'parts'/spec['case'];r=json.loads((dest/'result.json').read_text())
|
||||
assert r['label_arrays_decoded']==[] and r['status']=='COMPLETE'
|
||||
assert not list(dest.rglob('*label*'))
|
||||
if r['eligible_unique']<=0 or r['eligible_slots']<=0:
|
||||
save(dest/'eligibility-failure.json',dict(status='NO_COMMON_ELIGIBLE_QUERIES',labels_decoded=False))
|
||||
return 3
|
||||
return 0
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args()
|
||||
raise SystemExit(run(a.root,a.index))
|
||||
@@ -0,0 +1,132 @@
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from zipfile import ZipFile
|
||||
import numpy as np
|
||||
import run_s2_main_preparation as coordinator
|
||||
import s2_main_prepare as worker
|
||||
from s2_preprocess_pilot import load_selected,GEOMETRY_MEMBERS
|
||||
from g1_candidate_fingerprints import read_tets
|
||||
from freeze_s2_main_roster import assemble
|
||||
|
||||
|
||||
def synthetic_contract(root):
|
||||
return dict(cases=[dict(case=str(i),attempt=dict(wall_limit_seconds=3600 if i<6 else 1800)) for i in range(131)],
|
||||
prior_worker_cpu_seconds=125134.74286,limits=dict(worker_cpu_hours=20,workers=16,disk_bytes=100*2**30),budget_roots=[str(root)])
|
||||
|
||||
|
||||
class MainPreparationChecks(unittest.TestCase):
|
||||
def test_label_free_adapter_accepts_test_case_without_label_io(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);case='synthetic_sealed_test'
|
||||
(root/'contract.json').write_text(json.dumps(dict(cases=[dict(case=case,attempt=dict(decode_labels=False,fold0_role='test'))])))
|
||||
def geometric_run(r,i):
|
||||
d=r/'parts'/case;d.mkdir(parents=True)
|
||||
(d/'result.json').write_text(json.dumps(dict(status='COMPLETE',label_arrays_decoded=[],eligible_unique=1,eligible_slots=2000)))
|
||||
return 0
|
||||
with patch.object(worker.s2_preprocess_pilot,'run',side_effect=geometric_run) as numerical:
|
||||
self.assertEqual(worker.run(root,0),0)
|
||||
numerical.assert_called_once_with(root,0)
|
||||
self.assertFalse(list(root.rglob('*label*')))
|
||||
|
||||
def test_actual_geometry_cache_whitelist_never_opens_label(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
path=Path(t)/'mixed.npz';np.savez(path,points_raw=np.ones((2,3)),fill_time_raw=np.array([999.]))
|
||||
opened=[];original=ZipFile.open
|
||||
def spy(z,name,*a,**kw):opened.append(name);return original(z,name,*a,**kw)
|
||||
with patch.object(ZipFile,'open',spy):load_selected(path,['points_raw'],GEOMETRY_MEMBERS)
|
||||
self.assertEqual(opened,['points_raw.npy'])
|
||||
with self.assertRaises(ValueError):load_selected(path,['fill_time_raw'],GEOMETRY_MEMBERS)
|
||||
|
||||
def test_zero_eligible_queries_fail_without_fabricating_ready(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);case='empty'
|
||||
(root/'contract.json').write_text(json.dumps(dict(cases=[dict(case=case,attempt=dict(decode_labels=False))])))
|
||||
d=root/'parts'/case;d.mkdir(parents=True)
|
||||
(d/'result.json').write_text(json.dumps(dict(status='COMPLETE',label_arrays_decoded=[],eligible_unique=0,eligible_slots=0)))
|
||||
with patch.object(worker.s2_preprocess_pilot,'run',return_value=0):self.assertEqual(worker.run(root,0),3)
|
||||
self.assertEqual(json.loads((d/'eligibility-failure.json').read_text())['status'],'NO_COMMON_ELIGIBLE_QUERIES')
|
||||
|
||||
def test_geometry_prefix_rejects_field_before_geometry_without_decode(self):
|
||||
xml=b'<VTKFile byte_order="LittleEndian" compressor="vtkZLibDataCompressor"><UnstructuredGrid><Piece><PointData><DataArray Name="Fill_time"/></PointData></Piece></UnstructuredGrid></VTKFile>'
|
||||
with patch('g1_candidate_fingerprints.decode_array',side_effect=AssertionError('label decode')) as decode:
|
||||
with self.assertRaisesRegex(ValueError,'Field association'):read_tets(io.BytesIO(xml))
|
||||
decode.assert_not_called()
|
||||
|
||||
def test_guards_apply_60_and_30_minutes_to_label_free_worker(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);c=synthetic_contract(root);calls=[]
|
||||
def supervise(command,job,name,limits,**kw):
|
||||
calls.append((command,limits));return dict(status='PASS')
|
||||
with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator,'supervise',supervise):
|
||||
self.assertEqual(coordinator.guard(root,0),0)
|
||||
self.assertEqual(coordinator.guard(root,6),0)
|
||||
self.assertEqual([x[1]['job_wall_seconds'] for x in calls],[3600,1800])
|
||||
self.assertTrue(all(x[0][1]=='tools/s2_main_prepare.py' and x[1]['tree_rss_bytes']==1342177280 for x in calls))
|
||||
|
||||
def run_fake_batch(self,root,*,over_cpu=False,over_disk=False,monitor_fault=False):
|
||||
c=synthetic_contract(root);(root/'contract.json').write_text(json.dumps(c))
|
||||
spawned=[];fault=False;terminated=[];peak=0
|
||||
class FakeProcess:
|
||||
def __init__(self,i):self.index=i;self.pid=1000000+i;self.polls=0
|
||||
def poll(self):
|
||||
self.polls+=1
|
||||
if self.polls==1:return None
|
||||
p=root/'jobs'/f'job-{self.index:04d}'/'guard-result.json'
|
||||
p.write_text(json.dumps(dict(worker=dict(status='PASS',returncode=0,stop_reason=None,cpu_seconds=.01),supervisor_cpu_seconds=.001)))
|
||||
return 0
|
||||
def wait(self):return 0
|
||||
def spawn(command,**kwargs):
|
||||
nonlocal peak
|
||||
p=FakeProcess(int(command[-1]));spawned.append(p)
|
||||
peak=max(peak,sum(p.polls<2 for p in spawned));return p
|
||||
def disk(_):
|
||||
nonlocal fault
|
||||
if monitor_fault and spawned and not fault:fault=True;raise OSError('atomic replacement transient')
|
||||
return 100*2**30 if over_disk else 100
|
||||
with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator.subprocess,'Popen',spawn),\
|
||||
patch.object(coordinator,'disk_usage',disk),patch.object(coordinator,'tree_usage',return_value=(5000 if over_cpu else .01,100,1)),\
|
||||
patch.object(coordinator.time,'sleep'),patch.object(coordinator,'terminate',side_effect=lambda a:terminated.append(list(a))),\
|
||||
patch('freeze_s2_main_roster.freeze') as freeze,patch('builtins.print'):
|
||||
coordinator.run(root)
|
||||
return json.loads((root/'batch-complete.json').read_text()),spawned,peak,freeze.call_count
|
||||
|
||||
def test_131_terminal_and_transient_monitor_failure_without_retry(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
r,spawned,peak,freezes=self.run_fake_batch(Path(t),monitor_fault=True)
|
||||
self.assertEqual(r['status'],'ATTEMPTS_COMPLETE');self.assertEqual(len(spawned),131)
|
||||
self.assertLessEqual(peak,16);self.assertEqual(r['monitor_errors'],1);self.assertEqual(freezes,1)
|
||||
self.assertAlmostEqual(r['worker_cpu_seconds'],1.31)
|
||||
self.assertAlmostEqual(r['cumulative_worker_cpu_seconds'],125134.74286+1.31)
|
||||
|
||||
def test_new_20h_cap_stops_and_does_not_charge_prior_34h(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
r,spawned,_,freezes=self.run_fake_batch(Path(t),over_cpu=True)
|
||||
self.assertEqual(r['stop_reason'],'GLOBAL_CPU_LIMIT');self.assertEqual(len(spawned),16)
|
||||
self.assertEqual(len(r['not_run_indices']),115);self.assertEqual(freezes,0)
|
||||
|
||||
def test_cumulative_disk_cap_blocks_dispatch(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
r,spawned,_,freezes=self.run_fake_batch(Path(t),over_disk=True)
|
||||
self.assertEqual(r['stop_reason'],'GLOBAL_DISK_LIMIT');self.assertEqual(spawned,[]);self.assertEqual(freezes,0)
|
||||
|
||||
def test_existing_batch_marker_refuses_restart(self):
|
||||
with tempfile.TemporaryDirectory() as t:
|
||||
root=Path(t);(root/'batch-started.json').write_text('{}')
|
||||
with patch.object(coordinator,'contract',return_value=synthetic_contract(root)),patch.object(coordinator.subprocess,'Popen') as spawn:
|
||||
with self.assertRaises(AssertionError):coordinator.run(root)
|
||||
spawn.assert_not_called()
|
||||
|
||||
def test_failed_part_excluded_globally_original_component_and_nested_pools(self):
|
||||
parts=[dict(case=str(i),nodes=i+10,component_id='unchanged-'+str(i),assignments=[dict(fold=f,role='learning',learning_pools=['50','100']) for f in range(5)]) for i in range(3)]
|
||||
jobs=[dict(case='1',worker=dict(status='PASS')),dict(case='2',worker=dict(status='STOPPED_OR_FAILED',stop_reason='wall_limit'))]
|
||||
rows,counts=assemble(parts,{'0':dict(nodes=10)},jobs,{'1':dict(nodes=11)})
|
||||
self.assertFalse(rows[2]['included']);self.assertEqual(rows[2]['component_id'],'unchanged-2')
|
||||
self.assertEqual(rows[2]['assignments'],parts[2]['assignments'])
|
||||
for row in counts:self.assertEqual(row['learning']['parts'],0 if row['pool']==25 else 2)
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user