Freeze approved 131 label-free preparation and guarded roster finalization

This commit is contained in:
Frank14f committed 2026-11-15 07:50:45 +08:00
1 parent 0af3dab8d1
commit bf5203c2ae
10 files changed
+16899

No files matched your search

File diff suppressed because it is too large. Load diff
@@ -0,0 +1,48 @@
{
"status": "PASS_FROZEN_NOT_STARTED",
"tests_passed": 10,
"test_log_sha256": "bfccebc343951402d9ba1fb1d8d0574a8b475e3f265a09fe267f7a1a9ea92976",
"contract_sha256": "0a3c628c81c008eab53e9337406b84ec2063edcd49529bcc7a82e3e4a3d719a3",
"code_sha256": {
"tools/analyze_support_capacity_conditions.py": "6bfc3ab91c48ff1c28a4b6d56092c7c8bd1f73e7f263c54bf857858cf84fd2c9",
"tools/audit_samples.py": "a3daf70ec3bcb77bf46d9dc4887d552cf7c422117880a169969780012278cd70",
"tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
"tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd",
"tools/freeze_s2_main_roster.py": "f2c8b844d9c1c1096d6bd395022b5f766f51c15bbb4659b495bee728e2af0f31",
"tools/g1_candidate_fingerprints.py": "2d027bd95f8da2cbfc0381d051f6a37949a5abe6b27e6b37d747d43c5b6612a0",
"tools/g1_registration_cost.py": "c3d722a8cfd067dfd726f8a8729e72c37dbc63e2c24ba1b0e3d767d47f7756ba",
"tools/o2_rep.py": "8c784ea04f6c14fd47ab79e1f2160c19ed48344bf24e6ab4255ac23ab3eb2003",
"tools/physical_support_probe.py": "4513745d0f2c00e5b5b545ef87382efb2cf8057b1166068b3c85b7c1854a91f7",
"tools/prepare_s2_main_inputs.py": "15043e599c8564075084ccb5c10fed41262d9d23594c9c5e5dc19a0410e6d6fd",
"tools/prepare_s2_pilot_inputs.py": "8a04afe82c0add692c7cd5182b3fd70919ed9979f417af7223fba64753ca3144",
"tools/prepare_s2_ten_part_measurement.py": "a6c8ec22ebd541a2e68f72f4c3049c893adc9f55e4a706f3e9c2d4681f1a8a6f",
"tools/projector_remesh_probe.py": "90411c11dde855d3a17d651c859a0a5c8d3241c47375a0487eea1c9631abc55a",
"tools/route_feasibility.py": "00e6b67d9079fc9c5d371064d5bfeb1a9950ce8b677bbe2c89c9976f45fb23d2",
"tools/run_e6_real_kernel.py": "45a3ade5d1cbc52828b83dcb4a1b884e83063c7eea3f68851e464f4d1e2ce68a",
"tools/run_s2_main_preparation.py": "705b798d5c1b56831e5af114818dbbbd4f5419002d55dc1a3ac4d715e740f5e0",
"tools/run_s2_pilot_preparation.py": "08f07593a5f5f434ff1bcfdf952b9079500e2378ea3afb2e4d7e152db1a58def",
"tools/run_s2_ten_part_measurement.py": "4108352a44368bd5581b21df029a84db5f00afad04de3ff73716ad4fa214296c",
"tools/run_volume_quadrature_trial.py": "eff0c71b4cb795b1b9992a1111b2f28c23b5f8f5e34b483f79512644008ac725",
"tools/s2_e1_supervisor.py": "6753c9ceb0b932047ba505bd15681f109e9407bdb58fd96c55e1431a3f1b4a02",
"tools/s2_main_prepare.py": "4a2083f7400091d2a344af378dfdb3c9a5b94a4e95d1897a5576ae535a94a875",
"tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4",
"tools/s2_pilot_data.py": "82c1fd18ec51757479c8fd7dda2db4d5f38c366d77fedcca3216a08998214c97",
"tools/s2_pilot_prepare.py": "0987a5fe49816557d2cd8ccaeaf01de5effcb5d080221f2c93eb834719d413d2",
"tools/s2_preprocess.py": "0bb59c6b5be166b379c924ca68aacfcaca6737386b71be61a43dc5433ca53419",
"tools/s2_preprocess_pilot.py": "46cec13e13c0b94e5a51444b5d5ee78579dec9a3aa42ed24294e0a0deec55833",
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
"tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
"tools/test_s2_main_preparation.py": "075c752b337e65690ad0694eae7a3102d46620c7925ac7f69ff6d42358c12bc9",
"tools/test_s2_pilot_data.py": "cd9a5f69a5207d0ea62e8c150df737992ab640a08d82b182d504c9559e729562",
"tools/test_s2_pilot_preparation.py": "71667a38b352ad64e588653b7d3545e71a473fb151de775e515e174b5358403c",
"tools/test_s2_preprocess.py": "6f70d4392d3800b8146fbdd92b3c06b08638e45167fefb7261bf7ecf30f3faf4"
},
"preparation_cases": 131,
"reuse_only_cases": 493,
"new_worker_cpu_hours": 20,
"prior_worker_cpu_hours": 34.75965079444441,
"current_cumulative_disk_bytes": 6513541120,
"labels_decoded": false,
"workers_started": false,
"planner_validation": "All131 source geometry/cache hashes and all493 successful geometry feature output hashes verified. Initial metadata schema mismatch corrected before root creation; no scientific attempt occurred."
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,15 @@
test_131_terminal_and_transient_monitor_failure_without_retry (__main__.MainPreparationChecks) ... ok
test_actual_geometry_cache_whitelist_never_opens_label (__main__.MainPreparationChecks) ... ok
test_cumulative_disk_cap_blocks_dispatch (__main__.MainPreparationChecks) ... ok
test_existing_batch_marker_refuses_restart (__main__.MainPreparationChecks) ... ok
test_failed_part_excluded_globally_original_component_and_nested_pools (__main__.MainPreparationChecks) ... ok
test_geometry_prefix_rejects_field_before_geometry_without_decode (__main__.MainPreparationChecks) ... ok
test_guards_apply_60_and_30_minutes_to_label_free_worker (__main__.MainPreparationChecks) ... ok
test_label_free_adapter_accepts_test_case_without_label_io (__main__.MainPreparationChecks) ... ok
test_new_20h_cap_stops_and_does_not_charge_prior_34h (__main__.MainPreparationChecks) ... ok
test_zero_eligible_queries_fail_without_fabricating_ready (__main__.MainPreparationChecks) ... ok
----------------------------------------------------------------------
Ran 10 tests in 0.099s
OK
@@ -0,0 +1,18 @@
# S2 主矩阵前的 131 件无标签准备
Claude [1794699027.366469](https://arcticnimbus.slack.com/archives/C0C78UFKGRM/p1794699027366469) 已批准准备预算和后续执行顺序。本次入口完成实现及验收,合同冻结后实际启动;实时状态以新目录的启动、进度和终态回执为准。
- 新目录:`artifacts/s2-main-preparation-2026-11-15`。合同 SHA256:`0a3c628c81c008eab53e9337406b84ec2063edcd49529bcc7a82e3e4a3d719a3`。
- 固定原计划 131 件及顺序:6 件已记录的超时实件各 60 分钟,125 件 fold0 test 几何各 30 分钟;每件进程树 RSS 1.25 GiB,最多 16 工作者,无重试。
- 新增 20 worker CPU-h 单独计账;历史 34.7596507944 worker CPU-h 另列,不再次扣入新增额度。运行中保守计入活动监督器 CPU;结束后工作者与监督器分列。旧准备、事故隔离、恢复、十件测量和新目录累计不超过 100 GiB。
- 直接调用原固定 `s2_preprocess_pilot.run` 数值入口,独立包装器只检查无标签和共同查询资格。没有修改旧源代码、旧失败记录或旧合同。所有 131 件都不解码标签、不拟合、不使用 GPU。
- 原 493 件成功结果仅复用。已核验其几何/特征输出哈希;未打开标签数组。复用清单单独固定为 `reuse-493.json`。
- 单批次锁与启动标记拒绝重复启动;每件监督锁拒绝重试。监测保留原恢复监督器的临时文件竞争容错。若监督器退出但无法取得计账回执,则停止并记录未知计账,不能把未知 CPU 当零。
10 项 CPU 合成检查通过(0.099 秒):测试角色的无标签入口、实际混合容器白名单、几何前出现场数据时拒绝解码、60/30 分钟限额、131 项有界调度及监测瞬态错误、独立新增 CPU 上限、累计磁盘上限、重复启动拒绝、失败跨折排除及组件身份/嵌套池保持、零资格查询失败。日志及预启动核验在新目录中。
首次组装时发现历史合并回执中旧批与恢复批顶层字段不同,已改为使用统一的 `worker.status`。此错误发生在创建新准备目录和启动任何工作者之前;未消耗任何科学尝试。
全部 131 项终态且没有全局停止后,监督入口自动合并原 493 成功结果,校验输出并生成唯一 `final-roster.json` 和 `roster-receipt.json`。失败实件跨五折统一排除,保留原组件、划分和原因;报告每折每个组件比例池的实际件数、节点数及组件数。若全局预算停止或计账未知,不生成可供训练的冻结名单。
后续 90 个神经模型均从头训练,不复用 pilot 权重;只有 roster 已冻结后方可执行。总 46 GPU-h 含等待/设置,每个 run 最多 30 分钟、最多四张 V100、N/E 相邻、首八 run 重估预算。R2 沿批准的单 fit 30 CPU-min、总 100 worker CPU-h,并记录留给 GPU 数据供给的 CPU 配置。全部 90+15 终态且检查点冻结后统一托管预测及评分;本准备入口不启动训练或解封。
+91
View File
@@ -0,0 +1,91 @@
"""One metadata-only roster across five folds, after the 131 terminal attempts."""
import argparse
import json
from pathlib import Path
from freeze_g1_components import sha
from s2_recovery_supervisor import save
def read(path):
return json.loads(Path(path).read_text())
def validate_preparation(path,expected_sha=None):
path=Path(path)
if expected_sha is not None:assert sha(path/'result.json')==expected_sha
r=read(path/'result.json')
assert r['case']==path.name
assert r['status']=='COMPLETE' and r['label_arrays_decoded']==[]
assert r['eligible_unique']>0 and r['eligible_slots']>0,'No common eligible query'
for name,digest in r['output_sha256'].items():
assert 'label' not in name and sha(path/name)==digest,(path,name)
return dict(preparation_root=str(path),result_sha256=sha(path/'result.json'),output_sha256=r['output_sha256'],
nodes=r['nodes'],t_set=r['scales']['t_set'],eligible_unique=r['eligible_unique'],eligible_slots=r['eligible_slots'])
def assemble(parts,reused,new_jobs,new_sources):
"""Preserve original component IDs and all assignments, including excluded parts."""
assert len({p['case'] for p in parts})==len(parts)
records={p['case']:dict(p,included=False,exclusion_reason='NOT_PREPARED') for p in parts}
assert not (set(reused)&{j['case'] for j in new_jobs})
for case,source in reused.items():
assert source['nodes']==records[case]['nodes']
records[case].update(included=True,exclusion_reason=None,preparation=source,attempt_kind='REUSED_PREVIOUS_PASS')
for job in new_jobs:
case=job['case'];r=records[case];w=job['worker']
if w['status']=='PASS' and case in new_sources:
assert new_sources[case]['nodes']==r['nodes']
r.update(included=True,exclusion_reason=None,preparation=new_sources[case],attempt_kind='NEW_LABEL_FREE_ATTEMPT')
else:
r.update(exclusion_reason=w.get('stop_reason') or w['status'],attempt_kind='NEW_LABEL_FREE_ATTEMPT',terminal_worker=w)
assert set(reused)|{j['case'] for j in new_jobs}==set(records)
rows=[records[p['case']] for p in parts]
counts=[]
for fold in range(5):
for pool in [25,50,100]:
groups={role:[] for role in ['learning','validation','test']}
for p in rows:
a=p['assignments'][fold]
assert a['fold']==fold
if not p['included']:continue
if a['role']=='learning' and str(pool) not in a['learning_pools']:continue
groups[a['role']].append(p)
counts.append(dict(fold=fold,pool=pool,**{role:dict(parts=len(items),nodes=sum(p['nodes'] for p in items),
components=len({p['component_id'] for p in items})) for role,items in groups.items()}))
return rows,counts
def freeze(root):
root=Path(root)
assert not (root/'final-roster.json').exists(),'Never replace a frozen roster'
from run_s2_main_preparation import contract
c=contract(root);batch=read(root/'batch-complete.json')
assert batch['status']=='ATTEMPTS_COMPLETE' and batch['stop_reason'] is None
assert len(batch['jobs'])==131 and not batch['not_run_indices']
assert sorted(j['index'] for j in batch['jobs'])==list(range(131))
sources={}
for j in batch['jobs']:
assert j['case']==c['cases'][j['index']]['case']
guard=root/'jobs'/f"job-{j['index']:04d}"/'guard-result.json'
assert read(guard)['worker']==j['worker']
if j['worker']['status']=='PASS':
sources[j['case']]=validate_preparation(root/'parts'/j['case'])
reuse=read(root/'reuse-493.json')['parts']
old={r['case']:validate_preparation(r['preparation_root'],r['result_sha256']) for r in reuse}
parts=read('artifacts/s2-g1-components-2026-11-15/custodian/part-splits.json')
assert len(parts)==624 and len(old)==493
rows,counts=assemble(parts,old,batch['jobs'],sources)
roster=dict(status='FROZEN',version='S2-main-five-fold-roster-v1',parts=rows,fold_pool_counts=counts,
included=sum(r['included'] for r in rows),excluded=[r['case'] for r in rows if not r['included']],
contract_sha256=sha(root/'contract.json'),batch_sha256=sha(root/'batch-complete.json'),
splits_sha256=sha('artifacts/s2-g1-components-2026-11-15/custodian/part-splits.json'),
labels_decoded=False,models_reused=False,neural_runs_required=90,r2_final_fits_required=15,
test_unsealing='Only unified custody after all90+15 terminal and checkpoints frozen')
save(root/'final-roster.json',roster)
save(root/'roster-receipt.json',dict(status='FROZEN',sha256=sha(root/'final-roster.json'),included=roster['included'],
excluded=roster['excluded'],neural_started=False,r2_started=False,labels_decoded=False))
print(json.dumps(read(root/'roster-receipt.json')),flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);freeze(p.parse_args().root)
+104
View File
@@ -0,0 +1,104 @@
"""Freeze the approved 131 geometry-only attempts; preserve the old 493 successes."""
import csv
import json
from pathlib import Path
from freeze_g1_components import sha
from s2_recovery_supervisor import save
ROOT = Path('artifacts/s2-main-preparation-2026-11-15')
PLAN = Path('artifacts/s2-main-engineering-2026-11-15/planned-preparation-131.json')
SPLITS = Path('artifacts/s2-g1-components-2026-11-15/custodian/part-splits.json')
OLD = Path('artifacts/s2-pilot-inputs-2026-11-15/contract.json')
MERGED = Path('artifacts/s2-preparation-incident-2026-11-15/merged-recovery-terminal.json')
SUCCESS = Path('artifacts/s2-pilot-training-2026-11-15/success-manifest.json')
def read(path):
return json.loads(Path(path).read_text())
def main():
assert not ROOT.exists(), 'Never overwrite an existing attempt'
plan=read(PLAN);old=read(OLD);merged=read(MERGED);success=read(SUCCESS)
parts={p['case']:p for p in read(SPLITS)}
prior={p['case']:p for p in old['cases']}
failed={p['case'] for p in merged['jobs'] if p['worker']['status']!='PASS'}
passed={p['case'] for p in success['ready_parts']}
expected=failed | {k for k,p in parts.items() if p['assignments'][0]['role']=='test'}
assert len(parts)==624 and len(failed)==6 and len(passed)==493
assert len(plan['jobs'])==131 and {p['case'] for p in plan['jobs']}==expected
assert not expected & passed and expected | passed == set(parts)
g1=read('artifacts/s2-g1-registration-2026-11-14/custodian/plan.json')['geometry_cache']
settings_path=Path('artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json')
settings={p['member'].removesuffix('.log'):p for p in read(settings_path)}
counts={p['case']:p for p in csv.DictReader(open('artifacts/s2-e1-2026-11-14/preflight/geometry_counts.csv'))}
cases=[]
for job in plan['jobs']:
case=job['case'];p=parts[case]
assert job['decode_labels'] is False and job['retries']==0
assert job['nodes']==p['nodes'] and job['component_id']==p['component_id']
assert job['wall_limit_seconds']==(3600 if case in failed else 1800)
if case in failed:
spec=json.loads(json.dumps(prior[case]))
assert not any(k in spec for k in ['reuse_preparation','prior_failed_preparation'])
else:
setting=settings[case]
spec=dict(case=case,numeric_prefix=int(case.split('_')[0]),source_key=p['source_key'],
component_id=p['component_id'],tet_nodes=p['nodes'],tet_cells=int(counts[case]['tet_cells']),
t_set=setting['setting']['seconds'],t_set_provenance=setting,g1_geometry=g1[case]['path'],
input_sha256={g1[case]['path']:g1[case]['sha256'],str(settings_path):sha(settings_path)})
cache=Path('artifacts/s1-panel-precheck-2026-11-14')/case
if (cache/'result.json').exists():
r=read(cache/'result.json');assert r['status']=='COUNT_COMPLETE'
spec.update(legacy_geometry=str(cache/'geometry.npz'),existing_distance=str(cache/'boundary_distance_raw.npy'))
for name in ['geometry.npz','boundary_distance_raw.npy']:
spec['input_sha256'][str(cache/name)]=r['output_sha256'][name]
draw=Path('artifacts/s1-sampled-draw-2026-11-14')/case/'draw.npz'
if draw.exists():
r=read(draw.parent/'receipt.json');assert r['status']=='DRAW_COMPLETE'
spec['existing_primary_draw']=str(draw);spec['input_sha256'][str(draw)]=r['output_sha256']['draw.npz']
o2=Path('artifacts/s1-sampled-o2-2026-11-14/panel')/(case+'_original')
if (o2/'result.json').exists():
r=read(o2/'result.json');assert r['status']=='COMPLETE'
spec.update(existing_sampled_o2_nodes=str(o2/'nodes.npz'),existing_sampled_o2_result=str(o2/'result.json'))
spec['input_sha256'][str(o2/'nodes.npz')]=r['output_sha256']['nodes.npz']
spec['input_sha256'][str(o2/'result.json')]=sha(o2/'result.json')
spec.update(attempt=job)
for path,digest in spec['input_sha256'].items():assert sha(path)==digest,path
cases.append(spec)
# Only hash geometry/features; neither open nor decode any labels.npz.
reused=[]
for row in success['ready_parts']:
source=Path(row['preparation_root']);r=read(source/'result.json')
assert r['status']=='COMPLETE' and r['label_arrays_decoded']==[]
assert r['case']==row['case'] and r['nodes']==parts[row['case']]['nodes']
for name,digest in r['output_sha256'].items():
assert 'label' not in name and sha(source/name)==digest,(source,name)
reused.append(dict(case=row['case'],preparation_root=str(source),
result_sha256=sha(source/'result.json'),output_sha256=r['output_sha256']))
base=read('artifacts/s2-ten-part-measurement-2026-11-15/contract.json')
for path,digest in old['code_sha256'].items():assert sha(path)==digest,path
paths=set(old['code_sha256']) | {'tools/prepare_s2_main_inputs.py','tools/run_s2_main_preparation.py',
'tools/freeze_s2_main_roster.py','tools/test_s2_main_preparation.py','tools/s2_main_prepare.py',
'tools/s2_recovery_monitoring.py','tools/s2_recovery_supervisor.py'}
sources={str(p):sha(p) for p in [PLAN,SPLITS,OLD,MERGED,SUCCESS,settings_path]}
ROOT.mkdir()
save(ROOT/'reuse-493.json',dict(status='VERIFIED_GEOMETRY_FEATURES_ONLY',parts=reused,labels_decoded=False))
sources[str(ROOT/'reuse-493.json')]=sha(ROOT/'reuse-493.json')
c={k:base[k] for k in ['fixed_split_commit','archive','archive_stat','scales','draw','eligibility']}
c.update(version='S2-main-131-label-free-v1',authorizing_rulings=['1794698734.967459','1794699027.366469'],
cases=cases,code_sha256={p:sha(p) for p in sorted(paths)},source_sha256=sources,
prior_worker_cpu_seconds=merged['worker_cpu_seconds'],
limits=dict(worker_cpu_hours=20,workers=16,disk_bytes=100*2**30,per_part_tree_rss_bytes=1342177280,retries=0),
budget_roots=['artifacts/s2-ten-part-measurement-2026-11-15','artifacts/s2-pilot-inputs-2026-11-15',
'artifacts/s2-preparation-incident-2026-11-15','artifacts/s2-pilot-recovery-inputs-2026-11-15',str(ROOT)],
input_policy='Geometry/features only for all 131; no labels decoded, no fit, no GPU. Original 493 reused without label reads.',
accounting='20 additional worker CPU-h; historical CPU recorded separately; live guard CPU conservatively included in cap.',
failure_policy='One labelled new attempt per planned case; no retry. Retain prior failures. Remaining failures excluded globally across all five folds with original component identity.',
dispatch='Frozen plan order, at most16; no first50 gate; freeze roster only after all131 terminal within approved budget.')
save(ROOT/'contract.json',c)
print(json.dumps(dict(status='FROZEN_NOT_STARTED',root=str(ROOT),contract_sha256=sha(ROOT/'contract.json'),
cases=len(cases),reused=len(reused),labels_decoded=False)))
if __name__=='__main__':main()
+141
View File
@@ -0,0 +1,141 @@
"""Approved 131 geometry-only attempts; independent stage CPU and cumulative disk."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
from freeze_g1_components import sha
from s2_recovery_supervisor import supervise,save,tree_usage
from s2_recovery_monitoring import disk_usage,Monitor,budget_reason
from run_s2_pilot_preparation import terminate
def contract(root):
c=json.loads((root/'contract.json').read_text())
for p,h in c['code_sha256'].items():assert sha(p)==h,p
for p,h in c['source_sha256'].items():assert sha(p)==h,p
assert len(c['cases'])==131 and len({s['case'] for s in c['cases']})==131
for spec in c['cases']:
a=spec['attempt']
assert a['decode_labels'] is False and a['retries']==0
expected=3600 if a['kind']=='labelled_timeout_repreparation' else 1800
assert a['wall_limit_seconds']==expected and a['tree_rss_bytes']==1342177280
roots=[Path(p).resolve() for p in c['budget_roots']]
assert all(a!=b and a not in b.parents and b not in a.parents for i,a in enumerate(roots) for b in roots[i+1:])
return c
def guard(root,index):
c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True)
wall=c['cases'][index]['attempt']['wall_limit_seconds']
r=supervise([sys.executable,'tools/s2_main_prepare.py',str(root),str(index)],job,'worker',
dict(total_cpu_seconds=wall+1,job_cpu_seconds=wall,job_wall_seconds=wall,tree_rss_bytes=1342177280,
disk_bytes=100*2**30,max_processes=1),poll=.25)
monitor=Monitor(job/'accounting-errors.jsonl')
monitor.retry('guard_receipt',lambda:save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
attempt=c['cases'][index]['attempt'],label_arrays_decoded=[])),.25)
return 0 if r['status']=='PASS' else 1
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
def run_locked(root):
c=contract(root)
assert not (root/'batch-started.json').exists(),'One main preparation batch only; never restart'
assert c['limits']['workers']==16
assert c['limits']['worker_cpu_hours']==20 and c['limits']['disk_bytes']==100*2**30
monitor=Monitor(root/'monitor-errors.jsonl');start=time.monotonic()
save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0)),
limits=c['limits'],labels_decoded=False))
active={};done=[];next_index=0;reason=None;live_cpu=live_rss=used=peak_rss=0;fresh=False
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
def interrupted(*_):
nonlocal reason
reason='MAIN_PREPARATION_COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
def read_account(path):
r=json.loads(path.read_text());w=r['worker']
assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0
assert math.isfinite(r['supervisor_cpu_seconds']) and r['supervisor_cpu_seconds']>=0
assert isinstance(w['returncode'],int) and isinstance(w['status'],str)
assert 'stop_reason' in w
return r
try:
while active or (next_index<len(c['cases']) and reason is None):
cycle_ok=True
for i,(p,log) in list(active.items()):
code,ok=monitor.sample(f'poll_{i}',p.poll,None);cycle_ok &= ok
if ok and code is not None:
path=root/'jobs'/f'job-{i:04d}'/'guard-result.json'
r,ok=monitor.sample(f'account_{i}',lambda path=path:read_account(path),None);cycle_ok &= ok
if not ok:
reason='GUARD_ACCOUNTING_UNAVAILABLE'
terminate(active)
save(root/'infrastructure-stop.json',dict(status=reason,index=i,guard_returncode=code,
active_indices=list(active),not_run_indices=list(range(next_index,len(c['cases']))),
cpu_accounting_complete=False,prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],
settled_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
contract_sha256=sha(root/'contract.json'),retry=False))
raise RuntimeError(reason)
done.append(dict(index=i,case=c['cases'][i]['case'],returncode=code,**r));del active[i];log.close()
if reason is None and r['worker']['returncode']<0 and r['worker']['stop_reason'] is None:reason='UNEXPLAINED_MAIN_PREPARATION_WORKER_SIGNAL'
print(json.dumps(dict(event='FINISHED',index=i,completed=len(done),status=r['worker']['status'])),flush=True)
def sample_trees():
values=[tree_usage(p.pid) for p,_ in active.values()]
return sum(v[0] for v in values),sum(v[1] for v in values)
(live_cpu,live_rss),ok=monitor.sample('aggregate_live_usage',sample_trees,(live_cpu,live_rss));cycle_ok &= ok
peak_rss=max(peak_rss,live_rss)
used,ok=monitor.sample('aggregate_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),used);cycle_ok &= ok
spent=sum(r['worker']['cpu_seconds'] for r in done)
over=budget_reason(0,spent,live_cpu,used,cpu_limit=c['limits']['worker_cpu_hours']*3600,disk_limit=c['limits']['disk_bytes'])
if over:reason=over
if reason:terminate(active)
progress=dict(completed=len(done),active=list(active),next_index=next_index,stop_reason=reason,
prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],stage_worker_cpu_seconds=spent,
cumulative_settled_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,
live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=used,
elapsed_seconds=time.monotonic()-start,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,
fresh_accounting=cycle_ok,no_new_first50_barrier=True)
_,ok=monitor.sample('progress_write',lambda:save(root/'progress.json',progress),None);cycle_ok &= ok
# Do not dispatch new work on a stale accounting cycle. Existing
# workers keep their independent wall/CPU/RSS guards while re-polling.
while cycle_ok and reason is None and len(active)<c['limits']['workers'] and next_index<len(c['cases']):
i=next_index;job=root/'jobs'/f'job-{i:04d}';job.mkdir(parents=True,exist_ok=True)
log=(job/'guard.log').open('x')
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[i]=(p,log);next_index+=1
if active or (not cycle_ok and reason is None):time.sleep(.5)
finally:
terminate(active)
for p,log in active.values():
p.wait();log.close()
used=monitor.retry('final_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),.5)
spent=sum(r['worker']['cpu_seconds'] for r in done)
if spent>c['limits']['worker_cpu_hours']*3600:reason='GLOBAL_CPU_LIMIT_AFTER_EXIT'
if used>100*2**30:reason='GLOBAL_DISK_LIMIT_AFTER_EXIT'
result=dict(status='ATTEMPTS_COMPLETE' if len(done)==len(c['cases']) and reason is None else 'STOPPED',stop_reason=reason,jobs=done,
not_run_indices=list(range(next_index,len(c['cases']))),prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],worker_cpu_seconds=spent,
cumulative_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
peak_aggregate_rss_bytes=peak_rss,disk_bytes=used,failed=sum(r['worker']['status']!='PASS' for r in done),
monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,retry=False,replacement=False,test_labels_read=False)
monitor.retry('batch_receipt',lambda:save(root/'batch-complete.json',result),.5)
print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True)
if result['status']=='ATTEMPTS_COMPLETE':
from freeze_s2_main_roster import freeze
freeze(root)
return 0 if result['status']=='ATTEMPTS_COMPLETE' else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args()
sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root))
+26
View File
@@ -0,0 +1,26 @@
"""Geometry-only main adapter; all numerical preparation remains pinned and unchanged."""
import argparse
import json
from pathlib import Path
import s2_preprocess_pilot
from s2_recovery_supervisor import save
def run(root,index):
spec=json.loads((root/'contract.json').read_text())['cases'][index]
assert spec['attempt']['decode_labels'] is False
assert 'reuse_preparation' not in spec and 'prior_failed_preparation' not in spec
code=s2_preprocess_pilot.run(root,index)
if code:return code
dest=root/'parts'/spec['case'];r=json.loads((dest/'result.json').read_text())
assert r['label_arrays_decoded']==[] and r['status']=='COMPLETE'
assert not list(dest.rglob('*label*'))
if r['eligible_unique']<=0 or r['eligible_slots']<=0:
save(dest/'eligibility-failure.json',dict(status='NO_COMMON_ELIGIBLE_QUERIES',labels_decoded=False))
return 3
return 0
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args()
raise SystemExit(run(a.root,a.index))
+132
View File
@@ -0,0 +1,132 @@
import io
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from zipfile import ZipFile
import numpy as np
import run_s2_main_preparation as coordinator
import s2_main_prepare as worker
from s2_preprocess_pilot import load_selected,GEOMETRY_MEMBERS
from g1_candidate_fingerprints import read_tets
from freeze_s2_main_roster import assemble
def synthetic_contract(root):
return dict(cases=[dict(case=str(i),attempt=dict(wall_limit_seconds=3600 if i<6 else 1800)) for i in range(131)],
prior_worker_cpu_seconds=125134.74286,limits=dict(worker_cpu_hours=20,workers=16,disk_bytes=100*2**30),budget_roots=[str(root)])
class MainPreparationChecks(unittest.TestCase):
def test_label_free_adapter_accepts_test_case_without_label_io(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);case='synthetic_sealed_test'
(root/'contract.json').write_text(json.dumps(dict(cases=[dict(case=case,attempt=dict(decode_labels=False,fold0_role='test'))])))
def geometric_run(r,i):
d=r/'parts'/case;d.mkdir(parents=True)
(d/'result.json').write_text(json.dumps(dict(status='COMPLETE',label_arrays_decoded=[],eligible_unique=1,eligible_slots=2000)))
return 0
with patch.object(worker.s2_preprocess_pilot,'run',side_effect=geometric_run) as numerical:
self.assertEqual(worker.run(root,0),0)
numerical.assert_called_once_with(root,0)
self.assertFalse(list(root.rglob('*label*')))
def test_actual_geometry_cache_whitelist_never_opens_label(self):
with tempfile.TemporaryDirectory() as t:
path=Path(t)/'mixed.npz';np.savez(path,points_raw=np.ones((2,3)),fill_time_raw=np.array([999.]))
opened=[];original=ZipFile.open
def spy(z,name,*a,**kw):opened.append(name);return original(z,name,*a,**kw)
with patch.object(ZipFile,'open',spy):load_selected(path,['points_raw'],GEOMETRY_MEMBERS)
self.assertEqual(opened,['points_raw.npy'])
with self.assertRaises(ValueError):load_selected(path,['fill_time_raw'],GEOMETRY_MEMBERS)
def test_zero_eligible_queries_fail_without_fabricating_ready(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);case='empty'
(root/'contract.json').write_text(json.dumps(dict(cases=[dict(case=case,attempt=dict(decode_labels=False))])))
d=root/'parts'/case;d.mkdir(parents=True)
(d/'result.json').write_text(json.dumps(dict(status='COMPLETE',label_arrays_decoded=[],eligible_unique=0,eligible_slots=0)))
with patch.object(worker.s2_preprocess_pilot,'run',return_value=0):self.assertEqual(worker.run(root,0),3)
self.assertEqual(json.loads((d/'eligibility-failure.json').read_text())['status'],'NO_COMMON_ELIGIBLE_QUERIES')
def test_geometry_prefix_rejects_field_before_geometry_without_decode(self):
xml=b'<VTKFile byte_order="LittleEndian" compressor="vtkZLibDataCompressor"><UnstructuredGrid><Piece><PointData><DataArray Name="Fill_time"/></PointData></Piece></UnstructuredGrid></VTKFile>'
with patch('g1_candidate_fingerprints.decode_array',side_effect=AssertionError('label decode')) as decode:
with self.assertRaisesRegex(ValueError,'Field association'):read_tets(io.BytesIO(xml))
decode.assert_not_called()
def test_guards_apply_60_and_30_minutes_to_label_free_worker(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);c=synthetic_contract(root);calls=[]
def supervise(command,job,name,limits,**kw):
calls.append((command,limits));return dict(status='PASS')
with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator,'supervise',supervise):
self.assertEqual(coordinator.guard(root,0),0)
self.assertEqual(coordinator.guard(root,6),0)
self.assertEqual([x[1]['job_wall_seconds'] for x in calls],[3600,1800])
self.assertTrue(all(x[0][1]=='tools/s2_main_prepare.py' and x[1]['tree_rss_bytes']==1342177280 for x in calls))
def run_fake_batch(self,root,*,over_cpu=False,over_disk=False,monitor_fault=False):
c=synthetic_contract(root);(root/'contract.json').write_text(json.dumps(c))
spawned=[];fault=False;terminated=[];peak=0
class FakeProcess:
def __init__(self,i):self.index=i;self.pid=1000000+i;self.polls=0
def poll(self):
self.polls+=1
if self.polls==1:return None
p=root/'jobs'/f'job-{self.index:04d}'/'guard-result.json'
p.write_text(json.dumps(dict(worker=dict(status='PASS',returncode=0,stop_reason=None,cpu_seconds=.01),supervisor_cpu_seconds=.001)))
return 0
def wait(self):return 0
def spawn(command,**kwargs):
nonlocal peak
p=FakeProcess(int(command[-1]));spawned.append(p)
peak=max(peak,sum(p.polls<2 for p in spawned));return p
def disk(_):
nonlocal fault
if monitor_fault and spawned and not fault:fault=True;raise OSError('atomic replacement transient')
return 100*2**30 if over_disk else 100
with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator.subprocess,'Popen',spawn),\
patch.object(coordinator,'disk_usage',disk),patch.object(coordinator,'tree_usage',return_value=(5000 if over_cpu else .01,100,1)),\
patch.object(coordinator.time,'sleep'),patch.object(coordinator,'terminate',side_effect=lambda a:terminated.append(list(a))),\
patch('freeze_s2_main_roster.freeze') as freeze,patch('builtins.print'):
coordinator.run(root)
return json.loads((root/'batch-complete.json').read_text()),spawned,peak,freeze.call_count
def test_131_terminal_and_transient_monitor_failure_without_retry(self):
with tempfile.TemporaryDirectory() as t:
r,spawned,peak,freezes=self.run_fake_batch(Path(t),monitor_fault=True)
self.assertEqual(r['status'],'ATTEMPTS_COMPLETE');self.assertEqual(len(spawned),131)
self.assertLessEqual(peak,16);self.assertEqual(r['monitor_errors'],1);self.assertEqual(freezes,1)
self.assertAlmostEqual(r['worker_cpu_seconds'],1.31)
self.assertAlmostEqual(r['cumulative_worker_cpu_seconds'],125134.74286+1.31)
def test_new_20h_cap_stops_and_does_not_charge_prior_34h(self):
with tempfile.TemporaryDirectory() as t:
r,spawned,_,freezes=self.run_fake_batch(Path(t),over_cpu=True)
self.assertEqual(r['stop_reason'],'GLOBAL_CPU_LIMIT');self.assertEqual(len(spawned),16)
self.assertEqual(len(r['not_run_indices']),115);self.assertEqual(freezes,0)
def test_cumulative_disk_cap_blocks_dispatch(self):
with tempfile.TemporaryDirectory() as t:
r,spawned,_,freezes=self.run_fake_batch(Path(t),over_disk=True)
self.assertEqual(r['stop_reason'],'GLOBAL_DISK_LIMIT');self.assertEqual(spawned,[]);self.assertEqual(freezes,0)
def test_existing_batch_marker_refuses_restart(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'batch-started.json').write_text('{}')
with patch.object(coordinator,'contract',return_value=synthetic_contract(root)),patch.object(coordinator.subprocess,'Popen') as spawn:
with self.assertRaises(AssertionError):coordinator.run(root)
spawn.assert_not_called()
def test_failed_part_excluded_globally_original_component_and_nested_pools(self):
parts=[dict(case=str(i),nodes=i+10,component_id='unchanged-'+str(i),assignments=[dict(fold=f,role='learning',learning_pools=['50','100']) for f in range(5)]) for i in range(3)]
jobs=[dict(case='1',worker=dict(status='PASS')),dict(case='2',worker=dict(status='STOPPED_OR_FAILED',stop_reason='wall_limit'))]
rows,counts=assemble(parts,{'0':dict(nodes=10)},jobs,{'1':dict(nodes=11)})
self.assertFalse(rows[2]['included']);self.assertEqual(rows[2]['component_id'],'unchanged-2')
self.assertEqual(rows[2]['assignments'],parts[2]['assignments'])
for row in counts:self.assertEqual(row['learning']['parts'],0 if row['pool']==25 else 2)
if __name__=='__main__':unittest.main(verbosity=2)