Freeze approved 131 label-free preparation and guarded roster finalization

This commit is contained in:
Frank14f committed 2026-11-15 07:50:45 +08:00
1 parent 0af3dab8d1
commit bf5203c2ae
10 files changed
+16899

No files matched your search

+91
View File
@@ -0,0 +1,91 @@
"""One metadata-only roster across five folds, after the 131 terminal attempts."""
import argparse
import json
from pathlib import Path
from freeze_g1_components import sha
from s2_recovery_supervisor import save
def read(path):
return json.loads(Path(path).read_text())
def validate_preparation(path,expected_sha=None):
path=Path(path)
if expected_sha is not None:assert sha(path/'result.json')==expected_sha
r=read(path/'result.json')
assert r['case']==path.name
assert r['status']=='COMPLETE' and r['label_arrays_decoded']==[]
assert r['eligible_unique']>0 and r['eligible_slots']>0,'No common eligible query'
for name,digest in r['output_sha256'].items():
assert 'label' not in name and sha(path/name)==digest,(path,name)
return dict(preparation_root=str(path),result_sha256=sha(path/'result.json'),output_sha256=r['output_sha256'],
nodes=r['nodes'],t_set=r['scales']['t_set'],eligible_unique=r['eligible_unique'],eligible_slots=r['eligible_slots'])
def assemble(parts,reused,new_jobs,new_sources):
"""Preserve original component IDs and all assignments, including excluded parts."""
assert len({p['case'] for p in parts})==len(parts)
records={p['case']:dict(p,included=False,exclusion_reason='NOT_PREPARED') for p in parts}
assert not (set(reused)&{j['case'] for j in new_jobs})
for case,source in reused.items():
assert source['nodes']==records[case]['nodes']
records[case].update(included=True,exclusion_reason=None,preparation=source,attempt_kind='REUSED_PREVIOUS_PASS')
for job in new_jobs:
case=job['case'];r=records[case];w=job['worker']
if w['status']=='PASS' and case in new_sources:
assert new_sources[case]['nodes']==r['nodes']
r.update(included=True,exclusion_reason=None,preparation=new_sources[case],attempt_kind='NEW_LABEL_FREE_ATTEMPT')
else:
r.update(exclusion_reason=w.get('stop_reason') or w['status'],attempt_kind='NEW_LABEL_FREE_ATTEMPT',terminal_worker=w)
assert set(reused)|{j['case'] for j in new_jobs}==set(records)
rows=[records[p['case']] for p in parts]
counts=[]
for fold in range(5):
for pool in [25,50,100]:
groups={role:[] for role in ['learning','validation','test']}
for p in rows:
a=p['assignments'][fold]
assert a['fold']==fold
if not p['included']:continue
if a['role']=='learning' and str(pool) not in a['learning_pools']:continue
groups[a['role']].append(p)
counts.append(dict(fold=fold,pool=pool,**{role:dict(parts=len(items),nodes=sum(p['nodes'] for p in items),
components=len({p['component_id'] for p in items})) for role,items in groups.items()}))
return rows,counts
def freeze(root):
root=Path(root)
assert not (root/'final-roster.json').exists(),'Never replace a frozen roster'
from run_s2_main_preparation import contract
c=contract(root);batch=read(root/'batch-complete.json')
assert batch['status']=='ATTEMPTS_COMPLETE' and batch['stop_reason'] is None
assert len(batch['jobs'])==131 and not batch['not_run_indices']
assert sorted(j['index'] for j in batch['jobs'])==list(range(131))
sources={}
for j in batch['jobs']:
assert j['case']==c['cases'][j['index']]['case']
guard=root/'jobs'/f"job-{j['index']:04d}"/'guard-result.json'
assert read(guard)['worker']==j['worker']
if j['worker']['status']=='PASS':
sources[j['case']]=validate_preparation(root/'parts'/j['case'])
reuse=read(root/'reuse-493.json')['parts']
old={r['case']:validate_preparation(r['preparation_root'],r['result_sha256']) for r in reuse}
parts=read('artifacts/s2-g1-components-2026-11-15/custodian/part-splits.json')
assert len(parts)==624 and len(old)==493
rows,counts=assemble(parts,old,batch['jobs'],sources)
roster=dict(status='FROZEN',version='S2-main-five-fold-roster-v1',parts=rows,fold_pool_counts=counts,
included=sum(r['included'] for r in rows),excluded=[r['case'] for r in rows if not r['included']],
contract_sha256=sha(root/'contract.json'),batch_sha256=sha(root/'batch-complete.json'),
splits_sha256=sha('artifacts/s2-g1-components-2026-11-15/custodian/part-splits.json'),
labels_decoded=False,models_reused=False,neural_runs_required=90,r2_final_fits_required=15,
test_unsealing='Only unified custody after all90+15 terminal and checkpoints frozen')
save(root/'final-roster.json',roster)
save(root/'roster-receipt.json',dict(status='FROZEN',sha256=sha(root/'final-roster.json'),included=roster['included'],
excluded=roster['excluded'],neural_started=False,r2_started=False,labels_decoded=False))
print(json.dumps(read(root/'roster-receipt.json')),flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);freeze(p.parse_args().root)
+104
View File
@@ -0,0 +1,104 @@
"""Freeze the approved 131 geometry-only attempts; preserve the old 493 successes."""
import csv
import json
from pathlib import Path
from freeze_g1_components import sha
from s2_recovery_supervisor import save
ROOT = Path('artifacts/s2-main-preparation-2026-11-15')
PLAN = Path('artifacts/s2-main-engineering-2026-11-15/planned-preparation-131.json')
SPLITS = Path('artifacts/s2-g1-components-2026-11-15/custodian/part-splits.json')
OLD = Path('artifacts/s2-pilot-inputs-2026-11-15/contract.json')
MERGED = Path('artifacts/s2-preparation-incident-2026-11-15/merged-recovery-terminal.json')
SUCCESS = Path('artifacts/s2-pilot-training-2026-11-15/success-manifest.json')
def read(path):
return json.loads(Path(path).read_text())
def main():
assert not ROOT.exists(), 'Never overwrite an existing attempt'
plan=read(PLAN);old=read(OLD);merged=read(MERGED);success=read(SUCCESS)
parts={p['case']:p for p in read(SPLITS)}
prior={p['case']:p for p in old['cases']}
failed={p['case'] for p in merged['jobs'] if p['worker']['status']!='PASS'}
passed={p['case'] for p in success['ready_parts']}
expected=failed | {k for k,p in parts.items() if p['assignments'][0]['role']=='test'}
assert len(parts)==624 and len(failed)==6 and len(passed)==493
assert len(plan['jobs'])==131 and {p['case'] for p in plan['jobs']}==expected
assert not expected & passed and expected | passed == set(parts)
g1=read('artifacts/s2-g1-registration-2026-11-14/custodian/plan.json')['geometry_cache']
settings_path=Path('artifacts/s2-e1-2026-11-14/log-flow-setting-check/fill-time-setting-context.json')
settings={p['member'].removesuffix('.log'):p for p in read(settings_path)}
counts={p['case']:p for p in csv.DictReader(open('artifacts/s2-e1-2026-11-14/preflight/geometry_counts.csv'))}
cases=[]
for job in plan['jobs']:
case=job['case'];p=parts[case]
assert job['decode_labels'] is False and job['retries']==0
assert job['nodes']==p['nodes'] and job['component_id']==p['component_id']
assert job['wall_limit_seconds']==(3600 if case in failed else 1800)
if case in failed:
spec=json.loads(json.dumps(prior[case]))
assert not any(k in spec for k in ['reuse_preparation','prior_failed_preparation'])
else:
setting=settings[case]
spec=dict(case=case,numeric_prefix=int(case.split('_')[0]),source_key=p['source_key'],
component_id=p['component_id'],tet_nodes=p['nodes'],tet_cells=int(counts[case]['tet_cells']),
t_set=setting['setting']['seconds'],t_set_provenance=setting,g1_geometry=g1[case]['path'],
input_sha256={g1[case]['path']:g1[case]['sha256'],str(settings_path):sha(settings_path)})
cache=Path('artifacts/s1-panel-precheck-2026-11-14')/case
if (cache/'result.json').exists():
r=read(cache/'result.json');assert r['status']=='COUNT_COMPLETE'
spec.update(legacy_geometry=str(cache/'geometry.npz'),existing_distance=str(cache/'boundary_distance_raw.npy'))
for name in ['geometry.npz','boundary_distance_raw.npy']:
spec['input_sha256'][str(cache/name)]=r['output_sha256'][name]
draw=Path('artifacts/s1-sampled-draw-2026-11-14')/case/'draw.npz'
if draw.exists():
r=read(draw.parent/'receipt.json');assert r['status']=='DRAW_COMPLETE'
spec['existing_primary_draw']=str(draw);spec['input_sha256'][str(draw)]=r['output_sha256']['draw.npz']
o2=Path('artifacts/s1-sampled-o2-2026-11-14/panel')/(case+'_original')
if (o2/'result.json').exists():
r=read(o2/'result.json');assert r['status']=='COMPLETE'
spec.update(existing_sampled_o2_nodes=str(o2/'nodes.npz'),existing_sampled_o2_result=str(o2/'result.json'))
spec['input_sha256'][str(o2/'nodes.npz')]=r['output_sha256']['nodes.npz']
spec['input_sha256'][str(o2/'result.json')]=sha(o2/'result.json')
spec.update(attempt=job)
for path,digest in spec['input_sha256'].items():assert sha(path)==digest,path
cases.append(spec)
# Only hash geometry/features; neither open nor decode any labels.npz.
reused=[]
for row in success['ready_parts']:
source=Path(row['preparation_root']);r=read(source/'result.json')
assert r['status']=='COMPLETE' and r['label_arrays_decoded']==[]
assert r['case']==row['case'] and r['nodes']==parts[row['case']]['nodes']
for name,digest in r['output_sha256'].items():
assert 'label' not in name and sha(source/name)==digest,(source,name)
reused.append(dict(case=row['case'],preparation_root=str(source),
result_sha256=sha(source/'result.json'),output_sha256=r['output_sha256']))
base=read('artifacts/s2-ten-part-measurement-2026-11-15/contract.json')
for path,digest in old['code_sha256'].items():assert sha(path)==digest,path
paths=set(old['code_sha256']) | {'tools/prepare_s2_main_inputs.py','tools/run_s2_main_preparation.py',
'tools/freeze_s2_main_roster.py','tools/test_s2_main_preparation.py','tools/s2_main_prepare.py',
'tools/s2_recovery_monitoring.py','tools/s2_recovery_supervisor.py'}
sources={str(p):sha(p) for p in [PLAN,SPLITS,OLD,MERGED,SUCCESS,settings_path]}
ROOT.mkdir()
save(ROOT/'reuse-493.json',dict(status='VERIFIED_GEOMETRY_FEATURES_ONLY',parts=reused,labels_decoded=False))
sources[str(ROOT/'reuse-493.json')]=sha(ROOT/'reuse-493.json')
c={k:base[k] for k in ['fixed_split_commit','archive','archive_stat','scales','draw','eligibility']}
c.update(version='S2-main-131-label-free-v1',authorizing_rulings=['1794698734.967459','1794699027.366469'],
cases=cases,code_sha256={p:sha(p) for p in sorted(paths)},source_sha256=sources,
prior_worker_cpu_seconds=merged['worker_cpu_seconds'],
limits=dict(worker_cpu_hours=20,workers=16,disk_bytes=100*2**30,per_part_tree_rss_bytes=1342177280,retries=0),
budget_roots=['artifacts/s2-ten-part-measurement-2026-11-15','artifacts/s2-pilot-inputs-2026-11-15',
'artifacts/s2-preparation-incident-2026-11-15','artifacts/s2-pilot-recovery-inputs-2026-11-15',str(ROOT)],
input_policy='Geometry/features only for all 131; no labels decoded, no fit, no GPU. Original 493 reused without label reads.',
accounting='20 additional worker CPU-h; historical CPU recorded separately; live guard CPU conservatively included in cap.',
failure_policy='One labelled new attempt per planned case; no retry. Retain prior failures. Remaining failures excluded globally across all five folds with original component identity.',
dispatch='Frozen plan order, at most16; no first50 gate; freeze roster only after all131 terminal within approved budget.')
save(ROOT/'contract.json',c)
print(json.dumps(dict(status='FROZEN_NOT_STARTED',root=str(ROOT),contract_sha256=sha(ROOT/'contract.json'),
cases=len(cases),reused=len(reused),labels_decoded=False)))
if __name__=='__main__':main()
+141
View File
@@ -0,0 +1,141 @@
"""Approved 131 geometry-only attempts; independent stage CPU and cumulative disk."""
import argparse
import fcntl
import json
import math
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
from freeze_g1_components import sha
from s2_recovery_supervisor import supervise,save,tree_usage
from s2_recovery_monitoring import disk_usage,Monitor,budget_reason
from run_s2_pilot_preparation import terminate
def contract(root):
c=json.loads((root/'contract.json').read_text())
for p,h in c['code_sha256'].items():assert sha(p)==h,p
for p,h in c['source_sha256'].items():assert sha(p)==h,p
assert len(c['cases'])==131 and len({s['case'] for s in c['cases']})==131
for spec in c['cases']:
a=spec['attempt']
assert a['decode_labels'] is False and a['retries']==0
expected=3600 if a['kind']=='labelled_timeout_repreparation' else 1800
assert a['wall_limit_seconds']==expected and a['tree_rss_bytes']==1342177280
roots=[Path(p).resolve() for p in c['budget_roots']]
assert all(a!=b and a not in b.parents and b not in a.parents for i,a in enumerate(roots) for b in roots[i+1:])
return c
def guard(root,index):
c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True)
wall=c['cases'][index]['attempt']['wall_limit_seconds']
r=supervise([sys.executable,'tools/s2_main_prepare.py',str(root),str(index)],job,'worker',
dict(total_cpu_seconds=wall+1,job_cpu_seconds=wall,job_wall_seconds=wall,tree_rss_bytes=1342177280,
disk_bytes=100*2**30,max_processes=1),poll=.25)
monitor=Monitor(job/'accounting-errors.jsonl')
monitor.retry('guard_receipt',lambda:save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
attempt=c['cases'][index]['attempt'],label_arrays_decoded=[])),.25)
return 0 if r['status']=='PASS' else 1
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
def run_locked(root):
c=contract(root)
assert not (root/'batch-started.json').exists(),'One main preparation batch only; never restart'
assert c['limits']['workers']==16
assert c['limits']['worker_cpu_hours']==20 and c['limits']['disk_bytes']==100*2**30
monitor=Monitor(root/'monitor-errors.jsonl');start=time.monotonic()
save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json'),cpu_affinity=sorted(os.sched_getaffinity(0)),
limits=c['limits'],labels_decoded=False))
active={};done=[];next_index=0;reason=None;live_cpu=live_rss=used=peak_rss=0;fresh=False
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
def interrupted(*_):
nonlocal reason
reason='MAIN_PREPARATION_COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
def read_account(path):
r=json.loads(path.read_text());w=r['worker']
assert math.isfinite(w['cpu_seconds']) and w['cpu_seconds']>=0
assert math.isfinite(r['supervisor_cpu_seconds']) and r['supervisor_cpu_seconds']>=0
assert isinstance(w['returncode'],int) and isinstance(w['status'],str)
assert 'stop_reason' in w
return r
try:
while active or (next_index<len(c['cases']) and reason is None):
cycle_ok=True
for i,(p,log) in list(active.items()):
code,ok=monitor.sample(f'poll_{i}',p.poll,None);cycle_ok &= ok
if ok and code is not None:
path=root/'jobs'/f'job-{i:04d}'/'guard-result.json'
r,ok=monitor.sample(f'account_{i}',lambda path=path:read_account(path),None);cycle_ok &= ok
if not ok:
reason='GUARD_ACCOUNTING_UNAVAILABLE'
terminate(active)
save(root/'infrastructure-stop.json',dict(status=reason,index=i,guard_returncode=code,
active_indices=list(active),not_run_indices=list(range(next_index,len(c['cases']))),
cpu_accounting_complete=False,prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],
settled_worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
contract_sha256=sha(root/'contract.json'),retry=False))
raise RuntimeError(reason)
done.append(dict(index=i,case=c['cases'][i]['case'],returncode=code,**r));del active[i];log.close()
if reason is None and r['worker']['returncode']<0 and r['worker']['stop_reason'] is None:reason='UNEXPLAINED_MAIN_PREPARATION_WORKER_SIGNAL'
print(json.dumps(dict(event='FINISHED',index=i,completed=len(done),status=r['worker']['status'])),flush=True)
def sample_trees():
values=[tree_usage(p.pid) for p,_ in active.values()]
return sum(v[0] for v in values),sum(v[1] for v in values)
(live_cpu,live_rss),ok=monitor.sample('aggregate_live_usage',sample_trees,(live_cpu,live_rss));cycle_ok &= ok
peak_rss=max(peak_rss,live_rss)
used,ok=monitor.sample('aggregate_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),used);cycle_ok &= ok
spent=sum(r['worker']['cpu_seconds'] for r in done)
over=budget_reason(0,spent,live_cpu,used,cpu_limit=c['limits']['worker_cpu_hours']*3600,disk_limit=c['limits']['disk_bytes'])
if over:reason=over
if reason:terminate(active)
progress=dict(completed=len(done),active=list(active),next_index=next_index,stop_reason=reason,
prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],stage_worker_cpu_seconds=spent,
cumulative_settled_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,
live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=used,
elapsed_seconds=time.monotonic()-start,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,
fresh_accounting=cycle_ok,no_new_first50_barrier=True)
_,ok=monitor.sample('progress_write',lambda:save(root/'progress.json',progress),None);cycle_ok &= ok
# Do not dispatch new work on a stale accounting cycle. Existing
# workers keep their independent wall/CPU/RSS guards while re-polling.
while cycle_ok and reason is None and len(active)<c['limits']['workers'] and next_index<len(c['cases']):
i=next_index;job=root/'jobs'/f'job-{i:04d}';job.mkdir(parents=True,exist_ok=True)
log=(job/'guard.log').open('x')
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[i]=(p,log);next_index+=1
if active or (not cycle_ok and reason is None):time.sleep(.5)
finally:
terminate(active)
for p,log in active.values():
p.wait();log.close()
used=monitor.retry('final_disk',lambda:sum(disk_usage(p) for p in c['budget_roots']),.5)
spent=sum(r['worker']['cpu_seconds'] for r in done)
if spent>c['limits']['worker_cpu_hours']*3600:reason='GLOBAL_CPU_LIMIT_AFTER_EXIT'
if used>100*2**30:reason='GLOBAL_DISK_LIMIT_AFTER_EXIT'
result=dict(status='ATTEMPTS_COMPLETE' if len(done)==len(c['cases']) and reason is None else 'STOPPED',stop_reason=reason,jobs=done,
not_run_indices=list(range(next_index,len(c['cases']))),prior_worker_cpu_seconds=c['prior_worker_cpu_seconds'],worker_cpu_seconds=spent,
cumulative_worker_cpu_seconds=c['prior_worker_cpu_seconds']+spent,guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),wall_seconds=time.monotonic()-start,
peak_aggregate_rss_bytes=peak_rss,disk_bytes=used,failed=sum(r['worker']['status']!='PASS' for r in done),
monitor_errors=monitor.errors,last_monitor_error=monitor.last_error,retry=False,replacement=False,test_labels_read=False)
monitor.retry('batch_receipt',lambda:save(root/'batch-complete.json',result),.5)
print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True)
if result['status']=='ATTEMPTS_COMPLETE':
from freeze_s2_main_roster import freeze
freeze(root)
return 0 if result['status']=='ATTEMPTS_COMPLETE' else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args()
sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root))
+26
View File
@@ -0,0 +1,26 @@
"""Geometry-only main adapter; all numerical preparation remains pinned and unchanged."""
import argparse
import json
from pathlib import Path
import s2_preprocess_pilot
from s2_recovery_supervisor import save
def run(root,index):
spec=json.loads((root/'contract.json').read_text())['cases'][index]
assert spec['attempt']['decode_labels'] is False
assert 'reuse_preparation' not in spec and 'prior_failed_preparation' not in spec
code=s2_preprocess_pilot.run(root,index)
if code:return code
dest=root/'parts'/spec['case'];r=json.loads((dest/'result.json').read_text())
assert r['label_arrays_decoded']==[] and r['status']=='COMPLETE'
assert not list(dest.rglob('*label*'))
if r['eligible_unique']<=0 or r['eligible_slots']<=0:
save(dest/'eligibility-failure.json',dict(status='NO_COMMON_ELIGIBLE_QUERIES',labels_decoded=False))
return 3
return 0
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args()
raise SystemExit(run(a.root,a.index))
+132
View File
@@ -0,0 +1,132 @@
import io
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from zipfile import ZipFile
import numpy as np
import run_s2_main_preparation as coordinator
import s2_main_prepare as worker
from s2_preprocess_pilot import load_selected,GEOMETRY_MEMBERS
from g1_candidate_fingerprints import read_tets
from freeze_s2_main_roster import assemble
def synthetic_contract(root):
return dict(cases=[dict(case=str(i),attempt=dict(wall_limit_seconds=3600 if i<6 else 1800)) for i in range(131)],
prior_worker_cpu_seconds=125134.74286,limits=dict(worker_cpu_hours=20,workers=16,disk_bytes=100*2**30),budget_roots=[str(root)])
class MainPreparationChecks(unittest.TestCase):
def test_label_free_adapter_accepts_test_case_without_label_io(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);case='synthetic_sealed_test'
(root/'contract.json').write_text(json.dumps(dict(cases=[dict(case=case,attempt=dict(decode_labels=False,fold0_role='test'))])))
def geometric_run(r,i):
d=r/'parts'/case;d.mkdir(parents=True)
(d/'result.json').write_text(json.dumps(dict(status='COMPLETE',label_arrays_decoded=[],eligible_unique=1,eligible_slots=2000)))
return 0
with patch.object(worker.s2_preprocess_pilot,'run',side_effect=geometric_run) as numerical:
self.assertEqual(worker.run(root,0),0)
numerical.assert_called_once_with(root,0)
self.assertFalse(list(root.rglob('*label*')))
def test_actual_geometry_cache_whitelist_never_opens_label(self):
with tempfile.TemporaryDirectory() as t:
path=Path(t)/'mixed.npz';np.savez(path,points_raw=np.ones((2,3)),fill_time_raw=np.array([999.]))
opened=[];original=ZipFile.open
def spy(z,name,*a,**kw):opened.append(name);return original(z,name,*a,**kw)
with patch.object(ZipFile,'open',spy):load_selected(path,['points_raw'],GEOMETRY_MEMBERS)
self.assertEqual(opened,['points_raw.npy'])
with self.assertRaises(ValueError):load_selected(path,['fill_time_raw'],GEOMETRY_MEMBERS)
def test_zero_eligible_queries_fail_without_fabricating_ready(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);case='empty'
(root/'contract.json').write_text(json.dumps(dict(cases=[dict(case=case,attempt=dict(decode_labels=False))])))
d=root/'parts'/case;d.mkdir(parents=True)
(d/'result.json').write_text(json.dumps(dict(status='COMPLETE',label_arrays_decoded=[],eligible_unique=0,eligible_slots=0)))
with patch.object(worker.s2_preprocess_pilot,'run',return_value=0):self.assertEqual(worker.run(root,0),3)
self.assertEqual(json.loads((d/'eligibility-failure.json').read_text())['status'],'NO_COMMON_ELIGIBLE_QUERIES')
def test_geometry_prefix_rejects_field_before_geometry_without_decode(self):
xml=b'<VTKFile byte_order="LittleEndian" compressor="vtkZLibDataCompressor"><UnstructuredGrid><Piece><PointData><DataArray Name="Fill_time"/></PointData></Piece></UnstructuredGrid></VTKFile>'
with patch('g1_candidate_fingerprints.decode_array',side_effect=AssertionError('label decode')) as decode:
with self.assertRaisesRegex(ValueError,'Field association'):read_tets(io.BytesIO(xml))
decode.assert_not_called()
def test_guards_apply_60_and_30_minutes_to_label_free_worker(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);c=synthetic_contract(root);calls=[]
def supervise(command,job,name,limits,**kw):
calls.append((command,limits));return dict(status='PASS')
with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator,'supervise',supervise):
self.assertEqual(coordinator.guard(root,0),0)
self.assertEqual(coordinator.guard(root,6),0)
self.assertEqual([x[1]['job_wall_seconds'] for x in calls],[3600,1800])
self.assertTrue(all(x[0][1]=='tools/s2_main_prepare.py' and x[1]['tree_rss_bytes']==1342177280 for x in calls))
def run_fake_batch(self,root,*,over_cpu=False,over_disk=False,monitor_fault=False):
c=synthetic_contract(root);(root/'contract.json').write_text(json.dumps(c))
spawned=[];fault=False;terminated=[];peak=0
class FakeProcess:
def __init__(self,i):self.index=i;self.pid=1000000+i;self.polls=0
def poll(self):
self.polls+=1
if self.polls==1:return None
p=root/'jobs'/f'job-{self.index:04d}'/'guard-result.json'
p.write_text(json.dumps(dict(worker=dict(status='PASS',returncode=0,stop_reason=None,cpu_seconds=.01),supervisor_cpu_seconds=.001)))
return 0
def wait(self):return 0
def spawn(command,**kwargs):
nonlocal peak
p=FakeProcess(int(command[-1]));spawned.append(p)
peak=max(peak,sum(p.polls<2 for p in spawned));return p
def disk(_):
nonlocal fault
if monitor_fault and spawned and not fault:fault=True;raise OSError('atomic replacement transient')
return 100*2**30 if over_disk else 100
with patch.object(coordinator,'contract',return_value=c),patch.object(coordinator.subprocess,'Popen',spawn),\
patch.object(coordinator,'disk_usage',disk),patch.object(coordinator,'tree_usage',return_value=(5000 if over_cpu else .01,100,1)),\
patch.object(coordinator.time,'sleep'),patch.object(coordinator,'terminate',side_effect=lambda a:terminated.append(list(a))),\
patch('freeze_s2_main_roster.freeze') as freeze,patch('builtins.print'):
coordinator.run(root)
return json.loads((root/'batch-complete.json').read_text()),spawned,peak,freeze.call_count
def test_131_terminal_and_transient_monitor_failure_without_retry(self):
with tempfile.TemporaryDirectory() as t:
r,spawned,peak,freezes=self.run_fake_batch(Path(t),monitor_fault=True)
self.assertEqual(r['status'],'ATTEMPTS_COMPLETE');self.assertEqual(len(spawned),131)
self.assertLessEqual(peak,16);self.assertEqual(r['monitor_errors'],1);self.assertEqual(freezes,1)
self.assertAlmostEqual(r['worker_cpu_seconds'],1.31)
self.assertAlmostEqual(r['cumulative_worker_cpu_seconds'],125134.74286+1.31)
def test_new_20h_cap_stops_and_does_not_charge_prior_34h(self):
with tempfile.TemporaryDirectory() as t:
r,spawned,_,freezes=self.run_fake_batch(Path(t),over_cpu=True)
self.assertEqual(r['stop_reason'],'GLOBAL_CPU_LIMIT');self.assertEqual(len(spawned),16)
self.assertEqual(len(r['not_run_indices']),115);self.assertEqual(freezes,0)
def test_cumulative_disk_cap_blocks_dispatch(self):
with tempfile.TemporaryDirectory() as t:
r,spawned,_,freezes=self.run_fake_batch(Path(t),over_disk=True)
self.assertEqual(r['stop_reason'],'GLOBAL_DISK_LIMIT');self.assertEqual(spawned,[]);self.assertEqual(freezes,0)
def test_existing_batch_marker_refuses_restart(self):
with tempfile.TemporaryDirectory() as t:
root=Path(t);(root/'batch-started.json').write_text('{}')
with patch.object(coordinator,'contract',return_value=synthetic_contract(root)),patch.object(coordinator.subprocess,'Popen') as spawn:
with self.assertRaises(AssertionError):coordinator.run(root)
spawn.assert_not_called()
def test_failed_part_excluded_globally_original_component_and_nested_pools(self):
parts=[dict(case=str(i),nodes=i+10,component_id='unchanged-'+str(i),assignments=[dict(fold=f,role='learning',learning_pools=['50','100']) for f in range(5)]) for i in range(3)]
jobs=[dict(case='1',worker=dict(status='PASS')),dict(case='2',worker=dict(status='STOPPED_OR_FAILED',stop_reason='wall_limit'))]
rows,counts=assemble(parts,{'0':dict(nodes=10)},jobs,{'1':dict(nodes=11)})
self.assertFalse(rows[2]['included']);self.assertEqual(rows[2]['component_id'],'unchanged-2')
self.assertEqual(rows[2]['assignments'],parts[2]['assignments'])
for row in counts:self.assertEqual(row['learning']['parts'],0 if row['pool']==25 else 2)
if __name__=='__main__':unittest.main(verbosity=2)