"""5002/5314 continuation: reused21, three explicit recoveries,407 new.""" import argparse,fcntl,json,os,resource,sys,time,subprocess,signal import numpy as np from pathlib import Path from phase2_b1213_gate import config,coverage from phase2_h1_worker_v2 import checked,save,sha from phase2_fixed_io import BASE,freeze from r4bc_guard import supervise from s2_recovery_supervisor import tree_usage from s2_recovery_monitoring import disk_usage def read(p):return json.loads(Path(p).read_text()) def validated(root): c=config(root) if c['continuation_approval']!='1794905002.631809' or c['explicit_recovery_indices']!=[16,21,22] or set(c['new_indices'])!=(set(range(431))-set(c['reused_indices'])):raise PermissionError('Only approved3 recoveries and407 never attempted') if len(c['reused_indices'])!=21 or len(c['new_indices'])!=410 or c['part_cpu_seconds']!=300 or c['part_wall_seconds']!=300 or c['total_cpu_cap']!=16000:raise PermissionError('Fixed scope/caps') if {role:len(rows) for role,rows in c['rows'].items()}!={'learning':320,'validation':111}:raise PermissionError('Original431 roles only') return c def guard(root,mode,index,slot,cap): c=validated(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]]);d=root/'guards'/f'{mode}-{index:04d}';start=time.process_time() cmd=[sys.executable,c['scientific_entry'],mode,str(root)]+([str(index)] if mode in ['field','prep'] else []) result=supervise(cmd,d,dict(cpu_seconds=cap,wall_seconds=c['part_wall_seconds'] if mode in ['field','prep'] else cap+60,rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes']),global_stop=root/'global-stop.json') save(d/'guard-result.json',dict(mode=mode,index=index,guard=result,guard_cpu_seconds=time.process_time()-start));return 0 if result['status']=='PASS' else 1 def resource_failure(guard,cap): return cap==300 and guard['status']=='FAILED' and not guard.get('monitor_errors') and guard.get('stop_reason') in ['cpu_limit','wall_limit','cpu_limit_after_exit','wall_limit_after_exit'] def run(root): c=validated(root);os.sched_setaffinity(0,[c['coordinator_cpu']]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (root/'started.json').exists():raise FileExistsError('Single labelled continuation') start=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN);settled=0.;records=[];active={};requested=[] def total(): u=resource.getrusage(resource.RUSAGE_CHILDREN);return c['prior_cpu_seconds']+60+max(settled,u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime)+time.process_time()-start+sum(tree_usage(p.pid)[0] for p,_,_ in active.values()) def batch(mode,ids,jobcap,geometry=False): nonlocal settled pending=list(ids) while pending or active: sample=dict(cpu_seconds=total(),disk_bytes=disk_usage(root),rss_bytes=sum(tree_usage(p.pid)[1] for p,_,_ in active.values()),signals=list(requested)) reasons=[] if requested:reasons.append('COORDINATOR_SIGNAL') if sample['cpu_seconds']>c['total_cpu_cap']-120:reasons.append('TOTAL_CPU_RESERVE') if geometry and sample['cpu_seconds']>c['geometry_cpu_cap']-120:reasons.append('GEOMETRY_CPU_RESERVE') if sample['disk_bytes']>c['disk_bytes']:reasons.append('DIRECTORY_DISK_LIMIT') if sample['rss_bytes']>32*2**30:reasons.append('TREE_RSS_LIMIT') if reasons: save(root/'stop-trigger.json',dict(reasons=reasons,sample=sample,unix=time.time()));raise RuntimeError('GLOBAL_RESOURCE_OR_SIGNAL_STOP') for slot,(p,index,cap) in list(active.items()): if p.poll() is not None: rr=read(root/'guards'/f'{mode}-{index:04d}'/'guard-result.json');settled+=rr['guard']['cpu_seconds']+rr['guard_cpu_seconds'];records.append(rr);del active[slot] if p.returncode or rr['guard']['status']!='PASS': if geometry and resource_failure(rr['guard'],cap): row=c['parts'][index];dest=root/'parts'/f'{index:04d}';np.save(dest/'extra.npy',np.full((row['nodes'],2),np.nan),allow_pickle=False) save(dest/'result.json',dict(status='RESOURCE_FAIL',reason=rr['guard']['stop_reason'],case=row['case'],component_id=row['component_id'],index=index,extra_sha256=sha(dest/'extra.npy'),partial_families_retained_not_merged=True,geometry_complete=False,labels_read=False,guard_receipt_sha256=sha(root/'guards'/f'{mode}-{index:04d}'/'guard-result.json'))) else:raise RuntimeError('WORKER_NOT_PASS') slots=range(len(c['worker_cpus'])) if geometry else range(1) for slot in slots: if pending and slot not in active: cap=min(jobcap,c['total_cpu_cap']-total()-sum(z[2] for z in active.values())-300) if geometry:cap=min(cap,c['geometry_cpu_cap']-total()-sum(z[2] for z in active.values())-120) if cap<1 or (geometry and cap