"""Sequential four-run pilot guard: process-tree RSS/wall plus device VRAM.""" import argparse import fcntl import json import os from pathlib import Path import subprocess import sys import threading import time import psutil from freeze_g1_components import sha from s2_recovery_gpu_supervisor import supervise,save,stop_group,disk_usage,tree_usage from s2_pilot_train import select_capacity def parse_device_usage(text): lines=text.strip().splitlines() if len(lines)!=1:raise ValueError('Expected exactly one GPU') fields=lines[0].split(',') if len(fields)!=2:raise ValueError('Malformed GPU usage row') memory,util=map(lambda s:int(s.strip()),fields) if memory<0 or not 0<=util<=100:raise ValueError('Invalid GPU counters') return memory*2**20,util def device_usage(uuid): r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=2) return parse_device_usage(r.stdout) def active_training_pids(parent,root,index): ids=[] try:children=psutil.Process(parent).children(recursive=True) except psutil.NoSuchProcess:return ids for p in children: try: cmd=p.cmdline() if str(root) in cmd and (('tools/run_s2_pilot_training.py' in cmd and str(index)==cmd[-1]) or (index==-1 and 'tools/run_s2_pilot_probe.py' in cmd)):ids.append(p.pid) except psutil.NoSuchProcess:pass return ids def guarded_worker(root,spec,i,uuid,wall_limit,command): run_start=time.monotonic();initial,util=device_usage(uuid) if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN') done=threading.Event();monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,max_poll_gap_seconds=0.,reason=None,samples=0) def watch(): previous=time.monotonic() while not done.is_set(): now=time.monotonic();monitor['max_poll_gap_seconds']=max(monitor['max_poll_gap_seconds'],now-previous);previous=now try: used,_=device_usage(uuid);monitor['samples']+=1;monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used) if used>15*2**30:monitor['reason']='DEVICE_VRAM_LIMIT' _,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss) if rss>16*2**30:monitor['reason']='WHOLE_RUN_HOST_RSS_LIMIT' if disk_usage(root)>=5*2**30:monitor['reason']='PILOT_DISK_LIMIT' except Exception as exc:monitor['reason']='GPU_MONITOR_FAILED';monitor['error']=str(exc) if monitor['reason']: for pid in active_training_pids(os.getpid(),root,i):stop_group(pid) done.wait(.5) thread=threading.Thread(target=watch,daemon=True);thread.start() try: wall=wall_limit-(time.monotonic()-run_start)-2.5 if wall<=0:raise RuntimeError('No budget left before spawn') r=supervise(command,root/'guards',spec['name'],dict(total_cpu_seconds=7200,job_cpu_seconds=wall_limit,job_wall_seconds=wall, tree_rss_bytes=16*2**30,disk_bytes=5*2**30,max_processes=2),poll=.1,gpu_uuid=uuid) finally: done.set();thread.join(timeout=3) total_wall=time.monotonic()-run_start result_path=root/spec['name']/'result.json' result=json.loads(result_path.read_text()) if result_path.exists() else None if monitor['reason']:r.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reason']) if total_wall>wall_limit:r.update(status='STOPPED_OR_FAILED',stop_reason='TOTAL_RUN_WALL_LIMIT') if result and max(result['peak_allocated_bytes'],result['peak_reserved_bytes'])>15*2**30:r.update(status='STOPPED_OR_FAILED',stop_reason='RECORDED_VRAM_LIMIT') r.update(gpu_visible=True,gpu_uuid=uuid,device_monitor=monitor,total_run_wall_seconds=total_wall, result_sha256=sha(result_path) if result else None) save(root/(spec['name']+'-guard-receipt.json'),r) return r,result def run(root,prior_wall_seconds=0.): with (root/'pilot.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) return run_locked(root,prior_wall_seconds) def run_locked(root,prior_wall_seconds=0.): c=json.loads((root/'training-contract.json').read_text());uuid=c['gpu_uuid'] prior_wall_seconds=max(prior_wall_seconds,c.get('probe_guard_wall_seconds',0.)) assert [(r['arm'],r['width'],r['seed']) for r in c['runs']]==[('N',128,0),('E',128,0),('N',256,0),('E',256,0)] for p,h in c['code_sha256'].items():assert sha(p)==h,p assert not (root/'pilot-started.json').exists(),'No automatic pilot retry' start=time.monotonic();save(root/'pilot-started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json'))) rows=[];scores={};reason=None for i,spec in enumerate(c['runs']): remaining=7200-prior_wall_seconds-(time.monotonic()-start) if remaining<=3:reason='TOTAL_PILOT_WALL_LIMIT';break command=[sys.executable,'tools/run_s2_pilot_training.py',str(root),str(i)] try:r,result=guarded_worker(root,spec,i,uuid,min(1800,remaining),command) except Exception as exc:reason=str(exc);break rows.append(dict(run=spec,**r)) if r['status']!='PASS' or not result or result['status']!='COMPLETE':reason='PILOT_RUN_FAILED_NO_RETRY';break scores[(spec['arm'],spec['width'])]=result['best_validation_score'] if prior_wall_seconds+time.monotonic()-start>7200:reason='TOTAL_PILOT_WALL_LIMIT_AFTER_EXIT' selected=select_capacity(scores) if len(scores)==4 and reason is None else None summary=dict(status='FOUR_RUNS_COMPLETE' if selected else 'PILOT_INCOMPLETE',reason=reason,runs=rows,selected_global_width=selected, selection_rule='mean of N/E validation error at each width, ties smaller; all four runs required', pilot_wall_seconds=prior_wall_seconds+time.monotonic()-start,probe_and_setup_wall_seconds=prior_wall_seconds,test_labels_read=False,test_predictions=False,retries=0) save(root/'pilot-complete.json',summary);print(json.dumps(summary),flush=True) if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)