115 lines
6.2 KiB
Python
115 lines
6.2 KiB
Python
"""Sequential four-run pilot guard: process-tree RSS/wall plus device VRAM."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import psutil
|
|
from freeze_g1_components import sha
|
|
from s2_gpu_process_supervisor import supervise,save,stop_group,disk_usage,tree_usage
|
|
from s2_pilot_train import select_capacity
|
|
|
|
|
|
def parse_device_usage(text):
|
|
lines=text.strip().splitlines()
|
|
if len(lines)!=1:raise ValueError('Expected exactly one GPU')
|
|
fields=lines[0].split(',')
|
|
if len(fields)!=2:raise ValueError('Malformed GPU usage row')
|
|
memory,util=map(lambda s:int(s.strip()),fields)
|
|
if memory<0 or not 0<=util<=100:raise ValueError('Invalid GPU counters')
|
|
return memory*2**20,util
|
|
|
|
|
|
def device_usage(uuid):
|
|
r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=2)
|
|
return parse_device_usage(r.stdout)
|
|
|
|
|
|
def active_training_pids(parent,root,index):
|
|
ids=[]
|
|
try:children=psutil.Process(parent).children(recursive=True)
|
|
except psutil.NoSuchProcess:return ids
|
|
for p in children:
|
|
try:
|
|
cmd=p.cmdline()
|
|
if str(root) in cmd and (('tools/run_s2_pilot_training.py' in cmd and str(index)==cmd[-1]) or (index==-1 and 'tools/run_s2_pilot_probe.py' in cmd)):ids.append(p.pid)
|
|
except psutil.NoSuchProcess:pass
|
|
return ids
|
|
|
|
|
|
def guarded_worker(root,spec,i,uuid,wall_limit,command):
|
|
run_start=time.monotonic();initial,util=device_usage(uuid)
|
|
if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN')
|
|
done=threading.Event();monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,max_poll_gap_seconds=0.,reason=None,samples=0)
|
|
def watch():
|
|
previous=time.monotonic()
|
|
while not done.is_set():
|
|
now=time.monotonic();monitor['max_poll_gap_seconds']=max(monitor['max_poll_gap_seconds'],now-previous);previous=now
|
|
try:
|
|
used,_=device_usage(uuid);monitor['samples']+=1;monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used)
|
|
if used>15*2**30:monitor['reason']='DEVICE_VRAM_LIMIT'
|
|
_,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss)
|
|
if rss>16*2**30:monitor['reason']='WHOLE_RUN_HOST_RSS_LIMIT'
|
|
if disk_usage(root)>=5*2**30:monitor['reason']='PILOT_DISK_LIMIT'
|
|
except Exception as exc:monitor['reason']='GPU_MONITOR_FAILED';monitor['error']=str(exc)
|
|
if monitor['reason']:
|
|
for pid in active_training_pids(os.getpid(),root,i):stop_group(pid)
|
|
done.wait(.5)
|
|
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
|
try:
|
|
wall=wall_limit-(time.monotonic()-run_start)-2.5
|
|
if wall<=0:raise RuntimeError('No budget left before spawn')
|
|
r=supervise(command,root/'guards',spec['name'],dict(total_cpu_seconds=7200,job_cpu_seconds=wall_limit,job_wall_seconds=wall,
|
|
tree_rss_bytes=16*2**30,disk_bytes=5*2**30,max_processes=2),poll=.1,gpu_uuid=uuid)
|
|
finally:
|
|
done.set();thread.join(timeout=3)
|
|
total_wall=time.monotonic()-run_start
|
|
result_path=root/spec['name']/'result.json'
|
|
result=json.loads(result_path.read_text()) if result_path.exists() else None
|
|
if monitor['reason']:r.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reason'])
|
|
if total_wall>wall_limit:r.update(status='STOPPED_OR_FAILED',stop_reason='TOTAL_RUN_WALL_LIMIT')
|
|
if result and max(result['peak_allocated_bytes'],result['peak_reserved_bytes'])>15*2**30:r.update(status='STOPPED_OR_FAILED',stop_reason='RECORDED_VRAM_LIMIT')
|
|
r.update(gpu_visible=True,gpu_uuid=uuid,device_monitor=monitor,total_run_wall_seconds=total_wall,
|
|
result_sha256=sha(result_path) if result else None)
|
|
save(root/(spec['name']+'-guard-receipt.json'),r)
|
|
return r,result
|
|
|
|
|
|
def run(root,prior_wall_seconds=0.):
|
|
with (root/'pilot.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
return run_locked(root,prior_wall_seconds)
|
|
|
|
|
|
def run_locked(root,prior_wall_seconds=0.):
|
|
c=json.loads((root/'training-contract.json').read_text());uuid=c['gpu_uuid']
|
|
prior_wall_seconds=max(prior_wall_seconds,c.get('probe_guard_wall_seconds',0.))
|
|
assert [(r['arm'],r['width'],r['seed']) for r in c['runs']]==[('N',128,0),('E',128,0),('N',256,0),('E',256,0)]
|
|
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
|
assert not (root/'pilot-started.json').exists(),'No automatic pilot retry'
|
|
start=time.monotonic();save(root/'pilot-started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
|
|
rows=[];scores={};reason=None
|
|
for i,spec in enumerate(c['runs']):
|
|
remaining=7200-prior_wall_seconds-(time.monotonic()-start)
|
|
if remaining<=3:reason='TOTAL_PILOT_WALL_LIMIT';break
|
|
command=[sys.executable,'tools/run_s2_pilot_training.py',str(root),str(i)]
|
|
try:r,result=guarded_worker(root,spec,i,uuid,min(1800,remaining),command)
|
|
except Exception as exc:reason=str(exc);break
|
|
rows.append(dict(run=spec,**r))
|
|
if r['status']!='PASS' or not result or result['status']!='COMPLETE':reason='PILOT_RUN_FAILED_NO_RETRY';break
|
|
scores[(spec['arm'],spec['width'])]=result['best_validation_score']
|
|
if prior_wall_seconds+time.monotonic()-start>7200:reason='TOTAL_PILOT_WALL_LIMIT_AFTER_EXIT'
|
|
selected=select_capacity(scores) if len(scores)==4 and reason is None else None
|
|
summary=dict(status='FOUR_RUNS_COMPLETE' if selected else 'PILOT_INCOMPLETE',reason=reason,runs=rows,selected_global_width=selected,
|
|
selection_rule='mean of N/E validation error at each width, ties smaller; all four runs required',
|
|
pilot_wall_seconds=prior_wall_seconds+time.monotonic()-start,probe_and_setup_wall_seconds=prior_wall_seconds,test_labels_read=False,test_predictions=False,retries=0)
|
|
save(root/'pilot-complete.json',summary);print(json.dumps(summary),flush=True)
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)
|