86 lines
5.0 KiB
Python
86 lines
5.0 KiB
Python
"""Track A postfit guard: bounded selected-GPU wait, no retries."""
|
|
import json,os,signal,sys,threading,time
|
|
import psutil
|
|
from phase2_h1_worker_v2 import save
|
|
from s2_main_memory_guard import PeakRecords
|
|
from s2_precision_continuation_guard import device_usage
|
|
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
|
|
|
|
def final_reasons(reasons,receipt,result,tracker,gpu_seconds,cpu_seconds,cpu_cap,expected_parts):
|
|
out=list(reasons)
|
|
def add(s):
|
|
if s not in out:out.append(s)
|
|
if not receipt or receipt['status']!='PASS':add('WORKER_NOT_PASS')
|
|
if receipt and receipt.get('monitor_errors'):add('WORKER_MONITOR_ERRORS')
|
|
if not result or result.get('status')!='COMPLETE' or result.get('parts')!=expected_parts:add('INCOMPLETE_FOLD')
|
|
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):add('MISSING_PEAKS')
|
|
if any(v>15*2**30 for v in tracker.maxima.values()):add('RECORDED_VRAM_LIMIT')
|
|
if gpu_seconds>900 or cpu_seconds>cpu_cap:add('FINAL_RESOURCE_LIMIT')
|
|
return out
|
|
|
|
def run_prediction(queue,c,pilot,fold,prior_gpu,cpu_cap):
|
|
root=queue/'jobs'/str(fold);gpu_uuid=c['gpu_uuid'];expected_parts=next(j['expected_parts'] for j in c['jobs'] if j['id']==fold)
|
|
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
|
|
def elapsed():return 60+time.monotonic()-start
|
|
def reason(x):
|
|
if x not in reasons:reasons.append(x)
|
|
command=[sys.executable,'tools/track_a_postfit_rebind_worker.py','predict',str(queue),str(fold)]
|
|
def kill():
|
|
for p in psutil.Process().children(recursive=True):
|
|
try:
|
|
if p.cmdline()==command:stop_group(p.pid)
|
|
except psutil.NoSuchProcess:pass
|
|
def scan():
|
|
if tracker.scan(root/'prediction-records'):reason('RECORDED_VRAM_LIMIT')
|
|
def watch():
|
|
while not done.is_set():
|
|
try:
|
|
scan();mem,util=device_usage(gpu_uuid);_,rss,_=tree_usage(os.getpid())
|
|
if mem>15*2**30:reason('DEVICE_VRAM_LIMIT')
|
|
if rss>16*2**30:reason('TREE_RSS_LIMIT')
|
|
if disk_usage(queue)>20*2**30:reason('DISK_LIMIT')
|
|
if elapsed()>=895:reason('PREDICTION_TIME_LIMIT')
|
|
for x in requested:reason(x)
|
|
save(root/'prediction-live.json',dict(gpu_seconds_charged=elapsed(),device_bytes=mem,device_utilization=util,rss_bytes=rss,peaks=tracker.maxima,reasons=reasons,unix=time.time()))
|
|
except Exception as exc:reason('MONITOR_ERROR');errors.append(str(exc))
|
|
if reasons:kill()
|
|
done.wait(.5)
|
|
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGTERM,signal.SIGINT]};thread=None
|
|
try:
|
|
wait_start=time.monotonic();waiting=[]
|
|
while True:
|
|
mem,util=device_usage(gpu_uuid);waiting.append(dict(unix=time.time(),bytes=mem,utilization=util))
|
|
save(root/'recovery-idle-wait.json',dict(samples=waiting,limit_seconds=c['idle_wait_seconds'],worker_started=False))
|
|
if requested:raise RuntimeError('Signal received during idle wait')
|
|
duration=time.monotonic()-wait_start
|
|
if duration>c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
|
|
if mem<=128*2**20 and util==0:break
|
|
if duration>=c['idle_wait_seconds']:raise RuntimeError('SELECTED_GPU_BUSY_AFTER_BOUNDED_WAIT_NO_RETRY')
|
|
time.sleep(min(.5,max(0.,c['idle_wait_seconds']-(time.monotonic()-wait_start))))
|
|
for i in range(3):
|
|
mem,util=device_usage(gpu_uuid);samples.append(dict(unix=time.time(),bytes=mem,utilization=util))
|
|
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY_NO_RETRY')
|
|
if i<2:time.sleep(2)
|
|
save(root/'prediction-idle-polls.json',samples)
|
|
if requested:raise RuntimeError('Signal received during setup')
|
|
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
|
remaining=900-elapsed()-10
|
|
if remaining<=0:raise RuntimeError('Setup exhausted prediction budget')
|
|
receipt=supervise(command,root/'prediction-guard','predict',dict(total_cpu_seconds=cpu_cap-60,job_cpu_seconds=cpu_cap-60,job_wall_seconds=remaining,tree_rss_bytes=16*2**30,disk_bytes=20*2**30,max_processes=1),poll=.2,gpu_uuid=gpu_uuid)
|
|
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill()
|
|
finally:
|
|
done.set()
|
|
if thread:thread.join(timeout=4)
|
|
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
|
|
try:scan()
|
|
except Exception as exc:reason('FINAL_SCAN_ERROR');errors.append(str(exc))
|
|
for s,h in old.items():signal.signal(s,h)
|
|
result=None;p=root/'prediction-records/result.json'
|
|
try:
|
|
if p.exists():result=json.loads(p.read_text())
|
|
except Exception as exc:reason('FINAL_RESULT_READ_ERROR');errors.append(str(exc))
|
|
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
|
|
duration=elapsed();reasons=final_reasons(reasons,receipt,result,tracker,duration,totalcpu,cpu_cap,expected_parts)
|
|
out=dict(status='PASS' if not reasons else 'STOPPED',reasons=reasons,monitor_errors=errors,worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=duration,cpu_seconds_charged=totalcpu,cumulative_track_a_gpu_seconds=prior_gpu+duration,no_retry=True)
|
|
save(root/'prediction-guard.json',out);return out
|