134 lines
7.9 KiB
Python
134 lines
7.9 KiB
Python
"""Bounded, no-retry held-out inference queue; first-fold cost checkpoint."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import psutil
|
|
from freeze_g1_components import sha
|
|
from s2_custody_common import contract,read_json,test_rows
|
|
from s2_recovery_supervisor import save,stop_group,tree_usage
|
|
from s2_recovery_gpu_supervisor import supervise
|
|
from s2_recovery_monitoring import disk_usage
|
|
from s2_main_memory_guard import PeakRecords,device_usage,active_command_pids
|
|
|
|
|
|
def guard(root,fold,slot):
|
|
c=contract(root);limits=c['limits'];uuid=c['gpu_uuids'][slot]
|
|
os.sched_setaffinity(0,c['cpu_affinities'][slot]);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True'
|
|
start=time.monotonic();dest=root/'guards'/str(fold);dest.mkdir(parents=True,exist_ok=False)
|
|
command=[sys.executable,'tools/run_s2_custody_predict.py',str(root),str(fold)]
|
|
done=threading.Event();tracker=PeakRecords();monitor=dict(reasons=[],child_processes={},peak_device_bytes=0,peak_tree_rss_bytes=0)
|
|
receipt=None
|
|
def reason(s):
|
|
if s not in monitor['reasons']:monitor['reasons'].append(s)
|
|
def watch():
|
|
while not done.is_set():
|
|
try:
|
|
used,_=device_usage(uuid);monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],used)
|
|
if used>limits['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
|
|
_,rss,_=tree_usage(os.getpid());monitor['peak_tree_rss_bytes']=max(monitor['peak_tree_rss_bytes'],rss)
|
|
if rss>limits['host_bytes']:reason('TREE_RSS_LIMIT')
|
|
if tracker.scan(root/'folds'/str(fold)):reason('RECORDED_VRAM_LIMIT')
|
|
if disk_usage(root/'predictions'/str(fold))+disk_usage(root/'folds'/str(fold))>limits['fold_disk_bytes']:reason('FOLD_DISK_LIMIT')
|
|
for p in psutil.Process().children(recursive=True):
|
|
try:monitor['child_processes'][str(p.pid)]=dict(ppid=p.ppid(),cmdline=p.cmdline())
|
|
except psutil.NoSuchProcess:pass
|
|
except Exception as exc:reason('MONITOR_ERROR:'+str(exc))
|
|
if monitor['reasons']:
|
|
for pid in active_command_pids(command):stop_group(pid)
|
|
done.wait(1)
|
|
try:
|
|
for _ in range(3):
|
|
used,util=device_usage(uuid)
|
|
if used>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY')
|
|
time.sleep(1)
|
|
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
|
try:
|
|
receipt=supervise(command,dest,'worker',dict(total_cpu_seconds=86400,job_cpu_seconds=86400,
|
|
job_wall_seconds=limits['fold_wall_seconds']-(time.monotonic()-start)-5,tree_rss_bytes=limits['host_bytes'],
|
|
disk_bytes=limits['fold_disk_bytes'],max_processes=2),poll=.2,gpu_uuid=uuid)
|
|
finally:done.set();thread.join(timeout=5)
|
|
if thread.is_alive():reason('MONITOR_DID_NOT_STOP')
|
|
if tracker.scan(root/'folds'/str(fold)):reason('RECORDED_VRAM_LIMIT')
|
|
result=root/'folds'/str(fold)/'result.json'
|
|
if not result.exists() or read_json(result).get('status')!='COMPLETE':reason('MISSING_COMPLETE_RESULT')
|
|
if time.monotonic()-start>limits['fold_wall_seconds']:reason('TOTAL_FOLD_WALL_LIMIT')
|
|
if monitor['reasons'] or receipt['status']!='PASS':return 1
|
|
return 0
|
|
except Exception as exc:reason(type(exc).__name__+':'+str(exc));return 1
|
|
finally:
|
|
save(dest/'guard-result.json',dict(status='PASS' if receipt and receipt['status']=='PASS' and not monitor['reasons'] else 'STOPPED_OR_FAILED',
|
|
worker=receipt,monitor=monitor,recorded_memory=tracker.as_dict(),wall_seconds=time.monotonic()-start,fold=fold))
|
|
|
|
|
|
def run(root):
|
|
c=contract(root);limits=c['limits']
|
|
approval=read_json(root/'main-review.json');assert approval['approved'] and approval['contract_sha256']==sha(root/'contract.json')
|
|
marker=root/'predictions-started.json'
|
|
with marker.open('x') as f:json.dump(dict(pid=os.getpid(),unix=time.time()),f)
|
|
active={};completed=[];next_fold=0;charged=0.;first=False;stop=None
|
|
def launch(fold,slot):
|
|
log=(root/f'fold-{fold}-guard.log').open('x')
|
|
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(fold),'--slot',str(slot)],stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
|
log.close();active[slot]=dict(process=p,fold=fold,start=time.monotonic())
|
|
try:
|
|
while next_fold<5 or active:
|
|
for slot,v in list(active.items()):
|
|
ret=v['process'].poll()
|
|
if ret is None:continue
|
|
cost=time.monotonic()-v['start'];charged+=cost
|
|
p=root/'guards'/str(v['fold'])/'guard-result.json'
|
|
if ret or not p.exists() or read_json(p)['status']!='PASS':stop='FOLD_FAILED_'+str(v['fold'])
|
|
completed.append(dict(fold=v['fold'],charged_gpu_seconds=cost,returncode=ret));del active[slot]
|
|
if completed and not first and not active and stop is None:
|
|
nodes=[sum(r['nodes'] for r in test_rows(c,f)) for f in range(5)]
|
|
projection=charged+1.25*charged*sum(nodes[1:])/nodes[0]
|
|
save(root/'first-fold-cost.json',dict(charged_gpu_seconds=charged,projection_gpu_seconds=projection,
|
|
heuristic='first-fold full cost per test node;25percent reserve',continue_within_cap=projection<=limits['gpu_seconds']))
|
|
if projection>limits['gpu_seconds']:stop='FIRST_FOLD_COST_EXCEEDS_CAP'
|
|
first=True
|
|
live=sum(time.monotonic()-v['start'] for v in active.values())
|
|
if charged+live>limits['gpu_seconds']:stop='AGGREGATE_GPU_LIMIT'
|
|
if disk_usage(root)>limits['disk_bytes']:stop='TOTAL_DISK_LIMIT'
|
|
if stop:break
|
|
for slot in range(4):
|
|
if next_fold>=5 or slot in active or (not first and next_fold>0):continue
|
|
# Every dispatched job reserves its entire approved lifetime.
|
|
if charged+(len(active)+1)*(limits['fold_wall_seconds']+10)>limits['gpu_seconds']:
|
|
stop='INSUFFICIENT_RESERVED_GPU_BUDGET';break
|
|
launch(next_fold,slot);next_fold+=1
|
|
save(root/'predictions-progress.json',dict(completed=completed,active={s:v['fold'] for s,v in active.items()},charged_gpu_seconds=charged,
|
|
live_gpu_seconds=live,first_fold_passed=first,stop_reason=stop))
|
|
time.sleep(1)
|
|
finally:
|
|
for v in active.values():
|
|
# Stop descendants before the guard so none can outlive the queue.
|
|
try:
|
|
children=psutil.Process(v['process'].pid).children(recursive=True)
|
|
for p in reversed(children):
|
|
try:p.kill()
|
|
except psutil.NoSuchProcess:pass
|
|
except psutil.NoSuchProcess:pass
|
|
stop_group(v['process'].pid);v['process'].wait();charged+=time.monotonic()-v['start']
|
|
if stop:
|
|
save(root/'predictions-stopped.json',dict(reason=stop,completed=completed,charged_gpu_seconds=charged,labels_decoded=False));return 1
|
|
index={}
|
|
for f in range(5):
|
|
p=root/'folds'/str(f)/'index.json';d=read_json(root/'folds'/str(f)/'result.json');assert sha(p)==d['index_sha256']
|
|
index.update(read_json(p))
|
|
for p,h in index.items():assert sha(p)==h
|
|
save(root/'prediction-index.json',index)
|
|
save(root/'predictions-complete.json',dict(status='ALL_PREDICTIONS_FROZEN',contract_sha256=sha(root/'contract.json'),
|
|
prediction_index_sha256=sha(root/'prediction-index.json'),files=len(index),charged_gpu_seconds=charged,
|
|
neural_models=89,neural_NA=1,r2_models=15,B0=True,labels_decoded=False,model_fit=False))
|
|
return 0
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int);a=p.parse_args()
|
|
raise SystemExit(guard(a.root,a.guard,a.slot) if a.guard is not None else run(a.root))
|