Files
MeltMind/tools/run_s2_custody_predictions.py
T

134 lines
7.9 KiB
Python

"""Bounded, no-retry held-out inference queue; first-fold cost checkpoint."""
import argparse
import json
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
import psutil
from freeze_g1_components import sha
from s2_custody_common import contract,read_json,test_rows
from s2_recovery_supervisor import save,stop_group,tree_usage
from s2_recovery_gpu_supervisor import supervise
from s2_recovery_monitoring import disk_usage
from s2_main_memory_guard import PeakRecords,device_usage,active_command_pids
def guard(root,fold,slot):
c=contract(root);limits=c['limits'];uuid=c['gpu_uuids'][slot]
os.sched_setaffinity(0,c['cpu_affinities'][slot]);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True'
start=time.monotonic();dest=root/'guards'/str(fold);dest.mkdir(parents=True,exist_ok=False)
command=[sys.executable,'tools/run_s2_custody_predict.py',str(root),str(fold)]
done=threading.Event();tracker=PeakRecords();monitor=dict(reasons=[],child_processes={},peak_device_bytes=0,peak_tree_rss_bytes=0)
receipt=None
def reason(s):
if s not in monitor['reasons']:monitor['reasons'].append(s)
def watch():
while not done.is_set():
try:
used,_=device_usage(uuid);monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],used)
if used>limits['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
_,rss,_=tree_usage(os.getpid());monitor['peak_tree_rss_bytes']=max(monitor['peak_tree_rss_bytes'],rss)
if rss>limits['host_bytes']:reason('TREE_RSS_LIMIT')
if tracker.scan(root/'folds'/str(fold)):reason('RECORDED_VRAM_LIMIT')
if disk_usage(root/'predictions'/str(fold))+disk_usage(root/'folds'/str(fold))>limits['fold_disk_bytes']:reason('FOLD_DISK_LIMIT')
for p in psutil.Process().children(recursive=True):
try:monitor['child_processes'][str(p.pid)]=dict(ppid=p.ppid(),cmdline=p.cmdline())
except psutil.NoSuchProcess:pass
except Exception as exc:reason('MONITOR_ERROR:'+str(exc))
if monitor['reasons']:
for pid in active_command_pids(command):stop_group(pid)
done.wait(1)
try:
for _ in range(3):
used,util=device_usage(uuid)
if used>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY')
time.sleep(1)
thread=threading.Thread(target=watch,daemon=True);thread.start()
try:
receipt=supervise(command,dest,'worker',dict(total_cpu_seconds=86400,job_cpu_seconds=86400,
job_wall_seconds=limits['fold_wall_seconds']-(time.monotonic()-start)-5,tree_rss_bytes=limits['host_bytes'],
disk_bytes=limits['fold_disk_bytes'],max_processes=2),poll=.2,gpu_uuid=uuid)
finally:done.set();thread.join(timeout=5)
if thread.is_alive():reason('MONITOR_DID_NOT_STOP')
if tracker.scan(root/'folds'/str(fold)):reason('RECORDED_VRAM_LIMIT')
result=root/'folds'/str(fold)/'result.json'
if not result.exists() or read_json(result).get('status')!='COMPLETE':reason('MISSING_COMPLETE_RESULT')
if time.monotonic()-start>limits['fold_wall_seconds']:reason('TOTAL_FOLD_WALL_LIMIT')
if monitor['reasons'] or receipt['status']!='PASS':return 1
return 0
except Exception as exc:reason(type(exc).__name__+':'+str(exc));return 1
finally:
save(dest/'guard-result.json',dict(status='PASS' if receipt and receipt['status']=='PASS' and not monitor['reasons'] else 'STOPPED_OR_FAILED',
worker=receipt,monitor=monitor,recorded_memory=tracker.as_dict(),wall_seconds=time.monotonic()-start,fold=fold))
def run(root):
c=contract(root);limits=c['limits']
approval=read_json(root/'main-review.json');assert approval['approved'] and approval['contract_sha256']==sha(root/'contract.json')
marker=root/'predictions-started.json'
with marker.open('x') as f:json.dump(dict(pid=os.getpid(),unix=time.time()),f)
active={};completed=[];next_fold=0;charged=0.;first=False;stop=None
def launch(fold,slot):
log=(root/f'fold-{fold}-guard.log').open('x')
p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(fold),'--slot',str(slot)],stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
log.close();active[slot]=dict(process=p,fold=fold,start=time.monotonic())
try:
while next_fold<5 or active:
for slot,v in list(active.items()):
ret=v['process'].poll()
if ret is None:continue
cost=time.monotonic()-v['start'];charged+=cost
p=root/'guards'/str(v['fold'])/'guard-result.json'
if ret or not p.exists() or read_json(p)['status']!='PASS':stop='FOLD_FAILED_'+str(v['fold'])
completed.append(dict(fold=v['fold'],charged_gpu_seconds=cost,returncode=ret));del active[slot]
if completed and not first and not active and stop is None:
nodes=[sum(r['nodes'] for r in test_rows(c,f)) for f in range(5)]
projection=charged+1.25*charged*sum(nodes[1:])/nodes[0]
save(root/'first-fold-cost.json',dict(charged_gpu_seconds=charged,projection_gpu_seconds=projection,
heuristic='first-fold full cost per test node;25percent reserve',continue_within_cap=projection<=limits['gpu_seconds']))
if projection>limits['gpu_seconds']:stop='FIRST_FOLD_COST_EXCEEDS_CAP'
first=True
live=sum(time.monotonic()-v['start'] for v in active.values())
if charged+live>limits['gpu_seconds']:stop='AGGREGATE_GPU_LIMIT'
if disk_usage(root)>limits['disk_bytes']:stop='TOTAL_DISK_LIMIT'
if stop:break
for slot in range(4):
if next_fold>=5 or slot in active or (not first and next_fold>0):continue
# Every dispatched job reserves its entire approved lifetime.
if charged+(len(active)+1)*(limits['fold_wall_seconds']+10)>limits['gpu_seconds']:
stop='INSUFFICIENT_RESERVED_GPU_BUDGET';break
launch(next_fold,slot);next_fold+=1
save(root/'predictions-progress.json',dict(completed=completed,active={s:v['fold'] for s,v in active.items()},charged_gpu_seconds=charged,
live_gpu_seconds=live,first_fold_passed=first,stop_reason=stop))
time.sleep(1)
finally:
for v in active.values():
# Stop descendants before the guard so none can outlive the queue.
try:
children=psutil.Process(v['process'].pid).children(recursive=True)
for p in reversed(children):
try:p.kill()
except psutil.NoSuchProcess:pass
except psutil.NoSuchProcess:pass
stop_group(v['process'].pid);v['process'].wait();charged+=time.monotonic()-v['start']
if stop:
save(root/'predictions-stopped.json',dict(reason=stop,completed=completed,charged_gpu_seconds=charged,labels_decoded=False));return 1
index={}
for f in range(5):
p=root/'folds'/str(f)/'index.json';d=read_json(root/'folds'/str(f)/'result.json');assert sha(p)==d['index_sha256']
index.update(read_json(p))
for p,h in index.items():assert sha(p)==h
save(root/'prediction-index.json',index)
save(root/'predictions-complete.json',dict(status='ALL_PREDICTIONS_FROZEN',contract_sha256=sha(root/'contract.json'),
prediction_index_sha256=sha(root/'prediction-index.json'),files=len(index),charged_gpu_seconds=charged,
neural_models=89,neural_NA=1,r2_models=15,B0=True,labels_decoded=False,model_fit=False))
return 0
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int);a=p.parse_args()
raise SystemExit(guard(a.root,a.guard,a.slot) if a.guard is not None else run(a.root))