"""Bounded, no-retry held-out inference queue; first-fold cost checkpoint.""" import argparse import json import os from pathlib import Path import subprocess import sys import threading import time import psutil from freeze_g1_components import sha from s2_custody_common import contract,read_json,test_rows from s2_recovery_supervisor import save,stop_group,tree_usage from s2_recovery_gpu_supervisor import supervise from s2_recovery_monitoring import disk_usage from s2_main_memory_guard import PeakRecords,device_usage,active_command_pids def guard(root,fold,slot): c=contract(root);limits=c['limits'];uuid=c['gpu_uuids'][slot] os.sched_setaffinity(0,c['cpu_affinities'][slot]);os.environ['PYTORCH_CUDA_ALLOC_CONF']='expandable_segments:True' start=time.monotonic();dest=root/'guards'/str(fold);dest.mkdir(parents=True,exist_ok=False) command=[sys.executable,'tools/run_s2_custody_predict.py',str(root),str(fold)] done=threading.Event();tracker=PeakRecords();monitor=dict(reasons=[],child_processes={},peak_device_bytes=0,peak_tree_rss_bytes=0) receipt=None def reason(s): if s not in monitor['reasons']:monitor['reasons'].append(s) def watch(): while not done.is_set(): try: used,_=device_usage(uuid);monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],used) if used>limits['vram_bytes']:reason('DEVICE_VRAM_LIMIT') _,rss,_=tree_usage(os.getpid());monitor['peak_tree_rss_bytes']=max(monitor['peak_tree_rss_bytes'],rss) if rss>limits['host_bytes']:reason('TREE_RSS_LIMIT') if tracker.scan(root/'folds'/str(fold)):reason('RECORDED_VRAM_LIMIT') if disk_usage(root/'predictions'/str(fold))+disk_usage(root/'folds'/str(fold))>limits['fold_disk_bytes']:reason('FOLD_DISK_LIMIT') for p in psutil.Process().children(recursive=True): try:monitor['child_processes'][str(p.pid)]=dict(ppid=p.ppid(),cmdline=p.cmdline()) except psutil.NoSuchProcess:pass except Exception as exc:reason('MONITOR_ERROR:'+str(exc)) if monitor['reasons']: for pid in active_command_pids(command):stop_group(pid) done.wait(1) try: for _ in range(3): used,util=device_usage(uuid) if used>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY') time.sleep(1) thread=threading.Thread(target=watch,daemon=True);thread.start() try: receipt=supervise(command,dest,'worker',dict(total_cpu_seconds=86400,job_cpu_seconds=86400, job_wall_seconds=limits['fold_wall_seconds']-(time.monotonic()-start)-5,tree_rss_bytes=limits['host_bytes'], disk_bytes=limits['fold_disk_bytes'],max_processes=2),poll=.2,gpu_uuid=uuid) finally:done.set();thread.join(timeout=5) if thread.is_alive():reason('MONITOR_DID_NOT_STOP') if tracker.scan(root/'folds'/str(fold)):reason('RECORDED_VRAM_LIMIT') result=root/'folds'/str(fold)/'result.json' if not result.exists() or read_json(result).get('status')!='COMPLETE':reason('MISSING_COMPLETE_RESULT') if time.monotonic()-start>limits['fold_wall_seconds']:reason('TOTAL_FOLD_WALL_LIMIT') if monitor['reasons'] or receipt['status']!='PASS':return 1 return 0 except Exception as exc:reason(type(exc).__name__+':'+str(exc));return 1 finally: save(dest/'guard-result.json',dict(status='PASS' if receipt and receipt['status']=='PASS' and not monitor['reasons'] else 'STOPPED_OR_FAILED', worker=receipt,monitor=monitor,recorded_memory=tracker.as_dict(),wall_seconds=time.monotonic()-start,fold=fold)) def run(root): c=contract(root);limits=c['limits'] approval=read_json(root/'main-review.json');assert approval['approved'] and approval['contract_sha256']==sha(root/'contract.json') marker=root/'predictions-started.json' with marker.open('x') as f:json.dump(dict(pid=os.getpid(),unix=time.time()),f) active={};completed=[];next_fold=0;charged=0.;first=False;stop=None def launch(fold,slot): log=(root/f'fold-{fold}-guard.log').open('x') p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(fold),'--slot',str(slot)],stdout=log,stderr=subprocess.STDOUT,start_new_session=True) log.close();active[slot]=dict(process=p,fold=fold,start=time.monotonic()) try: while next_fold<5 or active: for slot,v in list(active.items()): ret=v['process'].poll() if ret is None:continue cost=time.monotonic()-v['start'];charged+=cost p=root/'guards'/str(v['fold'])/'guard-result.json' if ret or not p.exists() or read_json(p)['status']!='PASS':stop='FOLD_FAILED_'+str(v['fold']) completed.append(dict(fold=v['fold'],charged_gpu_seconds=cost,returncode=ret));del active[slot] if completed and not first and not active and stop is None: nodes=[sum(r['nodes'] for r in test_rows(c,f)) for f in range(5)] projection=charged+1.25*charged*sum(nodes[1:])/nodes[0] save(root/'first-fold-cost.json',dict(charged_gpu_seconds=charged,projection_gpu_seconds=projection, heuristic='first-fold full cost per test node;25percent reserve',continue_within_cap=projection<=limits['gpu_seconds'])) if projection>limits['gpu_seconds']:stop='FIRST_FOLD_COST_EXCEEDS_CAP' first=True live=sum(time.monotonic()-v['start'] for v in active.values()) if charged+live>limits['gpu_seconds']:stop='AGGREGATE_GPU_LIMIT' if disk_usage(root)>limits['disk_bytes']:stop='TOTAL_DISK_LIMIT' if stop:break for slot in range(4): if next_fold>=5 or slot in active or (not first and next_fold>0):continue # Every dispatched job reserves its entire approved lifetime. if charged+(len(active)+1)*(limits['fold_wall_seconds']+10)>limits['gpu_seconds']: stop='INSUFFICIENT_RESERVED_GPU_BUDGET';break launch(next_fold,slot);next_fold+=1 save(root/'predictions-progress.json',dict(completed=completed,active={s:v['fold'] for s,v in active.items()},charged_gpu_seconds=charged, live_gpu_seconds=live,first_fold_passed=first,stop_reason=stop)) time.sleep(1) finally: for v in active.values(): # Stop descendants before the guard so none can outlive the queue. try: children=psutil.Process(v['process'].pid).children(recursive=True) for p in reversed(children): try:p.kill() except psutil.NoSuchProcess:pass except psutil.NoSuchProcess:pass stop_group(v['process'].pid);v['process'].wait();charged+=time.monotonic()-v['start'] if stop: save(root/'predictions-stopped.json',dict(reason=stop,completed=completed,charged_gpu_seconds=charged,labels_decoded=False));return 1 index={} for f in range(5): p=root/'folds'/str(f)/'index.json';d=read_json(root/'folds'/str(f)/'result.json');assert sha(p)==d['index_sha256'] index.update(read_json(p)) for p,h in index.items():assert sha(p)==h save(root/'prediction-index.json',index) save(root/'predictions-complete.json',dict(status='ALL_PREDICTIONS_FROZEN',contract_sha256=sha(root/'contract.json'), prediction_index_sha256=sha(root/'prediction-index.json'),files=len(index),charged_gpu_seconds=charged, neural_models=89,neural_NA=1,r2_models=15,B0=True,labels_decoded=False,model_fit=False)) return 0 if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int);a=p.parse_args() raise SystemExit(guard(a.root,a.guard,a.slot) if a.guard is not None else run(a.root))