"""Independent future-run guard; no matrix launcher or training entry point. Preserves the old pilot guard and its receipts. Checks recorded cumulative CUDA peaks while the worker is alive and again after any exit, including failure. """ import hashlib import json import math import os from pathlib import Path import threading import time import psutil from s2_precision_continuation_guard import device_usage from s2_recovery_gpu_supervisor import supervise, save, stop_group, disk_usage, tree_usage ALLOCATOR_CONF='expandable_segments:True' VRAM_BYTES=15*2**30 HOST_BYTES=16*2**30 DISK_BYTES=5*2**30 PEAK_KEYS=('peak_allocated_bytes','peak_reserved_bytes') def peaks_in(value): found={} if isinstance(value,dict): for key,child in value.items(): if key in PEAK_KEYS: if isinstance(child,bool) or not isinstance(child,(int,float)) or not math.isfinite(child) or child<0: raise ValueError('Invalid recorded CUDA peak: '+key) found[key]=child nested=peaks_in(child) for k,v in nested.items():found[k]=max(found.get(k,0),v) elif isinstance(value,list): for child in value: for k,v in peaks_in(child).items():found[k]=max(found.get(k,0),v) return found class PeakRecords: def __init__(self): self.maxima={k:0 for k in PEAK_KEYS};self.witnesses={};self.cache={};self.observations=[] def scan(self,root): # Includes progress, result and failure records, plus epoch/history JSON. # Running maxima survive file replacement, regression and deletion. for path in sorted(Path(root).rglob('*.json')): try: stat=path.stat();version=(stat.st_mtime_ns,stat.st_size,stat.st_ino) if self.cache.get(str(path))==version:continue raw=path.read_bytes() except FileNotFoundError:continue peaks=peaks_in(json.loads(raw));self.cache[str(path)]=version if not peaks:continue record=dict(path=str(path),sha256=hashlib.sha256(raw).hexdigest(),observed_unix=time.time(),peaks=peaks) self.observations.append(record) for key,value in peaks.items(): if key not in self.witnesses or value>self.maxima[key]: self.maxima[key]=value;self.witnesses[key]=record return max(self.maxima.values())>VRAM_BYTES def as_dict(self): return dict(maxima=self.maxima,witnesses=self.witnesses,observations=self.observations, limit_bytes=VRAM_BYTES,breach=max(self.maxima.values())>VRAM_BYTES) def active_command_pids(command): result=[] try:children=psutil.Process(os.getpid()).children(recursive=True) except psutil.NoSuchProcess:return result for p in children: try: if p.cmdline()==command:result.append(p.pid) except (psutil.NoSuchProcess,psutil.AccessDenied):pass return result def guarded_worker(root,name,uuid,wall_limit,command,*,cumulative_worker_cpu_limit_seconds): root=Path(root) if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!=ALLOCATOR_CONF: raise ValueError('Explicit PYTORCH_CUDA_ALLOC_CONF='+ALLOCATOR_CONF+' required before spawn') assert 00 begin=time.monotonic();initial,util=device_usage(uuid) if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN') tracker=PeakRecords();done=threading.Event() monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,reasons=[],errors=[],samples=0) def reason(value): if value not in monitor['reasons']:monitor['reasons'].append(value) def scan_records(): try: if tracker.scan(root/name):reason('RECORDED_VRAM_LIMIT') except Exception as exc: reason('RECORDED_MEMORY_READ_FAILED');monitor['errors'].append(str(exc)) def watch(): while not done.is_set(): scan_records() try: used,_=device_usage(uuid);monitor['samples']+=1 monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used) if used>VRAM_BYTES:reason('DEVICE_VRAM_LIMIT') _,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss) if rss>HOST_BYTES:reason('WHOLE_RUN_HOST_RSS_LIMIT') if disk_usage(root)>=DISK_BYTES:reason('RUN_DISK_LIMIT') except Exception as exc: reason('GPU_MONITOR_FAILED');monitor['errors'].append(str(exc)) if monitor['reasons']: for pid in active_command_pids(command):stop_group(pid) done.wait(.5) thread=threading.Thread(target=watch,daemon=True);thread.start() try: remaining=wall_limit-(time.monotonic()-begin)-2.5 if remaining<=0:raise RuntimeError('No wall budget left before spawn') receipt=supervise(command,root/'guards',name,dict(total_cpu_seconds=cumulative_worker_cpu_limit_seconds, job_cpu_seconds=wall_limit,job_wall_seconds=remaining,tree_rss_bytes=HOST_BYTES, disk_bytes=DISK_BYTES,max_processes=2),poll=.1,gpu_uuid=uuid) finally: done.set();thread.join(timeout=3) if thread.is_alive():reason('GPU_MONITOR_DID_NOT_STOP') else:scan_records() # Mandatory even if the worker failed or wrote no result. result_path=root/name/'result.json';result=None if result_path.exists(): try:result=json.loads(result_path.read_text()) except Exception as exc:reason('RESULT_READ_FAILED');monitor['errors'].append(str(exc)) if receipt['status']=='PASS' and (not result or result.get('status')!='COMPLETE'):reason('MISSING_COMPLETE_RESULT') if receipt['status']=='PASS' and not all(k in tracker.witnesses for k in PEAK_KEYS):reason('MISSING_RECORDED_MEMORY_PEAKS') elapsed=time.monotonic()-begin if elapsed>wall_limit:reason('TOTAL_RUN_WALL_LIMIT') receipt['underlying_worker_status']=receipt['status'];receipt['underlying_stop_reason']=receipt.get('stop_reason') if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0]) receipt.update(device_monitor=monitor,recorded_memory=tracker.as_dict(),total_run_wall_seconds=elapsed, allocator_conf=os.environ['PYTORCH_CUDA_ALLOC_CONF'],gpu_uuid=uuid, result_sha256=hashlib.sha256(result_path.read_bytes()).hexdigest() if result else None) save(root/(name+'-guard-receipt.json'),receipt) return receipt,result