134 lines
6.5 KiB
Python
134 lines
6.5 KiB
Python
"""Independent future-run guard; no matrix launcher or training entry point.
|
|
|
|
Preserves the old pilot guard and its receipts. Checks recorded cumulative CUDA
|
|
peaks while the worker is alive and again after any exit, including failure.
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import threading
|
|
import time
|
|
import psutil
|
|
|
|
from s2_precision_continuation_guard import device_usage
|
|
from s2_recovery_gpu_supervisor import supervise, save, stop_group, disk_usage, tree_usage
|
|
|
|
ALLOCATOR_CONF='expandable_segments:True'
|
|
VRAM_BYTES=15*2**30
|
|
HOST_BYTES=16*2**30
|
|
DISK_BYTES=5*2**30
|
|
PEAK_KEYS=('peak_allocated_bytes','peak_reserved_bytes')
|
|
|
|
|
|
def peaks_in(value):
|
|
found={}
|
|
if isinstance(value,dict):
|
|
for key,child in value.items():
|
|
if key in PEAK_KEYS:
|
|
if isinstance(child,bool) or not isinstance(child,(int,float)) or not math.isfinite(child) or child<0:
|
|
raise ValueError('Invalid recorded CUDA peak: '+key)
|
|
found[key]=child
|
|
nested=peaks_in(child)
|
|
for k,v in nested.items():found[k]=max(found.get(k,0),v)
|
|
elif isinstance(value,list):
|
|
for child in value:
|
|
for k,v in peaks_in(child).items():found[k]=max(found.get(k,0),v)
|
|
return found
|
|
|
|
|
|
class PeakRecords:
|
|
def __init__(self):
|
|
self.maxima={k:0 for k in PEAK_KEYS};self.witnesses={};self.cache={};self.observations=[]
|
|
def scan(self,root):
|
|
# Includes progress, result and failure records, plus epoch/history JSON.
|
|
# Running maxima survive file replacement, regression and deletion.
|
|
for path in sorted(Path(root).rglob('*.json')):
|
|
try:
|
|
stat=path.stat();version=(stat.st_mtime_ns,stat.st_size,stat.st_ino)
|
|
if self.cache.get(str(path))==version:continue
|
|
raw=path.read_bytes()
|
|
except FileNotFoundError:continue
|
|
peaks=peaks_in(json.loads(raw));self.cache[str(path)]=version
|
|
if not peaks:continue
|
|
record=dict(path=str(path),sha256=hashlib.sha256(raw).hexdigest(),observed_unix=time.time(),peaks=peaks)
|
|
self.observations.append(record)
|
|
for key,value in peaks.items():
|
|
if key not in self.witnesses or value>self.maxima[key]:
|
|
self.maxima[key]=value;self.witnesses[key]=record
|
|
return max(self.maxima.values())>VRAM_BYTES
|
|
def as_dict(self):
|
|
return dict(maxima=self.maxima,witnesses=self.witnesses,observations=self.observations,
|
|
limit_bytes=VRAM_BYTES,breach=max(self.maxima.values())>VRAM_BYTES)
|
|
|
|
|
|
def active_command_pids(command):
|
|
result=[]
|
|
try:children=psutil.Process(os.getpid()).children(recursive=True)
|
|
except psutil.NoSuchProcess:return result
|
|
for p in children:
|
|
try:
|
|
if p.cmdline()==command:result.append(p.pid)
|
|
except (psutil.NoSuchProcess,psutil.AccessDenied):pass
|
|
return result
|
|
|
|
|
|
def guarded_worker(root,name,uuid,wall_limit,command,*,cumulative_worker_cpu_limit_seconds):
|
|
root=Path(root)
|
|
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!=ALLOCATOR_CONF:
|
|
raise ValueError('Explicit PYTORCH_CUDA_ALLOC_CONF='+ALLOCATOR_CONF+' required before spawn')
|
|
assert 0<wall_limit<=1800 and cumulative_worker_cpu_limit_seconds>0
|
|
begin=time.monotonic();initial,util=device_usage(uuid)
|
|
if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN')
|
|
tracker=PeakRecords();done=threading.Event()
|
|
monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,reasons=[],errors=[],samples=0)
|
|
def reason(value):
|
|
if value not in monitor['reasons']:monitor['reasons'].append(value)
|
|
def scan_records():
|
|
try:
|
|
if tracker.scan(root/name):reason('RECORDED_VRAM_LIMIT')
|
|
except Exception as exc:
|
|
reason('RECORDED_MEMORY_READ_FAILED');monitor['errors'].append(str(exc))
|
|
def watch():
|
|
while not done.is_set():
|
|
scan_records()
|
|
try:
|
|
used,_=device_usage(uuid);monitor['samples']+=1
|
|
monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used)
|
|
if used>VRAM_BYTES:reason('DEVICE_VRAM_LIMIT')
|
|
_,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss)
|
|
if rss>HOST_BYTES:reason('WHOLE_RUN_HOST_RSS_LIMIT')
|
|
if disk_usage(root)>=DISK_BYTES:reason('RUN_DISK_LIMIT')
|
|
except Exception as exc:
|
|
reason('GPU_MONITOR_FAILED');monitor['errors'].append(str(exc))
|
|
if monitor['reasons']:
|
|
for pid in active_command_pids(command):stop_group(pid)
|
|
done.wait(.5)
|
|
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
|
try:
|
|
remaining=wall_limit-(time.monotonic()-begin)-2.5
|
|
if remaining<=0:raise RuntimeError('No wall budget left before spawn')
|
|
receipt=supervise(command,root/'guards',name,dict(total_cpu_seconds=cumulative_worker_cpu_limit_seconds,
|
|
job_cpu_seconds=wall_limit,job_wall_seconds=remaining,tree_rss_bytes=HOST_BYTES,
|
|
disk_bytes=DISK_BYTES,max_processes=2),poll=.1,gpu_uuid=uuid)
|
|
finally:
|
|
done.set();thread.join(timeout=3)
|
|
if thread.is_alive():reason('GPU_MONITOR_DID_NOT_STOP')
|
|
else:scan_records() # Mandatory even if the worker failed or wrote no result.
|
|
result_path=root/name/'result.json';result=None
|
|
if result_path.exists():
|
|
try:result=json.loads(result_path.read_text())
|
|
except Exception as exc:reason('RESULT_READ_FAILED');monitor['errors'].append(str(exc))
|
|
if receipt['status']=='PASS' and (not result or result.get('status')!='COMPLETE'):reason('MISSING_COMPLETE_RESULT')
|
|
if receipt['status']=='PASS' and not all(k in tracker.witnesses for k in PEAK_KEYS):reason('MISSING_RECORDED_MEMORY_PEAKS')
|
|
elapsed=time.monotonic()-begin
|
|
if elapsed>wall_limit:reason('TOTAL_RUN_WALL_LIMIT')
|
|
receipt['underlying_worker_status']=receipt['status'];receipt['underlying_stop_reason']=receipt.get('stop_reason')
|
|
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
|
|
receipt.update(device_monitor=monitor,recorded_memory=tracker.as_dict(),total_run_wall_seconds=elapsed,
|
|
allocator_conf=os.environ['PYTORCH_CUDA_ALLOC_CONF'],gpu_uuid=uuid,
|
|
result_sha256=hashlib.sha256(result_path.read_bytes()).hexdigest() if result else None)
|
|
save(root/(name+'-guard-receipt.json'),receipt)
|
|
return receipt,result
|