Add independent recorded-peak guard and freeze fresh main-matrix engineering plan

This commit is contained in:
Frank14f committed 2026-11-15 07:29:21 +08:00
1 parent 59fd40adba
commit 0af3dab8d1
16 files changed
+3342

No files matched your search

+133
View File
@@ -0,0 +1,133 @@
"""Independent future-run guard; no matrix launcher or training entry point.
Preserves the old pilot guard and its receipts. Checks recorded cumulative CUDA
peaks while the worker is alive and again after any exit, including failure.
"""
import hashlib
import json
import math
import os
from pathlib import Path
import threading
import time
import psutil
from s2_precision_continuation_guard import device_usage
from s2_recovery_gpu_supervisor import supervise, save, stop_group, disk_usage, tree_usage
ALLOCATOR_CONF='expandable_segments:True'
VRAM_BYTES=15*2**30
HOST_BYTES=16*2**30
DISK_BYTES=5*2**30
PEAK_KEYS=('peak_allocated_bytes','peak_reserved_bytes')
def peaks_in(value):
found={}
if isinstance(value,dict):
for key,child in value.items():
if key in PEAK_KEYS:
if isinstance(child,bool) or not isinstance(child,(int,float)) or not math.isfinite(child) or child<0:
raise ValueError('Invalid recorded CUDA peak: '+key)
found[key]=child
nested=peaks_in(child)
for k,v in nested.items():found[k]=max(found.get(k,0),v)
elif isinstance(value,list):
for child in value:
for k,v in peaks_in(child).items():found[k]=max(found.get(k,0),v)
return found
class PeakRecords:
def __init__(self):
self.maxima={k:0 for k in PEAK_KEYS};self.witnesses={};self.cache={};self.observations=[]
def scan(self,root):
# Includes progress, result and failure records, plus epoch/history JSON.
# Running maxima survive file replacement, regression and deletion.
for path in sorted(Path(root).rglob('*.json')):
try:
stat=path.stat();version=(stat.st_mtime_ns,stat.st_size,stat.st_ino)
if self.cache.get(str(path))==version:continue
raw=path.read_bytes()
except FileNotFoundError:continue
peaks=peaks_in(json.loads(raw));self.cache[str(path)]=version
if not peaks:continue
record=dict(path=str(path),sha256=hashlib.sha256(raw).hexdigest(),observed_unix=time.time(),peaks=peaks)
self.observations.append(record)
for key,value in peaks.items():
if key not in self.witnesses or value>self.maxima[key]:
self.maxima[key]=value;self.witnesses[key]=record
return max(self.maxima.values())>VRAM_BYTES
def as_dict(self):
return dict(maxima=self.maxima,witnesses=self.witnesses,observations=self.observations,
limit_bytes=VRAM_BYTES,breach=max(self.maxima.values())>VRAM_BYTES)
def active_command_pids(command):
result=[]
try:children=psutil.Process(os.getpid()).children(recursive=True)
except psutil.NoSuchProcess:return result
for p in children:
try:
if p.cmdline()==command:result.append(p.pid)
except (psutil.NoSuchProcess,psutil.AccessDenied):pass
return result
def guarded_worker(root,name,uuid,wall_limit,command,*,cumulative_worker_cpu_limit_seconds):
root=Path(root)
if os.environ.get('PYTORCH_CUDA_ALLOC_CONF')!=ALLOCATOR_CONF:
raise ValueError('Explicit PYTORCH_CUDA_ALLOC_CONF='+ALLOCATOR_CONF+' required before spawn')
assert 0<wall_limit<=1800 and cumulative_worker_cpu_limit_seconds>0
begin=time.monotonic();initial,util=device_usage(uuid)
if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN')
tracker=PeakRecords();done=threading.Event()
monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,reasons=[],errors=[],samples=0)
def reason(value):
if value not in monitor['reasons']:monitor['reasons'].append(value)
def scan_records():
try:
if tracker.scan(root/name):reason('RECORDED_VRAM_LIMIT')
except Exception as exc:
reason('RECORDED_MEMORY_READ_FAILED');monitor['errors'].append(str(exc))
def watch():
while not done.is_set():
scan_records()
try:
used,_=device_usage(uuid);monitor['samples']+=1
monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used)
if used>VRAM_BYTES:reason('DEVICE_VRAM_LIMIT')
_,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss)
if rss>HOST_BYTES:reason('WHOLE_RUN_HOST_RSS_LIMIT')
if disk_usage(root)>=DISK_BYTES:reason('RUN_DISK_LIMIT')
except Exception as exc:
reason('GPU_MONITOR_FAILED');monitor['errors'].append(str(exc))
if monitor['reasons']:
for pid in active_command_pids(command):stop_group(pid)
done.wait(.5)
thread=threading.Thread(target=watch,daemon=True);thread.start()
try:
remaining=wall_limit-(time.monotonic()-begin)-2.5
if remaining<=0:raise RuntimeError('No wall budget left before spawn')
receipt=supervise(command,root/'guards',name,dict(total_cpu_seconds=cumulative_worker_cpu_limit_seconds,
job_cpu_seconds=wall_limit,job_wall_seconds=remaining,tree_rss_bytes=HOST_BYTES,
disk_bytes=DISK_BYTES,max_processes=2),poll=.1,gpu_uuid=uuid)
finally:
done.set();thread.join(timeout=3)
if thread.is_alive():reason('GPU_MONITOR_DID_NOT_STOP')
else:scan_records() # Mandatory even if the worker failed or wrote no result.
result_path=root/name/'result.json';result=None
if result_path.exists():
try:result=json.loads(result_path.read_text())
except Exception as exc:reason('RESULT_READ_FAILED');monitor['errors'].append(str(exc))
if receipt['status']=='PASS' and (not result or result.get('status')!='COMPLETE'):reason('MISSING_COMPLETE_RESULT')
if receipt['status']=='PASS' and not all(k in tracker.witnesses for k in PEAK_KEYS):reason('MISSING_RECORDED_MEMORY_PEAKS')
elapsed=time.monotonic()-begin
if elapsed>wall_limit:reason('TOTAL_RUN_WALL_LIMIT')
receipt['underlying_worker_status']=receipt['status'];receipt['underlying_stop_reason']=receipt.get('stop_reason')
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
receipt.update(device_monitor=monitor,recorded_memory=tracker.as_dict(),total_run_wall_seconds=elapsed,
allocator_conf=os.environ['PYTORCH_CUDA_ALLOC_CONF'],gpu_uuid=uuid,
result_sha256=hashlib.sha256(result_path.read_bytes()).hexdigest() if result else None)
save(root/(name+'-guard-receipt.json'),receipt)
return receipt,result
+61
View File
@@ -0,0 +1,61 @@
import json
import os
import sys
from pathlib import Path
import tempfile
import threading
import unittest
from unittest.mock import patch
import s2_main_memory_guard as guard
class RecordedPeakTests(unittest.TestCase):
def test_only_progress_reserved_breach_stops_live_worker(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);(root/'run').mkdir();killed=threading.Event()
(root/'run/progress.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=guard.VRAM_BYTES+1)))
def supervise(*args,**kwargs):
self.assertTrue(killed.wait(2));return dict(status='STOPPED_OR_FAILED',stop_reason=None,returncode=-9)
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)),patch.object(guard,'supervise',side_effect=supervise),patch.object(guard,'active_command_pids',return_value=[123456789]),patch.object(guard,'stop_group',side_effect=lambda pid:killed.set()):
receipt,result=guard.guarded_worker(root,'run','synthetic',20,['synthetic-command'],cumulative_worker_cpu_limit_seconds=20)
self.assertIsNone(result);self.assertEqual(receipt['stop_reason'],'RECORDED_VRAM_LIMIT')
self.assertEqual(receipt['recorded_memory']['maxima']['peak_reserved_bytes'],guard.VRAM_BYTES+1)
def test_failed_worker_final_scan_catches_breach_without_result(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);(root/'run').mkdir()
def supervise(*args,**kwargs):
(root/'run/progress.json').write_text(json.dumps(dict(peak_allocated_bytes=100,peak_reserved_bytes=guard.VRAM_BYTES+3)))
return dict(status='STOPPED_OR_FAILED',stop_reason=None,returncode=1)
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)),patch.object(guard,'supervise',side_effect=supervise),patch.object(guard,'active_command_pids',return_value=[]):
receipt,result=guard.guarded_worker(root,'run','synthetic',20,['synthetic-command'],cumulative_worker_cpu_limit_seconds=20)
self.assertEqual(receipt['stop_reason'],'RECORDED_VRAM_LIMIT');self.assertEqual(receipt['returncode'],1)
self.assertIsNone(result)
def test_maximum_survives_lower_replacement_and_deletion(self):
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp)/'progress.json';tracker=guard.PeakRecords()
p.write_text(json.dumps(dict(peak_reserved_bytes=guard.VRAM_BYTES+1)));self.assertTrue(tracker.scan(tmp))
p.write_text(json.dumps(dict(peak_reserved_bytes=7)));self.assertTrue(tracker.scan(tmp))
p.unlink();self.assertTrue(tracker.scan(tmp));self.assertEqual(tracker.maxima['peak_reserved_bytes'],guard.VRAM_BYTES+1)
def test_nested_history_and_result_both_contribute(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);(root/'history.json').write_text(json.dumps([dict(peak_reserved_bytes=20)]))
(root/'result.json').write_text(json.dumps(dict(peak_allocated_bytes=30,peak_reserved_bytes=10)))
tracker=guard.PeakRecords();self.assertFalse(tracker.scan(root));self.assertEqual(tracker.maxima,dict(peak_allocated_bytes=30,peak_reserved_bytes=20))
def test_invalid_recorded_peak_rejected(self):
for value in [float('nan'),-1,True,'100']:
with self.subTest(value=value),self.assertRaises(ValueError):guard.peaks_in(dict(peak_reserved_bytes=value))
def test_allocator_is_required_before_device_or_worker_call(self):
with patch.dict(os.environ,{},clear=True),patch.object(guard,'device_usage') as device:
with self.assertRaises(ValueError):guard.guarded_worker(Path('/unused'),'run','synthetic',20,[],cumulative_worker_cpu_limit_seconds=20)
device.assert_not_called()
def test_allocator_reaches_supervised_cpu_child_without_cuda_work(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp)
code="import os,json,pathlib; assert os.environ['PYTORCH_CUDA_ALLOC_CONF']=='expandable_segments:True'; assert os.environ['CUDA_VISIBLE_DEVICES']=='synthetic'; p=pathlib.Path('"+str(root/ 'run')+"'); p.mkdir(); (p/'result.json').write_text(json.dumps(dict(status='COMPLETE',peak_allocated_bytes=0,peak_reserved_bytes=0)))"
with patch.dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=guard.ALLOCATOR_CONF),patch.object(guard,'device_usage',return_value=(5*2**20,0)),patch.object(guard,'tree_usage',return_value=(0,1024,1)):
receipt,result=guard.guarded_worker(root,'run','synthetic',20,[sys.executable,'-c',code],cumulative_worker_cpu_limit_seconds=20)
self.assertEqual(receipt['status'],'PASS');self.assertEqual(result['status'],'COMPLETE')
self.assertEqual(receipt['allocator_conf'],guard.ALLOCATOR_CONF)
if __name__=='__main__':unittest.main(verbosity=2)