Freeze hash-bound zero-run v2 continuation with recorded idle settling

This commit is contained in:
Frank14f committed 2026-11-15 06:33:44 +08:00
1 parent a15f2882ee
commit ea5ac05191
8 files changed
+366

No files matched your search

+57
View File
@@ -0,0 +1,57 @@
"""Engineering continuation of zero-run v2; reuse probe, preserve both stops."""
import argparse
import fcntl
import json
from pathlib import Path
import time
import traceback
from freeze_g1_components import sha
from s2_e1_supervisor import save
from s2_gpu_idle_gate import inventory, recorded_wait
from s2_precision_continuation_guard import run as train_runs
def run(root):
with (root/'continuation.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
assert not (root/'started.json').exists(),'No automatic continuation retry'
contract=json.loads((root/'continuation-contract.json').read_text())
for p,h in contract['code_sha256'].items():assert sha(p)==h,p
for p,h in contract['source_sha256'].items():assert sha(p)==h,p
source=Path(contract['source']);old=json.loads((source/'training-contract.json').read_text())
ended=json.loads((source/'pilot-complete.json').read_text())
assert ended['status']=='PILOT_INCOMPLETE' and ended['reason']=='SELECTED_GPU_BUSY_BEFORE_RUN' and ended['runs']==[]
previous=json.loads((source/'orchestrator-complete.json').read_text())['total_wall_seconds']
assert previous==contract['prior_wall_seconds'] and old['E_max']==old['cosine_T_max']==42
assert sha(old['timing_probe_receipt'])==old['timing_probe_sha256']
started=time.monotonic();save(root/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'continuation-contract.json'),prior_wall_seconds=previous))
try:
names=inventory();selected=old['gpu_uuid'];assert 'V100' in names[selected]
gate=recorded_wait(root,'initial-selected',selected,min(600,7200-previous-(time.monotonic()-started)-5))
switched=False
if gate['status']!='IDLE_CONFIRMED':
if not gate['last'].get('occupants'):raise RuntimeError('Selected GPU failed idle gate without identified foreign occupant; no switch')
for uuid,name in sorted(names.items()):
if uuid==selected or name!=names[selected]:continue
remaining=7200-previous-(time.monotonic()-started)-5
if remaining<=20:break
alternative=recorded_wait(root,'alternative-'+uuid,uuid,min(600,remaining))
if alternative['status']=='IDLE_CONFIRMED':selected=uuid;switched=True;break
if not switched:raise RuntimeError('No identical idle V100 after foreign-occupant timeout')
c=dict(old);c['gpu_uuid']=selected;c['code_sha256']=dict(old['code_sha256'],**contract['code_sha256'])
c['engineering_continuation']=dict(source=str(source),source_sha256=contract['source_sha256'],
prior_wall_seconds=previous,probe_reused=True,probe_rerun=False,gpu_switched=switched,
original_gpu_uuid=old['gpu_uuid'],selected_gpu_uuid=selected,ruling='1794695421.177269')
save(root/'training-contract.json',c)
train_runs(root,prior_wall_seconds=previous+time.monotonic()-started)
result=json.loads((root/'pilot-complete.json').read_text())
save(root/'complete.json',dict(status=result['status'],selected_global_width=result['selected_global_width'],
cumulative_v2_wall_seconds=previous+time.monotonic()-started,prior_wall_seconds=previous,
limit_seconds=7200,probe_reused=True,probe_rerun=False,test_labels_read=False))
except Exception as exc:
save(root/'failure.json',dict(status='CONTINUATION_STOPPED_NO_RETRY',error=str(exc),traceback=traceback.format_exc(),
cumulative_v2_wall_seconds=previous+time.monotonic()-started,prior_wall_seconds=previous,limit_seconds=7200));raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)
+53
View File
@@ -0,0 +1,53 @@
"""Bounded, recorded idle observations; never signal foreign processes."""
import csv
import json
import subprocess
import time
from pathlib import Path
import psutil
def inventory():
r=subprocess.run(['nvidia-smi','--query-gpu=uuid,name','--format=csv,noheader'],check=True,capture_output=True,text=True,timeout=3)
return {row[0].strip():row[1].strip() for row in csv.reader(r.stdout.splitlines())}
def snapshot(uuid):
r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3)
rows=list(csv.reader(r.stdout.splitlines()));assert len(rows)==1
memory,util=[int(v.strip()) for v in rows[0]];assert memory>=0 and 0<=util<=100
r=subprocess.run(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,process_name,used_memory','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=3)
occupants=[]
for row in csv.reader(r.stdout.splitlines()):
if not row or row[0].strip()!=uuid:continue
pid=int(row[1]);user=None
try:user=psutil.Process(pid).username()
except (psutil.NoSuchProcess,psutil.AccessDenied):pass
occupants.append(dict(pid=pid,user=user,process_name=row[2].strip(),memory_MiB=row[3].strip()))
return dict(unix=time.time(),uuid=uuid,memory_MiB=memory,utilization_percent=util,occupants=occupants,
idle=memory<=128 and util==0 and not occupants)
def wait_idle(uuid,limit_seconds,record,*,sample=snapshot,clock=time.monotonic,sleep=time.sleep):
begin=clock();consecutive=0;count=0;last=None
while True:
try:last=sample(uuid)
except Exception as exc:
last=dict(uuid=uuid,unix=time.time(),idle=False,error=str(exc),occupants=[])
count+=1;consecutive=consecutive+1 if last['idle'] else 0
record(dict(**last,poll=count,consecutive_idle=consecutive,elapsed_seconds=clock()-begin))
if clock()-begin>limit_seconds:break
if consecutive>=3:return dict(status='IDLE_CONFIRMED',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last)
remaining=limit_seconds-(clock()-begin)
if remaining<=0:break
sleep(min(10,remaining))
return dict(status='IDLE_TIMEOUT',uuid=uuid,polls=count,consecutive_idle=consecutive,wall_seconds=clock()-begin,last=last)
def recorded_wait(root,name,uuid,limit_seconds):
path=Path(root)/'idle-gates';path.mkdir(exist_ok=True)
def record(row):
with (path/(name+'.jsonl')).open('a') as f:f.write(json.dumps(row)+'\n')
result=wait_idle(uuid,limit_seconds,record)
with (path/(name+'.json')).open('x') as f:json.dump(result,f,indent=2);f.write('\n')
return result
+122
View File
@@ -0,0 +1,122 @@
"""Sequential four-run pilot guard: process-tree RSS/wall plus device VRAM."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
import psutil
from freeze_g1_components import sha
from s2_recovery_gpu_supervisor import supervise,save,stop_group,disk_usage,tree_usage
from s2_pilot_train import select_capacity
from s2_gpu_idle_gate import recorded_wait
def parse_device_usage(text):
lines=text.strip().splitlines()
if len(lines)!=1:raise ValueError('Expected exactly one GPU')
fields=lines[0].split(',')
if len(fields)!=2:raise ValueError('Malformed GPU usage row')
memory,util=map(lambda s:int(s.strip()),fields)
if memory<0 or not 0<=util<=100:raise ValueError('Invalid GPU counters')
return memory*2**20,util
def device_usage(uuid):
r=subprocess.run(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],check=True,capture_output=True,text=True,timeout=2)
return parse_device_usage(r.stdout)
def active_training_pids(parent,root,index):
ids=[]
try:children=psutil.Process(parent).children(recursive=True)
except psutil.NoSuchProcess:return ids
for p in children:
try:
cmd=p.cmdline()
if str(root) in cmd and (('tools/run_s2_pilot_training_v2.py' in cmd and str(index)==cmd[-1]) or (index==-1 and 'tools/run_s2_pilot_probe_v2.py' in cmd)):ids.append(p.pid)
except psutil.NoSuchProcess:pass
return ids
def guarded_worker(root,spec,i,uuid,wall_limit,command):
run_start=time.monotonic();initial,util=device_usage(uuid)
if initial>128*2**20 or util>0:raise RuntimeError('SELECTED_GPU_BUSY_BEFORE_RUN')
done=threading.Event();monitor=dict(peak_device_memory_bytes=initial,peak_whole_run_tree_rss_bytes=0,max_poll_gap_seconds=0.,reason=None,samples=0)
def watch():
previous=time.monotonic()
while not done.is_set():
now=time.monotonic();monitor['max_poll_gap_seconds']=max(monitor['max_poll_gap_seconds'],now-previous);previous=now
try:
used,_=device_usage(uuid);monitor['samples']+=1;monitor['peak_device_memory_bytes']=max(monitor['peak_device_memory_bytes'],used)
if used>15*2**30:monitor['reason']='DEVICE_VRAM_LIMIT'
_,rss,_=tree_usage(os.getpid());monitor['peak_whole_run_tree_rss_bytes']=max(monitor['peak_whole_run_tree_rss_bytes'],rss)
if rss>16*2**30:monitor['reason']='WHOLE_RUN_HOST_RSS_LIMIT'
if disk_usage(root)>=5*2**30:monitor['reason']='PILOT_DISK_LIMIT'
except Exception as exc:monitor['reason']='GPU_MONITOR_FAILED';monitor['error']=str(exc)
if monitor['reason']:
for pid in active_training_pids(os.getpid(),root,i):stop_group(pid)
done.wait(.5)
thread=threading.Thread(target=watch,daemon=True);thread.start()
try:
wall=wall_limit-(time.monotonic()-run_start)-2.5
if wall<=0:raise RuntimeError('No budget left before spawn')
r=supervise(command,root/'guards',spec['name'],dict(total_cpu_seconds=7200,job_cpu_seconds=wall_limit,job_wall_seconds=wall,
tree_rss_bytes=16*2**30,disk_bytes=5*2**30,max_processes=2),poll=.1,gpu_uuid=uuid)
finally:
done.set();thread.join(timeout=3)
total_wall=time.monotonic()-run_start
result_path=root/spec['name']/'result.json'
result=json.loads(result_path.read_text()) if result_path.exists() else None
if monitor['reason']:r.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reason'])
if total_wall>wall_limit:r.update(status='STOPPED_OR_FAILED',stop_reason='TOTAL_RUN_WALL_LIMIT')
if result and max(result['peak_allocated_bytes'],result['peak_reserved_bytes'])>15*2**30:r.update(status='STOPPED_OR_FAILED',stop_reason='RECORDED_VRAM_LIMIT')
r.update(gpu_visible=True,gpu_uuid=uuid,device_monitor=monitor,total_run_wall_seconds=total_wall,
result_sha256=sha(result_path) if result else None)
save(root/(spec['name']+'-guard-receipt.json'),r)
return r,result
def run(root,prior_wall_seconds=0.):
with (root/'pilot.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return run_locked(root,prior_wall_seconds)
def run_locked(root,prior_wall_seconds=0.):
c=json.loads((root/'training-contract.json').read_text());uuid=c['gpu_uuid']
prior_wall_seconds=max(prior_wall_seconds,c.get('probe_guard_wall_seconds',0.))
assert [(r['arm'],r['width'],r['seed']) for r in c['runs']]==[('N',128,0),('E',128,0),('N',256,0),('E',256,0)]
for p,h in c['code_sha256'].items():assert sha(p)==h,p
assert not (root/'pilot-started.json').exists(),'No automatic pilot retry'
start=time.monotonic();save(root/'pilot-started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
rows=[];scores={};reason=None
for i,spec in enumerate(c['runs']):
remaining=7200-prior_wall_seconds-(time.monotonic()-start)
if remaining<=3:reason='TOTAL_PILOT_WALL_LIMIT';break
# Initial readiness was proved by the continuation entry; subsequent
# contexts also need a settling window. All waits consume the same budget.
if i>0:
gate=recorded_wait(root,spec['name'],uuid,min(600,remaining-3))
if gate['status']!='IDLE_CONFIRMED':reason='GPU_IDLE_TIMEOUT_BEFORE_RUN';break
remaining=7200-prior_wall_seconds-(time.monotonic()-start)
if remaining<=3:reason='TOTAL_PILOT_WALL_LIMIT';break
command=[sys.executable,'tools/run_s2_pilot_training_v2.py',str(root),str(i)]
try:r,result=guarded_worker(root,spec,i,uuid,min(1800,remaining),command)
except Exception as exc:reason=str(exc);break
rows.append(dict(run=spec,**r))
if r['status']!='PASS' or not result or result['status']!='COMPLETE':reason='PILOT_RUN_FAILED_NO_RETRY';break
scores[(spec['arm'],spec['width'])]=result['best_validation_score']
if prior_wall_seconds+time.monotonic()-start>7200:reason='TOTAL_PILOT_WALL_LIMIT_AFTER_EXIT'
selected=select_capacity(scores) if len(scores)==4 and reason is None else None
summary=dict(status='FOUR_RUNS_COMPLETE' if selected else 'PILOT_INCOMPLETE',reason=reason,runs=rows,selected_global_width=selected,
selection_rule='mean of N/E validation error at each width, ties smaller; all four runs required',
pilot_wall_seconds=prior_wall_seconds+time.monotonic()-start,probe_and_setup_wall_seconds=prior_wall_seconds,test_labels_read=False,test_predictions=False,retries=0)
save(root/'pilot-complete.json',summary);print(json.dumps(summary),flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);a=p.parse_args();run(a.root)
+26
View File
@@ -0,0 +1,26 @@
import unittest
from s2_gpu_idle_gate import wait_idle
class IdleGateTests(unittest.TestCase):
def exercise(self,states,limit):
now=[0.];events=[];iterator=iter(states);last=[states[-1]]
def sample(uuid):
value=next(iterator,last[0]);last[0]=value
return dict(uuid=uuid,idle=value,occupants=[] if value else [dict(pid=123,user='foreign',memory_MiB='1000')])
result=wait_idle('synthetic',limit,events.append,sample=sample,clock=lambda:now[0],sleep=lambda x:now.__setitem__(0,now[0]+x))
return result,events
def test_busy_then_three_consecutive_free(self):
result,events=self.exercise([False,True,False,True,True,True],600)
self.assertEqual(result['status'],'IDLE_CONFIRMED');self.assertEqual(result['wall_seconds'],50)
self.assertEqual([x['consecutive_idle'] for x in events],[0,1,0,1,2,3])
def test_foreign_occupant_times_out_without_killing(self):
result,events=self.exercise([False],600)
self.assertEqual(result['status'],'IDLE_TIMEOUT');self.assertEqual(result['wall_seconds'],600)
self.assertEqual(result['last']['occupants'][0]['pid'],123);self.assertEqual(len(events),61)
def test_short_remaining_budget_cannot_start_after_two_idle_polls(self):
result,_=self.exercise([True],10)
self.assertEqual(result['status'],'IDLE_TIMEOUT')
if __name__=='__main__':unittest.main(verbosity=2)