127 lines
8.7 KiB
Python
127 lines
8.7 KiB
Python
"""Isolated recovery fold: 10s GPU samples, three failures; fast local guards stay live."""
|
|
import fcntl,json,math,os,resource,signal,sys,threading,time
|
|
from pathlib import Path
|
|
import psutil
|
|
from phase2_h1_worker_v2 import save,sha
|
|
from track_a_recovery_common import config
|
|
from track_a_recovery_monitor import Monitor,device_usage
|
|
from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage
|
|
from s2_main_memory_guard import PeakRecords
|
|
|
|
|
|
|
|
def require_prestart_window(elapsed,cap,next_seconds=0):
|
|
"""Reserve bounded query/sleep plus20s cleanup inside the remaining own cap."""
|
|
if any(not math.isfinite(v) or v<0 for v in [elapsed,cap,next_seconds]):
|
|
raise ValueError('Invalid prestart deadline')
|
|
if elapsed+next_seconds+20>=cap:
|
|
raise RuntimeError('PRESTART_OWN_DEADLINE_NO_WORKER_STARTED')
|
|
|
|
|
|
def run(queue,fold):
|
|
contract,_=config(queue)
|
|
if fold not in contract['recovery_folds']:raise PermissionError('Only fresh folds3/4')
|
|
root=queue/'jobs'/str(fold);job=dict(contract['jobs'][fold]);assignment=json.loads((root/'assignment.json').read_text())
|
|
if assignment['id']!=fold or assignment['contract_sha256']!=sha(queue/'contract.json') or assignment['gpu_uuid']!=job['gpu_uuid']:raise ValueError('Fixed assignment')
|
|
wait=assignment['pre_dispatch_wait_seconds']
|
|
if not 0<=wait<job['gpu_seconds']-120:raise ValueError('No remaining per-fold window')
|
|
job['gpu_seconds']-=wait
|
|
os.sched_setaffinity(0,job['guard_cpus'])
|
|
with (root/'fold.lock').open('a') as lock,Path('/tmp/meltmind-track-a-'+job['gpu_uuid']+'.lock').open('a') as gpu_lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'started.json').exists():raise FileExistsError('One recovery attempt per fold')
|
|
start=time.monotonic();cpu=time.process_time();u0=resource.getrusage(resource.RUSAGE_CHILDREN)
|
|
reasons=[];requested=[];done=threading.Event();tracker=PeakRecords();monitor=Monitor(root/'monitor-errors.jsonl');receipt=None;threads=[]
|
|
command=[sys.executable,'tools/track_a_recovery_worker.py',str(queue),str(fold)]
|
|
def elapsed():return 60+time.monotonic()-start
|
|
def reason(s):
|
|
if s not in reasons:reasons.append(s)
|
|
def kill():
|
|
for p in psutil.Process().children(recursive=True):
|
|
try:
|
|
if p.cmdline()==command:stop_group(p.pid)
|
|
except psutil.NoSuchProcess:pass
|
|
def evaluate_errors():
|
|
if monitor.failed:reason('THREE_CONSECUTIVE_MONITOR_FAILURES')
|
|
if reasons:kill()
|
|
def scan():
|
|
if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT')
|
|
def local_watch():
|
|
while not done.is_set():
|
|
monitor.sample('recorded_peaks',scan)
|
|
usage,ok=monitor.sample('tree_usage',lambda:tree_usage(os.getpid()))
|
|
if ok and usage[1]>job['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT')
|
|
disk,ok=monitor.sample('disk_usage',lambda:disk_usage(root))
|
|
if ok and disk>job['disk_bytes']:reason('OUTPUT_DISK_LIMIT')
|
|
if elapsed()>=job['gpu_seconds']-15:reason('TRACK_A_FOLD_TIME_LIMIT')
|
|
if (root/'stop.json').exists():reason('OWN_FOLD_STOP')
|
|
if (queue/'global-stop.json').exists():reason('CUMULATIVE_OR_OPERATOR_STOP')
|
|
for s in requested:reason(s)
|
|
monitor.sample('local_record',lambda:save(root/'live-progress.json',dict(unix=time.time(),elapsed_gpu_seconds=elapsed(),pre_dispatch_wait_seconds=wait,recorded_peaks=tracker.maxima,reasons=list(reasons),monitor_policy=monitor.policy())))
|
|
evaluate_errors();done.wait(.5)
|
|
def gpu_watch():
|
|
while not done.is_set():
|
|
begin=time.time();value,ok=monitor.sample('gpu_usage',lambda:device_usage(job['gpu_uuid']))
|
|
if ok and value[0]>job['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
|
|
monitor.sample('gpu_record',lambda:save(root/'gpu-sample.json',dict(start_unix=begin,end_unix=time.time(),valid=ok,memory_bytes=value[0] if ok else None,utilization=value[1] if ok else None,monitor_policy=monitor.policy())))
|
|
evaluate_errors();done.wait(.5)
|
|
handlers={s:signal.signal(s,lambda n,f:requested.append('COORDINATOR_SIGNAL_'+str(n))) for s in [signal.SIGINT,signal.SIGTERM]}
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(queue/'contract.json'),gpu_uuid=job['gpu_uuid'],pre_dispatch_wait_seconds=wait,remaining_gpu_seconds=job['gpu_seconds'],fresh_seed=0))
|
|
try:
|
|
samples=[];success=0
|
|
while success<3:
|
|
if requested or (root/'stop.json').exists() or (queue/'global-stop.json').exists():raise RuntimeError('STOP_BEFORE_START')
|
|
require_prestart_window(elapsed(),job['gpu_seconds'],10)
|
|
value,ok=monitor.sample('gpu_usage',lambda:device_usage(job['gpu_uuid']))
|
|
require_prestart_window(elapsed(),job['gpu_seconds'])
|
|
if not ok:
|
|
success=0
|
|
if monitor.failed:raise RuntimeError('THREE_CONSECUTIVE_IDLE_SAMPLE_FAILURES')
|
|
require_prestart_window(elapsed(),job['gpu_seconds'],.5)
|
|
time.sleep(.5);continue
|
|
samples.append(dict(unix=time.time(),memory_bytes=value[0],utilization=value[1]))
|
|
save(root/'idle-polls.json',samples)
|
|
if value[0]>128*2**20 or value[1]!=0:raise RuntimeError('SELECTED_GPU_BECAME_BUSY_NO_WORKER_STARTED')
|
|
success+=1
|
|
if success<3:
|
|
require_prestart_window(elapsed(),job['gpu_seconds'],2)
|
|
time.sleep(2)
|
|
require_prestart_window(elapsed(),job['gpu_seconds'])
|
|
for fn in [local_watch,gpu_watch]:
|
|
t=threading.Thread(target=fn,daemon=True);t.start();threads.append(t)
|
|
remaining=job['gpu_seconds']-elapsed()-20
|
|
if remaining<=0:raise RuntimeError('SETUP_EXHAUSTED_WINDOW')
|
|
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=job['cpu_seconds']-60,job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid'])
|
|
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill()
|
|
finally:
|
|
done.set()
|
|
for t in threads:t.join(timeout=12)
|
|
if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT')
|
|
try:monitor.retry('recorded_peaks',scan,.5)
|
|
except Exception as exc:reason('FINAL_PEAK_SCAN_FAILED: '+str(exc))
|
|
for s,h in handlers.items():signal.signal(s,h)
|
|
evaluate_errors()
|
|
result=None;p=root/'worker/result.json'
|
|
if p.exists():
|
|
try:result=json.loads(p.read_text())
|
|
except Exception as exc:reason('RESULT_READ_FAILED: '+str(exc))
|
|
if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS')
|
|
if receipt and (type(receipt['monitor_errors']) is not int or receipt['monitor_policy']['failed']):reason('WORKER_MONITOR_FAILED')
|
|
if not result or result.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT')
|
|
elif result['job']!=fold or result['fold']!=fold or result['completed_epochs']!=200 or result['optimizer_steps']!=job['target_steps']:reason('INCOMPLETE_FINAL_IDENTITY')
|
|
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_RECORDED_PEAKS')
|
|
if any(v>job['vram_bytes'] for v in tracker.maxima.values()):reason('RECORDED_VRAM_LIMIT')
|
|
u=resource.getrusage(resource.RUSAGE_CHILDREN);childcpu=u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime
|
|
totalcpu=60+time.process_time()-cpu+max(childcpu,receipt['cpu_seconds'] if receipt else 0)
|
|
if elapsed()>job['gpu_seconds'] or totalcpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
|
try:config(queue)
|
|
except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc))
|
|
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons,
|
|
monitor_errors=monitor.records,monitor_policy=monitor.policy(),worker=receipt,recorded_memory=tracker.as_dict(),
|
|
gpu_seconds_charged=elapsed(),pre_dispatch_wait_seconds=wait,cpu_seconds_charged=totalcpu,
|
|
child_cpu_seconds_observed=childcpu,worker_result_sha256=sha(p) if p.exists() else None,
|
|
no_retry=True,no_further_recovery=True,prediction_started=False,scoring_started=False))
|
|
return 0 if not reasons else 1
|
|
|
|
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2])))
|