"""Isolated recovery fold: 10s GPU samples, three failures; fast local guards stay live.""" import fcntl,json,math,os,resource,signal,sys,threading,time from pathlib import Path import psutil from phase2_h1_worker_v2 import save,sha from track_a_recovery_common import config from track_a_recovery_monitor import Monitor,device_usage from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage from s2_main_memory_guard import PeakRecords def require_prestart_window(elapsed,cap,next_seconds=0): """Reserve bounded query/sleep plus20s cleanup inside the remaining own cap.""" if any(not math.isfinite(v) or v<0 for v in [elapsed,cap,next_seconds]): raise ValueError('Invalid prestart deadline') if elapsed+next_seconds+20>=cap: raise RuntimeError('PRESTART_OWN_DEADLINE_NO_WORKER_STARTED') def run(queue,fold): contract,_=config(queue) if fold not in contract['recovery_folds']:raise PermissionError('Only fresh folds3/4') root=queue/'jobs'/str(fold);job=dict(contract['jobs'][fold]);assignment=json.loads((root/'assignment.json').read_text()) if assignment['id']!=fold or assignment['contract_sha256']!=sha(queue/'contract.json') or assignment['gpu_uuid']!=job['gpu_uuid']:raise ValueError('Fixed assignment') wait=assignment['pre_dispatch_wait_seconds'] if not 0<=waitjob['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT') disk,ok=monitor.sample('disk_usage',lambda:disk_usage(root)) if ok and disk>job['disk_bytes']:reason('OUTPUT_DISK_LIMIT') if elapsed()>=job['gpu_seconds']-15:reason('TRACK_A_FOLD_TIME_LIMIT') if (root/'stop.json').exists():reason('OWN_FOLD_STOP') if (queue/'global-stop.json').exists():reason('CUMULATIVE_OR_OPERATOR_STOP') for s in requested:reason(s) monitor.sample('local_record',lambda:save(root/'live-progress.json',dict(unix=time.time(),elapsed_gpu_seconds=elapsed(),pre_dispatch_wait_seconds=wait,recorded_peaks=tracker.maxima,reasons=list(reasons),monitor_policy=monitor.policy()))) evaluate_errors();done.wait(.5) def gpu_watch(): while not done.is_set(): begin=time.time();value,ok=monitor.sample('gpu_usage',lambda:device_usage(job['gpu_uuid'])) if ok and value[0]>job['vram_bytes']:reason('DEVICE_VRAM_LIMIT') monitor.sample('gpu_record',lambda:save(root/'gpu-sample.json',dict(start_unix=begin,end_unix=time.time(),valid=ok,memory_bytes=value[0] if ok else None,utilization=value[1] if ok else None,monitor_policy=monitor.policy()))) evaluate_errors();done.wait(.5) handlers={s:signal.signal(s,lambda n,f:requested.append('COORDINATOR_SIGNAL_'+str(n))) for s in [signal.SIGINT,signal.SIGTERM]} save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(queue/'contract.json'),gpu_uuid=job['gpu_uuid'],pre_dispatch_wait_seconds=wait,remaining_gpu_seconds=job['gpu_seconds'],fresh_seed=0)) try: samples=[];success=0 while success<3: if requested or (root/'stop.json').exists() or (queue/'global-stop.json').exists():raise RuntimeError('STOP_BEFORE_START') require_prestart_window(elapsed(),job['gpu_seconds'],10) value,ok=monitor.sample('gpu_usage',lambda:device_usage(job['gpu_uuid'])) require_prestart_window(elapsed(),job['gpu_seconds']) if not ok: success=0 if monitor.failed:raise RuntimeError('THREE_CONSECUTIVE_IDLE_SAMPLE_FAILURES') require_prestart_window(elapsed(),job['gpu_seconds'],.5) time.sleep(.5);continue samples.append(dict(unix=time.time(),memory_bytes=value[0],utilization=value[1])) save(root/'idle-polls.json',samples) if value[0]>128*2**20 or value[1]!=0:raise RuntimeError('SELECTED_GPU_BECAME_BUSY_NO_WORKER_STARTED') success+=1 if success<3: require_prestart_window(elapsed(),job['gpu_seconds'],2) time.sleep(2) require_prestart_window(elapsed(),job['gpu_seconds']) for fn in [local_watch,gpu_watch]: t=threading.Thread(target=fn,daemon=True);t.start();threads.append(t) remaining=job['gpu_seconds']-elapsed()-20 if remaining<=0:raise RuntimeError('SETUP_EXHAUSTED_WINDOW') receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=job['cpu_seconds']-60,job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid']) except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill() finally: done.set() for t in threads:t.join(timeout=12) if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT') try:monitor.retry('recorded_peaks',scan,.5) except Exception as exc:reason('FINAL_PEAK_SCAN_FAILED: '+str(exc)) for s,h in handlers.items():signal.signal(s,h) evaluate_errors() result=None;p=root/'worker/result.json' if p.exists(): try:result=json.loads(p.read_text()) except Exception as exc:reason('RESULT_READ_FAILED: '+str(exc)) if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS') if receipt and (type(receipt['monitor_errors']) is not int or receipt['monitor_policy']['failed']):reason('WORKER_MONITOR_FAILED') if not result or result.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT') elif result['job']!=fold or result['fold']!=fold or result['completed_epochs']!=200 or result['optimizer_steps']!=job['target_steps']:reason('INCOMPLETE_FINAL_IDENTITY') if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_RECORDED_PEAKS') if any(v>job['vram_bytes'] for v in tracker.maxima.values()):reason('RECORDED_VRAM_LIMIT') u=resource.getrusage(resource.RUSAGE_CHILDREN);childcpu=u.ru_utime+u.ru_stime-u0.ru_utime-u0.ru_stime totalcpu=60+time.process_time()-cpu+max(childcpu,receipt['cpu_seconds'] if receipt else 0) if elapsed()>job['gpu_seconds'] or totalcpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT') try:config(queue) except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc)) save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons, monitor_errors=monitor.records,monitor_policy=monitor.policy(),worker=receipt,recorded_memory=tracker.as_dict(), gpu_seconds_charged=elapsed(),pre_dispatch_wait_seconds=wait,cpu_seconds_charged=totalcpu, child_cpu_seconds_observed=childcpu,worker_result_sha256=sha(p) if p.exists() else None, no_retry=True,no_further_recovery=True,prediction_started=False,scoring_started=False)) return 0 if not reasons else 1 if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2])))