"""Independent bounded ITEM1 fold guard; shared global stop and immutable S1.""" import fcntl import json import os from pathlib import Path import signal import sys import threading import time import psutil from phase3_data_sufficiency_v2_common import config from phase2_h1_worker_v2 import save,sha from s2_main_memory_guard import PeakRecords from s2_precision_continuation_guard import device_usage from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage def run(queue,fold_id): c,_=config(queue);job=c['jobs'][fold_id];c=dict(c,**job);root=queue/'jobs'/str(fold_id);os.sched_setaffinity(0,c['guard_cpus']) with (root/'fold.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (root/'started.json').exists():raise FileExistsError('Single ITEM1 fold launch, no retry') start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(queue/'contract.json'),gpu_uuid=c['gpu_uuid'],setup_gpu_seconds=c['setup_gpu_seconds'])) def elapsed():return c['setup_gpu_seconds']+time.monotonic()-start def reason(x): if x not in reasons:reasons.append(x) command=[sys.executable,'tools/phase3_data_sufficiency_v2_worker.py',str(queue),str(fold_id)] def kill_worker(): for p in psutil.Process().children(recursive=True): try: if p.cmdline()==command:stop_group(p.pid) except psutil.NoSuchProcess:pass old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]} def scan(): if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT') def watch(): while not done.is_set(): try: scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid()) if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT') if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT') if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT') if elapsed()>=c['gpu_seconds']-5:reason('ITEM1_FOLD_TIME_LIMIT') if (queue/'global-stop.json').exists():reason('ITEM1_GLOBAL_STOP') for x in requested:reason(x) save(root/'live-progress.json',dict(stage='ITEM1_TRAINING',fold=fold_id,elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time())) except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc))) if reasons:kill_worker() done.wait(.5) thread=None try: for i in range(3): mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util)) if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY') if i<2:time.sleep(2) save(root/'idle-polls.json',samples) thread=threading.Thread(target=watch,daemon=True);thread.start() remaining=c['gpu_seconds']-elapsed()-10 receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid']) except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker() finally: done.set() if thread:thread.join(timeout=4) if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT') try:scan() except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc))) for s,handler in old.items():signal.signal(s,handler) value=None;p=root/'worker/result.json' try: if p.exists():value=json.loads(p.read_text()) except Exception as exc:reason('RESULT_READ_ERROR');errors.append(dict(reason=str(exc))) if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS') if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS') if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_TRAIN_RESULT') if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS') totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT') try:config(queue) except Exception as exc:reason('FINAL_SOURCE_IDENTITY: '+str(exc)) if value and (value.get('optimizer_steps')!=c['target_steps'] or value.get('job')!=fold_id):reason('INCOMPLETE_EPOCH_OR_FOLD_COUNT') save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision='TRAIN_COMPLETE_CHECKPOINT_FREEZE_PENDING' if not reasons else 'ITEM1_FOLD_INCOMPLETE',reasons=reasons,monitor_errors=errors, worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),fold=fold_id,cpu_seconds_charged=totalcpu, worker_result_sha256=sha(p) if p.exists() else None,prediction_started=False,scoring_started=False,s2_started=True,no_retry=True)) return 0 if not reasons else 1 if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2])))