"""One item b attempt, scoped process memory plus failed-worker peak checks.""" import ctypes import json import os from pathlib import Path import resource import signal import subprocess import sys import threading import time import psutil from phase2_h1_worker_v2 import save,sha from phase3_item_b_shared_common import config,assignment,read from phase3_shared_capacity import validate_lease,alive from track_a_recovery_monitor import Monitor from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage from s2_main_memory_guard import PeakRecords def run(queue,index,digest): parent=os.getppid() if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGTERM,0,0,0)!=0: raise OSError('PDEATHSIG') requested=[] for sig in (signal.SIGTERM,signal.SIGINT): signal.signal(sig,lambda n,f:requested.append('SIGNAL_'+str(n))) if os.getppid()!=parent: raise RuntimeError('Queue lost before setup') c,_=config(queue);job=assignment(queue,c,index,digest) root=queue/'jobs'/str(index);start=time.monotonic();cpu=time.process_time() original=read(root/'assignment.json');wait=original['pre_dispatch_wait_seconds'] remaining=c['runaway_gpu_seconds']-60 if remaining<90: raise RuntimeError('No cleanup window before worker') deadline=start+remaining # Queue writes ready only after atomically transferring reservation ownership. while not (root/'lease-ready.json').exists(): if requested or not alive(job['queue_identity']) or time.monotonic()-start>30: raise RuntimeError('Lease transfer timeout') time.sleep(.1) validate_lease(job['token'],queue,index,sha(queue/'contract.json')) if os.getppid()!=job['queue_identity']['pid'] or not alive(job['queue_identity']): raise RuntimeError('Pinned queue lost before guard import completed') os.sched_setaffinity(0,[26,27]) tracker=PeakRecords();done=threading.Event();monitor=Monitor(root/'monitor-errors.jsonl') reasons=[];receipt=None command=[sys.executable,'tools/phase3_item_b_shared_worker.py',str(queue),str(index),digest] def reason(s): if s not in reasons:reasons.append(s) def workers(): result=[] for p in psutil.Process().children(recursive=True): try: if p.cmdline()==command:result.append(p.pid) except psutil.NoSuchProcess:pass return result def kill(): for pid in workers():stop_group(pid) def scan(): tracker.scan(root/'worker') if max(tracker.maxima.values())>job['recorded_vram_bytes']:reason('RECORDED_VRAM_LIMIT') def local_watch(): while not done.is_set(): monitor.sample('recorded_peaks',scan) usage,ok=monitor.sample('tree',lambda:tree_usage(os.getpid())) if ok and (usage[0]>job['cpu_seconds']-30 or usage[1]>job['rss_bytes']):reason('GUARD_TREE_LIMIT') disk,ok=monitor.sample('disk',lambda:disk_usage(root)) if ok and disk>job['disk_bytes']:reason('OUTPUT_LIMIT') if time.monotonic()>deadline-30:reason('OWN_ACTUAL_TIME_LIMIT') queue_live,ok=monitor.sample('queue_identity',lambda:alive(job['queue_identity'])) if ok and not queue_live:reason('QUEUE_LOST') if (queue/'global-stop.json').exists():reason('GLOBAL_STOP') for s in requested:reason(s) if monitor.failed:reason('MONITOR_FAILED') if reasons:kill() done.wait(.5) def gpu_watch(): def sample(): text=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,used_gpu_memory','--format=csv,noheader,nounits'],text=True,timeout=10) rows=[] for line in text.strip().splitlines(): if not line:continue u,p,m=[x.strip() for x in line.split(',')] if u==job['gpu_uuid']:rows.append(dict(pid=int(p),bytes=int(m)*2**20)) if len(rows)>3:reason('HOST_GPU_SLOT_LIMIT') own=set(workers()) if sum(r['bytes'] for r in rows if r['pid'] in own)>job['process_vram_bytes']:reason('OWN_GPU_PROCESS_MEMORY_LIMIT') save(root/'gpu-sample.json',dict(unix=time.time(),rows=rows,own_pids=sorted(own))) while not done.is_set(): monitor.sample('gpu_processes',sample) if monitor.failed:reason('MONITOR_FAILED') if reasons:kill() done.wait(1) threads=[threading.Thread(target=f,daemon=True) for f in (local_watch,gpu_watch)] try: save(root/'started.json',dict(unix=time.time(),assignment_sha256=digest)) for t in threads:t.start() if reasons or requested or time.monotonic()>deadline-60:raise RuntimeError('Prelaunch guard rejected') receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=job['cpu_seconds']-60, job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=deadline-time.monotonic()-40, tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid']) except BaseException as exc: reason(type(exc).__name__+': '+str(exc));kill() finally: done.set() for t in threads: if t.ident is not None:t.join(timeout=12) if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT') try:monitor.retry('final_peaks',scan,.1) except Exception as exc:reason('FINAL_PEAK_SCAN: '+str(exc)) for s in requested:reason(s) if monitor.failed:reason('MONITOR_FAILED') p=root/'worker/result.json';result=read(p) if p.exists() else None if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS') if not result or result.get('status')!='COMPLETE' or result.get('fold')!=index or result.get('completed_epochs')!=200 or result.get('optimizer_steps')!=job['target_steps']:reason('NO_COMPLETE_IDENTITY') if result: from phase2_h1_worker_v2 import checked checked(result['checkpoint'],result['checkpoint_sha256']) if len(tracker.witnesses)!=2:reason('MISSING_RECORDED_PEAKS') elapsed=time.monotonic()-start child=resource.getrusage(resource.RUSAGE_CHILDREN) charged_cpu=time.process_time()-cpu+child.ru_utime+child.ru_stime if 60+elapsed>c['runaway_gpu_seconds'] or charged_cpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT') try:config(queue) except Exception as exc:reason('FINAL_PINS: '+str(exc)) save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons, worker=receipt,worker_result_sha256=sha(p) if p.exists() else None, cpu_seconds_charged=charged_cpu,pre_dispatch_wait_seconds=wait,guard_wall_seconds=elapsed, recorded_memory=dict(maxima=tracker.maxima,witnesses=tracker.witnesses,observations_count=len(tracker.observations)), monitor_policy=monitor.policy(),monitor_errors=monitor.records,no_retry=True)) return 0 if not reasons else 1 if __name__=='__main__': raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))