"""Shared Track A prediction guard with b’s lease, monitoring and process limits.""" import ctypes import json import os from pathlib import Path import resource import signal import subprocess import sys import threading import time import psutil from phase2_h1_worker_v2 import save,sha from track_a_postfit_shared_common import config,assignment,read from phase3_shared_capacity import validate_lease,alive from track_a_recovery_monitor import Monitor from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage from s2_main_memory_guard import PeakRecords def run(queue,index,digest): parent=os.getppid() if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGTERM,0,0,0)!=0: raise OSError('PDEATHSIG') requested=[] for sig in (signal.SIGTERM,signal.SIGINT): signal.signal(sig,lambda n,f:requested.append('SIGNAL_'+str(n))) if os.getppid()!=parent: raise RuntimeError('Queue lost before setup') c,_,_=config(queue);job=assignment(queue,c,index,digest) root=queue/'jobs'/str(index);start=time.monotonic();cpu=time.process_time() original=read(root/'assignment.json');wait=original['pre_dispatch_wait_seconds'] remaining=job['gpu_seconds']-60 if remaining<90: raise RuntimeError('No cleanup window before worker') deadline=start+remaining # Queue writes ready only after atomically transferring reservation ownership. while not (root/'lease-ready.json').exists(): if requested or not alive(job['queue_identity']) or time.monotonic()-start>30: raise RuntimeError('Lease transfer timeout') time.sleep(.1) validate_lease(job['token'],queue,index,sha(queue/'contract.json')) if os.getppid()!=job['queue_identity']['pid'] or not alive(job['queue_identity']): raise RuntimeError('Pinned queue lost before guard import completed') os.sched_setaffinity(0,[26,27]) tracker=PeakRecords();done=threading.Event();monitor=Monitor(root/'monitor-errors.jsonl') reasons=[];receipt=None command=[sys.executable,'tools/track_a_postfit_shared_worker.py','predict',str(queue),str(index),digest] def reason(s): if s not in reasons:reasons.append(s) def workers(): result=[] for p in psutil.Process().children(recursive=True): try: if p.cmdline()==command:result.append(p.pid) except psutil.NoSuchProcess:pass return result def kill(): for pid in workers():stop_group(pid) def scan(): tracker.scan(root/'prediction-records') if max(tracker.maxima.values())>job['recorded_vram_bytes']:reason('RECORDED_VRAM_LIMIT') def local_watch(): while not done.is_set(): monitor.sample('recorded_peaks',scan) usage,ok=monitor.sample('tree',lambda:tree_usage(os.getpid())) if ok and (usage[0]>job['cpu_seconds']-30 or usage[1]>job['rss_bytes']):reason('GUARD_TREE_LIMIT') disk,ok=monitor.sample('disk',lambda:disk_usage(queue)) if ok and disk>job['disk_bytes']:reason('OUTPUT_LIMIT') if time.monotonic()>deadline-30:reason('OWN_ACTUAL_TIME_LIMIT') queue_live,ok=monitor.sample('queue_identity',lambda:alive(job['queue_identity'])) if ok and not queue_live:reason('QUEUE_LOST') if (queue/'global-stop.json').exists():reason('GLOBAL_STOP') for s in requested:reason(s) if monitor.failed:reason('MONITOR_FAILED') if reasons:kill() done.wait(.5) def gpu_watch(): def sample(): text=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,used_gpu_memory','--format=csv,noheader,nounits'],text=True,timeout=10) rows=[] for line in text.strip().splitlines(): if not line:continue u,p,m=[x.strip() for x in line.split(',')] if u==job['gpu_uuid']:rows.append(dict(pid=int(p),bytes=int(m)*2**20)) if len(rows)>3:reason('HOST_GPU_SLOT_LIMIT') own=set(workers()) if sum(r['bytes'] for r in rows if r['pid'] in own)>job['process_vram_bytes']:reason('OWN_GPU_PROCESS_MEMORY_LIMIT') save(root/'gpu-sample.json',dict(unix=time.time(),rows=rows,own_pids=sorted(own))) while not done.is_set(): monitor.sample('gpu_processes',sample) if monitor.failed:reason('MONITOR_FAILED') if reasons:kill() done.wait(1) threads=[threading.Thread(target=f,daemon=True) for f in (local_watch,gpu_watch)] try: save(root/'started.json',dict(unix=time.time(),assignment_sha256=digest)) for t in threads:t.start() if reasons or requested or time.monotonic()>deadline-60:raise RuntimeError('Prelaunch guard rejected') receipt=supervise(command,root/'guards','predict',dict(total_cpu_seconds=job['cpu_seconds']-60, job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=deadline-time.monotonic()-40, tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid']) except BaseException as exc: reason(type(exc).__name__+': '+str(exc));kill() finally: done.set() for t in threads: if t.ident is not None:t.join(timeout=12) if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT') try:monitor.retry('final_peaks',scan,.1) except Exception as exc:reason('FINAL_PEAK_SCAN: '+str(exc)) for s in requested:reason(s) if monitor.failed:reason('MONITOR_FAILED') p=root/'prediction-records/result.json';result=read(p) if p.exists() else None expected=next(j['expected_parts'] for j in c['jobs'] if j['id']==index) model=next(m for m in c['training_evidence']['models'] if m['job']==index) if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS') if (not result or result.get('status')!='COMPLETE' or result.get('parts')!=expected or result.get('checkpoint_sha256')!=model['sha256'] or result.get('test_targets_read') is not False): reason('INCOMPLETE_PREDICTION_IDENTITY') if len(tracker.witnesses)!=2:reason('MISSING_RECORDED_PEAKS') elapsed=time.monotonic()-start child=resource.getrusage(resource.RUSAGE_CHILDREN) charged_cpu=time.process_time()-cpu+child.ru_utime+child.ru_stime if 60+elapsed>job['gpu_seconds'] or charged_cpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT') try:config(queue) except Exception as exc:reason('FINAL_PINS: '+str(exc)) save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons, worker=receipt,worker_result_sha256=sha(p) if p.exists() else None,gpu_seconds_charged=60+elapsed, cpu_seconds_charged=charged_cpu,pre_dispatch_wait_seconds=wait,guard_wall_seconds=elapsed, recorded_memory=dict(maxima=tracker.maxima,witnesses=tracker.witnesses,observations_count=len(tracker.observations)), monitor_policy=monitor.policy(),monitor_errors=monitor.records,no_retry=True)) return 0 if not reasons else 1 if __name__=='__main__': raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))