138 lines
7.1 KiB
Python
138 lines
7.1 KiB
Python
"""Shared Track A prediction guard with b’s lease, monitoring and process limits."""
|
||
import ctypes
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import resource
|
||
import signal
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import psutil
|
||
from phase2_h1_worker_v2 import save,sha
|
||
from track_a_postfit_shared_common import config,assignment,read
|
||
from phase3_shared_capacity import validate_lease,alive
|
||
from track_a_recovery_monitor import Monitor
|
||
from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage
|
||
from s2_main_memory_guard import PeakRecords
|
||
|
||
|
||
def run(queue,index,digest):
|
||
parent=os.getppid()
|
||
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGTERM,0,0,0)!=0:
|
||
raise OSError('PDEATHSIG')
|
||
requested=[]
|
||
for sig in (signal.SIGTERM,signal.SIGINT):
|
||
signal.signal(sig,lambda n,f:requested.append('SIGNAL_'+str(n)))
|
||
if os.getppid()!=parent:
|
||
raise RuntimeError('Queue lost before setup')
|
||
c,_,_=config(queue);job=assignment(queue,c,index,digest)
|
||
root=queue/'jobs'/str(index);start=time.monotonic();cpu=time.process_time()
|
||
original=read(root/'assignment.json');wait=original['pre_dispatch_wait_seconds']
|
||
remaining=job['gpu_seconds']-60
|
||
if remaining<90:
|
||
raise RuntimeError('No cleanup window before worker')
|
||
deadline=start+remaining
|
||
# Queue writes ready only after atomically transferring reservation ownership.
|
||
while not (root/'lease-ready.json').exists():
|
||
if requested or not alive(job['queue_identity']) or time.monotonic()-start>30:
|
||
raise RuntimeError('Lease transfer timeout')
|
||
time.sleep(.1)
|
||
validate_lease(job['token'],queue,index,sha(queue/'contract.json'))
|
||
if os.getppid()!=job['queue_identity']['pid'] or not alive(job['queue_identity']):
|
||
raise RuntimeError('Pinned queue lost before guard import completed')
|
||
os.sched_setaffinity(0,[26,27])
|
||
tracker=PeakRecords();done=threading.Event();monitor=Monitor(root/'monitor-errors.jsonl')
|
||
reasons=[];receipt=None
|
||
command=[sys.executable,'tools/track_a_postfit_shared_worker.py','predict',str(queue),str(index),digest]
|
||
def reason(s):
|
||
if s not in reasons:reasons.append(s)
|
||
def workers():
|
||
result=[]
|
||
for p in psutil.Process().children(recursive=True):
|
||
try:
|
||
if p.cmdline()==command:result.append(p.pid)
|
||
except psutil.NoSuchProcess:pass
|
||
return result
|
||
def kill():
|
||
for pid in workers():stop_group(pid)
|
||
def scan():
|
||
tracker.scan(root/'prediction-records')
|
||
if max(tracker.maxima.values())>job['recorded_vram_bytes']:reason('RECORDED_VRAM_LIMIT')
|
||
def local_watch():
|
||
while not done.is_set():
|
||
monitor.sample('recorded_peaks',scan)
|
||
usage,ok=monitor.sample('tree',lambda:tree_usage(os.getpid()))
|
||
if ok and (usage[0]>job['cpu_seconds']-30 or usage[1]>job['rss_bytes']):reason('GUARD_TREE_LIMIT')
|
||
disk,ok=monitor.sample('disk',lambda:disk_usage(queue))
|
||
if ok and disk>job['disk_bytes']:reason('OUTPUT_LIMIT')
|
||
if time.monotonic()>deadline-30:reason('OWN_ACTUAL_TIME_LIMIT')
|
||
queue_live,ok=monitor.sample('queue_identity',lambda:alive(job['queue_identity']))
|
||
if ok and not queue_live:reason('QUEUE_LOST')
|
||
if (queue/'global-stop.json').exists():reason('GLOBAL_STOP')
|
||
for s in requested:reason(s)
|
||
if monitor.failed:reason('MONITOR_FAILED')
|
||
if reasons:kill()
|
||
done.wait(.5)
|
||
def gpu_watch():
|
||
def sample():
|
||
text=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,used_gpu_memory','--format=csv,noheader,nounits'],text=True,timeout=10)
|
||
rows=[]
|
||
for line in text.strip().splitlines():
|
||
if not line:continue
|
||
u,p,m=[x.strip() for x in line.split(',')]
|
||
if u==job['gpu_uuid']:rows.append(dict(pid=int(p),bytes=int(m)*2**20))
|
||
if len(rows)>3:reason('HOST_GPU_SLOT_LIMIT')
|
||
own=set(workers())
|
||
if sum(r['bytes'] for r in rows if r['pid'] in own)>job['process_vram_bytes']:reason('OWN_GPU_PROCESS_MEMORY_LIMIT')
|
||
save(root/'gpu-sample.json',dict(unix=time.time(),rows=rows,own_pids=sorted(own)))
|
||
while not done.is_set():
|
||
monitor.sample('gpu_processes',sample)
|
||
if monitor.failed:reason('MONITOR_FAILED')
|
||
if reasons:kill()
|
||
done.wait(1)
|
||
threads=[threading.Thread(target=f,daemon=True) for f in (local_watch,gpu_watch)]
|
||
try:
|
||
save(root/'started.json',dict(unix=time.time(),assignment_sha256=digest))
|
||
for t in threads:t.start()
|
||
if reasons or requested or time.monotonic()>deadline-60:raise RuntimeError('Prelaunch guard rejected')
|
||
receipt=supervise(command,root/'guards','predict',dict(total_cpu_seconds=job['cpu_seconds']-60,
|
||
job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=deadline-time.monotonic()-40,
|
||
tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid'])
|
||
except BaseException as exc:
|
||
reason(type(exc).__name__+': '+str(exc));kill()
|
||
finally:
|
||
done.set()
|
||
for t in threads:
|
||
if t.ident is not None:t.join(timeout=12)
|
||
if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT')
|
||
try:monitor.retry('final_peaks',scan,.1)
|
||
except Exception as exc:reason('FINAL_PEAK_SCAN: '+str(exc))
|
||
for s in requested:reason(s)
|
||
if monitor.failed:reason('MONITOR_FAILED')
|
||
p=root/'prediction-records/result.json';result=read(p) if p.exists() else None
|
||
expected=next(j['expected_parts'] for j in c['jobs'] if j['id']==index)
|
||
model=next(m for m in c['training_evidence']['models'] if m['job']==index)
|
||
if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS')
|
||
if (not result or result.get('status')!='COMPLETE' or result.get('parts')!=expected
|
||
or result.get('checkpoint_sha256')!=model['sha256'] or result.get('test_targets_read') is not False):
|
||
reason('INCOMPLETE_PREDICTION_IDENTITY')
|
||
if len(tracker.witnesses)!=2:reason('MISSING_RECORDED_PEAKS')
|
||
elapsed=time.monotonic()-start
|
||
child=resource.getrusage(resource.RUSAGE_CHILDREN)
|
||
charged_cpu=time.process_time()-cpu+child.ru_utime+child.ru_stime
|
||
if 60+elapsed>job['gpu_seconds'] or charged_cpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
||
try:config(queue)
|
||
except Exception as exc:reason('FINAL_PINS: '+str(exc))
|
||
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons,
|
||
worker=receipt,worker_result_sha256=sha(p) if p.exists() else None,gpu_seconds_charged=60+elapsed,
|
||
cpu_seconds_charged=charged_cpu,pre_dispatch_wait_seconds=wait,guard_wall_seconds=elapsed,
|
||
recorded_memory=dict(maxima=tracker.maxima,witnesses=tracker.witnesses,observations_count=len(tracker.observations)),
|
||
monitor_policy=monitor.policy(),monitor_errors=monitor.records,no_retry=True))
|
||
return 0 if not reasons else 1
|
||
|
||
|
||
if __name__=='__main__':
|
||
raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))
|