Files
MeltMind/tools/track_a_postfit_shared_guard.py
T

138 lines
7.1 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Shared Track A prediction guard with b’s lease, monitoring and process limits."""
import ctypes
import json
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import threading
import time
import psutil
from phase2_h1_worker_v2 import save,sha
from track_a_postfit_shared_common import config,assignment,read
from phase3_shared_capacity import validate_lease,alive
from track_a_recovery_monitor import Monitor
from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage
from s2_main_memory_guard import PeakRecords
def run(queue,index,digest):
parent=os.getppid()
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGTERM,0,0,0)!=0:
raise OSError('PDEATHSIG')
requested=[]
for sig in (signal.SIGTERM,signal.SIGINT):
signal.signal(sig,lambda n,f:requested.append('SIGNAL_'+str(n)))
if os.getppid()!=parent:
raise RuntimeError('Queue lost before setup')
c,_,_=config(queue);job=assignment(queue,c,index,digest)
root=queue/'jobs'/str(index);start=time.monotonic();cpu=time.process_time()
original=read(root/'assignment.json');wait=original['pre_dispatch_wait_seconds']
remaining=job['gpu_seconds']-60
if remaining<90:
raise RuntimeError('No cleanup window before worker')
deadline=start+remaining
# Queue writes ready only after atomically transferring reservation ownership.
while not (root/'lease-ready.json').exists():
if requested or not alive(job['queue_identity']) or time.monotonic()-start>30:
raise RuntimeError('Lease transfer timeout')
time.sleep(.1)
validate_lease(job['token'],queue,index,sha(queue/'contract.json'))
if os.getppid()!=job['queue_identity']['pid'] or not alive(job['queue_identity']):
raise RuntimeError('Pinned queue lost before guard import completed')
os.sched_setaffinity(0,[26,27])
tracker=PeakRecords();done=threading.Event();monitor=Monitor(root/'monitor-errors.jsonl')
reasons=[];receipt=None
command=[sys.executable,'tools/track_a_postfit_shared_worker.py','predict',str(queue),str(index),digest]
def reason(s):
if s not in reasons:reasons.append(s)
def workers():
result=[]
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:result.append(p.pid)
except psutil.NoSuchProcess:pass
return result
def kill():
for pid in workers():stop_group(pid)
def scan():
tracker.scan(root/'prediction-records')
if max(tracker.maxima.values())>job['recorded_vram_bytes']:reason('RECORDED_VRAM_LIMIT')
def local_watch():
while not done.is_set():
monitor.sample('recorded_peaks',scan)
usage,ok=monitor.sample('tree',lambda:tree_usage(os.getpid()))
if ok and (usage[0]>job['cpu_seconds']-30 or usage[1]>job['rss_bytes']):reason('GUARD_TREE_LIMIT')
disk,ok=monitor.sample('disk',lambda:disk_usage(queue))
if ok and disk>job['disk_bytes']:reason('OUTPUT_LIMIT')
if time.monotonic()>deadline-30:reason('OWN_ACTUAL_TIME_LIMIT')
queue_live,ok=monitor.sample('queue_identity',lambda:alive(job['queue_identity']))
if ok and not queue_live:reason('QUEUE_LOST')
if (queue/'global-stop.json').exists():reason('GLOBAL_STOP')
for s in requested:reason(s)
if monitor.failed:reason('MONITOR_FAILED')
if reasons:kill()
done.wait(.5)
def gpu_watch():
def sample():
text=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,used_gpu_memory','--format=csv,noheader,nounits'],text=True,timeout=10)
rows=[]
for line in text.strip().splitlines():
if not line:continue
u,p,m=[x.strip() for x in line.split(',')]
if u==job['gpu_uuid']:rows.append(dict(pid=int(p),bytes=int(m)*2**20))
if len(rows)>3:reason('HOST_GPU_SLOT_LIMIT')
own=set(workers())
if sum(r['bytes'] for r in rows if r['pid'] in own)>job['process_vram_bytes']:reason('OWN_GPU_PROCESS_MEMORY_LIMIT')
save(root/'gpu-sample.json',dict(unix=time.time(),rows=rows,own_pids=sorted(own)))
while not done.is_set():
monitor.sample('gpu_processes',sample)
if monitor.failed:reason('MONITOR_FAILED')
if reasons:kill()
done.wait(1)
threads=[threading.Thread(target=f,daemon=True) for f in (local_watch,gpu_watch)]
try:
save(root/'started.json',dict(unix=time.time(),assignment_sha256=digest))
for t in threads:t.start()
if reasons or requested or time.monotonic()>deadline-60:raise RuntimeError('Prelaunch guard rejected')
receipt=supervise(command,root/'guards','predict',dict(total_cpu_seconds=job['cpu_seconds']-60,
job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=deadline-time.monotonic()-40,
tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid'])
except BaseException as exc:
reason(type(exc).__name__+': '+str(exc));kill()
finally:
done.set()
for t in threads:
if t.ident is not None:t.join(timeout=12)
if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT')
try:monitor.retry('final_peaks',scan,.1)
except Exception as exc:reason('FINAL_PEAK_SCAN: '+str(exc))
for s in requested:reason(s)
if monitor.failed:reason('MONITOR_FAILED')
p=root/'prediction-records/result.json';result=read(p) if p.exists() else None
expected=next(j['expected_parts'] for j in c['jobs'] if j['id']==index)
model=next(m for m in c['training_evidence']['models'] if m['job']==index)
if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS')
if (not result or result.get('status')!='COMPLETE' or result.get('parts')!=expected
or result.get('checkpoint_sha256')!=model['sha256'] or result.get('test_targets_read') is not False):
reason('INCOMPLETE_PREDICTION_IDENTITY')
if len(tracker.witnesses)!=2:reason('MISSING_RECORDED_PEAKS')
elapsed=time.monotonic()-start
child=resource.getrusage(resource.RUSAGE_CHILDREN)
charged_cpu=time.process_time()-cpu+child.ru_utime+child.ru_stime
if 60+elapsed>job['gpu_seconds'] or charged_cpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
try:config(queue)
except Exception as exc:reason('FINAL_PINS: '+str(exc))
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons,
worker=receipt,worker_result_sha256=sha(p) if p.exists() else None,gpu_seconds_charged=60+elapsed,
cpu_seconds_charged=charged_cpu,pre_dispatch_wait_seconds=wait,guard_wall_seconds=elapsed,
recorded_memory=dict(maxima=tracker.maxima,witnesses=tracker.witnesses,observations_count=len(tracker.observations)),
monitor_policy=monitor.policy(),monitor_errors=monitor.records,no_retry=True))
return 0 if not reasons else 1
if __name__=='__main__':
raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))