137 lines
7.0 KiB
Python
137 lines
7.0 KiB
Python
"""One item c attempt, scoped process memory plus failed-worker peak checks."""
|
|
import ctypes
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import psutil
|
|
from phase2_h1_worker_v2 import save,sha
|
|
from phase3_item_c_common import config,assignment,read
|
|
from phase3_shared_capacity import validate_lease,alive
|
|
from track_a_recovery_monitor import Monitor
|
|
from track_a_recovery_supervisor import supervise,tree_usage,stop_group,disk_usage
|
|
from s2_main_memory_guard import PeakRecords
|
|
|
|
|
|
def run(queue,index,digest):
|
|
parent=os.getppid()
|
|
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGTERM,0,0,0)!=0:
|
|
raise OSError('PDEATHSIG')
|
|
requested=[]
|
|
for sig in (signal.SIGTERM,signal.SIGINT):
|
|
signal.signal(sig,lambda n,f:requested.append('SIGNAL_'+str(n)))
|
|
if os.getppid()!=parent:
|
|
raise RuntimeError('Queue lost before setup')
|
|
c,_=config(queue);job=assignment(queue,c,index,digest)
|
|
root=queue/'jobs'/str(index);start=time.monotonic();cpu=time.process_time()
|
|
original=read(root/'assignment.json');wait=original['pre_dispatch_wait_seconds']
|
|
remaining=c['runaway_gpu_seconds']-wait-60
|
|
if remaining<90:
|
|
raise RuntimeError('No cleanup window before worker')
|
|
deadline=start+remaining
|
|
# Queue writes ready only after atomically transferring reservation ownership.
|
|
while not (root/'lease-ready.json').exists():
|
|
if requested or not alive(job['queue_identity']) or time.monotonic()-start>30:
|
|
raise RuntimeError('Lease transfer timeout')
|
|
time.sleep(.1)
|
|
validate_lease(job['token'],queue,index,sha(queue/'contract.json'))
|
|
if os.getppid()!=job['queue_identity']['pid'] or not alive(job['queue_identity']):
|
|
raise RuntimeError('Pinned queue lost before guard import completed')
|
|
os.sched_setaffinity(0,[26,27])
|
|
tracker=PeakRecords();done=threading.Event();monitor=Monitor(root/'monitor-errors.jsonl')
|
|
reasons=[];receipt=None
|
|
command=[sys.executable,'tools/phase3_item_c_worker.py',str(queue),str(index),digest]
|
|
def reason(s):
|
|
if s not in reasons:reasons.append(s)
|
|
def workers():
|
|
result=[]
|
|
for p in psutil.Process().children(recursive=True):
|
|
try:
|
|
if p.cmdline()==command:result.append(p.pid)
|
|
except psutil.NoSuchProcess:pass
|
|
return result
|
|
def kill():
|
|
for pid in workers():stop_group(pid)
|
|
def scan():
|
|
tracker.scan(root/'worker')
|
|
if max(tracker.maxima.values())>job['recorded_vram_bytes']:reason('RECORDED_VRAM_LIMIT')
|
|
def local_watch():
|
|
while not done.is_set():
|
|
monitor.sample('recorded_peaks',scan)
|
|
usage,ok=monitor.sample('tree',lambda:tree_usage(os.getpid()))
|
|
if ok and (usage[0]>job['cpu_seconds']-30 or usage[1]>job['rss_bytes']):reason('GUARD_TREE_LIMIT')
|
|
disk,ok=monitor.sample('disk',lambda:disk_usage(root))
|
|
if ok and disk>job['disk_bytes']:reason('OUTPUT_LIMIT')
|
|
if time.monotonic()>deadline-30:reason('OWN_ACTUAL_TIME_LIMIT')
|
|
queue_live,ok=monitor.sample('queue_identity',lambda:alive(job['queue_identity']))
|
|
if ok and not queue_live:reason('QUEUE_LOST')
|
|
if (queue/'global-stop.json').exists():reason('GLOBAL_STOP')
|
|
for s in requested:reason(s)
|
|
if monitor.failed:reason('MONITOR_FAILED')
|
|
if reasons:kill()
|
|
done.wait(.5)
|
|
def gpu_watch():
|
|
def sample():
|
|
text=subprocess.check_output(['nvidia-smi','--query-compute-apps=gpu_uuid,pid,used_gpu_memory','--format=csv,noheader,nounits'],text=True,timeout=10)
|
|
rows=[]
|
|
for line in text.strip().splitlines():
|
|
if not line:continue
|
|
u,p,m=[x.strip() for x in line.split(',')]
|
|
if u==job['gpu_uuid']:rows.append(dict(pid=int(p),bytes=int(m)*2**20))
|
|
if len(rows)>3:reason('HOST_GPU_SLOT_LIMIT')
|
|
own=set(workers())
|
|
if sum(r['bytes'] for r in rows if r['pid'] in own)>job['process_vram_bytes']:reason('OWN_GPU_PROCESS_MEMORY_LIMIT')
|
|
save(root/'gpu-sample.json',dict(unix=time.time(),rows=rows,own_pids=sorted(own)))
|
|
while not done.is_set():
|
|
monitor.sample('gpu_processes',sample)
|
|
if monitor.failed:reason('MONITOR_FAILED')
|
|
if reasons:kill()
|
|
done.wait(1)
|
|
threads=[threading.Thread(target=f,daemon=True) for f in (local_watch,gpu_watch)]
|
|
try:
|
|
save(root/'started.json',dict(unix=time.time(),assignment_sha256=digest))
|
|
for t in threads:t.start()
|
|
if reasons or requested or time.monotonic()>deadline-60:raise RuntimeError('Prelaunch guard rejected')
|
|
receipt=supervise(command,root/'guards','train',dict(total_cpu_seconds=job['cpu_seconds']-60,
|
|
job_cpu_seconds=job['cpu_seconds']-60,job_wall_seconds=deadline-time.monotonic()-40,
|
|
tree_rss_bytes=job['rss_bytes'],disk_bytes=job['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=job['gpu_uuid'])
|
|
except BaseException as exc:
|
|
reason(type(exc).__name__+': '+str(exc));kill()
|
|
finally:
|
|
done.set()
|
|
for t in threads:
|
|
if t.ident is not None:t.join(timeout=12)
|
|
if any(t.is_alive() for t in threads):reason('MONITOR_DID_NOT_EXIT')
|
|
try:monitor.retry('final_peaks',scan,.1)
|
|
except Exception as exc:reason('FINAL_PEAK_SCAN: '+str(exc))
|
|
for s in requested:reason(s)
|
|
if monitor.failed:reason('MONITOR_FAILED')
|
|
p=root/'worker/result.json';result=read(p) if p.exists() else None
|
|
if not receipt or receipt['status']!='PASS' or receipt['returncode']!=0:reason('WORKER_NOT_PASS')
|
|
if not result or result.get('status')!='COMPLETE' or result.get('fold')!=index or result.get('completed_epochs')!=200 or result.get('optimizer_steps')!=job['target_steps']:reason('NO_COMPLETE_IDENTITY')
|
|
if result:
|
|
from phase2_h1_worker_v2 import checked
|
|
checked(result['checkpoint'],result['checkpoint_sha256'])
|
|
if len(tracker.witnesses)!=2:reason('MISSING_RECORDED_PEAKS')
|
|
elapsed=time.monotonic()-start
|
|
child=resource.getrusage(resource.RUSAGE_CHILDREN)
|
|
charged_cpu=time.process_time()-cpu+child.ru_utime+child.ru_stime
|
|
if wait+60+elapsed>c['runaway_gpu_seconds'] or charged_cpu>job['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
|
try:config(queue)
|
|
except Exception as exc:reason('FINAL_PINS: '+str(exc))
|
|
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',reasons=reasons,
|
|
worker=receipt,worker_result_sha256=sha(p) if p.exists() else None,
|
|
cpu_seconds_charged=charged_cpu,pre_dispatch_wait_seconds=wait,guard_wall_seconds=elapsed,
|
|
recorded_memory=dict(maxima=tracker.maxima,witnesses=tracker.witnesses,observations_count=len(tracker.observations)),
|
|
monitor_policy=monitor.policy(),monitor_errors=monitor.records,no_retry=True))
|
|
return 0 if not reasons else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
raise SystemExit(run(Path(sys.argv[1]),int(sys.argv[2]),sys.argv[3]))
|