84 lines
5.3 KiB
Python
84 lines
5.3 KiB
Python
"""Unique H2 pilot; includes setup/idle, recorded failure peaks and final CPU."""
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import sys
|
|
import threading
|
|
import time
|
|
import psutil
|
|
from phase2_h2_data import config,projection
|
|
from phase2_h1_worker_v2 import save,sha
|
|
from s2_main_memory_guard import PeakRecords
|
|
from s2_precision_continuation_guard import device_usage
|
|
from s2_recovery_gpu_supervisor import supervise,tree_usage,stop_group,disk_usage
|
|
|
|
def run(root):
|
|
c=config(root);os.sched_setaffinity(0,c['guard_cpus'])
|
|
with (root/'pilot.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'started.json').exists():raise FileExistsError('Single pilot launch, no retry')
|
|
start=time.monotonic();cpu=time.process_time();requested=[];reasons=[];errors=[];samples=[];done=threading.Event();tracker=PeakRecords();receipt=None
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json'),gpu_uuid=c['gpu_uuid'],pilot_setup_charge_seconds=c['pilot_setup_charge_seconds']))
|
|
def elapsed():return c['pilot_setup_charge_seconds']+time.monotonic()-start
|
|
def reason(x):
|
|
if x not in reasons:reasons.append(x)
|
|
command=[sys.executable,'tools/phase2_h2_pilot_worker.py',str(root)]
|
|
def kill_worker():
|
|
for p in psutil.Process().children(recursive=True):
|
|
try:
|
|
if p.cmdline()==command:stop_group(p.pid)
|
|
except psutil.NoSuchProcess:pass
|
|
old={s:signal.signal(s,lambda signum,frame:requested.append('COORDINATOR_SIGNAL_'+str(signum))) for s in [signal.SIGINT,signal.SIGTERM]}
|
|
def scan():
|
|
if tracker.scan(root/'worker'):reason('RECORDED_VRAM_LIMIT')
|
|
def watch():
|
|
while not done.is_set():
|
|
try:
|
|
scan();mem,util=device_usage(c['gpu_uuid']);_,rss,_=tree_usage(os.getpid())
|
|
if mem>c['vram_bytes']:reason('DEVICE_VRAM_LIMIT')
|
|
if rss>c['rss_bytes']:reason('WHOLE_TREE_RSS_LIMIT')
|
|
if disk_usage(root)>c['disk_bytes']:reason('OUTPUT_DISK_LIMIT')
|
|
if elapsed()>=c['gpu_seconds']-5:reason('PILOT_TIME_LIMIT')
|
|
for x in requested:reason(x)
|
|
save(root/'live-progress.json',dict(stage='PILOT_RUNNING',elapsed_gpu_seconds=elapsed(),device_bytes=mem,device_utilization=util,tree_rss_bytes=rss,recorded_peaks=tracker.maxima,reasons=reasons,unix=time.time()))
|
|
except Exception as exc:reason('MONITOR_ERROR');errors.append(dict(type=type(exc).__name__,reason=str(exc)))
|
|
if reasons:kill_worker()
|
|
done.wait(.5)
|
|
thread=None
|
|
try:
|
|
for i in range(3):
|
|
mem,util=device_usage(c['gpu_uuid']);samples.append(dict(unix=time.time(),memory_bytes=mem,utilization=util))
|
|
if mem>128*2**20 or util!=0:raise RuntimeError('SELECTED_GPU_BUSY')
|
|
if i<2:time.sleep(2)
|
|
save(root/'idle-polls.json',samples)
|
|
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
|
remaining=c['gpu_seconds']-elapsed()-10
|
|
receipt=supervise(command,root/'guards','pilot',dict(total_cpu_seconds=c['cpu_seconds']-60,job_cpu_seconds=c['cpu_seconds']-60,job_wall_seconds=remaining,tree_rss_bytes=c['rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=1),poll=.2,gpu_uuid=c['gpu_uuid'])
|
|
except BaseException as exc:reason(type(exc).__name__+': '+str(exc));kill_worker()
|
|
finally:
|
|
done.set()
|
|
if thread:thread.join(timeout=4)
|
|
if thread and thread.is_alive():reason('MONITOR_DID_NOT_EXIT')
|
|
try:scan()
|
|
except Exception as exc:reason('FINAL_PEAK_SCAN_ERROR');errors.append(dict(reason=str(exc)))
|
|
for s,handler in old.items():signal.signal(s,handler)
|
|
value=None;p=root/'worker/result.json'
|
|
if p.exists():value=json.loads(p.read_text())
|
|
if not receipt or receipt['status']!='PASS':reason('WORKER_NOT_PASS')
|
|
if receipt and receipt.get('monitor_errors'):reason('WORKER_MONITOR_ERRORS')
|
|
if not value or value.get('status')!='COMPLETE':reason('NO_COMPLETE_PILOT_RESULT')
|
|
if not all(k in tracker.witnesses for k in ['peak_allocated_bytes','peak_reserved_bytes']):reason('MISSING_CUDA_PEAK_RECORDS')
|
|
totalcpu=60+(receipt['cpu_seconds'] if receipt else 0)+time.process_time()-cpu
|
|
if elapsed()>c['gpu_seconds'] or totalcpu>c['cpu_seconds']:reason('FINAL_RESOURCE_LIMIT')
|
|
projected=None
|
|
if value:
|
|
projected=projection(c,value['full_epoch_seconds'],value['validation_seconds'],elapsed());save(root/'projection.json',projected)
|
|
save(root/'complete.json',dict(status='PASS' if not reasons else 'STOPPED_OR_FAILED',decision=projected['decision'] if projected and not reasons else 'PILOT_INCOMPLETE',reasons=reasons,monitor_errors=errors,
|
|
worker=receipt,recorded_memory=tracker.as_dict(),gpu_seconds_charged=elapsed(),cpu_seconds_charged=totalcpu,projection=projected,
|
|
worker_result_sha256=sha(p) if p.exists() else None,full_run_started=False,no_retry=True))
|
|
return 0 if not reasons else 1
|
|
|
|
if __name__=='__main__':raise SystemExit(run(Path(sys.argv[1])))
|