125 lines
7.5 KiB
Python
125 lines
7.5 KiB
Python
"""16 guarded workers, shared CPU/disk caps, exact first-50 terminal barrier."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import resource
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import psutil
|
|
from freeze_g1_components import sha
|
|
from s2_e1_supervisor import supervise,save,disk_usage,stop_group
|
|
|
|
|
|
def contract(root):
|
|
c=json.loads((root/'contract.json').read_text())
|
|
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
|
return c
|
|
|
|
|
|
def guard(root,index):
|
|
c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True)
|
|
r=supervise([sys.executable,'tools/s2_pilot_prepare.py',str(root),str(index)],job,'worker',
|
|
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,tree_rss_bytes=1342177280,
|
|
disk_bytes=100*2**30,max_processes=1),poll=.25)
|
|
save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
|
|
return 0 if r['status']=='PASS' else 1
|
|
|
|
|
|
def projection(c,done):
|
|
by_index={r['index']:r for r in done};observed={False:[0.,0],True:[0.,0]}
|
|
for i,r in by_index.items():
|
|
cached=bool(c['cases'][i].get('reuse_preparation'));observed[cached][0]+=r['worker']['cpu_seconds'];observed[cached][1]+=c['cases'][i]['tet_nodes']
|
|
rate={key:(cpu/nodes if nodes else c['initial_cpu_seconds_per_node']) for key,(cpu,nodes) in observed.items()}
|
|
spent=sum(r['worker']['cpu_seconds'] for r in done)
|
|
remaining=sum(rate[bool(s.get('reuse_preparation'))]*s['tet_nodes'] for i,s in enumerate(c['cases']) if i not in by_index)
|
|
return dict(projected_worker_cpu_seconds=spent+remaining,spent_worker_cpu_seconds=spent,
|
|
observed_rate_seconds_per_node={str(k):v for k,v in rate.items()},method='Separate reused-complete and newly prepared observed node rates; initial rate only for unseen category')
|
|
|
|
|
|
def terminate(active):
|
|
for process,_ in active.values():
|
|
try:
|
|
for child in psutil.Process(process.pid).children():stop_group(child.pid)
|
|
except psutil.NoSuchProcess:pass
|
|
|
|
|
|
def run(root):
|
|
with (root/'batch.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
return run_locked(root)
|
|
|
|
|
|
def run_locked(root):
|
|
c=contract(root)
|
|
assert not (root/'batch-started.json').exists(),'Batch already attempted; inspect receipts, do not restart'
|
|
assert c['initial_projection_cpu_hours']<=60 and c['initial_projection_gib']<=100
|
|
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
|
start=time.monotonic();save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
|
|
active={};done=[];next_index=0;stop_reason=None;checkpoint=False;peak_rss=0
|
|
def interrupted(*_):
|
|
nonlocal stop_reason
|
|
stop_reason='COORDINATOR_INTERRUPTED'
|
|
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
|
try:
|
|
while active or (next_index<len(c['cases']) and stop_reason is None):
|
|
limit=len(c['cases']) if checkpoint else min(50,len(c['cases']))
|
|
while stop_reason is None and len(active)<16 and next_index<limit:
|
|
i=next_index;next_index+=1;job=root/'jobs'/f'job-{i:04d}';job.mkdir(parents=True,exist_ok=True)
|
|
log=(job/'guard.log').open('x');p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
|
active[i]=(p,log)
|
|
for i,(p,log) in list(active.items()):
|
|
code=p.poll()
|
|
if code is not None:
|
|
log.close();path=root/'jobs'/f'job-{i:04d}'/'guard-result.json'
|
|
if not path.exists():
|
|
stop_reason='GUARD_MISSING_ACCOUNTING_RECEIPT'
|
|
result=dict(worker=dict(status='GUARD_FAILED_ACCOUNTING_UNKNOWN',cpu_seconds=0.,wall_seconds=None,peak_tree_rss_bytes=None),supervisor_cpu_seconds=0.)
|
|
else:result=json.loads(path.read_text())
|
|
done.append(dict(index=i,case=c['cases'][i]['case'],returncode=code,**result));del active[i]
|
|
print(json.dumps(dict(event='FINISHED',index=i,completed=len(done),status=result['worker']['status'])),flush=True)
|
|
live_cpu=live_rss=0
|
|
for p,_ in active.values():
|
|
try:
|
|
for q in [psutil.Process(p.pid)]+psutil.Process(p.pid).children(recursive=True):
|
|
try:
|
|
t=q.cpu_times();live_cpu+=t.user+t.system+t.children_user+t.children_system;live_rss+=q.memory_info().rss
|
|
except psutil.NoSuchProcess:pass
|
|
except psutil.NoSuchProcess:pass
|
|
peak_rss=max(peak_rss,live_rss)
|
|
used=disk_usage(root);cpu=sum(r['worker']['cpu_seconds'] for r in done)
|
|
# Conservatively includes live guard CPU; completed guard CPU is separately reported.
|
|
if cpu+live_cpu>=60*3600:stop_reason='GLOBAL_CPU_LIMIT'
|
|
if used>=100*2**30:stop_reason='GLOBAL_DISK_LIMIT'
|
|
if len(done)==min(50,len(c['cases'])) and not active and not checkpoint and stop_reason is None:
|
|
proj=projection(c,done);finished_nodes=sum(c['cases'][r['index']]['tet_nodes'] for r in done)
|
|
projected_disk=used/finished_nodes*sum(s['tet_nodes'] for s in c['cases'])
|
|
checkpoint=True
|
|
if proj['projected_worker_cpu_seconds']>60*3600 or projected_disk>100*2**30:stop_reason='FIRST_50_PROJECTION_OVER_CAP'
|
|
save(root/'first-50-checkpoint.json',dict(**proj,projected_disk_bytes=projected_disk,continue_within_cap=stop_reason is None,completed=len(done),failed=sum(r['worker']['status']!='PASS' for r in done)))
|
|
print(json.dumps(dict(event='FIRST_50_CHECKPOINT',**proj,projected_disk_bytes=projected_disk,stop_reason=stop_reason)),flush=True)
|
|
if stop_reason:terminate(active)
|
|
save(root/'progress.json',dict(completed=len(done),active=list(active),next_index=next_index,first50_complete=checkpoint,worker_cpu_seconds=cpu,
|
|
live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=used,elapsed_seconds=time.monotonic()-start,stop_reason=stop_reason))
|
|
if active:time.sleep(.5)
|
|
finally:
|
|
terminate(active)
|
|
for p,log in active.values():
|
|
try:p.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:stop_group(p.pid);p.wait()
|
|
log.close()
|
|
result=dict(status='ATTEMPTS_COMPLETE' if len(done)==len(c['cases']) else 'STOPPED',stop_reason=stop_reason,jobs=done,
|
|
not_run_indices=list(range(next_index,len(c['cases']))),worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
|
|
guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
|
wall_seconds=time.monotonic()-start,peak_aggregate_rss_bytes=peak_rss,disk_bytes=disk_usage(root),
|
|
failed=sum(r['worker']['status']!='PASS' for r in done),retry=False,replacement=False,test_labels_read=False)
|
|
save(root/'batch-complete.json',result);print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True)
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args()
|
|
sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root))
|