Files
MeltMind/tools/run_s2_pilot_preparation.py
T

125 lines
7.5 KiB
Python

"""16 guarded workers, shared CPU/disk caps, exact first-50 terminal barrier."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
import psutil
from freeze_g1_components import sha
from s2_e1_supervisor import supervise,save,disk_usage,stop_group
def contract(root):
c=json.loads((root/'contract.json').read_text())
for p,h in c['code_sha256'].items():assert sha(p)==h,p
return c
def guard(root,index):
c=contract(root);job=root/'jobs'/f'job-{index:04d}';job.mkdir(parents=True,exist_ok=True)
r=supervise([sys.executable,'tools/s2_pilot_prepare.py',str(root),str(index)],job,'worker',
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,tree_rss_bytes=1342177280,
disk_bytes=100*2**30,max_processes=1),poll=.25)
save(job/'guard-result.json',dict(worker=r,supervisor_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
return 0 if r['status']=='PASS' else 1
def projection(c,done):
by_index={r['index']:r for r in done};observed={False:[0.,0],True:[0.,0]}
for i,r in by_index.items():
cached=bool(c['cases'][i].get('reuse_preparation'));observed[cached][0]+=r['worker']['cpu_seconds'];observed[cached][1]+=c['cases'][i]['tet_nodes']
rate={key:(cpu/nodes if nodes else c['initial_cpu_seconds_per_node']) for key,(cpu,nodes) in observed.items()}
spent=sum(r['worker']['cpu_seconds'] for r in done)
remaining=sum(rate[bool(s.get('reuse_preparation'))]*s['tet_nodes'] for i,s in enumerate(c['cases']) if i not in by_index)
return dict(projected_worker_cpu_seconds=spent+remaining,spent_worker_cpu_seconds=spent,
observed_rate_seconds_per_node={str(k):v for k,v in rate.items()},method='Separate reused-complete and newly prepared observed node rates; initial rate only for unseen category')
def terminate(active):
for process,_ in active.values():
try:
for child in psutil.Process(process.pid).children():stop_group(child.pid)
except psutil.NoSuchProcess:pass
def run(root):
with (root/'batch.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return run_locked(root)
def run_locked(root):
c=contract(root)
assert not (root/'batch-started.json').exists(),'Batch already attempted; inspect receipts, do not restart'
assert c['initial_projection_cpu_hours']<=60 and c['initial_projection_gib']<=100
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
start=time.monotonic();save(root/'batch-started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
active={};done=[];next_index=0;stop_reason=None;checkpoint=False;peak_rss=0
def interrupted(*_):
nonlocal stop_reason
stop_reason='COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
try:
while active or (next_index<len(c['cases']) and stop_reason is None):
limit=len(c['cases']) if checkpoint else min(50,len(c['cases']))
while stop_reason is None and len(active)<16 and next_index<limit:
i=next_index;next_index+=1;job=root/'jobs'/f'job-{i:04d}';job.mkdir(parents=True,exist_ok=True)
log=(job/'guard.log').open('x');p=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[i]=(p,log)
for i,(p,log) in list(active.items()):
code=p.poll()
if code is not None:
log.close();path=root/'jobs'/f'job-{i:04d}'/'guard-result.json'
if not path.exists():
stop_reason='GUARD_MISSING_ACCOUNTING_RECEIPT'
result=dict(worker=dict(status='GUARD_FAILED_ACCOUNTING_UNKNOWN',cpu_seconds=0.,wall_seconds=None,peak_tree_rss_bytes=None),supervisor_cpu_seconds=0.)
else:result=json.loads(path.read_text())
done.append(dict(index=i,case=c['cases'][i]['case'],returncode=code,**result));del active[i]
print(json.dumps(dict(event='FINISHED',index=i,completed=len(done),status=result['worker']['status'])),flush=True)
live_cpu=live_rss=0
for p,_ in active.values():
try:
for q in [psutil.Process(p.pid)]+psutil.Process(p.pid).children(recursive=True):
try:
t=q.cpu_times();live_cpu+=t.user+t.system+t.children_user+t.children_system;live_rss+=q.memory_info().rss
except psutil.NoSuchProcess:pass
except psutil.NoSuchProcess:pass
peak_rss=max(peak_rss,live_rss)
used=disk_usage(root);cpu=sum(r['worker']['cpu_seconds'] for r in done)
# Conservatively includes live guard CPU; completed guard CPU is separately reported.
if cpu+live_cpu>=60*3600:stop_reason='GLOBAL_CPU_LIMIT'
if used>=100*2**30:stop_reason='GLOBAL_DISK_LIMIT'
if len(done)==min(50,len(c['cases'])) and not active and not checkpoint and stop_reason is None:
proj=projection(c,done);finished_nodes=sum(c['cases'][r['index']]['tet_nodes'] for r in done)
projected_disk=used/finished_nodes*sum(s['tet_nodes'] for s in c['cases'])
checkpoint=True
if proj['projected_worker_cpu_seconds']>60*3600 or projected_disk>100*2**30:stop_reason='FIRST_50_PROJECTION_OVER_CAP'
save(root/'first-50-checkpoint.json',dict(**proj,projected_disk_bytes=projected_disk,continue_within_cap=stop_reason is None,completed=len(done),failed=sum(r['worker']['status']!='PASS' for r in done)))
print(json.dumps(dict(event='FIRST_50_CHECKPOINT',**proj,projected_disk_bytes=projected_disk,stop_reason=stop_reason)),flush=True)
if stop_reason:terminate(active)
save(root/'progress.json',dict(completed=len(done),active=list(active),next_index=next_index,first50_complete=checkpoint,worker_cpu_seconds=cpu,
live_guard_and_worker_cpu_seconds=live_cpu,active_tree_rss_bytes=live_rss,disk_bytes=used,elapsed_seconds=time.monotonic()-start,stop_reason=stop_reason))
if active:time.sleep(.5)
finally:
terminate(active)
for p,log in active.values():
try:p.wait(timeout=10)
except subprocess.TimeoutExpired:stop_group(p.pid);p.wait()
log.close()
result=dict(status='ATTEMPTS_COMPLETE' if len(done)==len(c['cases']) else 'STOPPED',stop_reason=stop_reason,jobs=done,
not_run_indices=list(range(next_index,len(c['cases']))),worker_cpu_seconds=sum(r['worker']['cpu_seconds'] for r in done),
guard_cpu_seconds=sum(r['supervisor_cpu_seconds'] for r in done),coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
wall_seconds=time.monotonic()-start,peak_aggregate_rss_bytes=peak_rss,disk_bytes=disk_usage(root),
failed=sum(r['worker']['status']!='PASS' for r in done),retry=False,replacement=False,test_labels_read=False)
save(root/'batch-complete.json',result);print(json.dumps({k:v for k,v in result.items() if k!='jobs'}),flush=True)
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);a=p.parse_args()
sys.exit(guard(a.root,a.guard) if a.guard is not None else run(a.root))