Files
MeltMind/tools/run_g1_copy_calibration.py
T

101 lines
6.3 KiB
Python

"""Four guarded calibration workers; all CPU/failures retained, no retries."""
from pathlib import Path
import argparse
import fcntl
import json
import os
import resource
import signal
import subprocess
import sys
import time
import psutil
from g1_copy_calibration import ROOT, check_contract
from s2_e1_supervisor import supervise, save, disk_usage, stop_group
def guard(index):
c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text());n=len(plan['jobs'][index]['pairs'])
job=ROOT/'jobs'/f'job-{index:02d}';job.mkdir(parents=True,exist_ok=True)
limit=n*c['per_pair_wall_seconds']
r=supervise([sys.executable,'tools/g1_copy_calibration.py',str(index)],job,'worker',
dict(total_cpu_seconds=limit+1,job_cpu_seconds=limit,job_wall_seconds=limit,
tree_rss_bytes=c['per_worker_rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=2),poll=.25)
save(job/'guard-result.json',{'worker':r,'supervisor_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2])})
return 0 if r['status']=='PASS' else 1
def terminate_workers(active):
# Guards stay alive to reap killed workers and write failure receipts.
for p,_ in active.values():
try:
for child in psutil.Process(p.pid).children():stop_group(child.pid)
except psutil.NoSuchProcess:pass
def run():
c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text())
lock=(ROOT/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (ROOT/'batch-started.json').exists():raise FileExistsError('Batch already attempted; no automatic restart')
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
save(ROOT/'batch-started.json',{'pid':os.getpid(),'started_unix':time.time(),'jobs':len(plan['jobs']),'pairs':sum(len(j['pairs']) for j in plan['jobs']),'workers':c['workers'],'contract_sha256':__import__('hashlib').sha256((ROOT/'contract.json').read_bytes()).hexdigest()})
started=time.monotonic();active={};done=[];next_index=0;reason=None;peak_rss=0
def interrupted(*_):
nonlocal reason
reason='coordinator_interrupted'
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
try:
while active or (next_index<len(plan['jobs']) and reason is None):
while reason is None and len(active)<c['workers'] and next_index<len(plan['jobs']):
i=next_index;next_index+=1;path=ROOT/'jobs'/f'job-{i:02d}';path.mkdir(parents=True,exist_ok=True)
log=(path/'guard.log').open('x')
p=subprocess.Popen([sys.executable,__file__,'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[i]=(p,log)
print(json.dumps({'event':'STARTED','job':i,'guard_pid':p.pid}),flush=True)
for i,(p,log) in list(active.items()):
code=p.poll()
if code is not None:
log.close();path=ROOT/'jobs'/f'job-{i:02d}'/'guard-result.json'
result=json.loads(path.read_text()) if path.exists() else {'worker':{'status':'GUARD_FAILED','cpu_seconds':0},'supervisor_cpu_seconds':0}
done.append({'index':i,'returncode':code,**result});del active[i]
print(json.dumps({'event':'FINISHED','job':i,'status':result['worker']['status'],'completed':len(done)}),flush=True)
# Calibration has 22 jobs: its scheduled failure-rate checkpoint is batch end.
if time.monotonic()-started>=c['wall_seconds']:reason='global_wall_limit'
if disk_usage(ROOT)>=c['disk_bytes']:reason='global_disk_limit'
live_cpu=live_rss=0
for p,_ in active.values():
try:
for q in [psutil.Process(p.pid)]+psutil.Process(p.pid).children(recursive=True):
try:
t=q.cpu_times();live_cpu+=t.user+t.system+t.children_user+t.children_system;live_rss+=q.memory_info().rss
except psutil.NoSuchProcess:pass
except psutil.NoSuchProcess:pass
peak_rss=max(peak_rss,live_rss)
completed_cpu=sum(r['worker']['cpu_seconds']+r['supervisor_cpu_seconds'] for r in done)
if completed_cpu+live_cpu>=c['worker_cpu_budget_seconds']:reason='global_cpu_limit'
if live_rss>=c['aggregate_rss_bytes']:reason='aggregate_rss_limit'
if reason:terminate_workers(active)
save(ROOT/'progress.json',{'completed_jobs':len(done),'active_jobs':list(active),'next_job':next_index,
'completed_worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done),
'live_tree_cpu_seconds':live_cpu,'active_tree_rss_bytes':live_rss,'stop_reason':reason,'elapsed_seconds':time.monotonic()-started})
if active:time.sleep(.5)
finally:
terminate_workers(active)
for p,log in active.values():
try:p.wait(timeout=10)
except subprocess.TimeoutExpired:stop_group(p.pid);p.wait()
log.close()
failures=sum(r['worker']['status']!='PASS' for r in done)
passed=reason is None and len(done)==len(plan['jobs']) and failures==0
summary={'status':'CALIBRATION_RESIDUALS_COMPLETE' if passed else 'CALIBRATION_INCOMPLETE',
'jobs':done,'failed_jobs':failures,'failure_rate_at_final_checkpoint':failures/len(done) if done else None,
'stop_reason':reason,'worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done),
'guard_cpu_seconds':sum(r['supervisor_cpu_seconds'] for r in done),
'coordinator_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
'wall_seconds':time.monotonic()-started,'peak_aggregate_tree_rss_bytes':peak_rss,
'disk_bytes':disk_usage(ROOT),'thresholds_frozen':False,'candidate_generation':False,'no_retries':True}
save(ROOT/'batch-complete.json' if passed else ROOT/'batch-incomplete.json',summary)
print(json.dumps({k:v for k,v in summary.items() if k!='jobs'}),flush=True)
return 0 if passed else 1
if __name__=='__main__':
ap=argparse.ArgumentParser();ap.add_argument('--guard',type=int);a=ap.parse_args()
sys.exit(guard(a.guard) if a.guard is not None else run())