Freeze completed G1 calibration and start note16 fingerprint candidate count
This commit is contained in:
1 parent
2ba7800d39
commit
ff439e81fc
76 files changed
+170909
No files matched your search
@@ -0,0 +1,104 @@
|
||||
"""Guarded missing fingerprint extraction; no retries; count follows a complete corpus."""
|
||||
from pathlib import Path
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from g1_candidate_fingerprints import ROOT, check_contract
|
||||
from s2_e1_supervisor import supervise, save, disk_usage, stop_group
|
||||
|
||||
def guard(index):
|
||||
c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text());n=1
|
||||
job=ROOT/'jobs'/f'job-{index:02d}';job.mkdir(parents=True,exist_ok=True)
|
||||
limit=n*c['per_pair_wall_seconds']
|
||||
r=supervise([sys.executable,'tools/g1_candidate_fingerprints.py',str(index)],job,'worker',
|
||||
dict(total_cpu_seconds=limit+1,job_cpu_seconds=limit,job_wall_seconds=limit,
|
||||
tree_rss_bytes=c['per_worker_rss_bytes'],disk_bytes=c['disk_bytes'],max_processes=2),poll=.25)
|
||||
save(job/'guard-result.json',{'worker':r,'supervisor_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2])})
|
||||
return 0 if r['status']=='PASS' else 1
|
||||
|
||||
def terminate_workers(active):
|
||||
# Guards stay alive to reap killed workers and write failure receipts.
|
||||
for p,_ in active.values():
|
||||
try:
|
||||
for child in psutil.Process(p.pid).children():stop_group(child.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
|
||||
def run():
|
||||
c=check_contract(ROOT);plan=json.loads((ROOT/'custodian/plan.json').read_text())
|
||||
lock=(ROOT/'batch.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (ROOT/'batch-started.json').exists():raise FileExistsError('Batch already attempted; no automatic restart')
|
||||
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
||||
save(ROOT/'batch-started.json',{'pid':os.getpid(),'started_unix':time.time(),'jobs':len(plan['jobs']),'parts':len(plan['jobs']),'workers':c['workers'],'contract_sha256':__import__('hashlib').sha256((ROOT/'contract.json').read_bytes()).hexdigest()})
|
||||
started=time.monotonic();active={};done=[];next_index=0;reason=None;peak_rss=0
|
||||
def interrupted(*_):
|
||||
nonlocal reason
|
||||
reason='coordinator_interrupted'
|
||||
signal.signal(signal.SIGTERM,interrupted);signal.signal(signal.SIGINT,interrupted)
|
||||
try:
|
||||
while active or (next_index<len(plan['jobs']) and reason is None):
|
||||
while reason is None and len(active)<c['workers'] and next_index<len(plan['jobs']):
|
||||
i=next_index;next_index+=1;path=ROOT/'jobs'/f'job-{i:02d}';path.mkdir(parents=True,exist_ok=True)
|
||||
log=(path/'guard.log').open('x')
|
||||
p=subprocess.Popen([sys.executable,__file__,'--guard',str(i)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
active[i]=(p,log)
|
||||
print(json.dumps({'event':'STARTED','job':i,'guard_pid':p.pid}),flush=True)
|
||||
for i,(p,log) in list(active.items()):
|
||||
code=p.poll()
|
||||
if code is not None:
|
||||
log.close();path=ROOT/'jobs'/f'job-{i:02d}'/'guard-result.json'
|
||||
result=json.loads(path.read_text()) if path.exists() else {'worker':{'status':'GUARD_FAILED','cpu_seconds':0},'supervisor_cpu_seconds':0}
|
||||
done.append({'index':i,'returncode':code,**result});del active[i]
|
||||
if len(done)%100==0 and sum(r['worker']['status']!='PASS' for r in done)/len(done)>.02:
|
||||
reason='failure_rate_over_2_percent'
|
||||
print(json.dumps({'event':'FINISHED','job':i,'status':result['worker']['status'],'completed':len(done)}),flush=True)
|
||||
# Evaluate failures at every 100 terminal jobs and at the final partial batch.
|
||||
if done and len(done)==len(plan['jobs']):
|
||||
if sum(r['worker']['status']!='PASS' for r in done)/len(done)>.02:reason='failure_rate_over_2_percent'
|
||||
if time.monotonic()-started>=c['wall_seconds']:reason='global_wall_limit'
|
||||
if disk_usage(ROOT)>=c['disk_bytes']:reason='global_disk_limit'
|
||||
live_cpu=live_rss=0
|
||||
for p,_ in active.values():
|
||||
try:
|
||||
for q in [psutil.Process(p.pid)]+psutil.Process(p.pid).children(recursive=True):
|
||||
try:
|
||||
t=q.cpu_times();live_cpu+=t.user+t.system+t.children_user+t.children_system;live_rss+=q.memory_info().rss
|
||||
except psutil.NoSuchProcess:pass
|
||||
except psutil.NoSuchProcess:pass
|
||||
peak_rss=max(peak_rss,live_rss)
|
||||
completed_cpu=sum(r['worker']['cpu_seconds']+r['supervisor_cpu_seconds'] for r in done)
|
||||
if completed_cpu+live_cpu>=c['worker_cpu_budget_seconds']:reason='global_cpu_limit'
|
||||
if live_rss>=c['aggregate_rss_bytes']:reason='aggregate_rss_limit'
|
||||
if reason:terminate_workers(active)
|
||||
save(ROOT/'progress.json',{'completed_jobs':len(done),'active_jobs':list(active),'next_job':next_index,
|
||||
'completed_worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done),
|
||||
'live_tree_cpu_seconds':live_cpu,'active_tree_rss_bytes':live_rss,'stop_reason':reason,'elapsed_seconds':time.monotonic()-started})
|
||||
if active:time.sleep(.5)
|
||||
finally:
|
||||
terminate_workers(active)
|
||||
for p,log in active.values():
|
||||
try:p.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:stop_group(p.pid);p.wait()
|
||||
log.close()
|
||||
failures=sum(r['worker']['status']!='PASS' for r in done)
|
||||
passed=reason is None and len(done)==len(plan['jobs']) and failures==0
|
||||
summary={'status':'ALL_FINGERPRINTS_COMPLETE' if passed else 'FINGERPRINTS_INCOMPLETE',
|
||||
'jobs':done,'failed_jobs':failures,'failure_rate_at_final_checkpoint':failures/len(done) if done else None,
|
||||
'stop_reason':reason,'worker_cpu_seconds':sum(r['worker']['cpu_seconds'] for r in done),
|
||||
'guard_cpu_seconds':sum(r['supervisor_cpu_seconds'] for r in done),
|
||||
'coordinator_cpu_seconds':sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
|
||||
'wall_seconds':time.monotonic()-started,'peak_aggregate_tree_rss_bytes':peak_rss,
|
||||
'disk_bytes':disk_usage(ROOT),'thresholds_frozen_before_extraction':True,'candidate_generation':False,'no_retries':True}
|
||||
save(ROOT/'batch-complete.json' if passed else ROOT/'batch-incomplete.json',summary)
|
||||
print(json.dumps({k:v for k,v in summary.items() if k!='jobs'}),flush=True)
|
||||
return 0 if passed else 1
|
||||
|
||||
if __name__=='__main__':
|
||||
ap=argparse.ArgumentParser();ap.add_argument('--guard',type=int);a=ap.parse_args()
|
||||
sys.exit(guard(a.guard) if a.guard is not None else run())
|
||||
Reference in new issue
Block a user