Files
MeltMind/tools/run_r4a.py
T

142 lines
8.0 KiB
Python

"""One detached R4(a) queue: fail-stop, no retry, no scoring after a failure."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import resource
import signal
import subprocess
import sys
import time
import traceback
import psutil
from gre24_prepare_v2 import sha
from r4a_part import contract
from r4a_core import may_dispatch, scoring_allowed
from s2_recovery_supervisor import save, supervise, stop_group
from s2_recovery_monitoring import disk_usage
def guard(root,index,slot):
c=contract(root);os.sched_setaffinity(0,[c['worker_cpus'][slot]])
dest=root/'parts'/c['parts'][index]['case']
receipt=supervise([sys.executable,'tools/r4a_part.py',str(root),str(index)],dest,'worker',
dict(total_cpu_seconds=1801,job_cpu_seconds=1800,job_wall_seconds=1800,
tree_rss_bytes=4*2**30,disk_bytes=c['disk_bytes'],max_processes=1),poll=.2)
result=dest/'output/result.json'
if receipt['status']=='PASS' and (not result.exists() or json.loads(result.read_text())['status']!='PASS'):
receipt.update(status='STOPPED_OR_FAILED',stop_reason='MISSING_PASS_RESULT')
save(dest/'guard-result.json',dict(index=index,case=c['parts'][index]['case'],worker=receipt,
result_path=str(result) if result.exists() else None,result_sha256=sha(result) if result.exists() else None,
guard_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
return 0 if receipt['status']=='PASS' else 1
def kill_guard_worker(pid):
# Keep the supervisor alive to reap its worker and write the true failure receipt.
try:
for child in psutil.Process(pid).children():stop_group(child.pid)
except psutil.NoSuchProcess:pass
def fresh(root):
if (root/'started.json').exists() or (root/'complete.json').exists():
raise FileExistsError('Existing attempt: never restart R4 queue')
def run(root):
with (root/'queue.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return run_locked(root)
def run_locked(root):
c=contract(root);fresh(root);os.sched_setaffinity(0,[25])
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),contract_sha256=sha(root/'contract.json')))
active={};done=[];next_i=0;reason=None;start=time.monotonic();error=None
def interrupted(*_):
nonlocal reason
reason=reason or 'COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGINT,interrupted);signal.signal(signal.SIGTERM,interrupted)
env=dict(os.environ,OMP_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
try:
while active or (next_i<len(c['parts']) and reason is None):
for slot,item in list(active.items()):
if item['process'].poll() is None:continue
receipt=root/'parts'/c['parts'][item['index']]['case']/'guard-result.json'
if not receipt.exists():raise RuntimeError('MISSING_GUARD_RECEIPT')
r=json.loads(receipt.read_text());done.append(r);item['log'].close();del active[slot]
if r['worker']['status']!='PASS':reason=reason or 'PART_FAILED_NO_AUTOMATIC_SCORING'
spent=c['initial_worker_cpu_seconds']+sum(x['worker']['cpu_seconds'] for x in done)
used=disk_usage(root)
if used>=c['disk_bytes']:reason=reason or 'GLOBAL_DISK_LIMIT'
if spent>=28800:reason=reason or 'COMPUTE_CPU_LIMIT'
while reason is None and len(active)<4 and next_i<len(c['parts']):
if not may_dispatch(spent,len(active)):
if not active:reason='INSUFFICIENT_COMPUTE_RESERVATION'
break
slot=next(s for s in range(4) if s not in active);index=next_i;next_i+=1
dest=root/'parts'/c['parts'][index]['case'];dest.mkdir(parents=True,exist_ok=False)
log=(dest/'guard.log').open('x')
proc=subprocess.Popen([sys.executable,__file__,str(root),'--guard',str(index),'--slot',str(slot)],
stdout=log,stderr=subprocess.STDOUT,env=env,start_new_session=True)
active[slot]=dict(index=index,process=proc,log=log)
if reason:
for item in active.values():kill_guard_worker(item['process'].pid)
save(root/'progress.json',dict(completed=len(done),attempted=next_i,total=len(c['parts']),
passed=sum(x['worker']['status']=='PASS' for x in done),active={str(s):v['index'] for s,v in active.items()},
settled_worker_cpu_seconds=spent,reserved_active_worker_cpu_seconds=len(active)*1802,
stop_reason=reason,disk_bytes=used,wall_seconds=time.monotonic()-start))
if active:time.sleep(.5)
except Exception as exc:
reason=reason or 'INFRASTRUCTURE_EXCEPTION';error=dict(message=str(exc),traceback=traceback.format_exc())
finally:
for item in active.values():kill_guard_worker(item['process'].pid)
for item in active.values():
item['process'].wait();item['log'].close()
p=root/'parts'/c['parts'][item['index']]['case']/'guard-result.json'
if p.exists():done.append(json.loads(p.read_text()))
active={}
done.sort(key=lambda r:r['index']);spent=c['initial_worker_cpu_seconds']+sum(x['worker']['cpu_seconds'] for x in done)
allowed=scoring_allowed(done,len(c['parts']),reason)
terminal=dict(status='ALL_PASS' if allowed else 'STOPPED',stop_reason=reason,records=done,
never_attempted=list(range(next_i,len(c['parts']))),attempted=next_i,
missing_receipt_indices=sorted(set(range(next_i))-{r['index'] for r in done}),
charged_worker_cpu_seconds=spent,initial_worker_cpu_seconds=c['initial_worker_cpu_seconds'],
error=error,accounting_complete=len(done)==next_i)
save(root/'preparation-complete.json',terminal)
post=None
if allowed:
parts={}
for r in done:
result=Path(r['result_path']);checked_result=json.loads(result.read_text())
if sha(result)!=r['result_sha256']:raise ValueError('Result changed before freeze')
p=result.parent/'query-predictions.npz';digest=checked_result['output_sha256'][p.name]
if sha(p)!=digest:raise ValueError('Prediction changed before freeze')
parts[r['case']]=dict(result=str(result),result_sha256=sha(result),predictions=str(p),predictions_sha256=digest)
save(root/'predictions-freeze.json',dict(contract_sha256=sha(root/'contract.json'),
preparation_complete_sha256=sha(root/'preparation-complete.json'),parts=parts,labels_read=False))
remaining_disk=c['disk_bytes']-disk_usage(root)
if remaining_disk<=0:reason='POSTPROCESS_DISK_EXHAUSTED'
else:
# Inherit a non-v4 CPU; one guarded process for all cache-only scoring.
os.sched_setaffinity(0,[24])
post=supervise([sys.executable,'tools/r4a_score.py',str(root)],root/'postfit','worker',
dict(total_cpu_seconds=7200,job_cpu_seconds=7200,job_wall_seconds=7200,
tree_rss_bytes=4*2**30,disk_bytes=remaining_disk,max_processes=1),poll=.2)
os.sched_setaffinity(0,[25]);spent+=post['cpu_seconds']
if post['status']!='PASS':reason='POSTPROCESS_FAILED'
save(root/'complete.json',dict(status='COMPLETE' if allowed and post and post['status']=='PASS' else 'STOPPED',
stop_reason=reason,charged_worker_cpu_seconds=spent,worker_cpu_cap=36000,
preparation_complete_sha256=sha(root/'preparation-complete.json'),postfit=post,
coordinator_cpu_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2]),
guard_cpu_seconds=sum(x.get('guard_cpu_seconds',0.) for x in done),
wall_seconds=time.monotonic()-start,retries=0,accounting_complete=len(done)==next_i))
return 0 if allowed and post and post['status']=='PASS' else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--guard',type=int);p.add_argument('--slot',type=int,default=0)
a=p.parse_args();raise SystemExit(run(a.root) if a.guard is None else guard(a.root,a.guard,a.slot))