Files
MeltMind/tools/run_gre24_models.py
T

143 lines
9.0 KiB
Python

"""One sequential fixed-model chain after terminal preparation, within shared caps."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import sys
import time
import psutil
from gre24_prepare_v2 import sha
from gre24_job_guard import supervise,save
from gre24_gpu_guard import gpu_state,gpu_fit
def freeze_preparation(root):
complete=json.loads((root/'complete.json').read_text());c=json.loads((root/'contract.json').read_text())
if complete['status']!='COMPLETE' or not complete['accounting_complete'] or len(complete['records'])!=len(c['parts']):
raise ValueError('Preparation globally stopped or missing terminal accounting')
good={};fail=[]
for r in complete['records']:
if r['worker']['status']!='PASS':
fail.append(dict(case=r['case'],guard_status=r['worker']['status'],reason=r['worker']['stop_reason']));continue
p=Path(r['result_path'])
if sha(p)!=r['result_sha256']:raise ValueError('Preparation result changed')
result=json.loads(p.read_text())
good[r['case']]=dict(output=str(p.parent),nodes=result['geometry']['tet_nodes'],
raw_nodes=result['geometry'].get('raw_nodes',result['geometry']['tet_nodes']),
t_scale_seconds=result['t_scale_seconds'],output_sha256=result['output_sha256'],result_sha256=sha(p),
outside_tet_nodes=result['geometry']['outside_tet_nodes'])
frozen=dict(successful=good,failed=fail,requested=len(c['parts']),
preparation_complete_sha256=sha(root/'complete.json'),replacement=False,
cumulative_initial_worker_cpu_seconds=complete['worker_cpu_seconds']+complete['initial_cpu_reserve_seconds'])
save(root/'preparation-freeze.json',frozen);return frozen
def run(root,plan_path):
root=Path(root);plan_path=Path(plan_path)
with (root/'model-chain.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return run_locked(root,plan_path)
def run_locked(root,plan_path):
plan=json.loads(plan_path.read_text())
for p,h in plan['code_sha256'].items():
if sha(p)!=h:raise ValueError('Model plan source changed: '+p)
started=root/'model-chain-started.json'
if started.exists():raise FileExistsError('Never restart model chain')
save(started,dict(pid=os.getpid(),unix=time.time(),plan_sha256=sha(plan_path),stage='WAIT_PREPARATION'))
missing_since=None
while not (root/'complete.json').exists():
parent=json.loads((root/'queue-started.json').read_text())
try:
process=psutil.Process(parent['pid'])
alive=process.create_time()<=parent['unix'] and process.status()!=psutil.STATUS_ZOMBIE and any('run_gre24_protocol_preparation.py' in a for a in process.cmdline())
except psutil.NoSuchProcess:alive=False
if not alive:
missing_since=missing_since or time.monotonic()
if time.monotonic()-missing_since>30:raise RuntimeError('Preparation exited without complete receipt; no restart')
else:missing_since=None
save(root/'model-chain-progress.json',dict(stage='WAIT_PREPARATION',unix=time.time()))
time.sleep(10)
frozen=freeze_preparation(root)
split_path=root/'splits.json';splits=json.loads(split_path.read_text())
c=dict(plan,preparation_freeze_sha256=sha(root/'preparation-freeze.json'),splits_sha256=sha(split_path))
save(root/'training-contract.json',c)
spent=frozen['cumulative_initial_worker_cpu_seconds'];fits={};gpu_started=None;gpu_uuid=None;gpu_lock=None
gpu_previous=plan.get('initial_gpu_seconds',0.)
os.sched_setaffinity(0,range(8))
for name in ['GBRT','FNN']:
for split in splits:
index=split['index'];key=f'{index}/{name}';dest=root/'fits'/f'split-{index}'/name
remain=plan['total_worker_cpu_seconds']-spent-plan['postfit_cpu_reserve_seconds']
if remain<=5:
fits[key]=dict(status='NOT_ATTEMPTED',reason='SHARED_CPU_CAP_RESERVE');continue
command=[sys.executable,'tools/gre24_fit.py',str(root),str(index),name]
if name=='GBRT':
cap=min(plan['xgb_per_fit_cpu_seconds'],remain)
receipt=supervise(command,dest,'worker',dict(total_cpu_seconds=cap,job_cpu_seconds=cap,
job_wall_seconds=7200,tree_rss_bytes=24*2**30,disk_bytes=3*2**30,max_processes=1),
poll=.1,numerical_threads=plan['xgb_threads'])
else:
if gpu_started is None:gpu_started=time.monotonic()
remaining_gpu=plan['total_gpu_seconds']-gpu_previous-(time.monotonic()-gpu_started)
if remaining_gpu<=20:
fits[key]=dict(status='NOT_ATTEMPTED',reason='SHARED_GPU_CAP');continue
# Fixed one device once selected, three idle polls; waiting counts toward 2 GPU-hours.
idle=0
while idle<3 and time.monotonic()-gpu_started+gpu_previous<plan['total_gpu_seconds']-20:
states=gpu_state(gpu_uuid)
if gpu_uuid is None:
candidates=sorted((s for s in states if s['used_bytes']<=128*2**20 and s['utilization']==0),key=lambda s:s['uuid'])
if candidates:
gpu_uuid=candidates[0]['uuid']
gpu_lock=Path('/tmp/meltmind-gre24-'+gpu_uuid+'.lock').open('a')
fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
states=[candidates[0]]
clean=bool(gpu_uuid) and states[0]['used_bytes']<=128*2**20 and states[0]['utilization']==0
idle=idle+1 if clean else 0
save(root/'gpu-idle-progress.json',dict(uuid=gpu_uuid,consecutive_idle=idle,
charged_gpu_seconds=gpu_previous+time.monotonic()-gpu_started,states=states))
if idle<3:time.sleep(5)
remaining_gpu=plan['total_gpu_seconds']-gpu_previous-(time.monotonic()-gpu_started)
if idle<3 or remaining_gpu<=20:
fits[key]=dict(status='NOT_ATTEMPTED',reason='NO_IDLE_GPU_WITHIN_SHARED_CAP');continue
receipt=gpu_fit(command,dest,dest,gpu_uuid,remaining_gpu-2,
min(remain,plan['fnn_per_fit_cpu_seconds']),plan['gpu_reserved_byte_cap'])
spent+=receipt['cpu_seconds']
save(dest/'final-guard-receipt.json',receipt)
result_path=dest/'result.json';result=json.loads(result_path.read_text()) if result_path.exists() else None
passed=receipt['status']=='PASS' and result and result['status']=='FIT_COMPLETE_NO_TEST_SCORES'
if passed and sha(result['checkpoint'])!=result['checkpoint_sha256']:raise ValueError('Checkpoint changed before freeze')
fits[key]=dict(status='PASS' if passed else 'FAILED',guard_receipt=str(dest/'final-guard-receipt.json'),
guard_receipt_sha256=sha(dest/'final-guard-receipt.json'),result=str(result_path) if result else None,
checkpoint=result['checkpoint'] if passed else None,checkpoint_sha256=result['checkpoint_sha256'] if passed else None,
failure_checkpoint_retained=bool(result) and not passed)
save(root/'model-chain-progress.json',dict(stage='FITTING',completed=len(fits),fits=fits,
charged_worker_cpu_seconds=spent,charged_gpu_seconds=gpu_previous+(time.monotonic()-gpu_started if gpu_started else 0.)))
gpu_seconds=gpu_previous+(time.monotonic()-gpu_started if gpu_started else 0.)
if gpu_lock:gpu_lock.close()
save(root/'models-freeze.json',dict(fits=fits,charged_worker_cpu_seconds=spent,charged_gpu_seconds=gpu_seconds,
uuid=gpu_uuid,training_contract_sha256=sha(root/'training-contract.json'),all_fits_terminal=True,test_scores_read=False))
postfit=[]
for phase in ['predict','score']:
remain=plan['total_worker_cpu_seconds']-spent
if remain<=5:
postfit.append(dict(phase=phase,status='NOT_ATTEMPTED_CPU_CAP'));break
receipt=supervise([sys.executable,'tools/gre24_score.py',str(root),phase],root/'postfit-guards'/phase,'worker',
dict(total_cpu_seconds=remain,job_cpu_seconds=remain,job_wall_seconds=7200,tree_rss_bytes=24*2**30,
disk_bytes=3*2**30,max_processes=1),poll=.2,numerical_threads=4)
spent+=receipt['cpu_seconds'];postfit.append(dict(phase=phase,receipt=receipt))
if receipt['status']!='PASS':break
save(root/'reproduction-complete.json',dict(status='SCORED' if (root/'score-summary.json').exists() else 'INCOMPLETE',
fits=fits,postfit=postfit,charged_worker_cpu_seconds=spent,charged_gpu_seconds=gpu_seconds,
total_worker_cpu_cap=plan['total_worker_cpu_seconds'],total_gpu_cap=plan['total_gpu_seconds'],
dataset_version=plan['dataset_version'],paper_numbers_diagnostic_only=True))
return 0 if (root/'score-summary.json').exists() else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('plan',type=Path)
a=p.parse_args();raise SystemExit(run(a.root,a.plan))