Freeze Gre24 model chain and retain synthetic acceptance evidence
This commit is contained in:
1 parent
8409910b1e
commit
eb9ed0c762
124 files changed
+5993
No files matched your search
@@ -0,0 +1,142 @@
|
||||
"""One sequential fixed-model chain after terminal preparation, within shared caps."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from gre24_prepare_v2 import sha
|
||||
from gre24_job_guard import supervise,save
|
||||
from gre24_gpu_guard import gpu_state,gpu_fit
|
||||
|
||||
|
||||
def freeze_preparation(root):
|
||||
complete=json.loads((root/'complete.json').read_text());c=json.loads((root/'contract.json').read_text())
|
||||
if complete['status']!='COMPLETE' or not complete['accounting_complete'] or len(complete['records'])!=len(c['parts']):
|
||||
raise ValueError('Preparation globally stopped or missing terminal accounting')
|
||||
good={};fail=[]
|
||||
for r in complete['records']:
|
||||
if r['worker']['status']!='PASS':
|
||||
fail.append(dict(case=r['case'],guard_status=r['worker']['status'],reason=r['worker']['stop_reason']));continue
|
||||
p=Path(r['result_path'])
|
||||
if sha(p)!=r['result_sha256']:raise ValueError('Preparation result changed')
|
||||
result=json.loads(p.read_text())
|
||||
good[r['case']]=dict(output=str(p.parent),nodes=result['geometry']['tet_nodes'],
|
||||
raw_nodes=result['geometry'].get('raw_nodes',result['geometry']['tet_nodes']),
|
||||
t_scale_seconds=result['t_scale_seconds'],output_sha256=result['output_sha256'],result_sha256=sha(p),
|
||||
outside_tet_nodes=result['geometry']['outside_tet_nodes'])
|
||||
frozen=dict(successful=good,failed=fail,requested=len(c['parts']),
|
||||
preparation_complete_sha256=sha(root/'complete.json'),replacement=False,
|
||||
cumulative_initial_worker_cpu_seconds=complete['worker_cpu_seconds']+complete['initial_cpu_reserve_seconds'])
|
||||
save(root/'preparation-freeze.json',frozen);return frozen
|
||||
|
||||
|
||||
def run(root,plan_path):
|
||||
root=Path(root);plan_path=Path(plan_path)
|
||||
with (root/'model-chain.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
return run_locked(root,plan_path)
|
||||
|
||||
|
||||
def run_locked(root,plan_path):
|
||||
plan=json.loads(plan_path.read_text())
|
||||
for p,h in plan['code_sha256'].items():
|
||||
if sha(p)!=h:raise ValueError('Model plan source changed: '+p)
|
||||
started=root/'model-chain-started.json'
|
||||
if started.exists():raise FileExistsError('Never restart model chain')
|
||||
save(started,dict(pid=os.getpid(),unix=time.time(),plan_sha256=sha(plan_path),stage='WAIT_PREPARATION'))
|
||||
missing_since=None
|
||||
while not (root/'complete.json').exists():
|
||||
parent=json.loads((root/'queue-started.json').read_text())
|
||||
try:
|
||||
process=psutil.Process(parent['pid'])
|
||||
alive=process.create_time()<=parent['unix'] and process.status()!=psutil.STATUS_ZOMBIE and any('run_gre24_protocol_preparation.py' in a for a in process.cmdline())
|
||||
except psutil.NoSuchProcess:alive=False
|
||||
if not alive:
|
||||
missing_since=missing_since or time.monotonic()
|
||||
if time.monotonic()-missing_since>30:raise RuntimeError('Preparation exited without complete receipt; no restart')
|
||||
else:missing_since=None
|
||||
save(root/'model-chain-progress.json',dict(stage='WAIT_PREPARATION',unix=time.time()))
|
||||
time.sleep(10)
|
||||
frozen=freeze_preparation(root)
|
||||
split_path=root/'splits.json';splits=json.loads(split_path.read_text())
|
||||
c=dict(plan,preparation_freeze_sha256=sha(root/'preparation-freeze.json'),splits_sha256=sha(split_path))
|
||||
save(root/'training-contract.json',c)
|
||||
spent=frozen['cumulative_initial_worker_cpu_seconds'];fits={};gpu_started=None;gpu_uuid=None;gpu_lock=None
|
||||
gpu_previous=plan.get('initial_gpu_seconds',0.)
|
||||
os.sched_setaffinity(0,range(8))
|
||||
for name in ['GBRT','FNN']:
|
||||
for split in splits:
|
||||
index=split['index'];key=f'{index}/{name}';dest=root/'fits'/f'split-{index}'/name
|
||||
remain=plan['total_worker_cpu_seconds']-spent-plan['postfit_cpu_reserve_seconds']
|
||||
if remain<=5:
|
||||
fits[key]=dict(status='NOT_ATTEMPTED',reason='SHARED_CPU_CAP_RESERVE');continue
|
||||
command=[sys.executable,'tools/gre24_fit.py',str(root),str(index),name]
|
||||
if name=='GBRT':
|
||||
cap=min(plan['xgb_per_fit_cpu_seconds'],remain)
|
||||
receipt=supervise(command,dest,'worker',dict(total_cpu_seconds=cap,job_cpu_seconds=cap,
|
||||
job_wall_seconds=7200,tree_rss_bytes=24*2**30,disk_bytes=3*2**30,max_processes=1),
|
||||
poll=.1,numerical_threads=plan['xgb_threads'])
|
||||
else:
|
||||
if gpu_started is None:gpu_started=time.monotonic()
|
||||
remaining_gpu=plan['total_gpu_seconds']-gpu_previous-(time.monotonic()-gpu_started)
|
||||
if remaining_gpu<=20:
|
||||
fits[key]=dict(status='NOT_ATTEMPTED',reason='SHARED_GPU_CAP');continue
|
||||
# Fixed one device once selected, three idle polls; waiting counts toward 2 GPU-hours.
|
||||
idle=0
|
||||
while idle<3 and time.monotonic()-gpu_started+gpu_previous<plan['total_gpu_seconds']-20:
|
||||
states=gpu_state(gpu_uuid)
|
||||
if gpu_uuid is None:
|
||||
candidates=sorted((s for s in states if s['used_bytes']<=128*2**20 and s['utilization']==0),key=lambda s:s['uuid'])
|
||||
if candidates:
|
||||
gpu_uuid=candidates[0]['uuid']
|
||||
gpu_lock=Path('/tmp/meltmind-gre24-'+gpu_uuid+'.lock').open('a')
|
||||
fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
states=[candidates[0]]
|
||||
clean=bool(gpu_uuid) and states[0]['used_bytes']<=128*2**20 and states[0]['utilization']==0
|
||||
idle=idle+1 if clean else 0
|
||||
save(root/'gpu-idle-progress.json',dict(uuid=gpu_uuid,consecutive_idle=idle,
|
||||
charged_gpu_seconds=gpu_previous+time.monotonic()-gpu_started,states=states))
|
||||
if idle<3:time.sleep(5)
|
||||
remaining_gpu=plan['total_gpu_seconds']-gpu_previous-(time.monotonic()-gpu_started)
|
||||
if idle<3 or remaining_gpu<=20:
|
||||
fits[key]=dict(status='NOT_ATTEMPTED',reason='NO_IDLE_GPU_WITHIN_SHARED_CAP');continue
|
||||
receipt=gpu_fit(command,dest,dest,gpu_uuid,remaining_gpu-2,
|
||||
min(remain,plan['fnn_per_fit_cpu_seconds']),plan['gpu_reserved_byte_cap'])
|
||||
spent+=receipt['cpu_seconds']
|
||||
save(dest/'final-guard-receipt.json',receipt)
|
||||
result_path=dest/'result.json';result=json.loads(result_path.read_text()) if result_path.exists() else None
|
||||
passed=receipt['status']=='PASS' and result and result['status']=='FIT_COMPLETE_NO_TEST_SCORES'
|
||||
if passed and sha(result['checkpoint'])!=result['checkpoint_sha256']:raise ValueError('Checkpoint changed before freeze')
|
||||
fits[key]=dict(status='PASS' if passed else 'FAILED',guard_receipt=str(dest/'final-guard-receipt.json'),
|
||||
guard_receipt_sha256=sha(dest/'final-guard-receipt.json'),result=str(result_path) if result else None,
|
||||
checkpoint=result['checkpoint'] if passed else None,checkpoint_sha256=result['checkpoint_sha256'] if passed else None,
|
||||
failure_checkpoint_retained=bool(result) and not passed)
|
||||
save(root/'model-chain-progress.json',dict(stage='FITTING',completed=len(fits),fits=fits,
|
||||
charged_worker_cpu_seconds=spent,charged_gpu_seconds=gpu_previous+(time.monotonic()-gpu_started if gpu_started else 0.)))
|
||||
gpu_seconds=gpu_previous+(time.monotonic()-gpu_started if gpu_started else 0.)
|
||||
if gpu_lock:gpu_lock.close()
|
||||
save(root/'models-freeze.json',dict(fits=fits,charged_worker_cpu_seconds=spent,charged_gpu_seconds=gpu_seconds,
|
||||
uuid=gpu_uuid,training_contract_sha256=sha(root/'training-contract.json'),all_fits_terminal=True,test_scores_read=False))
|
||||
postfit=[]
|
||||
for phase in ['predict','score']:
|
||||
remain=plan['total_worker_cpu_seconds']-spent
|
||||
if remain<=5:
|
||||
postfit.append(dict(phase=phase,status='NOT_ATTEMPTED_CPU_CAP'));break
|
||||
receipt=supervise([sys.executable,'tools/gre24_score.py',str(root),phase],root/'postfit-guards'/phase,'worker',
|
||||
dict(total_cpu_seconds=remain,job_cpu_seconds=remain,job_wall_seconds=7200,tree_rss_bytes=24*2**30,
|
||||
disk_bytes=3*2**30,max_processes=1),poll=.2,numerical_threads=4)
|
||||
spent+=receipt['cpu_seconds'];postfit.append(dict(phase=phase,receipt=receipt))
|
||||
if receipt['status']!='PASS':break
|
||||
save(root/'reproduction-complete.json',dict(status='SCORED' if (root/'score-summary.json').exists() else 'INCOMPLETE',
|
||||
fits=fits,postfit=postfit,charged_worker_cpu_seconds=spent,charged_gpu_seconds=gpu_seconds,
|
||||
total_worker_cpu_cap=plan['total_worker_cpu_seconds'],total_gpu_cap=plan['total_gpu_seconds'],
|
||||
dataset_version=plan['dataset_version'],paper_numbers_diagnostic_only=True))
|
||||
return 0 if (root/'score-summary.json').exists() else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('plan',type=Path)
|
||||
a=p.parse_args();raise SystemExit(run(a.root,a.plan))
|
||||
Reference in new issue
Block a user