"""One sequential fixed-model chain after terminal preparation, within shared caps.""" import argparse import fcntl import json import os from pathlib import Path import sys import time import psutil from gre24_prepare_v2 import sha from gre24_job_guard import supervise,save from gre24_gpu_guard import gpu_state,gpu_fit def freeze_preparation(root): complete=json.loads((root/'complete.json').read_text());c=json.loads((root/'contract.json').read_text()) if complete['status']!='COMPLETE' or not complete['accounting_complete'] or len(complete['records'])!=len(c['parts']): raise ValueError('Preparation globally stopped or missing terminal accounting') good={};fail=[] for r in complete['records']: if r['worker']['status']!='PASS': fail.append(dict(case=r['case'],guard_status=r['worker']['status'],reason=r['worker']['stop_reason']));continue p=Path(r['result_path']) if sha(p)!=r['result_sha256']:raise ValueError('Preparation result changed') result=json.loads(p.read_text()) good[r['case']]=dict(output=str(p.parent),nodes=result['geometry']['tet_nodes'], raw_nodes=result['geometry'].get('raw_nodes',result['geometry']['tet_nodes']), t_scale_seconds=result['t_scale_seconds'],output_sha256=result['output_sha256'],result_sha256=sha(p), outside_tet_nodes=result['geometry']['outside_tet_nodes']) frozen=dict(successful=good,failed=fail,requested=len(c['parts']), preparation_complete_sha256=sha(root/'complete.json'),replacement=False, cumulative_initial_worker_cpu_seconds=complete['worker_cpu_seconds']+complete['initial_cpu_reserve_seconds']) save(root/'preparation-freeze.json',frozen);return frozen def run(root,plan_path): root=Path(root);plan_path=Path(plan_path) with (root/'model-chain.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) return run_locked(root,plan_path) def run_locked(root,plan_path): plan=json.loads(plan_path.read_text()) for p,h in plan['code_sha256'].items(): if sha(p)!=h:raise ValueError('Model plan source changed: '+p) started=root/'model-chain-started.json' if started.exists():raise FileExistsError('Never restart model chain') save(started,dict(pid=os.getpid(),unix=time.time(),plan_sha256=sha(plan_path),stage='WAIT_PREPARATION')) missing_since=None while not (root/'complete.json').exists(): parent=json.loads((root/'queue-started.json').read_text()) try: process=psutil.Process(parent['pid']) alive=process.create_time()<=parent['unix'] and process.status()!=psutil.STATUS_ZOMBIE and any('run_gre24_protocol_preparation.py' in a for a in process.cmdline()) except psutil.NoSuchProcess:alive=False if not alive: missing_since=missing_since or time.monotonic() if time.monotonic()-missing_since>30:raise RuntimeError('Preparation exited without complete receipt; no restart') else:missing_since=None save(root/'model-chain-progress.json',dict(stage='WAIT_PREPARATION',unix=time.time())) time.sleep(10) frozen=freeze_preparation(root) split_path=root/'splits.json';splits=json.loads(split_path.read_text()) c=dict(plan,preparation_freeze_sha256=sha(root/'preparation-freeze.json'),splits_sha256=sha(split_path)) save(root/'training-contract.json',c) spent=frozen['cumulative_initial_worker_cpu_seconds'];fits={};gpu_started=None;gpu_uuid=None;gpu_lock=None gpu_previous=plan.get('initial_gpu_seconds',0.) os.sched_setaffinity(0,range(8)) for name in ['GBRT','FNN']: for split in splits: index=split['index'];key=f'{index}/{name}';dest=root/'fits'/f'split-{index}'/name remain=plan['total_worker_cpu_seconds']-spent-plan['postfit_cpu_reserve_seconds'] if remain<=5: fits[key]=dict(status='NOT_ATTEMPTED',reason='SHARED_CPU_CAP_RESERVE');continue command=[sys.executable,'tools/gre24_fit.py',str(root),str(index),name] if name=='GBRT': cap=min(plan['xgb_per_fit_cpu_seconds'],remain) receipt=supervise(command,dest,'worker',dict(total_cpu_seconds=cap,job_cpu_seconds=cap, job_wall_seconds=7200,tree_rss_bytes=24*2**30,disk_bytes=3*2**30,max_processes=1), poll=.1,numerical_threads=plan['xgb_threads']) else: if gpu_started is None:gpu_started=time.monotonic() remaining_gpu=plan['total_gpu_seconds']-gpu_previous-(time.monotonic()-gpu_started) if remaining_gpu<=20: fits[key]=dict(status='NOT_ATTEMPTED',reason='SHARED_GPU_CAP');continue # Fixed one device once selected, three idle polls; waiting counts toward 2 GPU-hours. idle=0 while idle<3 and time.monotonic()-gpu_started+gpu_previous