"""Four isolated GPU block workers; adjacent N/E, first-eight barrier, aggregate wall ledger.""" import argparse import fcntl import json import os from pathlib import Path import signal import subprocess import sys import time import traceback import psutil from freeze_g1_components import sha from s2_recovery_supervisor import save from s2_gpu_idle_gate import recorded_wait from s2_main_memory_guard import guarded_worker,ALLOCATOR_CONF BLOCK_WALL_CAP=5000. def contract(root): c=json.loads((root/'training-contract.json').read_text()) for p,h in {**c['code_sha256'],**c['source_sha256']}.items():assert sha(p)==h,p assert c['roster_sha256']==sha(c['roster']) and len(c['runs'])==90 and len(c['blocks'])==45 assert c['limits']['aggregate_gpu_seconds']==46*3600 and c['limits']['run_wall_seconds']==1800 assert c['E_max']==c['cosine_T_max']==42 assert len(set(c['gpu_uuids']))==len(c['gpu_uuids'])==4 for b in c['blocks']: n,e=[c['runs'][i] for i in b['run_indices']] assert (n['arm'],e['arm'])==('N','E') assert (n['fold'],n['pool'],n['seed'])==(e['fold'],e['pool'],e['seed']) return c def first_eight_projection(c,blocks,charged): assert len(blocks)==4 and sum(len(b['runs']) for b in blocks)==8 rates={arm:[] for arm in ['N','E']} for b in blocks: for row in b['runs']: run=c['runs'][row['index']] if row['status']=='PASS':rates[run['arm']].append(row['cell_wall_seconds']/run['cost_nodes']) if any(not v for v in rates.values()): return dict(continue_within_cap=False,reason='FIRST8_NO_VALID_COST_FOR_BOTH_ARMS',observed_charged_gpu_seconds=charged) rate={a:max(v) for a,v in rates.items()} completed={row['index'] for b in blocks for row in b['runs']} projected=charged+1.25*sum(rate[r['arm']]*r['cost_nodes'] for i,r in enumerate(c['runs']) if i not in completed) return dict(continue_within_cap=projected<=c['limits']['aggregate_gpu_seconds'],projected_gpu_seconds=projected, observed_charged_gpu_seconds=charged,seconds_per_node_by_arm=rate,reserve_multiplier=1.25, method='Maximum successful observed per-arm cell wall/node (idle/setup included), remaining cells plus25percent; all actual block wall already charged.') def can_dispatch(c,next_block,active_count,charged,checkpoint): ceiling=len(c['blocks']) if checkpoint else 4 return next_blockBLOCK_WALL_CAP-2405:raise RuntimeError('BLOCK_REMAINING_BUDGET_INSUFFICIENT') idle=recorded_wait(cell,'before-run',uuid,600) if idle['status']!='IDLE_CONFIRMED':raise RuntimeError('GPU_IDLE_TIMEOUT_NO_RETRY') if (root/'cancel.json').exists():raise RuntimeError('MATRIX_CANCELLED_AFTER_IDLE') receipt,result=guarded_worker(cell,run['name'],uuid,1800, [sys.executable,'tools/run_s2_main_training.py',str(root),str(i)],cumulative_worker_cpu_limit_seconds=1801) row.update(status='PASS' if receipt['status']=='PASS' else 'NA',reason=(receipt.get('stop_reason') or ('WORKER_EXIT_'+str(receipt['returncode']) if receipt['status']!='PASS' else None)), guard_receipt=str(cell/(run['name']+'-guard-receipt.json')),guard_sha256=sha(cell/(run['name']+'-guard-receipt.json')), worker_status=receipt['status'],worker_returncode=receipt['returncode'],worker_cpu_seconds=receipt['cpu_seconds'], checkpoint_sha256=result.get('checkpoint_sha256') if result else None, standardizer_fingerprint=result.get('standardizer_fingerprint') if result else None) except Exception as exc: row.update(status='NA',reason=type(exc).__name__+': '+str(exc),traceback=traceback.format_exc()) row['cell_wall_seconds']=time.monotonic()-cell_begin;rows.append(row) save(cell/'terminal.json',row);save(dest/'progress.json',dict(runs=rows,elapsed_seconds=time.monotonic()-begin)) finally: # A failed arm never causes its paired arm to be silently replaced or retried. present={r['index'] for r in rows} for i in spec['run_indices']: if i not in present:rows.append(dict(index=i,name=c['runs'][i]['name'],status='NA',reason='BLOCK_INTERRUPTED_BEFORE_RUN',cell_wall_seconds=0)) valid=all(r['status']=='PASS' for r in rows) if valid and rows[0]['standardizer_fingerprint']!=rows[1]['standardizer_fingerprint']:valid=False save(dest/'terminal.json',dict(status='TERMINAL',block=index,gpu_uuid=uuid,runs=rows,pair_valid=valid, pair_invalid_reason=None if valid else 'AT_LEAST_ONE_ARM_NA_OR_STANDARDIZER_MISMATCH', block_wall_seconds=time.monotonic()-begin,cpu_affinity=sorted(os.sched_getaffinity(0)),retry=False,test_predictions=False)) def stop_owned_workers(active): for item in active.values(): try:children=psutil.Process(item['process'].pid).children(recursive=True) except psutil.NoSuchProcess:continue for p in children: try: if 'tools/run_s2_main_training.py' in p.cmdline():os.killpg(p.pid,signal.SIGKILL) except (ProcessLookupError,psutil.NoSuchProcess):pass def run(root): with (root/'matrix.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root) def run_locked(root): c=contract(root);assert not (root/'matrix-started.json').exists(),'No repeated matrix launch' save(root/'matrix-started.json',dict(pid=os.getpid(),started_unix=time.time(),contract_sha256=sha(root/'training-contract.json'))) start=time.monotonic();active={};done=[];next_block=0;reason=None;checkpoint=False;charged=0. def interrupt(*_): nonlocal reason reason='MATRIX_COORDINATOR_INTERRUPTED' signal.signal(signal.SIGTERM,interrupt);signal.signal(signal.SIGINT,interrupt) env=dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=ALLOCATOR_CONF,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='') try: while active or (next_blockBLOCK_WALL_CAP and code is None:reason='BLOCK_WALL_LIMIT' if code is None:continue path=root/'blocks'/f"block-{item['index']:03d}"/'terminal.json' if not path.exists(): reason='BLOCK_EXIT_WITHOUT_TERMINAL_RECEIPT' result=dict(status='INFRASTRUCTURE_FAILED',block=item['index'],runs=[dict(index=i,name=c['runs'][i]['name'],status='NA',reason=reason,cell_wall_seconds=0) for i in c['blocks'][item['index']]['run_indices']],pair_valid=False) else:result=json.loads(path.read_text()) result.update(charged_gpu_seconds=elapsed,coordinator_observed_returncode=code) save(root/'blocks'/f"block-{item['index']:03d}"/'accounted.json',result) done.append(result);charged+=elapsed;item['log'].close();del active[slot] print(json.dumps(dict(event='BLOCK_TERMINAL',block=result['block'],run_statuses=[r['status'] for r in result['runs']],charged_gpu_hours=charged/3600)),flush=True) live=sum(time.monotonic()-x['begin'] for x in active.values()) if charged+live>=c['limits']['aggregate_gpu_seconds']:reason='AGGREGATE_GPU_WALL_LIMIT' if not checkpoint and len(done)==4 and not active and reason is None: projection=first_eight_projection(c,done,charged);save(root/'first-eight-checkpoint.json',projection) if projection['continue_within_cap']:checkpoint=True else:reason='FIRST_EIGHT_PROJECTION_STOP' if reason: if not (root/'cancel.json').exists():save(root/'cancel.json',dict(reason=reason,unix=time.time())) stop_owned_workers(active) while reason is None and can_dispatch(c,next_block,len(active),charged,checkpoint): slot=next(s for s in range(4) if s not in active);idx=next_block dest=root/'blocks'/f'block-{idx:03d}';dest.mkdir(parents=True,exist_ok=True) log=(dest/'block.log').open('x');begin=time.monotonic() p=subprocess.Popen([sys.executable,__file__,str(root),'--block',str(idx),'--gpu-slot',str(slot)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True) active[slot]=dict(process=p,log=log,begin=begin,index=idx);next_block+=1 if not active and next_block