Files
MeltMind/tools/run_s2_main_matrix.py
T

183 lines
12 KiB
Python

"""Four isolated GPU block workers; adjacent N/E, first-eight barrier, aggregate wall ledger."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import time
import traceback
import psutil
from freeze_g1_components import sha
from s2_recovery_supervisor import save
from s2_gpu_idle_gate import recorded_wait
from s2_main_memory_guard import guarded_worker,ALLOCATOR_CONF
BLOCK_WALL_CAP=5000.
def contract(root):
c=json.loads((root/'training-contract.json').read_text())
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():assert sha(p)==h,p
assert c['roster_sha256']==sha(c['roster']) and len(c['runs'])==90 and len(c['blocks'])==45
assert c['limits']['aggregate_gpu_seconds']==46*3600 and c['limits']['run_wall_seconds']==1800
assert c['E_max']==c['cosine_T_max']==42
assert len(set(c['gpu_uuids']))==len(c['gpu_uuids'])==4
for b in c['blocks']:
n,e=[c['runs'][i] for i in b['run_indices']]
assert (n['arm'],e['arm'])==('N','E')
assert (n['fold'],n['pool'],n['seed'])==(e['fold'],e['pool'],e['seed'])
return c
def first_eight_projection(c,blocks,charged):
assert len(blocks)==4 and sum(len(b['runs']) for b in blocks)==8
rates={arm:[] for arm in ['N','E']}
for b in blocks:
for row in b['runs']:
run=c['runs'][row['index']]
if row['status']=='PASS':rates[run['arm']].append(row['cell_wall_seconds']/run['cost_nodes'])
if any(not v for v in rates.values()):
return dict(continue_within_cap=False,reason='FIRST8_NO_VALID_COST_FOR_BOTH_ARMS',observed_charged_gpu_seconds=charged)
rate={a:max(v) for a,v in rates.items()}
completed={row['index'] for b in blocks for row in b['runs']}
projected=charged+1.25*sum(rate[r['arm']]*r['cost_nodes'] for i,r in enumerate(c['runs']) if i not in completed)
return dict(continue_within_cap=projected<=c['limits']['aggregate_gpu_seconds'],projected_gpu_seconds=projected,
observed_charged_gpu_seconds=charged,seconds_per_node_by_arm=rate,reserve_multiplier=1.25,
method='Maximum successful observed per-arm cell wall/node (idle/setup included), remaining cells plus25percent; all actual block wall already charged.')
def can_dispatch(c,next_block,active_count,charged,checkpoint):
ceiling=len(c['blocks']) if checkpoint else 4
return next_block<ceiling and active_count<4 and charged+(active_count+1)*BLOCK_WALL_CAP<=c['limits']['aggregate_gpu_seconds']
def block(root,index,gpu_slot):
c=contract(root);spec=c['blocks'][index];uuid=c['gpu_uuids'][gpu_slot]
dest=root/'blocks'/f'block-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
with (dest/'block.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
assert not (dest/'started.json').exists(),'No repeated block'
with Path('/tmp/meltmind-s2-'+uuid+'.lock').open('a') as gpu_lock:
fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return block_locked(root,c,spec,dest,index,gpu_slot)
def block_locked(root,c,spec,dest,index,gpu_slot):
begin=time.monotonic();uuid=c['gpu_uuids'][gpu_slot];rows=[]
os.sched_setaffinity(0,c['gpu_cpu_affinities'][gpu_slot])
os.environ['PYTORCH_CUDA_ALLOC_CONF']=ALLOCATOR_CONF
save(dest/'started.json',dict(pid=os.getpid(),started_unix=time.time(),gpu_uuid=uuid,block=index,
run_indices=spec['run_indices'],cpu_affinity=sorted(os.sched_getaffinity(0)),contract_sha256=sha(root/'training-contract.json')))
try:
for i in spec['run_indices']:
run=c['runs'][i];cell=root/'cells'/run['name'];cell.mkdir(parents=True,exist_ok=False)
cell_begin=time.monotonic();row=dict(index=i,name=run['name'],arm=run['arm'],status='NA',reason=None)
try:
if (root/'cancel.json').exists():raise RuntimeError('MATRIX_CANCELLED_BEFORE_RUN')
if time.monotonic()-begin>BLOCK_WALL_CAP-2405:raise RuntimeError('BLOCK_REMAINING_BUDGET_INSUFFICIENT')
idle=recorded_wait(cell,'before-run',uuid,600)
if idle['status']!='IDLE_CONFIRMED':raise RuntimeError('GPU_IDLE_TIMEOUT_NO_RETRY')
if (root/'cancel.json').exists():raise RuntimeError('MATRIX_CANCELLED_AFTER_IDLE')
receipt,result=guarded_worker(cell,run['name'],uuid,1800,
[sys.executable,'tools/run_s2_main_training.py',str(root),str(i)],cumulative_worker_cpu_limit_seconds=1801)
row.update(status='PASS' if receipt['status']=='PASS' else 'NA',reason=(receipt.get('stop_reason') or ('WORKER_EXIT_'+str(receipt['returncode']) if receipt['status']!='PASS' else None)),
guard_receipt=str(cell/(run['name']+'-guard-receipt.json')),guard_sha256=sha(cell/(run['name']+'-guard-receipt.json')),
worker_status=receipt['status'],worker_returncode=receipt['returncode'],worker_cpu_seconds=receipt['cpu_seconds'],
checkpoint_sha256=result.get('checkpoint_sha256') if result else None,
standardizer_fingerprint=result.get('standardizer_fingerprint') if result else None)
except Exception as exc:
row.update(status='NA',reason=type(exc).__name__+': '+str(exc),traceback=traceback.format_exc())
row['cell_wall_seconds']=time.monotonic()-cell_begin;rows.append(row)
save(cell/'terminal.json',row);save(dest/'progress.json',dict(runs=rows,elapsed_seconds=time.monotonic()-begin))
finally:
# A failed arm never causes its paired arm to be silently replaced or retried.
present={r['index'] for r in rows}
for i in spec['run_indices']:
if i not in present:rows.append(dict(index=i,name=c['runs'][i]['name'],status='NA',reason='BLOCK_INTERRUPTED_BEFORE_RUN',cell_wall_seconds=0))
valid=all(r['status']=='PASS' for r in rows)
if valid and rows[0]['standardizer_fingerprint']!=rows[1]['standardizer_fingerprint']:valid=False
save(dest/'terminal.json',dict(status='TERMINAL',block=index,gpu_uuid=uuid,runs=rows,pair_valid=valid,
pair_invalid_reason=None if valid else 'AT_LEAST_ONE_ARM_NA_OR_STANDARDIZER_MISMATCH',
block_wall_seconds=time.monotonic()-begin,cpu_affinity=sorted(os.sched_getaffinity(0)),retry=False,test_predictions=False))
def stop_owned_workers(active):
for item in active.values():
try:children=psutil.Process(item['process'].pid).children(recursive=True)
except psutil.NoSuchProcess:continue
for p in children:
try:
if 'tools/run_s2_main_training.py' in p.cmdline():os.killpg(p.pid,signal.SIGKILL)
except (ProcessLookupError,psutil.NoSuchProcess):pass
def run(root):
with (root/'matrix.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
def run_locked(root):
c=contract(root);assert not (root/'matrix-started.json').exists(),'No repeated matrix launch'
save(root/'matrix-started.json',dict(pid=os.getpid(),started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
start=time.monotonic();active={};done=[];next_block=0;reason=None;checkpoint=False;charged=0.
def interrupt(*_):
nonlocal reason
reason='MATRIX_COORDINATOR_INTERRUPTED'
signal.signal(signal.SIGTERM,interrupt);signal.signal(signal.SIGINT,interrupt)
env=dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=ALLOCATOR_CONF,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
try:
while active or (next_block<len(c['blocks']) and reason is None):
for slot,item in list(active.items()):
code=item['process'].poll();elapsed=time.monotonic()-item['begin']
if elapsed>BLOCK_WALL_CAP and code is None:reason='BLOCK_WALL_LIMIT'
if code is None:continue
path=root/'blocks'/f"block-{item['index']:03d}"/'terminal.json'
if not path.exists():
reason='BLOCK_EXIT_WITHOUT_TERMINAL_RECEIPT'
result=dict(status='INFRASTRUCTURE_FAILED',block=item['index'],runs=[dict(index=i,name=c['runs'][i]['name'],status='NA',reason=reason,cell_wall_seconds=0) for i in c['blocks'][item['index']]['run_indices']],pair_valid=False)
else:result=json.loads(path.read_text())
result.update(charged_gpu_seconds=elapsed,coordinator_observed_returncode=code)
save(root/'blocks'/f"block-{item['index']:03d}"/'accounted.json',result)
done.append(result);charged+=elapsed;item['log'].close();del active[slot]
print(json.dumps(dict(event='BLOCK_TERMINAL',block=result['block'],run_statuses=[r['status'] for r in result['runs']],charged_gpu_hours=charged/3600)),flush=True)
live=sum(time.monotonic()-x['begin'] for x in active.values())
if charged+live>=c['limits']['aggregate_gpu_seconds']:reason='AGGREGATE_GPU_WALL_LIMIT'
if not checkpoint and len(done)==4 and not active and reason is None:
projection=first_eight_projection(c,done,charged);save(root/'first-eight-checkpoint.json',projection)
if projection['continue_within_cap']:checkpoint=True
else:reason='FIRST_EIGHT_PROJECTION_STOP'
if reason:
if not (root/'cancel.json').exists():save(root/'cancel.json',dict(reason=reason,unix=time.time()))
stop_owned_workers(active)
while reason is None and can_dispatch(c,next_block,len(active),charged,checkpoint):
slot=next(s for s in range(4) if s not in active);idx=next_block
dest=root/'blocks'/f'block-{idx:03d}';dest.mkdir(parents=True,exist_ok=True)
log=(dest/'block.log').open('x');begin=time.monotonic()
p=subprocess.Popen([sys.executable,__file__,str(root),'--block',str(idx),'--gpu-slot',str(slot)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
active[slot]=dict(process=p,log=log,begin=begin,index=idx);next_block+=1
if not active and next_block<len(c['blocks']) and reason is None and checkpoint:reason='AGGREGATE_BUDGET_RESERVATION_STOP'
save(root/'matrix-progress.json',dict(completed_blocks=len(done),terminal_runs=2*len(done),active_blocks={str(k):v['index'] for k,v in active.items()},
next_block=next_block,settled_gpu_seconds=charged,active_gpu_seconds=live,first_eight_passed=checkpoint,stop_reason=reason,elapsed_seconds=time.monotonic()-start))
if active:time.sleep(.5)
finally:
if active:
save(root/'cancel.json',dict(reason=reason or 'COORDINATOR_EXIT',unix=time.time()));stop_owned_workers(active)
for item in active.values():item['process'].wait();item['log'].close()
rows=[row for b in done for row in b['runs']];seen={r['index'] for r in rows}
for i,r in enumerate(c['runs']):
if i not in seen:rows.append(dict(index=i,name=r['name'],status='NA',reason='NOT_RUN_'+str(reason)))
result=dict(status='ALL_NEURAL_TERMINAL',stop_reason=reason,runs=sorted(rows,key=lambda r:r['index']),blocks=done,
charged_gpu_seconds=charged,passed=sum(r['status']=='PASS' for r in rows),NA=sum(r['status']!='PASS' for r in rows),
roster_sha256=c['roster_sha256'],contract_sha256=sha(root/'training-contract.json'),retry=False,
test_predictions=False,test_scoring=False,r2_required_before_custody=True)
save(root/'matrix-complete.json',result)
return 0 if reason is None else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--block',type=int);p.add_argument('--gpu-slot',type=int);a=p.parse_args()
raise SystemExit(block(a.root,a.block,a.gpu_slot) if a.block is not None else run(a.root))