183 lines
12 KiB
Python
183 lines
12 KiB
Python
"""Four isolated GPU block workers; adjacent N/E, first-eight barrier, aggregate wall ledger."""
|
|
import argparse
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import psutil
|
|
from freeze_g1_components import sha
|
|
from s2_recovery_supervisor import save
|
|
from s2_gpu_idle_gate import recorded_wait
|
|
from s2_main_memory_guard import guarded_worker,ALLOCATOR_CONF
|
|
|
|
BLOCK_WALL_CAP=5000.
|
|
|
|
|
|
def contract(root):
|
|
c=json.loads((root/'training-contract.json').read_text())
|
|
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():assert sha(p)==h,p
|
|
assert c['roster_sha256']==sha(c['roster']) and len(c['runs'])==90 and len(c['blocks'])==45
|
|
assert c['limits']['aggregate_gpu_seconds']==46*3600 and c['limits']['run_wall_seconds']==1800
|
|
assert c['E_max']==c['cosine_T_max']==42
|
|
assert len(set(c['gpu_uuids']))==len(c['gpu_uuids'])==4
|
|
for b in c['blocks']:
|
|
n,e=[c['runs'][i] for i in b['run_indices']]
|
|
assert (n['arm'],e['arm'])==('N','E')
|
|
assert (n['fold'],n['pool'],n['seed'])==(e['fold'],e['pool'],e['seed'])
|
|
return c
|
|
|
|
|
|
def first_eight_projection(c,blocks,charged):
|
|
assert len(blocks)==4 and sum(len(b['runs']) for b in blocks)==8
|
|
rates={arm:[] for arm in ['N','E']}
|
|
for b in blocks:
|
|
for row in b['runs']:
|
|
run=c['runs'][row['index']]
|
|
if row['status']=='PASS':rates[run['arm']].append(row['cell_wall_seconds']/run['cost_nodes'])
|
|
if any(not v for v in rates.values()):
|
|
return dict(continue_within_cap=False,reason='FIRST8_NO_VALID_COST_FOR_BOTH_ARMS',observed_charged_gpu_seconds=charged)
|
|
rate={a:max(v) for a,v in rates.items()}
|
|
completed={row['index'] for b in blocks for row in b['runs']}
|
|
projected=charged+1.25*sum(rate[r['arm']]*r['cost_nodes'] for i,r in enumerate(c['runs']) if i not in completed)
|
|
return dict(continue_within_cap=projected<=c['limits']['aggregate_gpu_seconds'],projected_gpu_seconds=projected,
|
|
observed_charged_gpu_seconds=charged,seconds_per_node_by_arm=rate,reserve_multiplier=1.25,
|
|
method='Maximum successful observed per-arm cell wall/node (idle/setup included), remaining cells plus25percent; all actual block wall already charged.')
|
|
|
|
|
|
def can_dispatch(c,next_block,active_count,charged,checkpoint):
|
|
ceiling=len(c['blocks']) if checkpoint else 4
|
|
return next_block<ceiling and active_count<4 and charged+(active_count+1)*BLOCK_WALL_CAP<=c['limits']['aggregate_gpu_seconds']
|
|
|
|
|
|
def block(root,index,gpu_slot):
|
|
c=contract(root);spec=c['blocks'][index];uuid=c['gpu_uuids'][gpu_slot]
|
|
dest=root/'blocks'/f'block-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
|
|
with (dest/'block.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
assert not (dest/'started.json').exists(),'No repeated block'
|
|
with Path('/tmp/meltmind-s2-'+uuid+'.lock').open('a') as gpu_lock:
|
|
fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
return block_locked(root,c,spec,dest,index,gpu_slot)
|
|
|
|
|
|
def block_locked(root,c,spec,dest,index,gpu_slot):
|
|
begin=time.monotonic();uuid=c['gpu_uuids'][gpu_slot];rows=[]
|
|
os.sched_setaffinity(0,c['gpu_cpu_affinities'][gpu_slot])
|
|
os.environ['PYTORCH_CUDA_ALLOC_CONF']=ALLOCATOR_CONF
|
|
save(dest/'started.json',dict(pid=os.getpid(),started_unix=time.time(),gpu_uuid=uuid,block=index,
|
|
run_indices=spec['run_indices'],cpu_affinity=sorted(os.sched_getaffinity(0)),contract_sha256=sha(root/'training-contract.json')))
|
|
try:
|
|
for i in spec['run_indices']:
|
|
run=c['runs'][i];cell=root/'cells'/run['name'];cell.mkdir(parents=True,exist_ok=False)
|
|
cell_begin=time.monotonic();row=dict(index=i,name=run['name'],arm=run['arm'],status='NA',reason=None)
|
|
try:
|
|
if (root/'cancel.json').exists():raise RuntimeError('MATRIX_CANCELLED_BEFORE_RUN')
|
|
if time.monotonic()-begin>BLOCK_WALL_CAP-2405:raise RuntimeError('BLOCK_REMAINING_BUDGET_INSUFFICIENT')
|
|
idle=recorded_wait(cell,'before-run',uuid,600)
|
|
if idle['status']!='IDLE_CONFIRMED':raise RuntimeError('GPU_IDLE_TIMEOUT_NO_RETRY')
|
|
if (root/'cancel.json').exists():raise RuntimeError('MATRIX_CANCELLED_AFTER_IDLE')
|
|
receipt,result=guarded_worker(cell,run['name'],uuid,1800,
|
|
[sys.executable,'tools/run_s2_main_training.py',str(root),str(i)],cumulative_worker_cpu_limit_seconds=1801)
|
|
row.update(status='PASS' if receipt['status']=='PASS' else 'NA',reason=(receipt.get('stop_reason') or ('WORKER_EXIT_'+str(receipt['returncode']) if receipt['status']!='PASS' else None)),
|
|
guard_receipt=str(cell/(run['name']+'-guard-receipt.json')),guard_sha256=sha(cell/(run['name']+'-guard-receipt.json')),
|
|
worker_status=receipt['status'],worker_returncode=receipt['returncode'],worker_cpu_seconds=receipt['cpu_seconds'],
|
|
checkpoint_sha256=result.get('checkpoint_sha256') if result else None,
|
|
standardizer_fingerprint=result.get('standardizer_fingerprint') if result else None)
|
|
except Exception as exc:
|
|
row.update(status='NA',reason=type(exc).__name__+': '+str(exc),traceback=traceback.format_exc())
|
|
row['cell_wall_seconds']=time.monotonic()-cell_begin;rows.append(row)
|
|
save(cell/'terminal.json',row);save(dest/'progress.json',dict(runs=rows,elapsed_seconds=time.monotonic()-begin))
|
|
finally:
|
|
# A failed arm never causes its paired arm to be silently replaced or retried.
|
|
present={r['index'] for r in rows}
|
|
for i in spec['run_indices']:
|
|
if i not in present:rows.append(dict(index=i,name=c['runs'][i]['name'],status='NA',reason='BLOCK_INTERRUPTED_BEFORE_RUN',cell_wall_seconds=0))
|
|
valid=all(r['status']=='PASS' for r in rows)
|
|
if valid and rows[0]['standardizer_fingerprint']!=rows[1]['standardizer_fingerprint']:valid=False
|
|
save(dest/'terminal.json',dict(status='TERMINAL',block=index,gpu_uuid=uuid,runs=rows,pair_valid=valid,
|
|
pair_invalid_reason=None if valid else 'AT_LEAST_ONE_ARM_NA_OR_STANDARDIZER_MISMATCH',
|
|
block_wall_seconds=time.monotonic()-begin,cpu_affinity=sorted(os.sched_getaffinity(0)),retry=False,test_predictions=False))
|
|
|
|
|
|
def stop_owned_workers(active):
|
|
for item in active.values():
|
|
try:children=psutil.Process(item['process'].pid).children(recursive=True)
|
|
except psutil.NoSuchProcess:continue
|
|
for p in children:
|
|
try:
|
|
if 'tools/run_s2_main_training.py' in p.cmdline():os.killpg(p.pid,signal.SIGKILL)
|
|
except (ProcessLookupError,psutil.NoSuchProcess):pass
|
|
|
|
|
|
def run(root):
|
|
with (root/'matrix.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
|
|
|
|
|
|
def run_locked(root):
|
|
c=contract(root);assert not (root/'matrix-started.json').exists(),'No repeated matrix launch'
|
|
save(root/'matrix-started.json',dict(pid=os.getpid(),started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
|
|
start=time.monotonic();active={};done=[];next_block=0;reason=None;checkpoint=False;charged=0.
|
|
def interrupt(*_):
|
|
nonlocal reason
|
|
reason='MATRIX_COORDINATOR_INTERRUPTED'
|
|
signal.signal(signal.SIGTERM,interrupt);signal.signal(signal.SIGINT,interrupt)
|
|
env=dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=ALLOCATOR_CONF,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
|
try:
|
|
while active or (next_block<len(c['blocks']) and reason is None):
|
|
for slot,item in list(active.items()):
|
|
code=item['process'].poll();elapsed=time.monotonic()-item['begin']
|
|
if elapsed>BLOCK_WALL_CAP and code is None:reason='BLOCK_WALL_LIMIT'
|
|
if code is None:continue
|
|
path=root/'blocks'/f"block-{item['index']:03d}"/'terminal.json'
|
|
if not path.exists():
|
|
reason='BLOCK_EXIT_WITHOUT_TERMINAL_RECEIPT'
|
|
result=dict(status='INFRASTRUCTURE_FAILED',block=item['index'],runs=[dict(index=i,name=c['runs'][i]['name'],status='NA',reason=reason,cell_wall_seconds=0) for i in c['blocks'][item['index']]['run_indices']],pair_valid=False)
|
|
else:result=json.loads(path.read_text())
|
|
result.update(charged_gpu_seconds=elapsed,coordinator_observed_returncode=code)
|
|
save(root/'blocks'/f"block-{item['index']:03d}"/'accounted.json',result)
|
|
done.append(result);charged+=elapsed;item['log'].close();del active[slot]
|
|
print(json.dumps(dict(event='BLOCK_TERMINAL',block=result['block'],run_statuses=[r['status'] for r in result['runs']],charged_gpu_hours=charged/3600)),flush=True)
|
|
live=sum(time.monotonic()-x['begin'] for x in active.values())
|
|
if charged+live>=c['limits']['aggregate_gpu_seconds']:reason='AGGREGATE_GPU_WALL_LIMIT'
|
|
if not checkpoint and len(done)==4 and not active and reason is None:
|
|
projection=first_eight_projection(c,done,charged);save(root/'first-eight-checkpoint.json',projection)
|
|
if projection['continue_within_cap']:checkpoint=True
|
|
else:reason='FIRST_EIGHT_PROJECTION_STOP'
|
|
if reason:
|
|
if not (root/'cancel.json').exists():save(root/'cancel.json',dict(reason=reason,unix=time.time()))
|
|
stop_owned_workers(active)
|
|
while reason is None and can_dispatch(c,next_block,len(active),charged,checkpoint):
|
|
slot=next(s for s in range(4) if s not in active);idx=next_block
|
|
dest=root/'blocks'/f'block-{idx:03d}';dest.mkdir(parents=True,exist_ok=True)
|
|
log=(dest/'block.log').open('x');begin=time.monotonic()
|
|
p=subprocess.Popen([sys.executable,__file__,str(root),'--block',str(idx),'--gpu-slot',str(slot)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
|
active[slot]=dict(process=p,log=log,begin=begin,index=idx);next_block+=1
|
|
if not active and next_block<len(c['blocks']) and reason is None and checkpoint:reason='AGGREGATE_BUDGET_RESERVATION_STOP'
|
|
save(root/'matrix-progress.json',dict(completed_blocks=len(done),terminal_runs=2*len(done),active_blocks={str(k):v['index'] for k,v in active.items()},
|
|
next_block=next_block,settled_gpu_seconds=charged,active_gpu_seconds=live,first_eight_passed=checkpoint,stop_reason=reason,elapsed_seconds=time.monotonic()-start))
|
|
if active:time.sleep(.5)
|
|
finally:
|
|
if active:
|
|
save(root/'cancel.json',dict(reason=reason or 'COORDINATOR_EXIT',unix=time.time()));stop_owned_workers(active)
|
|
for item in active.values():item['process'].wait();item['log'].close()
|
|
rows=[row for b in done for row in b['runs']];seen={r['index'] for r in rows}
|
|
for i,r in enumerate(c['runs']):
|
|
if i not in seen:rows.append(dict(index=i,name=r['name'],status='NA',reason='NOT_RUN_'+str(reason)))
|
|
result=dict(status='ALL_NEURAL_TERMINAL',stop_reason=reason,runs=sorted(rows,key=lambda r:r['index']),blocks=done,
|
|
charged_gpu_seconds=charged,passed=sum(r['status']=='PASS' for r in rows),NA=sum(r['status']!='PASS' for r in rows),
|
|
roster_sha256=c['roster_sha256'],contract_sha256=sha(root/'training-contract.json'),retry=False,
|
|
test_predictions=False,test_scoring=False,r2_required_before_custody=True)
|
|
save(root/'matrix-complete.json',result)
|
|
return 0 if reason is None else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--block',type=int);p.add_argument('--gpu-slot',type=int);a=p.parse_args()
|
|
raise SystemExit(block(a.root,a.block,a.gpu_slot) if a.block is not None else run(a.root))
|