Implement role-scoped 90 fresh fits with paired four-GPU scheduling and budget gate
This commit is contained in:
1 parent
31012ae4d3
commit
e3fa0af79a
11 files changed
+9462
No files matched your search
@@ -0,0 +1,182 @@
|
||||
"""Four isolated GPU block workers; adjacent N/E, first-eight barrier, aggregate wall ledger."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import psutil
|
||||
from freeze_g1_components import sha
|
||||
from s2_recovery_supervisor import save
|
||||
from s2_gpu_idle_gate import recorded_wait
|
||||
from s2_main_memory_guard import guarded_worker,ALLOCATOR_CONF
|
||||
|
||||
BLOCK_WALL_CAP=5000.
|
||||
|
||||
|
||||
def contract(root):
|
||||
c=json.loads((root/'training-contract.json').read_text())
|
||||
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():assert sha(p)==h,p
|
||||
assert c['roster_sha256']==sha(c['roster']) and len(c['runs'])==90 and len(c['blocks'])==45
|
||||
assert c['limits']['aggregate_gpu_seconds']==46*3600 and c['limits']['run_wall_seconds']==1800
|
||||
assert c['E_max']==c['cosine_T_max']==42
|
||||
assert len(set(c['gpu_uuids']))==len(c['gpu_uuids'])==4
|
||||
for b in c['blocks']:
|
||||
n,e=[c['runs'][i] for i in b['run_indices']]
|
||||
assert (n['arm'],e['arm'])==('N','E')
|
||||
assert (n['fold'],n['pool'],n['seed'])==(e['fold'],e['pool'],e['seed'])
|
||||
return c
|
||||
|
||||
|
||||
def first_eight_projection(c,blocks,charged):
|
||||
assert len(blocks)==4 and sum(len(b['runs']) for b in blocks)==8
|
||||
rates={arm:[] for arm in ['N','E']}
|
||||
for b in blocks:
|
||||
for row in b['runs']:
|
||||
run=c['runs'][row['index']]
|
||||
if row['status']=='PASS':rates[run['arm']].append(row['cell_wall_seconds']/run['cost_nodes'])
|
||||
if any(not v for v in rates.values()):
|
||||
return dict(continue_within_cap=False,reason='FIRST8_NO_VALID_COST_FOR_BOTH_ARMS',observed_charged_gpu_seconds=charged)
|
||||
rate={a:max(v) for a,v in rates.items()}
|
||||
completed={row['index'] for b in blocks for row in b['runs']}
|
||||
projected=charged+1.25*sum(rate[r['arm']]*r['cost_nodes'] for i,r in enumerate(c['runs']) if i not in completed)
|
||||
return dict(continue_within_cap=projected<=c['limits']['aggregate_gpu_seconds'],projected_gpu_seconds=projected,
|
||||
observed_charged_gpu_seconds=charged,seconds_per_node_by_arm=rate,reserve_multiplier=1.25,
|
||||
method='Maximum successful observed per-arm cell wall/node (idle/setup included), remaining cells plus25percent; all actual block wall already charged.')
|
||||
|
||||
|
||||
def can_dispatch(c,next_block,active_count,charged,checkpoint):
|
||||
ceiling=len(c['blocks']) if checkpoint else 4
|
||||
return next_block<ceiling and active_count<4 and charged+(active_count+1)*BLOCK_WALL_CAP<=c['limits']['aggregate_gpu_seconds']
|
||||
|
||||
|
||||
def block(root,index,gpu_slot):
|
||||
c=contract(root);spec=c['blocks'][index];uuid=c['gpu_uuids'][gpu_slot]
|
||||
dest=root/'blocks'/f'block-{index:03d}';dest.mkdir(parents=True,exist_ok=True)
|
||||
with (dest/'block.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
assert not (dest/'started.json').exists(),'No repeated block'
|
||||
with Path('/tmp/meltmind-s2-'+uuid+'.lock').open('a') as gpu_lock:
|
||||
fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
return block_locked(root,c,spec,dest,index,gpu_slot)
|
||||
|
||||
|
||||
def block_locked(root,c,spec,dest,index,gpu_slot):
|
||||
begin=time.monotonic();uuid=c['gpu_uuids'][gpu_slot];rows=[]
|
||||
os.sched_setaffinity(0,c['gpu_cpu_affinities'][gpu_slot])
|
||||
os.environ['PYTORCH_CUDA_ALLOC_CONF']=ALLOCATOR_CONF
|
||||
save(dest/'started.json',dict(pid=os.getpid(),started_unix=time.time(),gpu_uuid=uuid,block=index,
|
||||
run_indices=spec['run_indices'],cpu_affinity=sorted(os.sched_getaffinity(0)),contract_sha256=sha(root/'training-contract.json')))
|
||||
try:
|
||||
for i in spec['run_indices']:
|
||||
run=c['runs'][i];cell=root/'cells'/run['name'];cell.mkdir(parents=True,exist_ok=False)
|
||||
cell_begin=time.monotonic();row=dict(index=i,name=run['name'],arm=run['arm'],status='NA',reason=None)
|
||||
try:
|
||||
if (root/'cancel.json').exists():raise RuntimeError('MATRIX_CANCELLED_BEFORE_RUN')
|
||||
if time.monotonic()-begin>BLOCK_WALL_CAP-2405:raise RuntimeError('BLOCK_REMAINING_BUDGET_INSUFFICIENT')
|
||||
idle=recorded_wait(cell,'before-run',uuid,600)
|
||||
if idle['status']!='IDLE_CONFIRMED':raise RuntimeError('GPU_IDLE_TIMEOUT_NO_RETRY')
|
||||
if (root/'cancel.json').exists():raise RuntimeError('MATRIX_CANCELLED_AFTER_IDLE')
|
||||
receipt,result=guarded_worker(cell,run['name'],uuid,1800,
|
||||
[sys.executable,'tools/run_s2_main_training.py',str(root),str(i)],cumulative_worker_cpu_limit_seconds=1801)
|
||||
row.update(status='PASS' if receipt['status']=='PASS' else 'NA',reason=(receipt.get('stop_reason') or ('WORKER_EXIT_'+str(receipt['returncode']) if receipt['status']!='PASS' else None)),
|
||||
guard_receipt=str(cell/(run['name']+'-guard-receipt.json')),guard_sha256=sha(cell/(run['name']+'-guard-receipt.json')),
|
||||
worker_status=receipt['status'],worker_returncode=receipt['returncode'],worker_cpu_seconds=receipt['cpu_seconds'],
|
||||
checkpoint_sha256=result.get('checkpoint_sha256') if result else None,
|
||||
standardizer_fingerprint=result.get('standardizer_fingerprint') if result else None)
|
||||
except Exception as exc:
|
||||
row.update(status='NA',reason=type(exc).__name__+': '+str(exc),traceback=traceback.format_exc())
|
||||
row['cell_wall_seconds']=time.monotonic()-cell_begin;rows.append(row)
|
||||
save(cell/'terminal.json',row);save(dest/'progress.json',dict(runs=rows,elapsed_seconds=time.monotonic()-begin))
|
||||
finally:
|
||||
# A failed arm never causes its paired arm to be silently replaced or retried.
|
||||
present={r['index'] for r in rows}
|
||||
for i in spec['run_indices']:
|
||||
if i not in present:rows.append(dict(index=i,name=c['runs'][i]['name'],status='NA',reason='BLOCK_INTERRUPTED_BEFORE_RUN',cell_wall_seconds=0))
|
||||
valid=all(r['status']=='PASS' for r in rows)
|
||||
if valid and rows[0]['standardizer_fingerprint']!=rows[1]['standardizer_fingerprint']:valid=False
|
||||
save(dest/'terminal.json',dict(status='TERMINAL',block=index,gpu_uuid=uuid,runs=rows,pair_valid=valid,
|
||||
pair_invalid_reason=None if valid else 'AT_LEAST_ONE_ARM_NA_OR_STANDARDIZER_MISMATCH',
|
||||
block_wall_seconds=time.monotonic()-begin,cpu_affinity=sorted(os.sched_getaffinity(0)),retry=False,test_predictions=False))
|
||||
|
||||
|
||||
def stop_owned_workers(active):
|
||||
for item in active.values():
|
||||
try:children=psutil.Process(item['process'].pid).children(recursive=True)
|
||||
except psutil.NoSuchProcess:continue
|
||||
for p in children:
|
||||
try:
|
||||
if 'tools/run_s2_main_training.py' in p.cmdline():os.killpg(p.pid,signal.SIGKILL)
|
||||
except (ProcessLookupError,psutil.NoSuchProcess):pass
|
||||
|
||||
|
||||
def run(root):
|
||||
with (root/'matrix.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB);return run_locked(root)
|
||||
|
||||
|
||||
def run_locked(root):
|
||||
c=contract(root);assert not (root/'matrix-started.json').exists(),'No repeated matrix launch'
|
||||
save(root/'matrix-started.json',dict(pid=os.getpid(),started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
|
||||
start=time.monotonic();active={};done=[];next_block=0;reason=None;checkpoint=False;charged=0.
|
||||
def interrupt(*_):
|
||||
nonlocal reason
|
||||
reason='MATRIX_COORDINATOR_INTERRUPTED'
|
||||
signal.signal(signal.SIGTERM,interrupt);signal.signal(signal.SIGINT,interrupt)
|
||||
env=dict(os.environ,PYTORCH_CUDA_ALLOC_CONF=ALLOCATOR_CONF,OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',OPENBLAS_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',CUDA_VISIBLE_DEVICES='')
|
||||
try:
|
||||
while active or (next_block<len(c['blocks']) and reason is None):
|
||||
for slot,item in list(active.items()):
|
||||
code=item['process'].poll();elapsed=time.monotonic()-item['begin']
|
||||
if elapsed>BLOCK_WALL_CAP and code is None:reason='BLOCK_WALL_LIMIT'
|
||||
if code is None:continue
|
||||
path=root/'blocks'/f"block-{item['index']:03d}"/'terminal.json'
|
||||
if not path.exists():
|
||||
reason='BLOCK_EXIT_WITHOUT_TERMINAL_RECEIPT'
|
||||
result=dict(status='INFRASTRUCTURE_FAILED',block=item['index'],runs=[dict(index=i,name=c['runs'][i]['name'],status='NA',reason=reason,cell_wall_seconds=0) for i in c['blocks'][item['index']]['run_indices']],pair_valid=False)
|
||||
else:result=json.loads(path.read_text())
|
||||
result.update(charged_gpu_seconds=elapsed,coordinator_observed_returncode=code)
|
||||
save(root/'blocks'/f"block-{item['index']:03d}"/'accounted.json',result)
|
||||
done.append(result);charged+=elapsed;item['log'].close();del active[slot]
|
||||
print(json.dumps(dict(event='BLOCK_TERMINAL',block=result['block'],run_statuses=[r['status'] for r in result['runs']],charged_gpu_hours=charged/3600)),flush=True)
|
||||
live=sum(time.monotonic()-x['begin'] for x in active.values())
|
||||
if charged+live>=c['limits']['aggregate_gpu_seconds']:reason='AGGREGATE_GPU_WALL_LIMIT'
|
||||
if not checkpoint and len(done)==4 and not active and reason is None:
|
||||
projection=first_eight_projection(c,done,charged);save(root/'first-eight-checkpoint.json',projection)
|
||||
if projection['continue_within_cap']:checkpoint=True
|
||||
else:reason='FIRST_EIGHT_PROJECTION_STOP'
|
||||
if reason:
|
||||
if not (root/'cancel.json').exists():save(root/'cancel.json',dict(reason=reason,unix=time.time()))
|
||||
stop_owned_workers(active)
|
||||
while reason is None and can_dispatch(c,next_block,len(active),charged,checkpoint):
|
||||
slot=next(s for s in range(4) if s not in active);idx=next_block
|
||||
dest=root/'blocks'/f'block-{idx:03d}';dest.mkdir(parents=True,exist_ok=True)
|
||||
log=(dest/'block.log').open('x');begin=time.monotonic()
|
||||
p=subprocess.Popen([sys.executable,__file__,str(root),'--block',str(idx),'--gpu-slot',str(slot)],env=env,stdout=log,stderr=subprocess.STDOUT,start_new_session=True)
|
||||
active[slot]=dict(process=p,log=log,begin=begin,index=idx);next_block+=1
|
||||
if not active and next_block<len(c['blocks']) and reason is None and checkpoint:reason='AGGREGATE_BUDGET_RESERVATION_STOP'
|
||||
save(root/'matrix-progress.json',dict(completed_blocks=len(done),terminal_runs=2*len(done),active_blocks={str(k):v['index'] for k,v in active.items()},
|
||||
next_block=next_block,settled_gpu_seconds=charged,active_gpu_seconds=live,first_eight_passed=checkpoint,stop_reason=reason,elapsed_seconds=time.monotonic()-start))
|
||||
if active:time.sleep(.5)
|
||||
finally:
|
||||
if active:
|
||||
save(root/'cancel.json',dict(reason=reason or 'COORDINATOR_EXIT',unix=time.time()));stop_owned_workers(active)
|
||||
for item in active.values():item['process'].wait();item['log'].close()
|
||||
rows=[row for b in done for row in b['runs']];seen={r['index'] for r in rows}
|
||||
for i,r in enumerate(c['runs']):
|
||||
if i not in seen:rows.append(dict(index=i,name=r['name'],status='NA',reason='NOT_RUN_'+str(reason)))
|
||||
result=dict(status='ALL_NEURAL_TERMINAL',stop_reason=reason,runs=sorted(rows,key=lambda r:r['index']),blocks=done,
|
||||
charged_gpu_seconds=charged,passed=sum(r['status']=='PASS' for r in rows),NA=sum(r['status']!='PASS' for r in rows),
|
||||
roster_sha256=c['roster_sha256'],contract_sha256=sha(root/'training-contract.json'),retry=False,
|
||||
test_predictions=False,test_scoring=False,r2_required_before_custody=True)
|
||||
save(root/'matrix-complete.json',result)
|
||||
return 0 if reason is None else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--block',type=int);p.add_argument('--gpu-slot',type=int);a=p.parse_args()
|
||||
raise SystemExit(block(a.root,a.block,a.gpu_slot) if a.block is not None else run(a.root))
|
||||
Reference in new issue
Block a user