Files
MeltMind/artifacts/phase3-item1b-launch-2026-11-18/launch-reviewed.py
T

53 lines
4.9 KiB
Python

"""One reviewed1b launch only after real shared capacity and older queue priority."""
import argparse,json,os,fcntl,socket,subprocess,time
from pathlib import Path
import psutil
from phase2_h1_worker_v2 import sha,checked,save
from phase2_gate_outcome_review import validate_review
from phase3_item1b_common import ROOT,config
from phase3_shared_capacity import SHARED,host_snapshot,admission,GPUS,alive,descendants
B=Path('/home/frank14f/optane/MeltMind');PY=B/'artifacts/gre24-reproduction-2026-11-15/environment/venv/bin/python';A=B/'artifacts/phase3-item1b-launch-2026-11-18'
p=argparse.ArgumentParser();p.add_argument('--launch',action='store_true');args=p.parse_args()
assert socket.gethostname()=='frank14f-Horsea'
c,_=config(ROOT);validate_review(ROOT)
assert sha(ROOT/'contract.json')=='d33f036adc48cde904fbb6bb0f615a5464f94b039906d0b45a4dbd15f0681bd3'
assert sha(ROOT/'review.json')=='4ea5b84df28d7dc2c72318bf315f9220cec45b27cfe46bfb058b8f9719d1a004'
assert not any((ROOT/f).exists() for f in ['started.json','complete.json','launch-attempt.json'])
for proc in psutil.process_iter(['pid','cmdline']):
assert not any(Path(x).name in {'phase3_item1b_queue.py','phase3_item1b_guard.py','phase3_item1b_worker.py'} for x in proc.info['cmdline'] or []),proc.info
for name in ['phase3-item-c-2026-11-18','phase3-seed1-v2-2026-11-18','track-a-recovery-2026-11-18']:
d=json.loads((ROOT.with_name(name)/'contract.json').read_text())
for q,h in d['code_sha256'].items():checked(q,h)
unit='meltmind-phase3-item1b-20261118.service'
old=subprocess.check_output(['systemctl','--user','show',unit,'-p','LoadState','-p','ActiveState','-p','MainPID'],text=True)
assert 'LoadState=not-found' in old,old
with (SHARED/'capacity.lock').open('r') as lock:
fcntl.flock(lock,fcntl.LOCK_SH);d=json.loads((SHARED/'registry.json').read_text());assert d['version']==2 and 'item1b' not in d['entries'];leases=[]
for l in d['leases'].values():
assert alive(l['owner']),'Await original owner collecting exited lease'
ids=descendants(l['owner']) if l['state']=='GUARD_BOUND' else set();rss=0
for pid in ids:
try:rss+=psutil.Process(pid).memory_info().rss
except psutil.NoSuchProcess:pass
leases.append(dict(l,observed_pids=sorted(ids),observed_rss=rss))
s=host_snapshot();decisions={u:admission(s,leases,u) for u in GPUS[1:3]}
used={x for l in leases for x in l['worker_cpus']};free=[x for x in range(16,26) if x not in used and x in os.sched_getaffinity(0)]
cr=ROOT.with_name('phase3-item-c-2026-11-18');cp=json.loads((cr/'live-progress.json').read_text())
assert (cr/'complete.json').exists() or time.time()-cp['unix']<120
# c's already-running queue takes shared slots first. Truly idle cards are left
# for old idle-only dispatch while either old queue still has pending folds.
c_pending=bool(cp['pending']) and not (cr/'complete.json').exists()
old_idle_pending=bool(s['old_pending']) and any(not any(x['gpu_uuid']==u for x in s['processes']) for u in GPUS[1:3])
ready=any(v[0] for v in decisions.values()) and bool(free) and not c_pending and not old_idle_pending and psutil.disk_usage(ROOT).free>=100*2**30
record=dict(unix=time.time(),ready=ready,snapshot=s,admission=decisions,free_worker_cpus=free,c_pending_priority=c_pending,old_idle_pending_priority=old_idle_pending,old_code_pins='130PASS',unit=unit,launched=False)
save(A/('prelaunch-check-'+str(time.time_ns())+'.json'),record)
if not ready or not args.launch:
print(json.dumps(dict(status='READY_NO_LAUNCH' if ready else 'WAIT_EXISTING_QUEUE_PRIORITY_OR_CAPACITY',admission=decisions,c_pending=c_pending,started=False)));raise SystemExit(0)
command=['systemd-run','--user','--collect','--unit='+unit,'--working-directory='+str(B),'--property=Restart=no','--property=KillMode=control-group','--property=TimeoutStopSec=60','--property=StandardOutput=append:'+str(ROOT/'service.log'),'--property=StandardError=inherit']
for k,v in dict(OPENBLAS_NUM_THREADS='1',OMP_NUM_THREADS='1',MKL_NUM_THREADS='1',NUMEXPR_NUM_THREADS='1',PYTHONPATH='tools',PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True',PYTHONUNBUFFERED='1').items():command.append('--setenv='+k+'='+v)
command += [str(PY),str(B/'tools/phase3_item1b_queue.py')]
save(ROOT/'host-prelaunch-refreshed.json',record)
save(ROOT/'launch-attempt.json',dict(unix=time.time(),command=command,claude_ruling='1795011859.371609',capacity_priority_ruling='1795011981.652149',review_sha256=sha(ROOT/'review.json'),contract_sha256=sha(ROOT/'contract.json'),new_fits=5,seed=0,epochs=100,gpu_cap=88560,postfit_reserve=7200,phase3_cap=720000,no_retry=True,direct_user_authorization=str(B/'artifacts/phase3-item-c-launch-2026-11-18/direct-user-authorization.json')))
r=subprocess.run(command,capture_output=True,text=True)
save(ROOT/'service-launch.json',dict(returncode=r.returncode,stdout=r.stdout,stderr=r.stderr,unit=unit,unix=time.time()));print(json.dumps(dict(returncode=r.returncode,stdout=r.stdout,stderr=r.stderr,unit=unit)));raise SystemExit(r.returncode)