21 lines
1.8 KiB
Python
21 lines
1.8 KiB
Python
"""Start the one published H2 S1 training after host/GPU duplicate checks."""
|
|
import hashlib,json,pathlib,subprocess,sys,time,psutil
|
|
B=pathlib.Path('/home/frank14f/optane/MeltMind');root=B/'artifacts/phase2-h2-s1-2026-11-17'
|
|
proof=json.loads((root/'launch-freeze.json').read_text());assert proof['remote_published']
|
|
for p,h in proof['files'].items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p
|
|
contract=json.loads((root/'contract.json').read_text())
|
|
for p,h in {**contract['code_sha256'],**contract['source_sha256']}.items():assert hashlib.sha256((B/p).read_bytes()).hexdigest()==h,p
|
|
for p in psutil.process_iter(['pid','cmdline']):
|
|
c=p.info['cmdline'] or []
|
|
assert not {'tools/phase2_h2_guard.py','tools/phase2_h2_pilot_worker.py','tools/phase2_h2_s1_guard.py','tools/phase2_h2_s1_worker.py'}.intersection(c),'Existing H2'
|
|
assert not (root/'started.json').exists() and not (root/'service-launch.json').exists()
|
|
uuid=contract['gpu_uuid']
|
|
raw=subprocess.check_output(['nvidia-smi','--id='+uuid,'--query-gpu=memory.used,utilization.gpu','--format=csv,noheader,nounits'],text=True)
|
|
mem,util=[int(x.strip()) for x in raw.strip().split(',')];assert mem<=128 and util==0
|
|
unit='meltmind-phase2-h2-s1-20261117'
|
|
cmd=['systemd-run','--user','--unit='+unit,'--collect','--property=Restart=no','--property=WorkingDirectory='+str(B),
|
|
'--setenv=CUDA_VISIBLE_DEVICES='+uuid,'--setenv=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True','--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1',sys.executable,'tools/phase2_h2_s1_guard.py',str(root)]
|
|
r=subprocess.run(cmd,check=True,capture_output=True,text=True)
|
|
(root/'service-launch.json').write_text(json.dumps(dict(unit=unit,unix=time.time(),commit=proof['commit'],stdout=r.stdout,stderr=r.stderr,gpu_uuid=uuid),indent=2)+'\n')
|
|
print('SERVICE_LAUNCHED',unit,flush=True)
|