Files
MeltMind/tools/phase2_local_launch_chain.py
T

62 lines
5.3 KiB
Python

"""6657 single replacement launcher and local-to-remote identity monitor."""
import fcntl,json,os,subprocess,sys,time
from pathlib import Path
import psutil,shutil
from phase2_h1_worker_v2 import sha,checked,save
from phase2_local_freeze_io import BASE,ENGINEERING,freeze
from phase2_continuation_launch_chain import processes
def read(p):return json.loads(p.read_text())
def run(root):
os.sched_setaffinity(0,[30,31]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'started.json').exists():raise FileExistsError('One labelled replacement')
e=read(ENGINEERING)
for p,h in e['engineering_sha256'].items():checked(BASE/p,h)
old=BASE/'artifacts/phase2-approved-continuation-launch-chain-2026-11-17'
if not (old/'retirement.json').exists() or psutil.pid_exists(604069):raise RuntimeError('Original waiter not retired')
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),approval=e['approval'],engineering_contract_sha256=sha(ENGINEERING),original_waiter_retired=True))
launched=[]
try:
for name in e['order']:
j=e['jobs'][name];r=BASE/'artifacts'/name;c=read(checked(r/'contract.json',j['contract_sha256']))
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(BASE/p,h)
if (r/'started.json').exists() or (r/'service-launch.json').exists():raise RuntimeError('Root already attempted: '+name)
for p in processes():
args=p['cmdline'] or []
if str(r) in args and (j['module'] in args or 'tools/'+j['module']+'.py' in args or 'tools/'+j['worker'] in args):raise RuntimeError('Existing root process')
if name=='phase2-b15-recovery-2026-11-17' and shutil.disk_usage(r).free<24*2**30:raise RuntimeError('B15 free space below24GiB')
freeze(r,'launch',[r/'contract.json',r/'acceptance.json',*[BASE/p for p in c['code_sha256']]])
proof=read(r/'launch-git-freeze.json');checked(r/'contract.json',j['contract_sha256']);commit=proof['commit']
cmd=['systemd-run','--user','--unit='+j['unit'],'--collect','--property=Restart=no','--property=WorkingDirectory='+str(BASE),'--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1',sys.executable,'tools/phase2_local_coordinator.py',j['module'],str(r)]
save(root/(name+'-launch-requested.json'),dict(unix=time.time(),contract_sha256=j['contract_sha256'],local_commit=commit,unit=j['unit']))
result=subprocess.run(cmd,check=True,capture_output=True,text=True);save(r/'service-launch.json',dict(unit=j['unit'],unix=time.time(),local_commit=commit,remote_required=False,engineering_contract_sha256=sha(ENGINEERING),stdout=result.stdout,stderr=result.stderr))
observed=None;end=time.monotonic()+300
while time.monotonic()<end:
actual=[p for p in processes() if str(r) in (p['cmdline'] or []) and 'tools/'+j['worker'] in p['cmdline'] and j['mode'] in p['cmdline']]
if actual:
for p in actual:p['cpu_affinity']=psutil.Process(p['pid']).cpu_affinity()
observed=dict(status='WORKER_OBSERVED',unix=time.time(),started=read(r/'started.json'),workers=actual,contract_sha256=sha(r/'contract.json'),source_pins_pass=True,local_commit=commit,remote_published=False,engineering_contract_sha256=sha(ENGINEERING));save(r/'actual-start-audit.json',observed);break
if (r/'complete.json').exists():raise RuntimeError('Root stopped before actual worker: '+name)
time.sleep(1)
if observed is None:raise RuntimeError('Worker not observed within5min')
launched.append(dict(root=name,pid=observed['started']['pid'],local_commit=commit,contract_sha256=j['contract_sha256'],workers=[p['pid'] for p in observed['workers']]))
save(root/'progress.json',dict(stage='ACTUAL_START_VERIFIED',launched=launched,unix=time.time()))
save(root/'launch-complete.json',dict(status='ALL_THREE_ACTUAL_STARTS_VERIFIED',launched=launched,unix=time.time(),science_results_pending=True))
mapping={};deadline=time.monotonic()+24*3600
while len(mapping)<len(e['order']):
if time.monotonic()>deadline:raise RuntimeError('Remote mapping wait expired; starts not repeated')
for name in e['order']:
j=e['jobs'][name];r=BASE/'artifacts'/name;p=r/'publication.json';mismatch=sha(r/'contract.json')!=j['contract_sha256']
if p.exists():
proof=read(p);rel=str((r/'contract.json').relative_to(BASE));mismatch=mismatch or proof['files'].get(rel)!=j['contract_sha256']
if proof.get('remote_published') and not mismatch:
mapping[name]=dict(local_commit=read(r/'launch-git-freeze.json')['commit'],remote_commit=proof['commit'],contract_sha256=j['contract_sha256'],hash_match=True,unix=time.time());save(r/'local-remote-mapping.json',mapping[name])
if mismatch:
save(root/(name+'-contract-mismatch.json'),dict(status='STOP_ROOT',expected=j['contract_sha256'],actual=sha(r/'contract.json'),unix=time.time()))
subprocess.run(['systemctl','--user','stop',j['unit']],check=False);raise RuntimeError('Contract changed; root stopped: '+name)
save(root/'progress.json',dict(stage='WAIT_REMOTE_IDENTITY_MAPPING',launched=launched,mapped=list(mapping),unix=time.time()));time.sleep(5)
save(root/'complete.json',dict(status='STARTS_AND_REMOTE_MAPPING_COMPLETE',launched=launched,mapping=mapping,unix=time.time()))
except BaseException as exc:
save(root/'complete.json',dict(status='ENGINEERING_STOPPED',reason=type(exc).__name__+': '+str(exc),launched=launched,unix=time.time(),no_retry=True));raise
if __name__=='__main__':run(Path(sys.argv[1]))