62 lines
5.3 KiB
Python
62 lines
5.3 KiB
Python
"""6657 single replacement launcher and local-to-remote identity monitor."""
|
|
import fcntl,json,os,subprocess,sys,time
|
|
from pathlib import Path
|
|
import psutil,shutil
|
|
from phase2_h1_worker_v2 import sha,checked,save
|
|
from phase2_local_freeze_io import BASE,ENGINEERING,freeze
|
|
from phase2_continuation_launch_chain import processes
|
|
|
|
def read(p):return json.loads(p.read_text())
|
|
def run(root):
|
|
os.sched_setaffinity(0,[30,31]);lock=(root/'run.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'started.json').exists():raise FileExistsError('One labelled replacement')
|
|
e=read(ENGINEERING)
|
|
for p,h in e['engineering_sha256'].items():checked(BASE/p,h)
|
|
old=BASE/'artifacts/phase2-approved-continuation-launch-chain-2026-11-17'
|
|
if not (old/'retirement.json').exists() or psutil.pid_exists(604069):raise RuntimeError('Original waiter not retired')
|
|
save(root/'started.json',dict(pid=os.getpid(),unix=time.time(),approval=e['approval'],engineering_contract_sha256=sha(ENGINEERING),original_waiter_retired=True))
|
|
launched=[]
|
|
try:
|
|
for name in e['order']:
|
|
j=e['jobs'][name];r=BASE/'artifacts'/name;c=read(checked(r/'contract.json',j['contract_sha256']))
|
|
for p,h in {**c['code_sha256'],**c['source_sha256']}.items():checked(BASE/p,h)
|
|
if (r/'started.json').exists() or (r/'service-launch.json').exists():raise RuntimeError('Root already attempted: '+name)
|
|
for p in processes():
|
|
args=p['cmdline'] or []
|
|
if str(r) in args and (j['module'] in args or 'tools/'+j['module']+'.py' in args or 'tools/'+j['worker'] in args):raise RuntimeError('Existing root process')
|
|
if name=='phase2-b15-recovery-2026-11-17' and shutil.disk_usage(r).free<24*2**30:raise RuntimeError('B15 free space below24GiB')
|
|
freeze(r,'launch',[r/'contract.json',r/'acceptance.json',*[BASE/p for p in c['code_sha256']]])
|
|
proof=read(r/'launch-git-freeze.json');checked(r/'contract.json',j['contract_sha256']);commit=proof['commit']
|
|
cmd=['systemd-run','--user','--unit='+j['unit'],'--collect','--property=Restart=no','--property=WorkingDirectory='+str(BASE),'--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1',sys.executable,'tools/phase2_local_coordinator.py',j['module'],str(r)]
|
|
save(root/(name+'-launch-requested.json'),dict(unix=time.time(),contract_sha256=j['contract_sha256'],local_commit=commit,unit=j['unit']))
|
|
result=subprocess.run(cmd,check=True,capture_output=True,text=True);save(r/'service-launch.json',dict(unit=j['unit'],unix=time.time(),local_commit=commit,remote_required=False,engineering_contract_sha256=sha(ENGINEERING),stdout=result.stdout,stderr=result.stderr))
|
|
observed=None;end=time.monotonic()+300
|
|
while time.monotonic()<end:
|
|
actual=[p for p in processes() if str(r) in (p['cmdline'] or []) and 'tools/'+j['worker'] in p['cmdline'] and j['mode'] in p['cmdline']]
|
|
if actual:
|
|
for p in actual:p['cpu_affinity']=psutil.Process(p['pid']).cpu_affinity()
|
|
observed=dict(status='WORKER_OBSERVED',unix=time.time(),started=read(r/'started.json'),workers=actual,contract_sha256=sha(r/'contract.json'),source_pins_pass=True,local_commit=commit,remote_published=False,engineering_contract_sha256=sha(ENGINEERING));save(r/'actual-start-audit.json',observed);break
|
|
if (r/'complete.json').exists():raise RuntimeError('Root stopped before actual worker: '+name)
|
|
time.sleep(1)
|
|
if observed is None:raise RuntimeError('Worker not observed within5min')
|
|
launched.append(dict(root=name,pid=observed['started']['pid'],local_commit=commit,contract_sha256=j['contract_sha256'],workers=[p['pid'] for p in observed['workers']]))
|
|
save(root/'progress.json',dict(stage='ACTUAL_START_VERIFIED',launched=launched,unix=time.time()))
|
|
save(root/'launch-complete.json',dict(status='ALL_THREE_ACTUAL_STARTS_VERIFIED',launched=launched,unix=time.time(),science_results_pending=True))
|
|
mapping={};deadline=time.monotonic()+24*3600
|
|
while len(mapping)<len(e['order']):
|
|
if time.monotonic()>deadline:raise RuntimeError('Remote mapping wait expired; starts not repeated')
|
|
for name in e['order']:
|
|
j=e['jobs'][name];r=BASE/'artifacts'/name;p=r/'publication.json';mismatch=sha(r/'contract.json')!=j['contract_sha256']
|
|
if p.exists():
|
|
proof=read(p);rel=str((r/'contract.json').relative_to(BASE));mismatch=mismatch or proof['files'].get(rel)!=j['contract_sha256']
|
|
if proof.get('remote_published') and not mismatch:
|
|
mapping[name]=dict(local_commit=read(r/'launch-git-freeze.json')['commit'],remote_commit=proof['commit'],contract_sha256=j['contract_sha256'],hash_match=True,unix=time.time());save(r/'local-remote-mapping.json',mapping[name])
|
|
if mismatch:
|
|
save(root/(name+'-contract-mismatch.json'),dict(status='STOP_ROOT',expected=j['contract_sha256'],actual=sha(r/'contract.json'),unix=time.time()))
|
|
subprocess.run(['systemctl','--user','stop',j['unit']],check=False);raise RuntimeError('Contract changed; root stopped: '+name)
|
|
save(root/'progress.json',dict(stage='WAIT_REMOTE_IDENTITY_MAPPING',launched=launched,mapped=list(mapping),unix=time.time()));time.sleep(5)
|
|
save(root/'complete.json',dict(status='STARTS_AND_REMOTE_MAPPING_COMPLETE',launched=launched,mapping=mapping,unix=time.time()))
|
|
except BaseException as exc:
|
|
save(root/'complete.json',dict(status='ENGINEERING_STOPPED',reason=type(exc).__name__+': '+str(exc),launched=launched,unix=time.time(),no_retry=True));raise
|
|
if __name__=='__main__':run(Path(sys.argv[1]))
|