109 lines
5.6 KiB
Python
109 lines
5.6 KiB
Python
"""Wait for the sole verified receiver, then start unchanged bounded services once."""
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import psutil
|
|
from gre24_prepare_v2 import sha
|
|
from make_gre24_v4_contract import ROOT, build
|
|
from s2_recovery_supervisor import save
|
|
|
|
PREP_UNIT='meltmind-gre24-v4-prep-20261116'
|
|
MODEL_UNIT='meltmind-gre24-v4-models-20261116'
|
|
|
|
|
|
def check_pins(root):
|
|
for p,h in json.loads((root/'continuation-source-freeze.json').read_text())['code_sha256'].items():
|
|
if sha(p)!=h:
|
|
raise ValueError('Continuation source changed: '+p)
|
|
|
|
|
|
def service_state(unit):
|
|
result=subprocess.run(['systemctl','--user','show',unit+'.service','-p','MainPID','-p','ActiveState',
|
|
'-p','SubState','-p','Restart'],capture_output=True,text=True)
|
|
return dict(line.split('=',1) for line in result.stdout.splitlines() if '=' in line)
|
|
|
|
|
|
def launch(unit, root, script, extra=(), preparation=False):
|
|
state=service_state(unit)
|
|
if state.get('ActiveState') in ('active','activating','deactivating') or int(state.get('MainPID',0)):
|
|
raise RuntimeError('Existing service must not be duplicated: '+unit)
|
|
command=['systemd-run','--user','--unit='+unit,'--working-directory='+str(Path.cwd()),
|
|
'--property=Restart=no','--property=KillMode=control-group','--property=TimeoutStopSec=30s',
|
|
'--property=StandardOutput=append:'+str(root.resolve()/(unit+'.log')),
|
|
'--property=StandardError=inherit','--setenv=PYTHONUNBUFFERED=1',
|
|
'--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1']
|
|
if preparation:
|
|
command.append('--setenv=CUDA_VISIBLE_DEVICES=')
|
|
command += [sys.executable,'-u',script,str(root.resolve()),*map(str,extra)]
|
|
result=subprocess.run(command,capture_output=True,text=True)
|
|
save(root/(unit+'-launch.json'),dict(command=command,returncode=result.returncode,
|
|
stdout=result.stdout,stderr=result.stderr,unix=time.time()))
|
|
if result.returncode:
|
|
raise RuntimeError('Service launch failed: '+unit)
|
|
|
|
|
|
def run(root=ROOT):
|
|
with (root/'continuation.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if (root/'continuation-started.json').exists():
|
|
raise FileExistsError('Never restart this continuation')
|
|
check_pins(root)
|
|
save(root/'continuation-started.json',dict(pid=os.getpid(),unix=time.time(),
|
|
source_freeze_sha256=sha(root/'continuation-source-freeze.json')))
|
|
try:
|
|
entry=json.loads((root/'data/receive-started.json').read_text())
|
|
complete=root/'data/receive-complete.json'
|
|
absent=None
|
|
while not complete.exists():
|
|
try:
|
|
p=psutil.Process(entry['pid'])
|
|
alive=p.create_time()<=entry['unix'] and p.status()!=psutil.STATUS_ZOMBIE and any('receive_gre24_v4.py' in a for a in p.cmdline())
|
|
except psutil.NoSuchProcess:
|
|
alive=False
|
|
absent=None if alive else (absent or time.monotonic())
|
|
if absent and time.monotonic()-absent>30:
|
|
raise RuntimeError('Receiver disappeared without completion; no restart')
|
|
save(root/'continuation-progress.json',dict(stage='WAIT_EXISTING_RECEIVER',receiver_pid=entry['pid'],
|
|
unix=time.time(),science_started=False))
|
|
time.sleep(10)
|
|
if json.loads(complete.read_text())['status']!='VERIFIED':
|
|
raise RuntimeError('Receiver did not verify official v4; no science launch')
|
|
check_pins(root)
|
|
protocol=build(root)
|
|
save(root/'continuation-progress.json',dict(stage='INPUTS_FROZEN',unix=time.time(),science_started=False,
|
|
input_freeze_sha256=sha(protocol/'input-freeze.json')))
|
|
launch(PREP_UNIT,protocol,'tools/run_gre24_protocol_preparation.py',preparation=True)
|
|
deadline=time.monotonic()+30
|
|
while not (protocol/'queue-started.json').exists():
|
|
if time.monotonic()>deadline:
|
|
raise RuntimeError('No preparation start receipt; no duplicate launch')
|
|
time.sleep(.5)
|
|
launch(MODEL_UNIT,protocol,'tools/run_gre24_models.py',[(protocol/'model-plan.json').resolve()])
|
|
deadline=time.monotonic()+30
|
|
while not (protocol/'model-chain-started.json').exists():
|
|
if time.monotonic()>deadline:
|
|
raise RuntimeError('No model waiter start receipt; no duplicate launch')
|
|
time.sleep(.5)
|
|
check_pins(root)
|
|
audit=dict(status='SERVICES_STARTED',unix=time.time(),
|
|
preparation=service_state(PREP_UNIT),model_chain=service_state(MODEL_UNIT),
|
|
preparation_start=json.loads((protocol/'queue-started.json').read_text()),
|
|
model_chain_start=json.loads((protocol/'model-chain-started.json').read_text()),
|
|
input_freeze_sha256=sha(protocol/'input-freeze.json'),source_pins_verified=True,
|
|
receiver_verification_sha256=sha(root/'data/archive-verification.json'))
|
|
save(root/'actual-service-start-audit.json',audit)
|
|
save(root/'continuation-complete.json',audit)
|
|
return 0
|
|
except Exception as error:
|
|
save(root/'continuation-complete.json',dict(status='STOPPED',error_type=type(error).__name__,
|
|
message=str(error),unix=time.time(),no_automatic_retry=True))
|
|
return 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
raise SystemExit(run())
|