Files
MeltMind/tools/continue_gre24_v4_after_receive.py
T

109 lines
5.6 KiB
Python

"""Wait for the sole verified receiver, then start unchanged bounded services once."""
import fcntl
import json
import os
from pathlib import Path
import subprocess
import sys
import time
import psutil
from gre24_prepare_v2 import sha
from make_gre24_v4_contract import ROOT, build
from s2_recovery_supervisor import save
PREP_UNIT='meltmind-gre24-v4-prep-20261116'
MODEL_UNIT='meltmind-gre24-v4-models-20261116'
def check_pins(root):
for p,h in json.loads((root/'continuation-source-freeze.json').read_text())['code_sha256'].items():
if sha(p)!=h:
raise ValueError('Continuation source changed: '+p)
def service_state(unit):
result=subprocess.run(['systemctl','--user','show',unit+'.service','-p','MainPID','-p','ActiveState',
'-p','SubState','-p','Restart'],capture_output=True,text=True)
return dict(line.split('=',1) for line in result.stdout.splitlines() if '=' in line)
def launch(unit, root, script, extra=(), preparation=False):
state=service_state(unit)
if state.get('ActiveState') in ('active','activating','deactivating') or int(state.get('MainPID',0)):
raise RuntimeError('Existing service must not be duplicated: '+unit)
command=['systemd-run','--user','--unit='+unit,'--working-directory='+str(Path.cwd()),
'--property=Restart=no','--property=KillMode=control-group','--property=TimeoutStopSec=30s',
'--property=StandardOutput=append:'+str(root.resolve()/(unit+'.log')),
'--property=StandardError=inherit','--setenv=PYTHONUNBUFFERED=1',
'--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1']
if preparation:
command.append('--setenv=CUDA_VISIBLE_DEVICES=')
command += [sys.executable,'-u',script,str(root.resolve()),*map(str,extra)]
result=subprocess.run(command,capture_output=True,text=True)
save(root/(unit+'-launch.json'),dict(command=command,returncode=result.returncode,
stdout=result.stdout,stderr=result.stderr,unix=time.time()))
if result.returncode:
raise RuntimeError('Service launch failed: '+unit)
def run(root=ROOT):
with (root/'continuation.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (root/'continuation-started.json').exists():
raise FileExistsError('Never restart this continuation')
check_pins(root)
save(root/'continuation-started.json',dict(pid=os.getpid(),unix=time.time(),
source_freeze_sha256=sha(root/'continuation-source-freeze.json')))
try:
entry=json.loads((root/'data/receive-started.json').read_text())
complete=root/'data/receive-complete.json'
absent=None
while not complete.exists():
try:
p=psutil.Process(entry['pid'])
alive=p.create_time()<=entry['unix'] and p.status()!=psutil.STATUS_ZOMBIE and any('receive_gre24_v4.py' in a for a in p.cmdline())
except psutil.NoSuchProcess:
alive=False
absent=None if alive else (absent or time.monotonic())
if absent and time.monotonic()-absent>30:
raise RuntimeError('Receiver disappeared without completion; no restart')
save(root/'continuation-progress.json',dict(stage='WAIT_EXISTING_RECEIVER',receiver_pid=entry['pid'],
unix=time.time(),science_started=False))
time.sleep(10)
if json.loads(complete.read_text())['status']!='VERIFIED':
raise RuntimeError('Receiver did not verify official v4; no science launch')
check_pins(root)
protocol=build(root)
save(root/'continuation-progress.json',dict(stage='INPUTS_FROZEN',unix=time.time(),science_started=False,
input_freeze_sha256=sha(protocol/'input-freeze.json')))
launch(PREP_UNIT,protocol,'tools/run_gre24_protocol_preparation.py',preparation=True)
deadline=time.monotonic()+30
while not (protocol/'queue-started.json').exists():
if time.monotonic()>deadline:
raise RuntimeError('No preparation start receipt; no duplicate launch')
time.sleep(.5)
launch(MODEL_UNIT,protocol,'tools/run_gre24_models.py',[(protocol/'model-plan.json').resolve()])
deadline=time.monotonic()+30
while not (protocol/'model-chain-started.json').exists():
if time.monotonic()>deadline:
raise RuntimeError('No model waiter start receipt; no duplicate launch')
time.sleep(.5)
check_pins(root)
audit=dict(status='SERVICES_STARTED',unix=time.time(),
preparation=service_state(PREP_UNIT),model_chain=service_state(MODEL_UNIT),
preparation_start=json.loads((protocol/'queue-started.json').read_text()),
model_chain_start=json.loads((protocol/'model-chain-started.json').read_text()),
input_freeze_sha256=sha(protocol/'input-freeze.json'),source_pins_verified=True,
receiver_verification_sha256=sha(root/'data/archive-verification.json'))
save(root/'actual-service-start-audit.json',audit)
save(root/'continuation-complete.json',audit)
return 0
except Exception as error:
save(root/'continuation-complete.json',dict(status='STOPPED',error_type=type(error).__name__,
message=str(error),unix=time.time(),no_automatic_retry=True))
return 1
if __name__=='__main__':
raise SystemExit(run())