"""Wait for the sole verified receiver, then start unchanged bounded services once.""" import fcntl import json import os from pathlib import Path import subprocess import sys import time import psutil from gre24_prepare_v2 import sha from make_gre24_v4_contract import ROOT, build from s2_recovery_supervisor import save PREP_UNIT='meltmind-gre24-v4-prep-20261116' MODEL_UNIT='meltmind-gre24-v4-models-20261116' def check_pins(root): for p,h in json.loads((root/'continuation-source-freeze.json').read_text())['code_sha256'].items(): if sha(p)!=h: raise ValueError('Continuation source changed: '+p) def service_state(unit): result=subprocess.run(['systemctl','--user','show',unit+'.service','-p','MainPID','-p','ActiveState', '-p','SubState','-p','Restart'],capture_output=True,text=True) return dict(line.split('=',1) for line in result.stdout.splitlines() if '=' in line) def launch(unit, root, script, extra=(), preparation=False): state=service_state(unit) if state.get('ActiveState') in ('active','activating','deactivating') or int(state.get('MainPID',0)): raise RuntimeError('Existing service must not be duplicated: '+unit) command=['systemd-run','--user','--unit='+unit,'--working-directory='+str(Path.cwd()), '--property=Restart=no','--property=KillMode=control-group','--property=TimeoutStopSec=30s', '--property=StandardOutput=append:'+str(root.resolve()/(unit+'.log')), '--property=StandardError=inherit','--setenv=PYTHONUNBUFFERED=1', '--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1'] if preparation: command.append('--setenv=CUDA_VISIBLE_DEVICES=') command += [sys.executable,'-u',script,str(root.resolve()),*map(str,extra)] result=subprocess.run(command,capture_output=True,text=True) save(root/(unit+'-launch.json'),dict(command=command,returncode=result.returncode, stdout=result.stdout,stderr=result.stderr,unix=time.time())) if result.returncode: raise RuntimeError('Service launch failed: '+unit) def run(root=ROOT): with (root/'continuation.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if (root/'continuation-started.json').exists(): raise FileExistsError('Never restart this continuation') check_pins(root) save(root/'continuation-started.json',dict(pid=os.getpid(),unix=time.time(), source_freeze_sha256=sha(root/'continuation-source-freeze.json'))) try: entry=json.loads((root/'data/receive-started.json').read_text()) complete=root/'data/receive-complete.json' absent=None while not complete.exists(): try: p=psutil.Process(entry['pid']) alive=p.create_time()<=entry['unix'] and p.status()!=psutil.STATUS_ZOMBIE and any('receive_gre24_v4.py' in a for a in p.cmdline()) except psutil.NoSuchProcess: alive=False absent=None if alive else (absent or time.monotonic()) if absent and time.monotonic()-absent>30: raise RuntimeError('Receiver disappeared without completion; no restart') save(root/'continuation-progress.json',dict(stage='WAIT_EXISTING_RECEIVER',receiver_pid=entry['pid'], unix=time.time(),science_started=False)) time.sleep(10) if json.loads(complete.read_text())['status']!='VERIFIED': raise RuntimeError('Receiver did not verify official v4; no science launch') check_pins(root) protocol=build(root) save(root/'continuation-progress.json',dict(stage='INPUTS_FROZEN',unix=time.time(),science_started=False, input_freeze_sha256=sha(protocol/'input-freeze.json'))) launch(PREP_UNIT,protocol,'tools/run_gre24_protocol_preparation.py',preparation=True) deadline=time.monotonic()+30 while not (protocol/'queue-started.json').exists(): if time.monotonic()>deadline: raise RuntimeError('No preparation start receipt; no duplicate launch') time.sleep(.5) launch(MODEL_UNIT,protocol,'tools/run_gre24_models.py',[(protocol/'model-plan.json').resolve()]) deadline=time.monotonic()+30 while not (protocol/'model-chain-started.json').exists(): if time.monotonic()>deadline: raise RuntimeError('No model waiter start receipt; no duplicate launch') time.sleep(.5) check_pins(root) audit=dict(status='SERVICES_STARTED',unix=time.time(), preparation=service_state(PREP_UNIT),model_chain=service_state(MODEL_UNIT), preparation_start=json.loads((protocol/'queue-started.json').read_text()), model_chain_start=json.loads((protocol/'model-chain-started.json').read_text()), input_freeze_sha256=sha(protocol/'input-freeze.json'),source_pins_verified=True, receiver_verification_sha256=sha(root/'data/archive-verification.json')) save(root/'actual-service-start-audit.json',audit) save(root/'continuation-complete.json',audit) return 0 except Exception as error: save(root/'continuation-complete.json',dict(status='STOPPED',error_type=type(error).__name__, message=str(error),unix=time.time(),no_automatic_retry=True)) return 1 if __name__=='__main__': raise SystemExit(run())