diff --git a/artifacts/gre24-v4-reproduction-2026-11-16/continuation-source-freeze.json b/artifacts/gre24-v4-reproduction-2026-11-16/continuation-source-freeze.json new file mode 100644 index 000000000..d091f49eb --- /dev/null +++ b/artifacts/gre24-v4-reproduction-2026-11-16/continuation-source-freeze.json @@ -0,0 +1,26 @@ +{ + "code_sha256": { + "tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf", + "tools/gre24_prepare_v2.py": "7637e55009005f6a8c77114159d7ac0d3c024c7add3e320afc486927537b78bc", + "tools/run_gre24_protocol_preparation.py": "74a95323ec43ed089e3768c6b0a5ebd4c369207476d81d03ab1f423d33753499", + "tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7", + "tools/audit_samples.py": "a3daf70ec3bcb77bf46d9dc4887d552cf7c422117880a169969780012278cd70", + "tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919", + "tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "tools/run_gre24_models.py": "103ecb6dc8a40481f72da02a02e0a41069a9a747ad0b7af4418c6ed06f0dcbc9", + "tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a", + "tools/gre24_fit.py": "63af0305a2c22b524d95ffee50e972848decdd042b1d24f0b3b946d9152f7688", + "tools/gre24_score.py": "dbb11b57b7b7b2178ac11fd7243c934adb899d666a4c717dfb6a51ee1c65c705", + "tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4", + "tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa", + "tools/make_gre24_v4_contract.py": "2303f801fb2ced6fe119149c03a8116b321970422abd34f27b4ff5e58faa4ed1", + "tools/continue_gre24_v4_after_receive.py": "b1be8f5d020d0b7919c2ac98cb023a6e708952bf1042ab767b0a2ff7dc579925", + "tools/receive_gre24_v4.py": "4e5b5ba676c9970dc0166ac93009b748dc44128a2b6f5c6179600f9ca5260729" + }, + "preflight_sha256": "4297c7c3358bcdd2cd7a164245f1de1ff13ab59c3ead7c8351fa5d172f8f0e51", + "acceptance_sha256": "455aec60e7fde24973608114ec84a9c8b49968ba9599a89b97e188b9955f0143", + "synthetic_cpu_seconds": 1.949516, + "setup_allowance_seconds": 60, + "original_scientific_sources_unchanged": true, + "unix": 1794804102.71015 +} diff --git a/artifacts/gre24-v4-reproduction-2026-11-16/continuation-synthetic-acceptance.json b/artifacts/gre24-v4-reproduction-2026-11-16/continuation-synthetic-acceptance.json new file mode 100644 index 000000000..14c07f699 --- /dev/null +++ b/artifacts/gre24-v4-reproduction-2026-11-16/continuation-synthetic-acceptance.json @@ -0,0 +1,13 @@ +{ + "status": "PASS", + "tests": 8, + "failures": 0, + "errors": 0, + "cpu_seconds": 1.9380359999999999, + "scientific_data_read": false, + "code_sha256": { + "tools/make_gre24_v4_contract.py": "2303f801fb2ced6fe119149c03a8116b321970422abd34f27b4ff5e58faa4ed1", + "tools/continue_gre24_v4_after_receive.py": "b1be8f5d020d0b7919c2ac98cb023a6e708952bf1042ab767b0a2ff7dc579925", + "tools/test_gre24_v4_continuation.py": "c412707bf120454f0daa4998e4205b272636626ff86a85695c90a4e40c656d65" + } +} diff --git a/artifacts/gre24-v4-reproduction-2026-11-16/data/private-entry-receipt.json b/artifacts/gre24-v4-reproduction-2026-11-16/data/private-entry-receipt.json new file mode 100644 index 000000000..6ffa5740c --- /dev/null +++ b/artifacts/gre24-v4-reproduction-2026-11-16/data/private-entry-receipt.json @@ -0,0 +1,12 @@ +{ + "status": "PRIVATE_ENTRY_RECEIVED", + "drive_file_id": "1I8SRSqY5d2WmUzc1Dz7GEqoATUbvL4Xn", + "ruling": "1794802690.164829", + "expiry": "2026-11-23 12:15:17 Asia/Shanghai", + "source_key": "greif-v4/data.zip", + "url_excluded": true, + "encoded_slash_identity_check": "PASS", + "receiver_sha256": "4e5b5ba676c9970dc0166ac93009b748dc44128a2b6f5c6179600f9ca5260729", + "unix": 1794803849.6359196, + "science_started": false +} diff --git a/artifacts/gre24-v4-reproduction-2026-11-16/data/receive-started.json b/artifacts/gre24-v4-reproduction-2026-11-16/data/receive-started.json new file mode 100644 index 000000000..7b8469120 --- /dev/null +++ b/artifacts/gre24-v4-reproduction-2026-11-16/data/receive-started.json @@ -0,0 +1,9 @@ +{ + "pid": 506357, + "unix": 1794803859.112114, + "source_host": "frank14f-private-hk.oss-cn-hongkong.aliyuncs.com", + "source_key": "greif-v4/data.zip", + "expected_bytes": 35925187752, + "expected_md5": "4973f24e12a38c87c31cf129be1a3e3b", + "private_url_not_recorded": true +} diff --git a/tools/continue_gre24_v4_after_receive.py b/tools/continue_gre24_v4_after_receive.py new file mode 100644 index 000000000..40768137d --- /dev/null +++ b/tools/continue_gre24_v4_after_receive.py @@ -0,0 +1,108 @@ +"""Wait for the sole verified receiver, then start unchanged bounded services once.""" +import fcntl +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import psutil +from gre24_prepare_v2 import sha +from make_gre24_v4_contract import ROOT, build +from s2_recovery_supervisor import save + +PREP_UNIT='meltmind-gre24-v4-prep-20261116' +MODEL_UNIT='meltmind-gre24-v4-models-20261116' + + +def check_pins(root): + for p,h in json.loads((root/'continuation-source-freeze.json').read_text())['code_sha256'].items(): + if sha(p)!=h: + raise ValueError('Continuation source changed: '+p) + + +def service_state(unit): + result=subprocess.run(['systemctl','--user','show',unit+'.service','-p','MainPID','-p','ActiveState', + '-p','SubState','-p','Restart'],capture_output=True,text=True) + return dict(line.split('=',1) for line in result.stdout.splitlines() if '=' in line) + + +def launch(unit, root, script, extra=(), preparation=False): + state=service_state(unit) + if state.get('ActiveState') in ('active','activating','deactivating') or int(state.get('MainPID',0)): + raise RuntimeError('Existing service must not be duplicated: '+unit) + command=['systemd-run','--user','--unit='+unit,'--working-directory='+str(Path.cwd()), + '--property=Restart=no','--property=KillMode=control-group','--property=TimeoutStopSec=30s', + '--property=StandardOutput=append:'+str(root.resolve()/(unit+'.log')), + '--property=StandardError=inherit','--setenv=PYTHONUNBUFFERED=1', + '--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1'] + if preparation: + command.append('--setenv=CUDA_VISIBLE_DEVICES=') + command += [sys.executable,'-u',script,str(root.resolve()),*map(str,extra)] + result=subprocess.run(command,capture_output=True,text=True) + save(root/(unit+'-launch.json'),dict(command=command,returncode=result.returncode, + stdout=result.stdout,stderr=result.stderr,unix=time.time())) + if result.returncode: + raise RuntimeError('Service launch failed: '+unit) + + +def run(root=ROOT): + with (root/'continuation.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + if (root/'continuation-started.json').exists(): + raise FileExistsError('Never restart this continuation') + check_pins(root) + save(root/'continuation-started.json',dict(pid=os.getpid(),unix=time.time(), + source_freeze_sha256=sha(root/'continuation-source-freeze.json'))) + try: + entry=json.loads((root/'data/receive-started.json').read_text()) + complete=root/'data/receive-complete.json' + absent=None + while not complete.exists(): + try: + p=psutil.Process(entry['pid']) + alive=p.create_time()<=entry['unix'] and p.status()!=psutil.STATUS_ZOMBIE and any('receive_gre24_v4.py' in a for a in p.cmdline()) + except psutil.NoSuchProcess: + alive=False + absent=None if alive else (absent or time.monotonic()) + if absent and time.monotonic()-absent>30: + raise RuntimeError('Receiver disappeared without completion; no restart') + save(root/'continuation-progress.json',dict(stage='WAIT_EXISTING_RECEIVER',receiver_pid=entry['pid'], + unix=time.time(),science_started=False)) + time.sleep(10) + if json.loads(complete.read_text())['status']!='VERIFIED': + raise RuntimeError('Receiver did not verify official v4; no science launch') + check_pins(root) + protocol=build(root) + save(root/'continuation-progress.json',dict(stage='INPUTS_FROZEN',unix=time.time(),science_started=False, + input_freeze_sha256=sha(protocol/'input-freeze.json'))) + launch(PREP_UNIT,protocol,'tools/run_gre24_protocol_preparation.py',preparation=True) + deadline=time.monotonic()+30 + while not (protocol/'queue-started.json').exists(): + if time.monotonic()>deadline: + raise RuntimeError('No preparation start receipt; no duplicate launch') + time.sleep(.5) + launch(MODEL_UNIT,protocol,'tools/run_gre24_models.py',[(protocol/'model-plan.json').resolve()]) + deadline=time.monotonic()+30 + while not (protocol/'model-chain-started.json').exists(): + if time.monotonic()>deadline: + raise RuntimeError('No model waiter start receipt; no duplicate launch') + time.sleep(.5) + check_pins(root) + audit=dict(status='SERVICES_STARTED',unix=time.time(), + preparation=service_state(PREP_UNIT),model_chain=service_state(MODEL_UNIT), + preparation_start=json.loads((protocol/'queue-started.json').read_text()), + model_chain_start=json.loads((protocol/'model-chain-started.json').read_text()), + input_freeze_sha256=sha(protocol/'input-freeze.json'),source_pins_verified=True, + receiver_verification_sha256=sha(root/'data/archive-verification.json')) + save(root/'actual-service-start-audit.json',audit) + save(root/'continuation-complete.json',audit) + return 0 + except Exception as error: + save(root/'continuation-complete.json',dict(status='STOPPED',error_type=type(error).__name__, + message=str(error),unix=time.time(),no_automatic_retry=True)) + return 1 + + +if __name__=='__main__': + raise SystemExit(run()) diff --git a/tools/make_gre24_v4_contract.py b/tools/make_gre24_v4_contract.py new file mode 100644 index 000000000..4bc328842 --- /dev/null +++ b/tools/make_gre24_v4_contract.py @@ -0,0 +1,149 @@ +"""Bind verified v4 input and preregistered splits to unchanged v5 science.""" +import copy +import json +from pathlib import Path +import re +import time +from xml.etree import ElementTree as ET +from zipfile import ZipFile +import numpy as np +from gre24_prepare_v2 import sha + +BASE = Path('artifacts/gre24-reproduction-2026-11-15') +ROOT = Path('artifacts/gre24-v4-reproduction-2026-11-16') +PREFLIGHT_SHA = '4297c7c3358bcdd2cd7a164245f1de1ff13ab59c3ead7c8351fa5d172f8f0e51' + + +def write(path, value): + with path.open('x') as output: + json.dump(value, output, indent=2, allow_nan=False) + output.write('\n') + + +def read_setting(stream): + """Read process settings only, stopping at the filling-phase heading.""" + lines = [] + for index, raw in enumerate(stream, 1): + text = raw.decode('utf-8').rstrip('\r\n') + lines.append((index, text)) + if 'Filling Phase:' in text: + break + if index >= 10000: + raise ValueError('No bounded pre-solver settings section') + else: + raise ValueError('Filling Phase heading absent') + phase = lines[-1][0] + process = [(i, s) for i, s in lines if s.strip() == 'Process Settings:'] + control = [(i, s) for i, s in lines if s.strip() == 'Filling Control:'] + matches = [(i, s, re.fullmatch(r'\s*Fill time\s*=\s*([0-9.eE+\-]+)\s+s\s*', s)) for i, s in lines] + matches = [(i, s, m) for i, s, m in matches if m] + if len(matches) != 1 or not process or not control: + raise ValueError('Ambiguous configured fill time') + line, text, match = matches[0] + seconds = float(match.group(1)) + if not (process[-1][0] < control[-1][0] < line < phase and np.isfinite(seconds) and seconds > 0): + raise ValueError('Invalid pre-solver fill setting') + return dict(setting=dict(line=line, text=text, seconds=seconds, + section=dict(line=control[-1][0], text=control[-1][1]), + context_before=[dict(line=i,text=s) for i,s in lines[max(0,line-5):line-1]]), + filling_phase=dict(line=phase,text=lines[-1][1]),setting_before_filling_phase=True, + no_time_series_values_read=True) + + +def make_splits(cases, seeds): + if len(cases) != 629 or len(set(cases)) != 629 or cases != sorted(cases): + raise ValueError('Expected 629 sorted unique geometries') + splits = [] + for index, seed in enumerate(seeds): + order = np.random.Generator(np.random.PCG64(seed)).permutation(len(cases)) + test = sorted(cases[i] for i in order[:315]) + train = sorted(cases[i] for i in order[315:]) + vi = np.random.Generator(np.random.PCG64(seed+1000)).permutation(len(train))[:32] + val = sorted(train[i] for i in vi) + learning = sorted(set(train)-set(val)) + assert len(train)==314 and len(test)==315 and len(learning)==282 + assert not set(train)&set(test) and not set(learning)&set(val) + splits.append(dict(index=index,seed=seed,train=train,test=test,fnn_validation=val, + fnn_learning=learning,fnn_validation_seed=seed+1000, + validation_rounding='ceil(10percent training geometries)')) + return splits + + +def build(root=ROOT): + start = time.process_time() + preflight_path = root/'preflight-and-seed-freeze.json' + if sha(preflight_path) != PREFLIGHT_SHA: + raise ValueError('Preregistered seeds or pins changed') + pre = json.loads(preflight_path.read_text()) + for p,h in pre['source_pins'].items(): + if sha(p) != h: + raise ValueError('Pinned science changed: '+p) + verified_path = root/'data/archive-verification.json' + verified = json.loads(verified_path.read_text()) + archive = Path(verified['archive']) + stat = archive.stat() + if verified['status']!='VERIFIED' or verified['md5']!=pre['expected_md5'] or verified['bytes']!=pre['expected_bytes']: + raise ValueError('Verified official v4 archive required') + if (stat.st_size,stat.st_mtime_ns,stat.st_ino)!=(verified['bytes'],verified['mtime_ns'],verified['inode']): + raise ValueError('Verified archive identity changed') + original_prep_path = BASE/'v5-protocol/contract.json' + original_plan_path = BASE/'v5-models-continuation-2026-11-16/model-plan.json' + if sha(original_prep_path)!=pre['original_prep_contract_sha256'] or sha(original_plan_path)!=pre['original_model_plan_sha256']: + raise ValueError('Fixed v5 contract or model plan changed') + original = json.loads(original_prep_path.read_text()) + plan = json.loads(original_plan_path.read_text()) + dest = root/'protocol' + dest.mkdir(exist_ok=False) + rows = [] + with ZipFile(archive) as z: + members = z.namelist() + cases = sorted(n[:-4] for n in members if n.endswith('.yml')) + splits = make_splits(cases,pre['split_seeds']) + for case in cases: + if case+'.log' not in members or case+'_sim.vtu' not in members: + raise ValueError('Incomplete archive triple') + with z.open(case+'.log') as stream: + setting = read_setting(stream) + with z.open(case+'_sim.vtu') as stream: + for _, element in ET.iterparse(stream,events=('start',)): + if element.tag=='Piece': + nodes=int(element.attrib['NumberOfPoints']) + if nodes<=0:raise ValueError('Invalid node count') + break + else:raise ValueError('VTU Piece header absent') + rows.append(dict(case=case,raw_nodes=nodes,member=case+'.log',**setting)) + metadata = dest/'pre-solver-metadata.json' + write(metadata,rows) + parts = [dict(case=r['case'],raw_nodes=r['raw_nodes'],t_scale_seconds=r['setting']['seconds'], + time_scale_source=dict(mode='t_set_substitute_Q_unavailable_declared_deviation',setting=r['setting'], + pre_solver_setting=True,source=str(metadata),source_sha256=sha(metadata))) for r in rows] + write(dest/'splits.json',splits) + limits = copy.deepcopy(original['limits']) + limits['preparation_initial_cpu_reserve_seconds']=60.0 + contract = dict(version='Gre24-v4-fixed-science-v1',dataset_version='v4', + authorizing_rulings=pre['authorizing_rulings'],archive=str(archive.resolve()),archive_stat=verified, + archive_verification_sha256=sha(verified_path),parts=parts,dispatch_order=list(range(629)), + worker_cpus=original['worker_cpus'],coordinator_cpus=original['coordinator_cpus'],limits=limits, + code_sha256=original['code_sha256'],splits_sha256=sha(dest/'splits.json'), + protocol='Five independent 315test/314train random geometry splits; original v4 dataset; NOT five-fold CV', + preparation_setup_reserve_note='60 CPU-s allowance for new metadata/synthetic setup; old v5 costs not recharged; transfer intake recorded separately', + failure_policy=original['failure_policy'],data_domain=original['data_domain'],features=original['features'], + preregistration_sha256=PREFLIGHT_SHA,timestamp=time.time()) + plan.update(version='Gre24-v4-model-chain-v1',dataset_version='v4',protocol=contract['protocol'], + authorizing_rulings=pre['authorizing_rulings'],initial_gpu_seconds=0., + limits_notes='New v4 ledger: preparation plus setup reserve, fits and postfit share60workerCPUh; all FNN waits/setup/runs share2GPUh; no retry/replacement', + timestamp=time.time()) + write(dest/'contract.json',contract) + write(dest/'model-plan.json',plan) + cpu=time.process_time()-start + if cpu>55: + raise ValueError('Metadata setup exceeded reserved CPU allowance; no launch') + write(dest/'input-freeze.json',dict(status='FROZEN',case_count=len(parts),splits=5, + contract_sha256=sha(dest/'contract.json'),splits_sha256=sha(dest/'splits.json'), + model_plan_sha256=sha(dest/'model-plan.json'),metadata_cpu_seconds=cpu, + original_scientific_sources_unchanged=True,unix=time.time())) + return dest + + +if __name__=='__main__': + print(build()) diff --git a/tools/receive_gre24_v4.py b/tools/receive_gre24_v4.py index 304acf883..3925a629f 100644 --- a/tools/receive_gre24_v4.py +++ b/tools/receive_gre24_v4.py @@ -6,7 +6,7 @@ import json import os from pathlib import Path import time -from urllib.parse import urlsplit +from urllib.parse import urlsplit, unquote import requests EXPECTED_BYTES = 35925187752 @@ -28,7 +28,7 @@ def receive(root, private_url_file): raise RuntimeError('Receiver already attempted; inspect receipts before continuation') url = private_url_file.read_text().strip() parsed = urlsplit(url) - if parsed.scheme != 'https' or parsed.hostname != HOST or parsed.path != '/greif-v4/data.zip': + if parsed.scheme != 'https' or parsed.hostname != HOST or unquote(parsed.path) != '/greif-v4/data.zip': raise ValueError('Unexpected private object identity') target = root / 'data.zip' partial = root / 'data.zip.partial' diff --git a/tools/test_gre24_v4_continuation.py b/tools/test_gre24_v4_continuation.py new file mode 100644 index 000000000..1b96ef498 --- /dev/null +++ b/tools/test_gre24_v4_continuation.py @@ -0,0 +1,112 @@ +"""Synthetic input/dispatch checks; no scientific data or fits.""" +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +from zipfile import ZipFile +import make_gre24_v4_contract as builder +import continue_gre24_v4_after_receive as chain + +LOG=b'Process Settings:\n Filling Control:\n Fill time = 0.40 s\nFilling Phase: Status\nUNREAD OUTCOME\n' + + +class Checks(unittest.TestCase): + def test_setting_stops_before_outcomes(self): + stream=io.BytesIO(LOG) + setting=builder.read_setting(stream) + self.assertEqual(setting['setting']['seconds'],.4) + self.assertEqual(stream.read(),b'UNREAD OUTCOME\n') + + def test_invalid_settings_stop(self): + for log in [LOG.replace(b'0.40',b'0'),LOG.replace(b'0.40',b'-1'), + LOG.replace(b'Filling Phase:',b'No phase:'), + LOG.replace(b'Filling Phase:',b'Fill time = 1 s\nFilling Phase:')]: + with self.assertRaises(ValueError):builder.read_setting(io.BytesIO(log)) + + def test_split_counts_disjoint_and_deterministic(self): + cases=[f'{i:08d}' for i in range(629)] + seeds=list(range(2026111601,2026111606)) + first=builder.make_splits(cases,seeds) + self.assertEqual(first,builder.make_splits(cases,seeds)) + self.assertEqual(len(first),5) + for s in first: + self.assertEqual([len(s[k]) for k in ['test','train','fnn_validation','fnn_learning']],[315,314,32,282]) + self.assertFalse(set(s['test'])&set(s['train'])) + self.assertEqual(set(s['fnn_learning'])|set(s['fnn_validation']),set(s['train'])) + + def test_wrong_population_rejected(self): + with self.assertRaises(ValueError):builder.make_splits(['x']*629,[1]) + with self.assertRaises(ValueError):builder.make_splits([str(i) for i in range(628)],[1]) + + def test_synthetic_contract_preserves_fixed_science_and_model(self): + with tempfile.TemporaryDirectory(prefix='gre24-v4-contract-test-') as td: + root=Path(td);(root/'data').mkdir();archive=root/'data/data.zip' + with ZipFile(archive,'w') as z: + for i in range(629): + case=f'{i:08d}' + z.writestr(case+'.yml','synthetic: true\n') + z.writestr(case+'.log',LOG) + z.writestr(case+'_sim.vtu','') + st=archive.stat() + prep_path=builder.BASE/'v5-protocol/contract.json' + plan_path=builder.BASE/'v5-models-continuation-2026-11-16/model-plan.json' + pre=dict(source_pins={},original_prep_contract_sha256=builder.sha(prep_path), + original_model_plan_sha256=builder.sha(plan_path),expected_md5='synthetic',expected_bytes=st.st_size, + split_seeds=list(range(2026111601,2026111606)),authorizing_rulings=['synthetic']) + builder.write(root/'preflight-and-seed-freeze.json',pre) + builder.write(root/'data/archive-verification.json',dict(status='VERIFIED',archive=str(archive), + md5='synthetic',bytes=st.st_size,mtime_ns=st.st_mtime_ns,inode=st.st_ino)) + with patch.object(builder,'PREFLIGHT_SHA',builder.sha(root/'preflight-and-seed-freeze.json')): + dest=builder.build(root) + c=json.loads((dest/'contract.json').read_text());old=json.loads(prep_path.read_text()) + self.assertEqual(len(c['parts']),629) + self.assertEqual(c['code_sha256'],old['code_sha256']) + self.assertFalse(any('reuse_feature_root' in p for p in c['parts'])) + self.assertEqual(c['limits']['total_worker_cpu_seconds'],144000) + self.assertEqual(c['limits']['per_part_wall_seconds'],1800) + newplan=json.loads((dest/'model-plan.json').read_text());oldplan=json.loads(plan_path.read_text()) + changes={k for k in newplan if newplan[k]!=oldplan[k]} + self.assertLessEqual(changes,{'version','dataset_version','protocol','authorizing_rulings','initial_gpu_seconds','limits_notes','timestamp'}) + self.assertEqual(newplan['total_worker_cpu_seconds'],216000) + self.assertEqual(newplan['total_gpu_seconds'],7200) + self.assertEqual(newplan['initial_gpu_seconds'],0.) + + def prepare_root(self,root,status): + (root/'data').mkdir() + builder.write(root/'continuation-source-freeze.json',dict(code_sha256={})) + builder.write(root/'data/receive-started.json',dict(pid=1,unix=0)) + builder.write(root/'data/receive-complete.json',dict(status=status)) + + def test_unverified_receiver_stops_without_launch(self): + with tempfile.TemporaryDirectory() as td: + root=Path(td);self.prepare_root(root,'STOPPED') + with patch.object(chain,'build') as build,patch.object(chain,'launch') as launch: + self.assertEqual(chain.run(root),1) + build.assert_not_called();launch.assert_not_called() + + def test_changed_source_stops(self): + with tempfile.TemporaryDirectory() as td: + root=Path(td);self.prepare_root(root,'VERIFIED') + source=root/'source.py';source.write_text('new') + (root/'continuation-source-freeze.json').write_text(json.dumps(dict(code_sha256={str(source):'wrong'}))) + with self.assertRaises(ValueError):chain.run(root) + + def test_verified_launches_one_prep_and_one_waiter(self): + with tempfile.TemporaryDirectory() as td: + root=Path(td);self.prepare_root(root,'VERIFIED');dest=root/'protocol';dest.mkdir() + builder.write(dest/'input-freeze.json',dict(synthetic=True)) + builder.write(root/'data/archive-verification.json',dict(synthetic=True)) + calls=[] + def launch(unit,protocol,script,extra=(),preparation=False): + calls.append((unit,script,preparation)) + builder.write(dest/('queue-started.json' if preparation else 'model-chain-started.json'),dict(pid=123,synthetic=True)) + with patch.object(chain,'build',return_value=dest),patch.object(chain,'launch',side_effect=launch),patch.object(chain,'service_state',return_value=dict(MainPID='123')): + self.assertEqual(chain.run(root),0) + self.assertEqual(calls,[(chain.PREP_UNIT,'tools/run_gre24_protocol_preparation.py',True), + (chain.MODEL_UNIT,'tools/run_gre24_models.py',False)]) + with self.assertRaises(FileExistsError):chain.run(root) + + +if __name__=='__main__':unittest.main()