Freeze v4 verified-input continuation with unchanged scientific workers
This commit is contained in:
1 parent
514235f5d8
commit
b01645c50a
8 files changed
+431
-2
No files matched your search
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"code_sha256": {
|
||||
"tools/gre24_features_v2.py": "7dcd5ac218ffdc1ca149edcd87e8658fb3c59e18c0fb9ce47985c7f1357d2cdf",
|
||||
"tools/gre24_prepare_v2.py": "7637e55009005f6a8c77114159d7ac0d3c024c7add3e320afc486927537b78bc",
|
||||
"tools/run_gre24_protocol_preparation.py": "74a95323ec43ed089e3768c6b0a5ebd4c369207476d81d03ab1f423d33753499",
|
||||
"tools/boundary_distance.py": "a45c1cbce92521a96eb814913b4f465b926c0b3883932ab0b7c37d11db113ed7",
|
||||
"tools/audit_samples.py": "a3daf70ec3bcb77bf46d9dc4887d552cf7c422117880a169969780012278cd70",
|
||||
"tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919",
|
||||
"tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e",
|
||||
"tools/run_gre24_models.py": "103ecb6dc8a40481f72da02a02e0a41069a9a747ad0b7af4418c6ed06f0dcbc9",
|
||||
"tools/gre24_models.py": "53b6fee8b700b9b982b06d29908ce2ce23e0c57512c46dbf74b2aade79e3ac3a",
|
||||
"tools/gre24_fit.py": "63af0305a2c22b524d95ffee50e972848decdd042b1d24f0b3b946d9152f7688",
|
||||
"tools/gre24_score.py": "dbb11b57b7b7b2178ac11fd7243c934adb899d666a4c717dfb6a51ee1c65c705",
|
||||
"tools/gre24_gpu_guard.py": "7c0fd523e917357c07b63983bb0af1653ece12b9f76d9e07dbbad8e7200c47b4",
|
||||
"tools/gre24_job_guard.py": "baff57c6748380ea739dde703b20fdd2c6958389da3d24dbf0df8742654052aa",
|
||||
"tools/make_gre24_v4_contract.py": "2303f801fb2ced6fe119149c03a8116b321970422abd34f27b4ff5e58faa4ed1",
|
||||
"tools/continue_gre24_v4_after_receive.py": "b1be8f5d020d0b7919c2ac98cb023a6e708952bf1042ab767b0a2ff7dc579925",
|
||||
"tools/receive_gre24_v4.py": "4e5b5ba676c9970dc0166ac93009b748dc44128a2b6f5c6179600f9ca5260729"
|
||||
},
|
||||
"preflight_sha256": "4297c7c3358bcdd2cd7a164245f1de1ff13ab59c3ead7c8351fa5d172f8f0e51",
|
||||
"acceptance_sha256": "455aec60e7fde24973608114ec84a9c8b49968ba9599a89b97e188b9955f0143",
|
||||
"synthetic_cpu_seconds": 1.949516,
|
||||
"setup_allowance_seconds": 60,
|
||||
"original_scientific_sources_unchanged": true,
|
||||
"unix": 1794804102.71015
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"status": "PASS",
|
||||
"tests": 8,
|
||||
"failures": 0,
|
||||
"errors": 0,
|
||||
"cpu_seconds": 1.9380359999999999,
|
||||
"scientific_data_read": false,
|
||||
"code_sha256": {
|
||||
"tools/make_gre24_v4_contract.py": "2303f801fb2ced6fe119149c03a8116b321970422abd34f27b4ff5e58faa4ed1",
|
||||
"tools/continue_gre24_v4_after_receive.py": "b1be8f5d020d0b7919c2ac98cb023a6e708952bf1042ab767b0a2ff7dc579925",
|
||||
"tools/test_gre24_v4_continuation.py": "c412707bf120454f0daa4998e4205b272636626ff86a85695c90a4e40c656d65"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"status": "PRIVATE_ENTRY_RECEIVED",
|
||||
"drive_file_id": "1I8SRSqY5d2WmUzc1Dz7GEqoATUbvL4Xn",
|
||||
"ruling": "1794802690.164829",
|
||||
"expiry": "2026-11-23 12:15:17 Asia/Shanghai",
|
||||
"source_key": "greif-v4/data.zip",
|
||||
"url_excluded": true,
|
||||
"encoded_slash_identity_check": "PASS",
|
||||
"receiver_sha256": "4e5b5ba676c9970dc0166ac93009b748dc44128a2b6f5c6179600f9ca5260729",
|
||||
"unix": 1794803849.6359196,
|
||||
"science_started": false
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"pid": 506357,
|
||||
"unix": 1794803859.112114,
|
||||
"source_host": "frank14f-private-hk.oss-cn-hongkong.aliyuncs.com",
|
||||
"source_key": "greif-v4/data.zip",
|
||||
"expected_bytes": 35925187752,
|
||||
"expected_md5": "4973f24e12a38c87c31cf129be1a3e3b",
|
||||
"private_url_not_recorded": true
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Wait for the sole verified receiver, then start unchanged bounded services once."""
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from gre24_prepare_v2 import sha
|
||||
from make_gre24_v4_contract import ROOT, build
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
PREP_UNIT='meltmind-gre24-v4-prep-20261116'
|
||||
MODEL_UNIT='meltmind-gre24-v4-models-20261116'
|
||||
|
||||
|
||||
def check_pins(root):
|
||||
for p,h in json.loads((root/'continuation-source-freeze.json').read_text())['code_sha256'].items():
|
||||
if sha(p)!=h:
|
||||
raise ValueError('Continuation source changed: '+p)
|
||||
|
||||
|
||||
def service_state(unit):
|
||||
result=subprocess.run(['systemctl','--user','show',unit+'.service','-p','MainPID','-p','ActiveState',
|
||||
'-p','SubState','-p','Restart'],capture_output=True,text=True)
|
||||
return dict(line.split('=',1) for line in result.stdout.splitlines() if '=' in line)
|
||||
|
||||
|
||||
def launch(unit, root, script, extra=(), preparation=False):
|
||||
state=service_state(unit)
|
||||
if state.get('ActiveState') in ('active','activating','deactivating') or int(state.get('MainPID',0)):
|
||||
raise RuntimeError('Existing service must not be duplicated: '+unit)
|
||||
command=['systemd-run','--user','--unit='+unit,'--working-directory='+str(Path.cwd()),
|
||||
'--property=Restart=no','--property=KillMode=control-group','--property=TimeoutStopSec=30s',
|
||||
'--property=StandardOutput=append:'+str(root.resolve()/(unit+'.log')),
|
||||
'--property=StandardError=inherit','--setenv=PYTHONUNBUFFERED=1',
|
||||
'--setenv=OMP_NUM_THREADS=1','--setenv=OPENBLAS_NUM_THREADS=1','--setenv=MKL_NUM_THREADS=1']
|
||||
if preparation:
|
||||
command.append('--setenv=CUDA_VISIBLE_DEVICES=')
|
||||
command += [sys.executable,'-u',script,str(root.resolve()),*map(str,extra)]
|
||||
result=subprocess.run(command,capture_output=True,text=True)
|
||||
save(root/(unit+'-launch.json'),dict(command=command,returncode=result.returncode,
|
||||
stdout=result.stdout,stderr=result.stderr,unix=time.time()))
|
||||
if result.returncode:
|
||||
raise RuntimeError('Service launch failed: '+unit)
|
||||
|
||||
|
||||
def run(root=ROOT):
|
||||
with (root/'continuation.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
if (root/'continuation-started.json').exists():
|
||||
raise FileExistsError('Never restart this continuation')
|
||||
check_pins(root)
|
||||
save(root/'continuation-started.json',dict(pid=os.getpid(),unix=time.time(),
|
||||
source_freeze_sha256=sha(root/'continuation-source-freeze.json')))
|
||||
try:
|
||||
entry=json.loads((root/'data/receive-started.json').read_text())
|
||||
complete=root/'data/receive-complete.json'
|
||||
absent=None
|
||||
while not complete.exists():
|
||||
try:
|
||||
p=psutil.Process(entry['pid'])
|
||||
alive=p.create_time()<=entry['unix'] and p.status()!=psutil.STATUS_ZOMBIE and any('receive_gre24_v4.py' in a for a in p.cmdline())
|
||||
except psutil.NoSuchProcess:
|
||||
alive=False
|
||||
absent=None if alive else (absent or time.monotonic())
|
||||
if absent and time.monotonic()-absent>30:
|
||||
raise RuntimeError('Receiver disappeared without completion; no restart')
|
||||
save(root/'continuation-progress.json',dict(stage='WAIT_EXISTING_RECEIVER',receiver_pid=entry['pid'],
|
||||
unix=time.time(),science_started=False))
|
||||
time.sleep(10)
|
||||
if json.loads(complete.read_text())['status']!='VERIFIED':
|
||||
raise RuntimeError('Receiver did not verify official v4; no science launch')
|
||||
check_pins(root)
|
||||
protocol=build(root)
|
||||
save(root/'continuation-progress.json',dict(stage='INPUTS_FROZEN',unix=time.time(),science_started=False,
|
||||
input_freeze_sha256=sha(protocol/'input-freeze.json')))
|
||||
launch(PREP_UNIT,protocol,'tools/run_gre24_protocol_preparation.py',preparation=True)
|
||||
deadline=time.monotonic()+30
|
||||
while not (protocol/'queue-started.json').exists():
|
||||
if time.monotonic()>deadline:
|
||||
raise RuntimeError('No preparation start receipt; no duplicate launch')
|
||||
time.sleep(.5)
|
||||
launch(MODEL_UNIT,protocol,'tools/run_gre24_models.py',[(protocol/'model-plan.json').resolve()])
|
||||
deadline=time.monotonic()+30
|
||||
while not (protocol/'model-chain-started.json').exists():
|
||||
if time.monotonic()>deadline:
|
||||
raise RuntimeError('No model waiter start receipt; no duplicate launch')
|
||||
time.sleep(.5)
|
||||
check_pins(root)
|
||||
audit=dict(status='SERVICES_STARTED',unix=time.time(),
|
||||
preparation=service_state(PREP_UNIT),model_chain=service_state(MODEL_UNIT),
|
||||
preparation_start=json.loads((protocol/'queue-started.json').read_text()),
|
||||
model_chain_start=json.loads((protocol/'model-chain-started.json').read_text()),
|
||||
input_freeze_sha256=sha(protocol/'input-freeze.json'),source_pins_verified=True,
|
||||
receiver_verification_sha256=sha(root/'data/archive-verification.json'))
|
||||
save(root/'actual-service-start-audit.json',audit)
|
||||
save(root/'continuation-complete.json',audit)
|
||||
return 0
|
||||
except Exception as error:
|
||||
save(root/'continuation-complete.json',dict(status='STOPPED',error_type=type(error).__name__,
|
||||
message=str(error),unix=time.time(),no_automatic_retry=True))
|
||||
return 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
raise SystemExit(run())
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Bind verified v4 input and preregistered splits to unchanged v5 science."""
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import time
|
||||
from xml.etree import ElementTree as ET
|
||||
from zipfile import ZipFile
|
||||
import numpy as np
|
||||
from gre24_prepare_v2 import sha
|
||||
|
||||
BASE = Path('artifacts/gre24-reproduction-2026-11-15')
|
||||
ROOT = Path('artifacts/gre24-v4-reproduction-2026-11-16')
|
||||
PREFLIGHT_SHA = '4297c7c3358bcdd2cd7a164245f1de1ff13ab59c3ead7c8351fa5d172f8f0e51'
|
||||
|
||||
|
||||
def write(path, value):
|
||||
with path.open('x') as output:
|
||||
json.dump(value, output, indent=2, allow_nan=False)
|
||||
output.write('\n')
|
||||
|
||||
|
||||
def read_setting(stream):
|
||||
"""Read process settings only, stopping at the filling-phase heading."""
|
||||
lines = []
|
||||
for index, raw in enumerate(stream, 1):
|
||||
text = raw.decode('utf-8').rstrip('\r\n')
|
||||
lines.append((index, text))
|
||||
if 'Filling Phase:' in text:
|
||||
break
|
||||
if index >= 10000:
|
||||
raise ValueError('No bounded pre-solver settings section')
|
||||
else:
|
||||
raise ValueError('Filling Phase heading absent')
|
||||
phase = lines[-1][0]
|
||||
process = [(i, s) for i, s in lines if s.strip() == 'Process Settings:']
|
||||
control = [(i, s) for i, s in lines if s.strip() == 'Filling Control:']
|
||||
matches = [(i, s, re.fullmatch(r'\s*Fill time\s*=\s*([0-9.eE+\-]+)\s+s\s*', s)) for i, s in lines]
|
||||
matches = [(i, s, m) for i, s, m in matches if m]
|
||||
if len(matches) != 1 or not process or not control:
|
||||
raise ValueError('Ambiguous configured fill time')
|
||||
line, text, match = matches[0]
|
||||
seconds = float(match.group(1))
|
||||
if not (process[-1][0] < control[-1][0] < line < phase and np.isfinite(seconds) and seconds > 0):
|
||||
raise ValueError('Invalid pre-solver fill setting')
|
||||
return dict(setting=dict(line=line, text=text, seconds=seconds,
|
||||
section=dict(line=control[-1][0], text=control[-1][1]),
|
||||
context_before=[dict(line=i,text=s) for i,s in lines[max(0,line-5):line-1]]),
|
||||
filling_phase=dict(line=phase,text=lines[-1][1]),setting_before_filling_phase=True,
|
||||
no_time_series_values_read=True)
|
||||
|
||||
|
||||
def make_splits(cases, seeds):
|
||||
if len(cases) != 629 or len(set(cases)) != 629 or cases != sorted(cases):
|
||||
raise ValueError('Expected 629 sorted unique geometries')
|
||||
splits = []
|
||||
for index, seed in enumerate(seeds):
|
||||
order = np.random.Generator(np.random.PCG64(seed)).permutation(len(cases))
|
||||
test = sorted(cases[i] for i in order[:315])
|
||||
train = sorted(cases[i] for i in order[315:])
|
||||
vi = np.random.Generator(np.random.PCG64(seed+1000)).permutation(len(train))[:32]
|
||||
val = sorted(train[i] for i in vi)
|
||||
learning = sorted(set(train)-set(val))
|
||||
assert len(train)==314 and len(test)==315 and len(learning)==282
|
||||
assert not set(train)&set(test) and not set(learning)&set(val)
|
||||
splits.append(dict(index=index,seed=seed,train=train,test=test,fnn_validation=val,
|
||||
fnn_learning=learning,fnn_validation_seed=seed+1000,
|
||||
validation_rounding='ceil(10percent training geometries)'))
|
||||
return splits
|
||||
|
||||
|
||||
def build(root=ROOT):
|
||||
start = time.process_time()
|
||||
preflight_path = root/'preflight-and-seed-freeze.json'
|
||||
if sha(preflight_path) != PREFLIGHT_SHA:
|
||||
raise ValueError('Preregistered seeds or pins changed')
|
||||
pre = json.loads(preflight_path.read_text())
|
||||
for p,h in pre['source_pins'].items():
|
||||
if sha(p) != h:
|
||||
raise ValueError('Pinned science changed: '+p)
|
||||
verified_path = root/'data/archive-verification.json'
|
||||
verified = json.loads(verified_path.read_text())
|
||||
archive = Path(verified['archive'])
|
||||
stat = archive.stat()
|
||||
if verified['status']!='VERIFIED' or verified['md5']!=pre['expected_md5'] or verified['bytes']!=pre['expected_bytes']:
|
||||
raise ValueError('Verified official v4 archive required')
|
||||
if (stat.st_size,stat.st_mtime_ns,stat.st_ino)!=(verified['bytes'],verified['mtime_ns'],verified['inode']):
|
||||
raise ValueError('Verified archive identity changed')
|
||||
original_prep_path = BASE/'v5-protocol/contract.json'
|
||||
original_plan_path = BASE/'v5-models-continuation-2026-11-16/model-plan.json'
|
||||
if sha(original_prep_path)!=pre['original_prep_contract_sha256'] or sha(original_plan_path)!=pre['original_model_plan_sha256']:
|
||||
raise ValueError('Fixed v5 contract or model plan changed')
|
||||
original = json.loads(original_prep_path.read_text())
|
||||
plan = json.loads(original_plan_path.read_text())
|
||||
dest = root/'protocol'
|
||||
dest.mkdir(exist_ok=False)
|
||||
rows = []
|
||||
with ZipFile(archive) as z:
|
||||
members = z.namelist()
|
||||
cases = sorted(n[:-4] for n in members if n.endswith('.yml'))
|
||||
splits = make_splits(cases,pre['split_seeds'])
|
||||
for case in cases:
|
||||
if case+'.log' not in members or case+'_sim.vtu' not in members:
|
||||
raise ValueError('Incomplete archive triple')
|
||||
with z.open(case+'.log') as stream:
|
||||
setting = read_setting(stream)
|
||||
with z.open(case+'_sim.vtu') as stream:
|
||||
for _, element in ET.iterparse(stream,events=('start',)):
|
||||
if element.tag=='Piece':
|
||||
nodes=int(element.attrib['NumberOfPoints'])
|
||||
if nodes<=0:raise ValueError('Invalid node count')
|
||||
break
|
||||
else:raise ValueError('VTU Piece header absent')
|
||||
rows.append(dict(case=case,raw_nodes=nodes,member=case+'.log',**setting))
|
||||
metadata = dest/'pre-solver-metadata.json'
|
||||
write(metadata,rows)
|
||||
parts = [dict(case=r['case'],raw_nodes=r['raw_nodes'],t_scale_seconds=r['setting']['seconds'],
|
||||
time_scale_source=dict(mode='t_set_substitute_Q_unavailable_declared_deviation',setting=r['setting'],
|
||||
pre_solver_setting=True,source=str(metadata),source_sha256=sha(metadata))) for r in rows]
|
||||
write(dest/'splits.json',splits)
|
||||
limits = copy.deepcopy(original['limits'])
|
||||
limits['preparation_initial_cpu_reserve_seconds']=60.0
|
||||
contract = dict(version='Gre24-v4-fixed-science-v1',dataset_version='v4',
|
||||
authorizing_rulings=pre['authorizing_rulings'],archive=str(archive.resolve()),archive_stat=verified,
|
||||
archive_verification_sha256=sha(verified_path),parts=parts,dispatch_order=list(range(629)),
|
||||
worker_cpus=original['worker_cpus'],coordinator_cpus=original['coordinator_cpus'],limits=limits,
|
||||
code_sha256=original['code_sha256'],splits_sha256=sha(dest/'splits.json'),
|
||||
protocol='Five independent 315test/314train random geometry splits; original v4 dataset; NOT five-fold CV',
|
||||
preparation_setup_reserve_note='60 CPU-s allowance for new metadata/synthetic setup; old v5 costs not recharged; transfer intake recorded separately',
|
||||
failure_policy=original['failure_policy'],data_domain=original['data_domain'],features=original['features'],
|
||||
preregistration_sha256=PREFLIGHT_SHA,timestamp=time.time())
|
||||
plan.update(version='Gre24-v4-model-chain-v1',dataset_version='v4',protocol=contract['protocol'],
|
||||
authorizing_rulings=pre['authorizing_rulings'],initial_gpu_seconds=0.,
|
||||
limits_notes='New v4 ledger: preparation plus setup reserve, fits and postfit share60workerCPUh; all FNN waits/setup/runs share2GPUh; no retry/replacement',
|
||||
timestamp=time.time())
|
||||
write(dest/'contract.json',contract)
|
||||
write(dest/'model-plan.json',plan)
|
||||
cpu=time.process_time()-start
|
||||
if cpu>55:
|
||||
raise ValueError('Metadata setup exceeded reserved CPU allowance; no launch')
|
||||
write(dest/'input-freeze.json',dict(status='FROZEN',case_count=len(parts),splits=5,
|
||||
contract_sha256=sha(dest/'contract.json'),splits_sha256=sha(dest/'splits.json'),
|
||||
model_plan_sha256=sha(dest/'model-plan.json'),metadata_cpu_seconds=cpu,
|
||||
original_scientific_sources_unchanged=True,unix=time.time()))
|
||||
return dest
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
print(build())
|
||||
@@ -6,7 +6,7 @@ import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
from urllib.parse import urlsplit, unquote
|
||||
import requests
|
||||
|
||||
EXPECTED_BYTES = 35925187752
|
||||
@@ -28,7 +28,7 @@ def receive(root, private_url_file):
|
||||
raise RuntimeError('Receiver already attempted; inspect receipts before continuation')
|
||||
url = private_url_file.read_text().strip()
|
||||
parsed = urlsplit(url)
|
||||
if parsed.scheme != 'https' or parsed.hostname != HOST or parsed.path != '/greif-v4/data.zip':
|
||||
if parsed.scheme != 'https' or parsed.hostname != HOST or unquote(parsed.path) != '/greif-v4/data.zip':
|
||||
raise ValueError('Unexpected private object identity')
|
||||
target = root / 'data.zip'
|
||||
partial = root / 'data.zip.partial'
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Synthetic input/dispatch checks; no scientific data or fits."""
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from zipfile import ZipFile
|
||||
import make_gre24_v4_contract as builder
|
||||
import continue_gre24_v4_after_receive as chain
|
||||
|
||||
LOG=b'Process Settings:\n Filling Control:\n Fill time = 0.40 s\nFilling Phase: Status\nUNREAD OUTCOME\n'
|
||||
|
||||
|
||||
class Checks(unittest.TestCase):
|
||||
def test_setting_stops_before_outcomes(self):
|
||||
stream=io.BytesIO(LOG)
|
||||
setting=builder.read_setting(stream)
|
||||
self.assertEqual(setting['setting']['seconds'],.4)
|
||||
self.assertEqual(stream.read(),b'UNREAD OUTCOME\n')
|
||||
|
||||
def test_invalid_settings_stop(self):
|
||||
for log in [LOG.replace(b'0.40',b'0'),LOG.replace(b'0.40',b'-1'),
|
||||
LOG.replace(b'Filling Phase:',b'No phase:'),
|
||||
LOG.replace(b'Filling Phase:',b'Fill time = 1 s\nFilling Phase:')]:
|
||||
with self.assertRaises(ValueError):builder.read_setting(io.BytesIO(log))
|
||||
|
||||
def test_split_counts_disjoint_and_deterministic(self):
|
||||
cases=[f'{i:08d}' for i in range(629)]
|
||||
seeds=list(range(2026111601,2026111606))
|
||||
first=builder.make_splits(cases,seeds)
|
||||
self.assertEqual(first,builder.make_splits(cases,seeds))
|
||||
self.assertEqual(len(first),5)
|
||||
for s in first:
|
||||
self.assertEqual([len(s[k]) for k in ['test','train','fnn_validation','fnn_learning']],[315,314,32,282])
|
||||
self.assertFalse(set(s['test'])&set(s['train']))
|
||||
self.assertEqual(set(s['fnn_learning'])|set(s['fnn_validation']),set(s['train']))
|
||||
|
||||
def test_wrong_population_rejected(self):
|
||||
with self.assertRaises(ValueError):builder.make_splits(['x']*629,[1])
|
||||
with self.assertRaises(ValueError):builder.make_splits([str(i) for i in range(628)],[1])
|
||||
|
||||
def test_synthetic_contract_preserves_fixed_science_and_model(self):
|
||||
with tempfile.TemporaryDirectory(prefix='gre24-v4-contract-test-') as td:
|
||||
root=Path(td);(root/'data').mkdir();archive=root/'data/data.zip'
|
||||
with ZipFile(archive,'w') as z:
|
||||
for i in range(629):
|
||||
case=f'{i:08d}'
|
||||
z.writestr(case+'.yml','synthetic: true\n')
|
||||
z.writestr(case+'.log',LOG)
|
||||
z.writestr(case+'_sim.vtu','<VTKFile><UnstructuredGrid><Piece NumberOfPoints="4"/></UnstructuredGrid></VTKFile>')
|
||||
st=archive.stat()
|
||||
prep_path=builder.BASE/'v5-protocol/contract.json'
|
||||
plan_path=builder.BASE/'v5-models-continuation-2026-11-16/model-plan.json'
|
||||
pre=dict(source_pins={},original_prep_contract_sha256=builder.sha(prep_path),
|
||||
original_model_plan_sha256=builder.sha(plan_path),expected_md5='synthetic',expected_bytes=st.st_size,
|
||||
split_seeds=list(range(2026111601,2026111606)),authorizing_rulings=['synthetic'])
|
||||
builder.write(root/'preflight-and-seed-freeze.json',pre)
|
||||
builder.write(root/'data/archive-verification.json',dict(status='VERIFIED',archive=str(archive),
|
||||
md5='synthetic',bytes=st.st_size,mtime_ns=st.st_mtime_ns,inode=st.st_ino))
|
||||
with patch.object(builder,'PREFLIGHT_SHA',builder.sha(root/'preflight-and-seed-freeze.json')):
|
||||
dest=builder.build(root)
|
||||
c=json.loads((dest/'contract.json').read_text());old=json.loads(prep_path.read_text())
|
||||
self.assertEqual(len(c['parts']),629)
|
||||
self.assertEqual(c['code_sha256'],old['code_sha256'])
|
||||
self.assertFalse(any('reuse_feature_root' in p for p in c['parts']))
|
||||
self.assertEqual(c['limits']['total_worker_cpu_seconds'],144000)
|
||||
self.assertEqual(c['limits']['per_part_wall_seconds'],1800)
|
||||
newplan=json.loads((dest/'model-plan.json').read_text());oldplan=json.loads(plan_path.read_text())
|
||||
changes={k for k in newplan if newplan[k]!=oldplan[k]}
|
||||
self.assertLessEqual(changes,{'version','dataset_version','protocol','authorizing_rulings','initial_gpu_seconds','limits_notes','timestamp'})
|
||||
self.assertEqual(newplan['total_worker_cpu_seconds'],216000)
|
||||
self.assertEqual(newplan['total_gpu_seconds'],7200)
|
||||
self.assertEqual(newplan['initial_gpu_seconds'],0.)
|
||||
|
||||
def prepare_root(self,root,status):
|
||||
(root/'data').mkdir()
|
||||
builder.write(root/'continuation-source-freeze.json',dict(code_sha256={}))
|
||||
builder.write(root/'data/receive-started.json',dict(pid=1,unix=0))
|
||||
builder.write(root/'data/receive-complete.json',dict(status=status))
|
||||
|
||||
def test_unverified_receiver_stops_without_launch(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
root=Path(td);self.prepare_root(root,'STOPPED')
|
||||
with patch.object(chain,'build') as build,patch.object(chain,'launch') as launch:
|
||||
self.assertEqual(chain.run(root),1)
|
||||
build.assert_not_called();launch.assert_not_called()
|
||||
|
||||
def test_changed_source_stops(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
root=Path(td);self.prepare_root(root,'VERIFIED')
|
||||
source=root/'source.py';source.write_text('new')
|
||||
(root/'continuation-source-freeze.json').write_text(json.dumps(dict(code_sha256={str(source):'wrong'})))
|
||||
with self.assertRaises(ValueError):chain.run(root)
|
||||
|
||||
def test_verified_launches_one_prep_and_one_waiter(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
root=Path(td);self.prepare_root(root,'VERIFIED');dest=root/'protocol';dest.mkdir()
|
||||
builder.write(dest/'input-freeze.json',dict(synthetic=True))
|
||||
builder.write(root/'data/archive-verification.json',dict(synthetic=True))
|
||||
calls=[]
|
||||
def launch(unit,protocol,script,extra=(),preparation=False):
|
||||
calls.append((unit,script,preparation))
|
||||
builder.write(dest/('queue-started.json' if preparation else 'model-chain-started.json'),dict(pid=123,synthetic=True))
|
||||
with patch.object(chain,'build',return_value=dest),patch.object(chain,'launch',side_effect=launch),patch.object(chain,'service_state',return_value=dict(MainPID='123')):
|
||||
self.assertEqual(chain.run(root),0)
|
||||
self.assertEqual(calls,[(chain.PREP_UNIT,'tools/run_gre24_protocol_preparation.py',True),
|
||||
(chain.MODEL_UNIT,'tools/run_gre24_models.py',False)])
|
||||
with self.assertRaises(FileExistsError):chain.run(root)
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in new issue
Block a user