150 lines
7.8 KiB
Python
150 lines
7.8 KiB
Python
"""Bind verified v4 input and preregistered splits to unchanged v5 science."""
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import time
|
|
from xml.etree import ElementTree as ET
|
|
from zipfile import ZipFile
|
|
import numpy as np
|
|
from gre24_prepare_v2 import sha
|
|
|
|
BASE = Path('artifacts/gre24-reproduction-2026-11-15')
|
|
ROOT = Path('artifacts/gre24-v4-reproduction-2026-11-16')
|
|
PREFLIGHT_SHA = '4297c7c3358bcdd2cd7a164245f1de1ff13ab59c3ead7c8351fa5d172f8f0e51'
|
|
|
|
|
|
def write(path, value):
|
|
with path.open('x') as output:
|
|
json.dump(value, output, indent=2, allow_nan=False)
|
|
output.write('\n')
|
|
|
|
|
|
def read_setting(stream):
|
|
"""Read process settings only, stopping at the filling-phase heading."""
|
|
lines = []
|
|
for index, raw in enumerate(stream, 1):
|
|
text = raw.decode('utf-8').rstrip('\r\n')
|
|
lines.append((index, text))
|
|
if 'Filling Phase:' in text:
|
|
break
|
|
if index >= 10000:
|
|
raise ValueError('No bounded pre-solver settings section')
|
|
else:
|
|
raise ValueError('Filling Phase heading absent')
|
|
phase = lines[-1][0]
|
|
process = [(i, s) for i, s in lines if s.strip() == 'Process Settings:']
|
|
control = [(i, s) for i, s in lines if s.strip() == 'Filling Control:']
|
|
matches = [(i, s, re.fullmatch(r'\s*Fill time\s*=\s*([0-9.eE+\-]+)\s+s\s*', s)) for i, s in lines]
|
|
matches = [(i, s, m) for i, s, m in matches if m]
|
|
if len(matches) != 1 or not process or not control:
|
|
raise ValueError('Ambiguous configured fill time')
|
|
line, text, match = matches[0]
|
|
seconds = float(match.group(1))
|
|
if not (process[-1][0] < control[-1][0] < line < phase and np.isfinite(seconds) and seconds > 0):
|
|
raise ValueError('Invalid pre-solver fill setting')
|
|
return dict(setting=dict(line=line, text=text, seconds=seconds,
|
|
section=dict(line=control[-1][0], text=control[-1][1]),
|
|
context_before=[dict(line=i,text=s) for i,s in lines[max(0,line-5):line-1]]),
|
|
filling_phase=dict(line=phase,text=lines[-1][1]),setting_before_filling_phase=True,
|
|
no_time_series_values_read=True)
|
|
|
|
|
|
def make_splits(cases, seeds):
|
|
if len(cases) != 629 or len(set(cases)) != 629 or cases != sorted(cases):
|
|
raise ValueError('Expected 629 sorted unique geometries')
|
|
splits = []
|
|
for index, seed in enumerate(seeds):
|
|
order = np.random.Generator(np.random.PCG64(seed)).permutation(len(cases))
|
|
test = sorted(cases[i] for i in order[:315])
|
|
train = sorted(cases[i] for i in order[315:])
|
|
vi = np.random.Generator(np.random.PCG64(seed+1000)).permutation(len(train))[:32]
|
|
val = sorted(train[i] for i in vi)
|
|
learning = sorted(set(train)-set(val))
|
|
assert len(train)==314 and len(test)==315 and len(learning)==282
|
|
assert not set(train)&set(test) and not set(learning)&set(val)
|
|
splits.append(dict(index=index,seed=seed,train=train,test=test,fnn_validation=val,
|
|
fnn_learning=learning,fnn_validation_seed=seed+1000,
|
|
validation_rounding='ceil(10percent training geometries)'))
|
|
return splits
|
|
|
|
|
|
def build(root=ROOT):
|
|
start = time.process_time()
|
|
preflight_path = root/'preflight-and-seed-freeze.json'
|
|
if sha(preflight_path) != PREFLIGHT_SHA:
|
|
raise ValueError('Preregistered seeds or pins changed')
|
|
pre = json.loads(preflight_path.read_text())
|
|
for p,h in pre['source_pins'].items():
|
|
if sha(p) != h:
|
|
raise ValueError('Pinned science changed: '+p)
|
|
verified_path = root/'data/archive-verification.json'
|
|
verified = json.loads(verified_path.read_text())
|
|
archive = Path(verified['archive'])
|
|
stat = archive.stat()
|
|
if verified['status']!='VERIFIED' or verified['md5']!=pre['expected_md5'] or verified['bytes']!=pre['expected_bytes']:
|
|
raise ValueError('Verified official v4 archive required')
|
|
if (stat.st_size,stat.st_mtime_ns,stat.st_ino)!=(verified['bytes'],verified['mtime_ns'],verified['inode']):
|
|
raise ValueError('Verified archive identity changed')
|
|
original_prep_path = BASE/'v5-protocol/contract.json'
|
|
original_plan_path = BASE/'v5-models-continuation-2026-11-16/model-plan.json'
|
|
if sha(original_prep_path)!=pre['original_prep_contract_sha256'] or sha(original_plan_path)!=pre['original_model_plan_sha256']:
|
|
raise ValueError('Fixed v5 contract or model plan changed')
|
|
original = json.loads(original_prep_path.read_text())
|
|
plan = json.loads(original_plan_path.read_text())
|
|
dest = root/'protocol'
|
|
dest.mkdir(exist_ok=False)
|
|
rows = []
|
|
with ZipFile(archive) as z:
|
|
members = z.namelist()
|
|
cases = sorted(n[:-4] for n in members if n.endswith('.yml'))
|
|
splits = make_splits(cases,pre['split_seeds'])
|
|
for case in cases:
|
|
if case+'.log' not in members or case+'_sim.vtu' not in members:
|
|
raise ValueError('Incomplete archive triple')
|
|
with z.open(case+'.log') as stream:
|
|
setting = read_setting(stream)
|
|
with z.open(case+'_sim.vtu') as stream:
|
|
for _, element in ET.iterparse(stream,events=('start',)):
|
|
if element.tag=='Piece':
|
|
nodes=int(element.attrib['NumberOfPoints'])
|
|
if nodes<=0:raise ValueError('Invalid node count')
|
|
break
|
|
else:raise ValueError('VTU Piece header absent')
|
|
rows.append(dict(case=case,raw_nodes=nodes,member=case+'.log',**setting))
|
|
metadata = dest/'pre-solver-metadata.json'
|
|
write(metadata,rows)
|
|
parts = [dict(case=r['case'],raw_nodes=r['raw_nodes'],t_scale_seconds=r['setting']['seconds'],
|
|
time_scale_source=dict(mode='t_set_substitute_Q_unavailable_declared_deviation',setting=r['setting'],
|
|
pre_solver_setting=True,source=str(metadata),source_sha256=sha(metadata))) for r in rows]
|
|
write(dest/'splits.json',splits)
|
|
limits = copy.deepcopy(original['limits'])
|
|
limits['preparation_initial_cpu_reserve_seconds']=60.0
|
|
contract = dict(version='Gre24-v4-fixed-science-v1',dataset_version='v4',
|
|
authorizing_rulings=pre['authorizing_rulings'],archive=str(archive.resolve()),archive_stat=verified,
|
|
archive_verification_sha256=sha(verified_path),parts=parts,dispatch_order=list(range(629)),
|
|
worker_cpus=original['worker_cpus'],coordinator_cpus=original['coordinator_cpus'],limits=limits,
|
|
code_sha256=original['code_sha256'],splits_sha256=sha(dest/'splits.json'),
|
|
protocol='Five independent 315test/314train random geometry splits; original v4 dataset; NOT five-fold CV',
|
|
preparation_setup_reserve_note='60 CPU-s allowance for new metadata/synthetic setup; old v5 costs not recharged; transfer intake recorded separately',
|
|
failure_policy=original['failure_policy'],data_domain=original['data_domain'],features=original['features'],
|
|
preregistration_sha256=PREFLIGHT_SHA,timestamp=time.time())
|
|
plan.update(version='Gre24-v4-model-chain-v1',dataset_version='v4',protocol=contract['protocol'],
|
|
authorizing_rulings=pre['authorizing_rulings'],initial_gpu_seconds=0.,
|
|
limits_notes='New v4 ledger: preparation plus setup reserve, fits and postfit share60workerCPUh; all FNN waits/setup/runs share2GPUh; no retry/replacement',
|
|
timestamp=time.time())
|
|
write(dest/'contract.json',contract)
|
|
write(dest/'model-plan.json',plan)
|
|
cpu=time.process_time()-start
|
|
if cpu>55:
|
|
raise ValueError('Metadata setup exceeded reserved CPU allowance; no launch')
|
|
write(dest/'input-freeze.json',dict(status='FROZEN',case_count=len(parts),splits=5,
|
|
contract_sha256=sha(dest/'contract.json'),splits_sha256=sha(dest/'splits.json'),
|
|
model_plan_sha256=sha(dest/'model-plan.json'),metadata_cpu_seconds=cpu,
|
|
original_scientific_sources_unchanged=True,unix=time.time()))
|
|
return dest
|
|
|
|
|
|
if __name__=='__main__':
|
|
print(build())
|