Freeze Gre24 model chain and retain synthetic acceptance evidence

This commit is contained in:
Frank14f committed 2026-11-15 23:54:50 +08:00
1 parent 8409910b1e
commit eb9ed0c762
124 files changed
+5993

No files matched your search

+139
View File
@@ -0,0 +1,139 @@
"""One fixed split/model fit; reads training-side targets only, no test scores."""
import argparse
import json
import os
from pathlib import Path
import time
import traceback
import numpy as np
from gre24_prepare_v2 import sha
from gre24_models import xgb_model,fnn_model,eligible_roles,require_training_role,early_stop_update
from s2_recovery_supervisor import save
def contract(root):
c=json.loads((root/'training-contract.json').read_text())
for p,h in c['code_sha256'].items():
if sha(p)!=h:raise ValueError('Pinned training source changed: '+p)
if sha(root/'preparation-freeze.json')!=c['preparation_freeze_sha256']:
raise ValueError('Preparation freeze changed')
if sha(root/'splits.json')!=c['splits_sha256']:raise ValueError('Splits changed')
return c,json.loads((root/'preparation-freeze.json').read_text()),json.loads((root/'splits.json').read_text())
def training_arrays(root,freeze,split,role,read_log):
selected=[case for case in split[role] if case in freeze['successful']]
n=sum(freeze['successful'][case]['nodes'] for case in selected)
if not n:raise ValueError('No successful training rows')
x=np.empty((n,19),dtype=np.float32);y=np.empty(n,dtype=np.float32);cursor=0
for case in selected:
require_training_role(case,split,role)
r=freeze['successful'][case];source=Path(r['output'])
for name in ['features.npy','target.npy']:
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Frozen preparation changed')
a=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
b=np.load(source/'target.npy',mmap_mode='r',allow_pickle=False)
count=len(b)
if a.shape!=(count,19) or count!=r['nodes']:raise ValueError('Training row identity mismatch')
x[cursor:cursor+count]=a;y[cursor:cursor+count]=b;cursor+=count
read_log.append(dict(case=case,role=role,nodes=count,source=str(source),
feature_sha256=r['output_sha256']['features.npy'],target_sha256=r['output_sha256']['target.npy']))
if not np.isfinite(x).all() or not np.isfinite(y).all():raise ValueError('Nonfinite training input')
return x,y
def fit(root,index,model_name):
c,f,splits=contract(root);split=splits[index]
dest=root/'fits'/f'split-{index}'/model_name;dest.mkdir(parents=True,exist_ok=True)
if (dest/'started.json').exists():raise FileExistsError('No fit restart or overwrite')
start=time.monotonic();cpu=time.process_time();read_log=[]
def progress(stage,**kw):
record=dict(stage=stage,pid=os.getpid(),unix=time.time(),split=index,model=model_name,
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start)
record.update(kw);save(dest/'progress.json',record)
save(dest/'started.json',dict(pid=os.getpid(),unix=time.time(),split=index,model=model_name,seed=split['seed'],
training_contract_sha256=sha(root/'training-contract.json')))
roles,missing=eligible_roles(split,f['successful'])
try:
if model_name=='GBRT':
progress('LOAD_TRAIN_ONLY')
x,y=training_arrays(root,f,split,'train',read_log)
save(dest/'training-target-access.json',read_log)
model=xgb_model(split['seed'],c['xgb_threads'])
progress('FIT',nodes=len(y),geometries=len(roles['train']))
model.fit(x,y,verbose=False)
checkpoint=dest/'model.ubj';model.save_model(checkpoint)
(dest/'xgboost-config.json').write_text(model.get_booster().save_config())
params=model.get_params()
# Preserve XGBoost's default missing=NaN as an explicit textual parameter;
# this is not a nonfinite observation or a change to the fitted estimator.
params={k:('NaN (library default missing marker)' if isinstance(v,float) and np.isnan(v) else v)
for k,v in params.items()}
extra=dict(train_nodes=len(y),train_geometries=len(roles['train']),parameters=params)
elif model_name=='FNN':
import torch
torch.set_num_threads(c['fnn_cpu_threads']);torch.manual_seed(split['seed'])
if not torch.cuda.is_available():raise RuntimeError('Selected CUDA GPU unavailable')
torch.cuda.set_device(0) # Initialise the selected device before resetting its allocator counters.
device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device)
torch.backends.cuda.matmul.allow_tf32=False
progress('LOAD_FNN_TRAIN_VALIDATION_ONLY')
x,y=training_arrays(root,f,split,'fnn_learning',read_log)
vx,vy=training_arrays(root,f,split,'fnn_validation',read_log)
save(dest/'training-target-access.json',read_log)
x=torch.as_tensor(x,device=device);y=torch.as_tensor(y,device=device)
vx=torch.as_tensor(vx,device=device);vy=torch.as_tensor(vy,device=device)
model=fnn_model().to(device);optimizer=torch.optim.Adam(model.parameters(),lr=.001)
best=float('inf');stale=0;best_epoch=None;history=[];checkpoint=dest/'model.pt'
for epoch in range(1,31):
model.train();order=torch.randperm(len(y),device=device);total=torch.zeros((),device=device)
for begin in range(0,len(y),1024):
ids=order[begin:begin+1024];optimizer.zero_grad(set_to_none=True)
prediction=model(x[ids]).squeeze(-1);loss=torch.mean((prediction-y[ids])**2)
if not torch.isfinite(loss):raise ValueError('Nonfinite training loss')
loss.backward();optimizer.step();total+=loss.detach()*len(ids)
model.eval();sse=torch.zeros((),device=device)
with torch.no_grad():
for begin in range(0,len(vy),65536):
sse+=torch.sum((model(vx[begin:begin+65536]).squeeze(-1)-vy[begin:begin+65536])**2)
validation=float((sse/len(vy)).item());train=float((total/len(y)).item())
best,stale,improved,stop=early_stop_update(best,stale,validation)
if improved:
torch.save({'state_dict':{k:v.detach().cpu() for k,v in model.state_dict().items()},
'epoch':epoch,'validation_mse':validation,'seed':split['seed']},checkpoint)
best_epoch=epoch
row=dict(epoch=epoch,training_mse=train,validation_mse=validation,best_epoch=best_epoch,
stale_epochs=stale,peak_allocated_bytes=torch.cuda.max_memory_allocated(device),
peak_reserved_bytes=torch.cuda.max_memory_reserved(device),wall_s=time.monotonic()-start)
history.append(row);save(dest/'history.json',history);progress('EPOCH_COMPLETE',**row)
if row['peak_reserved_bytes']>c['gpu_reserved_byte_cap']:raise RuntimeError('Recorded VRAM peak exceeds cap')
if stop:break
restored=torch.load(checkpoint,map_location='cpu',weights_only=True)
model.load_state_dict(restored['state_dict'])
extra=dict(epochs=len(history),best_epoch=best_epoch,best_validation_mse=best,
train_nodes=len(y),validation_nodes=len(vy),train_geometries=len(roles['fnn_learning']),
validation_geometries=len(roles['fnn_validation']),restored_best=True,amp=False,
input_standardization='none; use eq12 columns',peak_allocated_bytes=torch.cuda.max_memory_allocated(device),
peak_reserved_bytes=torch.cuda.max_memory_reserved(device))
else:raise ValueError('Unknown model')
save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name,
checkpoint=str(checkpoint),checkpoint_sha256=sha(checkpoint),missing_preparation=missing,
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,**extra))
progress('FIT_COMPLETE_NO_TEST_SCORES');return 0
except Exception as exc:
peaks={}
if model_name=='FNN':
try:
import torch
if torch.cuda.is_initialized():
peaks=dict(peak_allocated_bytes=torch.cuda.max_memory_allocated(),
peak_reserved_bytes=torch.cuda.max_memory_reserved())
except Exception as peak_error:peaks={'peak_read_error':str(peak_error)}
save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,automatic_retry=False,**peaks))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('split',type=int);p.add_argument('model',choices=['GBRT','FNN'])
a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model))
+82
View File
@@ -0,0 +1,82 @@
"""Single-GPU wall accounting and recorded/device VRAM checks including failure."""
import hashlib
import json
import math
from pathlib import Path
import subprocess
import threading
import time
import psutil
from gre24_job_guard import supervise,stop_group
def gpu_state(uuid=None):
args=['nvidia-smi','--query-gpu=uuid,memory.used,utilization.gpu','--format=csv,noheader,nounits']
if uuid:args+=['-i',uuid]
out=subprocess.run(args,text=True,capture_output=True,check=True,timeout=10).stdout
rows=[]
for line in out.splitlines():
u,mem,util=[v.strip() for v in line.split(',')]
rows.append(dict(uuid=u,used_bytes=int(mem)*2**20,utilization=int(util)))
if not rows:raise RuntimeError('No GPU state returned')
return rows
def recorded_peaks(root):
peaks={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
def walk(value,path,digest):
if isinstance(value,dict):
for k,v in value.items():
if k in peaks:
if isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or v<0:
raise ValueError('Invalid CUDA peak record')
if v>=peaks[k]:peaks[k]=v;witnesses[k]={'path':str(path),'sha256':digest}
else:walk(v,path,digest)
elif isinstance(value,list):
for v in value:walk(v,path,digest)
for path in sorted(Path(root).glob('*.json')):
raw=path.read_bytes();walk(json.loads(raw),path,hashlib.sha256(raw).hexdigest())
return peaks,witnesses
def gpu_fit(command,guard_root,fit_root,uuid,wall_limit,cpu_limit,byte_cap):
start=time.monotonic();done=threading.Event();monitor={'reasons':[],'peak_device_bytes':0,'samples':0}
maxima={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
def scan():
current,seen=recorded_peaks(fit_root)
for k,v in current.items():
if v>=maxima[k] and k in seen:maxima[k]=v;witnesses[k]=seen[k]
if max(maxima.values())>byte_cap and 'RECORDED_VRAM_LIMIT' not in monitor['reasons']:
monitor['reasons'].append('RECORDED_VRAM_LIMIT')
def watch():
while not done.is_set():
try:
scan();state=gpu_state(uuid)[0];monitor['samples']+=1
monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],state['used_bytes'])
if state['used_bytes']>byte_cap and 'DEVICE_VRAM_LIMIT' not in monitor['reasons']:
monitor['reasons'].append('DEVICE_VRAM_LIMIT')
except Exception as exc:monitor['reasons'].append('GPU_MONITOR_FAILURE:'+str(exc))
if monitor['reasons']:
for p in psutil.Process().children(recursive=True):
try:
if p.cmdline()==command:stop_group(p.pid)
except psutil.NoSuchProcess:pass
done.wait(.5)
thread=threading.Thread(target=watch,daemon=True);thread.start()
try:
receipt=supervise(command,guard_root,'worker',dict(total_cpu_seconds=cpu_limit,
job_cpu_seconds=cpu_limit,job_wall_seconds=max(.1,wall_limit-3),tree_rss_bytes=24*2**30,
disk_bytes=3*2**30,max_processes=1),poll=.1,cuda_device=uuid,numerical_threads=2)
finally:
done.set();thread.join(timeout=12)
if thread.is_alive():monitor['reasons'].append('GPU_MONITOR_DID_NOT_STOP')
try:scan()
except Exception as exc:monitor['reasons'].append('FINAL_MEMORY_RECORD_READ_FAILED:'+str(exc))
if receipt['status']=='PASS' and len(witnesses)!=2:monitor['reasons'].append('MISSING_FINAL_PEAKS')
elapsed=time.monotonic()-start
if elapsed>wall_limit:monitor['reasons'].append('TOTAL_GPU_WALL_LIMIT')
receipt['underlying_worker_status']=receipt['status']
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
receipt.update(device_monitor=monitor,recorded_peaks=maxima,peak_witnesses=witnesses,
total_gpu_wall_seconds=elapsed,peak_cap_bytes=byte_cap)
return receipt
+160
View File
@@ -0,0 +1,160 @@
"""Sequential E1 job guard: tree CPU/RSS, wall, process count and disk receipts.
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
"""
import fcntl
import ctypes
import json
import math
import os
from pathlib import Path
import signal
import subprocess
import time
import psutil
from s2_recovery_monitoring import disk_usage,Monitor
def save(path, data):
path = Path(path)
tmp = path.with_suffix(path.suffix + '.tmp')
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
tmp.replace(path)
def tree_usage(pid):
try:
root = psutil.Process(pid)
processes = [root] + root.children(recursive=True)
except psutil.NoSuchProcess:
return 0., 0, 0
cpu = 0.
rss = count = 0
for p in processes:
try:
t = p.cpu_times()
# Linux child times include reaped descendants, not live children.
cpu += t.user + t.system + t.children_user + t.children_system
rss += p.memory_info().rss
count += 1
except psutil.NoSuchProcess:
pass
return cpu, rss, count
def stop_group(pid):
try:
os.killpg(pid, signal.SIGKILL)
except ProcessLookupError:
pass
def supervise(command, root, name, limits, *, poll=.02, cuda_device='', numerical_threads=1):
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
root = Path(root)
root.mkdir(parents=True, exist_ok=True)
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
with (root/'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
started_path = root/(name+'.started.json')
receipt_path = root/(name+'.receipt.json')
if started_path.exists() or receipt_path.exists():
raise FileExistsError('Attempt already exists; no silent retry')
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
raise ValueError('Every limit must be finite and positive')
ledger_path = root/'ledger.json'
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
raise RuntimeError('Total CPU exhausted before spawn')
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
raise RuntimeError('Disk exhausted before spawn')
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
start = last = time.monotonic()
cpu_peak = rss_peak = max_processes = max_gap = 0
cpu=rss=count=used_disk=0
reason = None
exit_status = usage = None
env = dict(os.environ, OMP_NUM_THREADS=str(numerical_threads), OPENBLAS_NUM_THREADS=str(numerical_threads),
MKL_NUM_THREADS=str(numerical_threads), NUMEXPR_NUM_THREADS=str(numerical_threads),
CUDA_VISIBLE_DEVICES=cuda_device)
with (root/(name+'.log')).open('x') as log:
libc = ctypes.CDLL(None, use_errno=True)
previous_subreaper = ctypes.c_int()
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
env=env, start_new_session=True)
try:
while True:
now = time.monotonic()
max_gap = max(max_gap, now-last)
last = now
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
cpu_peak = max(cpu_peak,cpu)
rss_peak = max(rss_peak,rss)
max_processes = max(max_processes,count)
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
if now-start >= limits['job_wall_seconds']:
reason = 'wall_limit'
elif rss > limits['tree_rss_bytes']:
reason = 'tree_rss_limit'
elif count > limits['max_processes']:
reason = 'process_count_limit'
elif used_disk >= limits['disk_bytes']:
reason = 'disk_limit'
elif cpu_peak >= limits['job_cpu_seconds']:
reason = 'job_cpu_limit'
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
reason = 'total_cpu_limit'
if reason:
stop_group(proc.pid)
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
if pid:
exit_status, usage = status, child_usage
break
time.sleep(poll)
finally:
# No detached work may outlive a supervised attempt.
stop_group(proc.pid)
if exit_status is None:
_, exit_status, usage = os.wait4(proc.pid, 0)
proc.returncode = os.waitstatus_to_exitcode(exit_status)
orphan_cpu = 0.
orphan_peak = 0
# Killed descendants are adopted here rather than losing their
# final CPU usage when the root worker cannot reap them itself.
while True:
try:
_, _, orphan_usage = os.wait4(-proc.pid, 0)
except ChildProcessError:
break
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
elapsed = time.monotonic()-start
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
if reason is None:
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
command=command,returncode=proc.returncode,stop_reason=reason,
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
reaped_orphan_cpu_seconds=orphan_cpu,
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
gpu_visible=bool(cuda_device),cuda_device=cuda_device,
numerical_threads=numerical_threads,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error)
ledger['cpu_seconds_charged'] += charged
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll)
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
return receipt
+42
View File
@@ -0,0 +1,42 @@
"""Fixed Gre24 model choices; no test-driven selection or node subsampling."""
import numpy as np
def xgb_model(seed,threads=8):
from xgboost import XGBRegressor
return XGBRegressor(n_estimators=100,max_depth=8,learning_rate=.1,
objective='reg:squarederror',random_state=seed,n_jobs=threads)
def fnn_model():
import torch
layers=[];width=19
for _ in range(4):
layers.extend([torch.nn.Linear(width,100),torch.nn.ReLU()]);width=100
layers.append(torch.nn.Linear(100,1))
return torch.nn.Sequential(*layers)
def eligible_roles(split,successful):
successful=set(successful)
roles={role:[c for c in split[role] if c in successful]
for role in ['train','test','fnn_learning','fnn_validation']}
if set(split['train'])&set(split['test']):raise ValueError('Train/test overlap')
if set(split['fnn_learning'])&set(split['fnn_validation']):raise ValueError('FNN role overlap')
if set(split['fnn_learning'])|set(split['fnn_validation'])!=set(split['train']):
raise ValueError('FNN learning/validation do not partition train')
missing={role:[c for c in split[role] if c not in successful] for role in roles}
return roles,missing
def require_training_role(case,split,role):
if role not in ['train','fnn_learning','fnn_validation']:
raise PermissionError('Only frozen training-side roles may read targets')
if case in split['test'] or case not in split[role]:
raise PermissionError('Case is not in the requested training role')
def early_stop_update(best,stale,score,patience=5):
if not np.isfinite(score):raise ValueError('Nonfinite validation MSE')
improved=score<best
return (score if improved else best),(0 if improved else stale+1),improved,(not improved and stale+1>=patience)
+104
View File
@@ -0,0 +1,104 @@
"""Predictions frozen before test-target scoring, all successful tet nodes."""
import argparse
import json
from pathlib import Path
import time
import numpy as np
from gre24_prepare_v2 import sha
from gre24_features_v2 import per_geometry_metrics
from gre24_models import fnn_model
from gre24_fit import contract
from s2_recovery_supervisor import save
def predict(root):
c,prep,splits=contract(root)
models=json.loads((root/'models-freeze.json').read_text())
dest=root/'predictions';dest.mkdir(exist_ok=False)
start=time.process_time();rows=[]
for split in splits:
available={'naive':None}
for name in ['GBRT','FNN']:
r=models['fits'][f"{split['index']}/{name}"]
if r['status']!='PASS':continue
if sha(r['checkpoint'])!=r['checkpoint_sha256']:raise ValueError('Checkpoint changed after freeze')
if name=='GBRT':
from xgboost import XGBRegressor
m=XGBRegressor(n_jobs=4);m.load_model(r['checkpoint'])
else:
import torch
torch.set_num_threads(4);m=fnn_model()
m.load_state_dict(torch.load(r['checkpoint'],map_location='cpu',weights_only=True)['state_dict']);m.eval()
available[name]=m
for case in split['test']:
if case not in prep['successful']:
rows.append(dict(split=split['index'],case=case,status='PREPARATION_FAILED',predictions={}))
continue
r=prep['successful'][case];source=Path(r['output']);outputs={}
for name in ['features.npy','Vf.npy','raw_ids.npy']:
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Prediction geometry/feature changed')
x=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
# No seconds/target file is opened in this phase.
for name,m in available.items():
if name=='naive':pred=np.load(source/'Vf.npy',allow_pickle=False)
elif name=='GBRT':pred=m.predict(np.asarray(x,dtype=np.float32))
else:
chunks=[]
with torch.no_grad():
for a in range(0,len(x),65536):
chunks.append(m(torch.from_numpy(np.array(x[a:a+65536],dtype=np.float32))).squeeze(-1).numpy())
pred=np.concatenate(chunks)
if pred.shape!=(r['nodes'],) or not np.isfinite(pred).all():raise ValueError('Invalid full-node prediction')
folder=dest/f"split-{split['index']}"/name;folder.mkdir(parents=True,exist_ok=True)
path=folder/(case+'.npy');np.save(path,np.asarray(pred,dtype=np.float64)*r['t_scale_seconds'])
outputs[name]=dict(path=str(path),sha256=sha(path),nodes=len(pred),no_clipping=True)
rows.append(dict(split=split['index'],case=case,status='PREDICTED',predictions=outputs,
raw_ids_sha256=r['output_sha256']['raw_ids.npy']))
save(root/'prediction-progress.json',dict(test_case_visits=len(rows),worker_CPU_s=time.process_time()-start))
save(root/'predictions-freeze.json',dict(models_freeze_sha256=sha(root/'models-freeze.json'),rows=rows,
worker_CPU_s=time.process_time()-start,test_targets_read=False,device='CPU',prediction_units='seconds'))
def score(root):
c,prep,splits=contract(root);frozen=root/'predictions-freeze.json'
if not frozen.exists():raise ValueError('Predictions must all be frozen before scoring')
freeze=json.loads(frozen.read_text());rows=[];start=time.process_time()
for r in freeze['rows']:
if r['status']!='PREDICTED':
rows.append(dict(split=r['split'],case=r['case'],status=r['status'],scores={}));continue
p=prep['successful'][r['case']];source=Path(p['output'])
if sha(source/'seconds.npy')!=p['output_sha256']['seconds.npy']:raise ValueError('Frozen test targets changed')
y=np.load(source/'seconds.npy',allow_pickle=False);scores={}
for name,value in r['predictions'].items():
if sha(value['path'])!=value['sha256']:raise ValueError('Frozen predictions changed')
pred=np.load(value['path'],allow_pickle=False);scores[name]=per_geometry_metrics(y,pred)
rows.append(dict(split=r['split'],case=r['case'],status='SCORED',scores=scores))
summary={}
for name in ['naive','GBRT','FNN']:
summary[name]={}
for metric in ['rrmse_rms','rrmse_maximum','rrmse_mean']:
per=[]
for split in splits:
a=[r['scores'][name][metric] for r in rows if r['split']==split['index'] and name in r['scores']
and r['scores'][name][metric] is not None]
per.append(dict(split=split['index'],requested_test_geometries=len(split['test']),scored=len(a),
missing=len(split['test'])-len(a),min=float(np.min(a)) if a else None,
median=float(np.median(a)) if a else None,max=float(np.max(a)) if a else None))
# Do not average only whichever model splits happened to finish.
avg={k:float(np.mean([r[k] for r in per])) if all(r[k] is not None for r in per) else None for k in ['min','median','max']}
summary[name][metric]=dict(per_split=per,mean_of_five_split_summaries=avg,
full_requested_coverage=all(r['missing']==0 for r in per))
save(root/'scores.json',dict(rows=rows,predictions_freeze_sha256=sha(frozen)))
save(root/'score-summary.json',dict(dataset_version=c['dataset_version'],protocol=c['protocol'],
status='SCORED_WITH_EXPLICIT_COVERAGE',summary=summary,worker_CPU_s=time.process_time()-start,
node_domain='All retained tetrahedral nodes including walls, not S2 queries; auxiliary nodes separately reported',
outside_tet_node_counts={case:r.get('outside_tet_nodes',0) for case,r in prep['successful'].items() if r.get('outside_tet_nodes',0)},
scores_sha256=sha(root/'scores.json'),paper_numbers_diagnostic_only=True,
limitations=['v5 data instead of paper v4','R1 discretisation choices are explicit supplements',
't_set substitutes unavailable applied Q','FNN fill-only and early stopping are declared choices',
'Random geometry splits may leak near-duplicate geometry families','Failed preparations not replaced; coverage is explicit']))
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score'])
a=p.parse_args();(predict if a.phase=='predict' else score)(a.root)
+142
View File
@@ -0,0 +1,142 @@
"""One sequential fixed-model chain after terminal preparation, within shared caps."""
import argparse
import fcntl
import json
import os
from pathlib import Path
import sys
import time
import psutil
from gre24_prepare_v2 import sha
from gre24_job_guard import supervise,save
from gre24_gpu_guard import gpu_state,gpu_fit
def freeze_preparation(root):
complete=json.loads((root/'complete.json').read_text());c=json.loads((root/'contract.json').read_text())
if complete['status']!='COMPLETE' or not complete['accounting_complete'] or len(complete['records'])!=len(c['parts']):
raise ValueError('Preparation globally stopped or missing terminal accounting')
good={};fail=[]
for r in complete['records']:
if r['worker']['status']!='PASS':
fail.append(dict(case=r['case'],guard_status=r['worker']['status'],reason=r['worker']['stop_reason']));continue
p=Path(r['result_path'])
if sha(p)!=r['result_sha256']:raise ValueError('Preparation result changed')
result=json.loads(p.read_text())
good[r['case']]=dict(output=str(p.parent),nodes=result['geometry']['tet_nodes'],
raw_nodes=result['geometry'].get('raw_nodes',result['geometry']['tet_nodes']),
t_scale_seconds=result['t_scale_seconds'],output_sha256=result['output_sha256'],result_sha256=sha(p),
outside_tet_nodes=result['geometry']['outside_tet_nodes'])
frozen=dict(successful=good,failed=fail,requested=len(c['parts']),
preparation_complete_sha256=sha(root/'complete.json'),replacement=False,
cumulative_initial_worker_cpu_seconds=complete['worker_cpu_seconds']+complete['initial_cpu_reserve_seconds'])
save(root/'preparation-freeze.json',frozen);return frozen
def run(root,plan_path):
root=Path(root);plan_path=Path(plan_path)
with (root/'model-chain.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
return run_locked(root,plan_path)
def run_locked(root,plan_path):
plan=json.loads(plan_path.read_text())
for p,h in plan['code_sha256'].items():
if sha(p)!=h:raise ValueError('Model plan source changed: '+p)
started=root/'model-chain-started.json'
if started.exists():raise FileExistsError('Never restart model chain')
save(started,dict(pid=os.getpid(),unix=time.time(),plan_sha256=sha(plan_path),stage='WAIT_PREPARATION'))
missing_since=None
while not (root/'complete.json').exists():
parent=json.loads((root/'queue-started.json').read_text())
try:
process=psutil.Process(parent['pid'])
alive=process.create_time()<=parent['unix'] and process.status()!=psutil.STATUS_ZOMBIE and any('run_gre24_protocol_preparation.py' in a for a in process.cmdline())
except psutil.NoSuchProcess:alive=False
if not alive:
missing_since=missing_since or time.monotonic()
if time.monotonic()-missing_since>30:raise RuntimeError('Preparation exited without complete receipt; no restart')
else:missing_since=None
save(root/'model-chain-progress.json',dict(stage='WAIT_PREPARATION',unix=time.time()))
time.sleep(10)
frozen=freeze_preparation(root)
split_path=root/'splits.json';splits=json.loads(split_path.read_text())
c=dict(plan,preparation_freeze_sha256=sha(root/'preparation-freeze.json'),splits_sha256=sha(split_path))
save(root/'training-contract.json',c)
spent=frozen['cumulative_initial_worker_cpu_seconds'];fits={};gpu_started=None;gpu_uuid=None;gpu_lock=None
gpu_previous=plan.get('initial_gpu_seconds',0.)
os.sched_setaffinity(0,range(8))
for name in ['GBRT','FNN']:
for split in splits:
index=split['index'];key=f'{index}/{name}';dest=root/'fits'/f'split-{index}'/name
remain=plan['total_worker_cpu_seconds']-spent-plan['postfit_cpu_reserve_seconds']
if remain<=5:
fits[key]=dict(status='NOT_ATTEMPTED',reason='SHARED_CPU_CAP_RESERVE');continue
command=[sys.executable,'tools/gre24_fit.py',str(root),str(index),name]
if name=='GBRT':
cap=min(plan['xgb_per_fit_cpu_seconds'],remain)
receipt=supervise(command,dest,'worker',dict(total_cpu_seconds=cap,job_cpu_seconds=cap,
job_wall_seconds=7200,tree_rss_bytes=24*2**30,disk_bytes=3*2**30,max_processes=1),
poll=.1,numerical_threads=plan['xgb_threads'])
else:
if gpu_started is None:gpu_started=time.monotonic()
remaining_gpu=plan['total_gpu_seconds']-gpu_previous-(time.monotonic()-gpu_started)
if remaining_gpu<=20:
fits[key]=dict(status='NOT_ATTEMPTED',reason='SHARED_GPU_CAP');continue
# Fixed one device once selected, three idle polls; waiting counts toward 2 GPU-hours.
idle=0
while idle<3 and time.monotonic()-gpu_started+gpu_previous<plan['total_gpu_seconds']-20:
states=gpu_state(gpu_uuid)
if gpu_uuid is None:
candidates=sorted((s for s in states if s['used_bytes']<=128*2**20 and s['utilization']==0),key=lambda s:s['uuid'])
if candidates:
gpu_uuid=candidates[0]['uuid']
gpu_lock=Path('/tmp/meltmind-gre24-'+gpu_uuid+'.lock').open('a')
fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
states=[candidates[0]]
clean=bool(gpu_uuid) and states[0]['used_bytes']<=128*2**20 and states[0]['utilization']==0
idle=idle+1 if clean else 0
save(root/'gpu-idle-progress.json',dict(uuid=gpu_uuid,consecutive_idle=idle,
charged_gpu_seconds=gpu_previous+time.monotonic()-gpu_started,states=states))
if idle<3:time.sleep(5)
remaining_gpu=plan['total_gpu_seconds']-gpu_previous-(time.monotonic()-gpu_started)
if idle<3 or remaining_gpu<=20:
fits[key]=dict(status='NOT_ATTEMPTED',reason='NO_IDLE_GPU_WITHIN_SHARED_CAP');continue
receipt=gpu_fit(command,dest,dest,gpu_uuid,remaining_gpu-2,
min(remain,plan['fnn_per_fit_cpu_seconds']),plan['gpu_reserved_byte_cap'])
spent+=receipt['cpu_seconds']
save(dest/'final-guard-receipt.json',receipt)
result_path=dest/'result.json';result=json.loads(result_path.read_text()) if result_path.exists() else None
passed=receipt['status']=='PASS' and result and result['status']=='FIT_COMPLETE_NO_TEST_SCORES'
if passed and sha(result['checkpoint'])!=result['checkpoint_sha256']:raise ValueError('Checkpoint changed before freeze')
fits[key]=dict(status='PASS' if passed else 'FAILED',guard_receipt=str(dest/'final-guard-receipt.json'),
guard_receipt_sha256=sha(dest/'final-guard-receipt.json'),result=str(result_path) if result else None,
checkpoint=result['checkpoint'] if passed else None,checkpoint_sha256=result['checkpoint_sha256'] if passed else None,
failure_checkpoint_retained=bool(result) and not passed)
save(root/'model-chain-progress.json',dict(stage='FITTING',completed=len(fits),fits=fits,
charged_worker_cpu_seconds=spent,charged_gpu_seconds=gpu_previous+(time.monotonic()-gpu_started if gpu_started else 0.)))
gpu_seconds=gpu_previous+(time.monotonic()-gpu_started if gpu_started else 0.)
if gpu_lock:gpu_lock.close()
save(root/'models-freeze.json',dict(fits=fits,charged_worker_cpu_seconds=spent,charged_gpu_seconds=gpu_seconds,
uuid=gpu_uuid,training_contract_sha256=sha(root/'training-contract.json'),all_fits_terminal=True,test_scores_read=False))
postfit=[]
for phase in ['predict','score']:
remain=plan['total_worker_cpu_seconds']-spent
if remain<=5:
postfit.append(dict(phase=phase,status='NOT_ATTEMPTED_CPU_CAP'));break
receipt=supervise([sys.executable,'tools/gre24_score.py',str(root),phase],root/'postfit-guards'/phase,'worker',
dict(total_cpu_seconds=remain,job_cpu_seconds=remain,job_wall_seconds=7200,tree_rss_bytes=24*2**30,
disk_bytes=3*2**30,max_processes=1),poll=.2,numerical_threads=4)
spent+=receipt['cpu_seconds'];postfit.append(dict(phase=phase,receipt=receipt))
if receipt['status']!='PASS':break
save(root/'reproduction-complete.json',dict(status='SCORED' if (root/'score-summary.json').exists() else 'INCOMPLETE',
fits=fits,postfit=postfit,charged_worker_cpu_seconds=spent,charged_gpu_seconds=gpu_seconds,
total_worker_cpu_cap=plan['total_worker_cpu_seconds'],total_gpu_cap=plan['total_gpu_seconds'],
dataset_version=plan['dataset_version'],paper_numbers_diagnostic_only=True))
return 0 if (root/'score-summary.json').exists() else 1
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('plan',type=Path)
a=p.parse_args();raise SystemExit(run(a.root,a.plan))
+44
View File
@@ -0,0 +1,44 @@
"""Exercise the real chain with exhausted fit allocation: no model/GPU launches."""
import json
from pathlib import Path
import subprocess
import sys
from gre24_prepare_v2 import sha
from s2_recovery_supervisor import save
def main():
root=Path('artifacts/gre24-reproduction-2026-11-15/budget-chain-test');root.mkdir(exist_ok=False)
reference=Path('artifacts/gre24-reproduction-2026-11-15/integration-03')
good=json.loads((reference/'preparation-freeze.json').read_text())['successful'];records=[]
for case,p in good.items():
result=root/(case+'-prep-result.json')
save(result,dict(status='PASS',case=case,geometry=dict(tet_nodes=p['nodes'],outside_tet_nodes=0),
t_scale_seconds=2.,output_sha256=p['output_sha256']))
# Result's parent defines input root; use separate existing arrays via copied tiny files.
folder=root/case;folder.mkdir();result=folder/'result.json'
for file in Path(p['output']).glob('*.npy'):(folder/file.name).write_bytes(file.read_bytes())
save(result,dict(status='PASS',case=case,geometry=dict(tet_nodes=p['nodes'],outside_tet_nodes=0),
t_scale_seconds=2.,output_sha256=p['output_sha256']))
records.append(dict(case=case,worker={'status':'PASS'},result_path=str(result),result_sha256=sha(result)))
save(root/'contract.json',dict(parts=list(good)))
save(root/'complete.json',dict(status='COMPLETE',accounting_complete=True,records=records,
worker_cpu_seconds=0.,initial_cpu_reserve_seconds=0.))
(root/'splits.json').write_bytes((reference/'splits.json').read_bytes())
paths=['tools/run_gre24_models.py','tools/gre24_fit.py','tools/gre24_score.py','tools/gre24_models.py','tools/gre24_gpu_guard.py','tools/gre24_job_guard.py']
plan=dict(code_sha256={p:sha(p) for p in paths},total_worker_cpu_seconds=1000,postfit_cpu_reserve_seconds=998,
total_gpu_seconds=60,initial_gpu_seconds=0,dataset_version='synthetic_budget_only',protocol='synthetic',
xgb_threads=1,fnn_cpu_threads=1,xgb_per_fit_cpu_seconds=10,fnn_per_fit_cpu_seconds=10,gpu_reserved_byte_cap=15*2**30)
save(root/'plan.json',plan)
subprocess.run([sys.executable,'tools/run_gre24_models.py',str(root),str(root/'plan.json')],check=True)
result=json.loads((root/'reproduction-complete.json').read_text())
assert all(v['status']=='NOT_ATTEMPTED' for v in result['fits'].values())
assert result['charged_gpu_seconds']==0
assert not list((root/'fits').glob('**/started.json')) if (root/'fits').exists() else True
assert result['charged_worker_cpu_seconds']<1000
save(root/'test-acceptance.json',dict(status='PASS',checks=['CPU reserve prevents all fits','no GPU starts',
'terminal model freeze exists before prediction','naive score survives missing models','actual postfit CPU charged'],result=result))
print('PASS actual chain budget boundary, zero GPU fits')
if __name__=='__main__':main()
+78
View File
@@ -0,0 +1,78 @@
"""Actual fit/predict/score entry points on tiny synthetic data, one GPU only."""
import json
import os
from pathlib import Path
import resource
import sys
import time
import numpy as np
from gre24_prepare_v2 import sha
from gre24_job_guard import supervise,save
from gre24_gpu_guard import gpu_state,gpu_fit,recorded_peaks
def main():
root=Path(sys.argv[1]);root.mkdir(exist_ok=False)
os.sched_setaffinity(0,[0,1,2,3]);rng=np.random.default_rng(91);good={}
for case in ['learn','validation','test']:
d=root/'parts'/case;d.mkdir(parents=True)
x=rng.uniform(0,1,(128,19));y=.2+.1*x[:,0]+.1*x[:,1]
values={'features':x,'Vf':np.linspace(0,1,128,endpoint=False),'raw_ids':np.arange(128),'seconds':2*y}
if case!='test':values['target']=y # Deliberately absent for test: no fit/predict may need it.
for name,v in values.items():np.save(d/(name+'.npy'),v)
good[case]=dict(output=str(d),nodes=len(y),t_scale_seconds=2.,
output_sha256={p.name:sha(p) for p in d.glob('*.npy')})
splits=[dict(index=i,seed=91+i,train=['learn','validation'],test=['test'],fnn_learning=['learn'],fnn_validation=['validation']) for i in range(5)]
save(root/'splits.json',splits);save(root/'preparation-freeze.json',dict(successful=good,failed=[]))
paths=['tools/gre24_models.py','tools/gre24_fit.py','tools/gre24_score.py','tools/gre24_gpu_guard.py','tools/gre24_job_guard.py']
c=dict(code_sha256={p:sha(p) for p in paths},preparation_freeze_sha256=sha(root/'preparation-freeze.json'),
splits_sha256=sha(root/'splits.json'),dataset_version='synthetic_only',protocol='synthetic smoke, not scientific data',
xgb_threads=1,fnn_cpu_threads=2,gpu_reserved_byte_cap=15*2**30)
save(root/'training-contract.json',c)
receipts={};models={};gpu_seconds=0
for name in ['GBRT','FNN']:
dest=root/'fits'/'split-0'/name;cmd=[sys.executable,'tools/gre24_fit.py',str(root),'0',name]
if name=='GBRT':
r=supervise(cmd,dest,'worker',dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120,
tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1)
else:
begin=time.monotonic();available=sorted((s for s in gpu_state() if s['used_bytes']<128*2**20 and s['utilization']==0),key=lambda s:s['uuid'])
if not available:raise RuntimeError('No idle GPU for bounded synthetic acceptance')
uuid=available[0]['uuid']
r=gpu_fit(cmd,dest,dest,uuid,60.,120.,15*2**30);gpu_seconds=time.monotonic()-begin
receipts[name]=r;save(root/(name+'-guard.json'),r)
if r['status']!='PASS':
save(root/'acceptance.json',dict(status='FAILED',receipts=receipts,GPU_seconds=gpu_seconds));raise RuntimeError(name+' integration failed')
result=json.loads((dest/'result.json').read_text())
access=json.loads((dest/'training-target-access.json').read_text())
assert all(a['case']!='test' for a in access)
models['0/'+name]=dict(status='PASS',checkpoint=result['checkpoint'],checkpoint_sha256=result['checkpoint_sha256'])
for i in range(1,5):
for name in ['GBRT','FNN']:models[f'{i}/{name}']=dict(status='NOT_ATTEMPTED_SYNTHETIC')
save(root/'models-freeze.json',dict(fits=models))
test_seconds=root/'parts/test/seconds.npy';hidden=root/'parts/test/seconds.unavailable'
test_seconds.rename(hidden)
try:
r=supervise([sys.executable,'tools/gre24_score.py',str(root),'predict'],root/'predict-guard','worker',
dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120,tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1)
receipts['predict']=r
if r['status']!='PASS':raise RuntimeError('Prediction failed with test seconds unavailable')
finally:hidden.rename(test_seconds)
r=supervise([sys.executable,'tools/gre24_score.py',str(root),'score'],root/'score-guard','worker',
dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120,tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1)
receipts['score']=r;assert r['status']=='PASS'
summary=json.loads((root/'score-summary.json').read_text())['summary']
assert summary['naive']['rrmse_rms']['full_requested_coverage']
assert summary['FNN']['rrmse_rms']['mean_of_five_split_summaries']['median'] is None
# Failed-result records still contribute their peaks, independently of live memory.
fake=root/'peak-check';fake.mkdir();save(fake/'failure.json',dict(peak_reserved_bytes=16*2**30))
peaks,witness=recorded_peaks(fake);assert peaks['peak_reserved_bytes']==16*2**30
save(root/'acceptance.json',dict(status='PASS',receipts=receipts,GPU_seconds=gpu_seconds,
worker_CPU_seconds=sum(r['cpu_seconds'] for r in receipts.values()),
checks=['real GBRT and FNN entrypoints','test target absent during fits','test seconds unavailable during prediction',
'all predictions frozen before scoring','missing splits not silently averaged','failure VRAM peaks retained'],
scientific_data_read=False,coordinator_CPU_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
print(json.dumps(dict(status='PASS',GPU_seconds=gpu_seconds,worker_CPU_seconds=sum(r['cpu_seconds'] for r in receipts.values()))))
if __name__=='__main__':main()
+65
View File
@@ -0,0 +1,65 @@
"""Role isolation, full-row learning, early stopping and real guard receipts."""
import json
from pathlib import Path
import resource
import sys
import tempfile
import unittest
import numpy as np
from gre24_models import xgb_model,fnn_model,eligible_roles,require_training_role,early_stop_update
from gre24_job_guard import supervise
class ModelChecks(unittest.TestCase):
def test_role_isolation_and_missing_no_replacement(self):
s=dict(train=['a','b'],test=['c','d'],fnn_learning=['a'],fnn_validation=['b'])
roles,missing=eligible_roles(s,['a','c','d'])
self.assertEqual(roles['train'],['a']);self.assertEqual(missing['train'],['b'])
with self.assertRaises(PermissionError):require_training_role('c',s,'train')
with self.assertRaises(PermissionError):require_training_role('a',s,'test')
def test_earlystop_strict_improvement_patience(self):
best=1.;stale=0
for i in range(5):
best,stale,improved,stop=early_stop_update(best,stale,1.)
self.assertFalse(improved);self.assertEqual(stop,i==4)
best,stale,improved,stop=early_stop_update(best,stale,.9)
self.assertTrue(improved);self.assertEqual(stale,0);self.assertFalse(stop)
with self.assertRaises(ValueError):early_stop_update(best,stale,float('nan'))
def test_xgboost_fixed_recipe_learns_all_rows(self):
rng=np.random.default_rng(42);x=rng.normal(size=(512,19)).astype(np.float32);y=x[:,0]+.5*x[:,1]
m=xgb_model(42,threads=1);m.fit(x,y)
self.assertLess(np.mean((m.predict(x)-y)**2),.02)
self.assertEqual(m.n_estimators,100);self.assertEqual(m.max_depth,8)
def test_fnn_architecture_gradients_and_checkpoint_restore(self):
import torch
torch.set_num_threads(1);torch.manual_seed(42);m=fnn_model()
linear=[a for a in m if isinstance(a,torch.nn.Linear)]
self.assertEqual([(a.in_features,a.out_features) for a in linear],[(19,100),(100,100),(100,100),(100,100),(100,1)])
x=torch.randn(64,19);y=torch.ones(64,1);o=torch.optim.Adam(m.parameters(),lr=.001)
before=float(((m(x)-y)**2).mean())
for _ in range(15):
o.zero_grad();loss=((m(x)-y)**2).mean();loss.backward();o.step()
self.assertLess(float(((m(x)-y)**2).mean()),before)
with tempfile.TemporaryDirectory() as tmp:
p=Path(tmp)/'m.pt';torch.save(m.state_dict(),p);n=fnn_model();n.load_state_dict(torch.load(p,weights_only=True))
torch.testing.assert_close(m(x),n(x))
def test_guard_records_failure_cpu_and_refuses_repeat(self):
limits=dict(total_cpu_seconds=10,job_cpu_seconds=5,job_wall_seconds=5,tree_rss_bytes=256*2**20,disk_bytes=2**20,max_processes=1)
with tempfile.TemporaryDirectory() as tmp:
r=supervise([sys.executable,'-c','raise SystemExit(7)'],Path(tmp),'fail',limits,poll=.02)
self.assertEqual(r['returncode'],7);self.assertGreater(r['cpu_seconds'],0)
self.assertEqual(r['status'],'STOPPED_OR_FAILED')
with self.assertRaises(FileExistsError):supervise([sys.executable,'-c','pass'],Path(tmp),'fail',limits)
if __name__=='__main__':
result=unittest.TextTestRunner(verbosity=2).run(unittest.defaultTestLoader.loadTestsFromTestCase(ModelChecks))
u=resource.getrusage(resource.RUSAGE_SELF);ch=resource.getrusage(resource.RUSAGE_CHILDREN)
Path('artifacts/gre24-reproduction-2026-11-15/model-synthetic-receipt.json').write_text(json.dumps(dict(
tests=result.testsRun,passed=result.wasSuccessful(),failures=len(result.failures),errors=len(result.errors),
CPU_s=u.ru_utime+u.ru_stime+ch.ru_utime+ch.ru_stime,GPU_used=False),indent=2))
raise SystemExit(0 if result.wasSuccessful() else 1)