Freeze Gre24 model chain and retain synthetic acceptance evidence
This commit is contained in:
1 parent
8409910b1e
commit
eb9ed0c762
124 files changed
+5993
No files matched your search
@@ -0,0 +1,139 @@
|
||||
"""One fixed split/model fit; reads training-side targets only, no test scores."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
import traceback
|
||||
import numpy as np
|
||||
from gre24_prepare_v2 import sha
|
||||
from gre24_models import xgb_model,fnn_model,eligible_roles,require_training_role,early_stop_update
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
|
||||
def contract(root):
|
||||
c=json.loads((root/'training-contract.json').read_text())
|
||||
for p,h in c['code_sha256'].items():
|
||||
if sha(p)!=h:raise ValueError('Pinned training source changed: '+p)
|
||||
if sha(root/'preparation-freeze.json')!=c['preparation_freeze_sha256']:
|
||||
raise ValueError('Preparation freeze changed')
|
||||
if sha(root/'splits.json')!=c['splits_sha256']:raise ValueError('Splits changed')
|
||||
return c,json.loads((root/'preparation-freeze.json').read_text()),json.loads((root/'splits.json').read_text())
|
||||
|
||||
|
||||
def training_arrays(root,freeze,split,role,read_log):
|
||||
selected=[case for case in split[role] if case in freeze['successful']]
|
||||
n=sum(freeze['successful'][case]['nodes'] for case in selected)
|
||||
if not n:raise ValueError('No successful training rows')
|
||||
x=np.empty((n,19),dtype=np.float32);y=np.empty(n,dtype=np.float32);cursor=0
|
||||
for case in selected:
|
||||
require_training_role(case,split,role)
|
||||
r=freeze['successful'][case];source=Path(r['output'])
|
||||
for name in ['features.npy','target.npy']:
|
||||
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Frozen preparation changed')
|
||||
a=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
|
||||
b=np.load(source/'target.npy',mmap_mode='r',allow_pickle=False)
|
||||
count=len(b)
|
||||
if a.shape!=(count,19) or count!=r['nodes']:raise ValueError('Training row identity mismatch')
|
||||
x[cursor:cursor+count]=a;y[cursor:cursor+count]=b;cursor+=count
|
||||
read_log.append(dict(case=case,role=role,nodes=count,source=str(source),
|
||||
feature_sha256=r['output_sha256']['features.npy'],target_sha256=r['output_sha256']['target.npy']))
|
||||
if not np.isfinite(x).all() or not np.isfinite(y).all():raise ValueError('Nonfinite training input')
|
||||
return x,y
|
||||
|
||||
|
||||
def fit(root,index,model_name):
|
||||
c,f,splits=contract(root);split=splits[index]
|
||||
dest=root/'fits'/f'split-{index}'/model_name;dest.mkdir(parents=True,exist_ok=True)
|
||||
if (dest/'started.json').exists():raise FileExistsError('No fit restart or overwrite')
|
||||
start=time.monotonic();cpu=time.process_time();read_log=[]
|
||||
def progress(stage,**kw):
|
||||
record=dict(stage=stage,pid=os.getpid(),unix=time.time(),split=index,model=model_name,
|
||||
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start)
|
||||
record.update(kw);save(dest/'progress.json',record)
|
||||
save(dest/'started.json',dict(pid=os.getpid(),unix=time.time(),split=index,model=model_name,seed=split['seed'],
|
||||
training_contract_sha256=sha(root/'training-contract.json')))
|
||||
roles,missing=eligible_roles(split,f['successful'])
|
||||
try:
|
||||
if model_name=='GBRT':
|
||||
progress('LOAD_TRAIN_ONLY')
|
||||
x,y=training_arrays(root,f,split,'train',read_log)
|
||||
save(dest/'training-target-access.json',read_log)
|
||||
model=xgb_model(split['seed'],c['xgb_threads'])
|
||||
progress('FIT',nodes=len(y),geometries=len(roles['train']))
|
||||
model.fit(x,y,verbose=False)
|
||||
checkpoint=dest/'model.ubj';model.save_model(checkpoint)
|
||||
(dest/'xgboost-config.json').write_text(model.get_booster().save_config())
|
||||
params=model.get_params()
|
||||
# Preserve XGBoost's default missing=NaN as an explicit textual parameter;
|
||||
# this is not a nonfinite observation or a change to the fitted estimator.
|
||||
params={k:('NaN (library default missing marker)' if isinstance(v,float) and np.isnan(v) else v)
|
||||
for k,v in params.items()}
|
||||
extra=dict(train_nodes=len(y),train_geometries=len(roles['train']),parameters=params)
|
||||
elif model_name=='FNN':
|
||||
import torch
|
||||
torch.set_num_threads(c['fnn_cpu_threads']);torch.manual_seed(split['seed'])
|
||||
if not torch.cuda.is_available():raise RuntimeError('Selected CUDA GPU unavailable')
|
||||
torch.cuda.set_device(0) # Initialise the selected device before resetting its allocator counters.
|
||||
device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device)
|
||||
torch.backends.cuda.matmul.allow_tf32=False
|
||||
progress('LOAD_FNN_TRAIN_VALIDATION_ONLY')
|
||||
x,y=training_arrays(root,f,split,'fnn_learning',read_log)
|
||||
vx,vy=training_arrays(root,f,split,'fnn_validation',read_log)
|
||||
save(dest/'training-target-access.json',read_log)
|
||||
x=torch.as_tensor(x,device=device);y=torch.as_tensor(y,device=device)
|
||||
vx=torch.as_tensor(vx,device=device);vy=torch.as_tensor(vy,device=device)
|
||||
model=fnn_model().to(device);optimizer=torch.optim.Adam(model.parameters(),lr=.001)
|
||||
best=float('inf');stale=0;best_epoch=None;history=[];checkpoint=dest/'model.pt'
|
||||
for epoch in range(1,31):
|
||||
model.train();order=torch.randperm(len(y),device=device);total=torch.zeros((),device=device)
|
||||
for begin in range(0,len(y),1024):
|
||||
ids=order[begin:begin+1024];optimizer.zero_grad(set_to_none=True)
|
||||
prediction=model(x[ids]).squeeze(-1);loss=torch.mean((prediction-y[ids])**2)
|
||||
if not torch.isfinite(loss):raise ValueError('Nonfinite training loss')
|
||||
loss.backward();optimizer.step();total+=loss.detach()*len(ids)
|
||||
model.eval();sse=torch.zeros((),device=device)
|
||||
with torch.no_grad():
|
||||
for begin in range(0,len(vy),65536):
|
||||
sse+=torch.sum((model(vx[begin:begin+65536]).squeeze(-1)-vy[begin:begin+65536])**2)
|
||||
validation=float((sse/len(vy)).item());train=float((total/len(y)).item())
|
||||
best,stale,improved,stop=early_stop_update(best,stale,validation)
|
||||
if improved:
|
||||
torch.save({'state_dict':{k:v.detach().cpu() for k,v in model.state_dict().items()},
|
||||
'epoch':epoch,'validation_mse':validation,'seed':split['seed']},checkpoint)
|
||||
best_epoch=epoch
|
||||
row=dict(epoch=epoch,training_mse=train,validation_mse=validation,best_epoch=best_epoch,
|
||||
stale_epochs=stale,peak_allocated_bytes=torch.cuda.max_memory_allocated(device),
|
||||
peak_reserved_bytes=torch.cuda.max_memory_reserved(device),wall_s=time.monotonic()-start)
|
||||
history.append(row);save(dest/'history.json',history);progress('EPOCH_COMPLETE',**row)
|
||||
if row['peak_reserved_bytes']>c['gpu_reserved_byte_cap']:raise RuntimeError('Recorded VRAM peak exceeds cap')
|
||||
if stop:break
|
||||
restored=torch.load(checkpoint,map_location='cpu',weights_only=True)
|
||||
model.load_state_dict(restored['state_dict'])
|
||||
extra=dict(epochs=len(history),best_epoch=best_epoch,best_validation_mse=best,
|
||||
train_nodes=len(y),validation_nodes=len(vy),train_geometries=len(roles['fnn_learning']),
|
||||
validation_geometries=len(roles['fnn_validation']),restored_best=True,amp=False,
|
||||
input_standardization='none; use eq12 columns',peak_allocated_bytes=torch.cuda.max_memory_allocated(device),
|
||||
peak_reserved_bytes=torch.cuda.max_memory_reserved(device))
|
||||
else:raise ValueError('Unknown model')
|
||||
save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name,
|
||||
checkpoint=str(checkpoint),checkpoint_sha256=sha(checkpoint),missing_preparation=missing,
|
||||
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,**extra))
|
||||
progress('FIT_COMPLETE_NO_TEST_SCORES');return 0
|
||||
except Exception as exc:
|
||||
peaks={}
|
||||
if model_name=='FNN':
|
||||
try:
|
||||
import torch
|
||||
if torch.cuda.is_initialized():
|
||||
peaks=dict(peak_allocated_bytes=torch.cuda.max_memory_allocated(),
|
||||
peak_reserved_bytes=torch.cuda.max_memory_reserved())
|
||||
except Exception as peak_error:peaks={'peak_read_error':str(peak_error)}
|
||||
save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
|
||||
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,automatic_retry=False,**peaks))
|
||||
raise
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('split',type=int);p.add_argument('model',choices=['GBRT','FNN'])
|
||||
a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model))
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Single-GPU wall accounting and recorded/device VRAM checks including failure."""
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import psutil
|
||||
from gre24_job_guard import supervise,stop_group
|
||||
|
||||
|
||||
def gpu_state(uuid=None):
|
||||
args=['nvidia-smi','--query-gpu=uuid,memory.used,utilization.gpu','--format=csv,noheader,nounits']
|
||||
if uuid:args+=['-i',uuid]
|
||||
out=subprocess.run(args,text=True,capture_output=True,check=True,timeout=10).stdout
|
||||
rows=[]
|
||||
for line in out.splitlines():
|
||||
u,mem,util=[v.strip() for v in line.split(',')]
|
||||
rows.append(dict(uuid=u,used_bytes=int(mem)*2**20,utilization=int(util)))
|
||||
if not rows:raise RuntimeError('No GPU state returned')
|
||||
return rows
|
||||
|
||||
|
||||
def recorded_peaks(root):
|
||||
peaks={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
|
||||
def walk(value,path,digest):
|
||||
if isinstance(value,dict):
|
||||
for k,v in value.items():
|
||||
if k in peaks:
|
||||
if isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or v<0:
|
||||
raise ValueError('Invalid CUDA peak record')
|
||||
if v>=peaks[k]:peaks[k]=v;witnesses[k]={'path':str(path),'sha256':digest}
|
||||
else:walk(v,path,digest)
|
||||
elif isinstance(value,list):
|
||||
for v in value:walk(v,path,digest)
|
||||
for path in sorted(Path(root).glob('*.json')):
|
||||
raw=path.read_bytes();walk(json.loads(raw),path,hashlib.sha256(raw).hexdigest())
|
||||
return peaks,witnesses
|
||||
|
||||
|
||||
def gpu_fit(command,guard_root,fit_root,uuid,wall_limit,cpu_limit,byte_cap):
|
||||
start=time.monotonic();done=threading.Event();monitor={'reasons':[],'peak_device_bytes':0,'samples':0}
|
||||
maxima={'peak_allocated_bytes':0,'peak_reserved_bytes':0};witnesses={}
|
||||
def scan():
|
||||
current,seen=recorded_peaks(fit_root)
|
||||
for k,v in current.items():
|
||||
if v>=maxima[k] and k in seen:maxima[k]=v;witnesses[k]=seen[k]
|
||||
if max(maxima.values())>byte_cap and 'RECORDED_VRAM_LIMIT' not in monitor['reasons']:
|
||||
monitor['reasons'].append('RECORDED_VRAM_LIMIT')
|
||||
def watch():
|
||||
while not done.is_set():
|
||||
try:
|
||||
scan();state=gpu_state(uuid)[0];monitor['samples']+=1
|
||||
monitor['peak_device_bytes']=max(monitor['peak_device_bytes'],state['used_bytes'])
|
||||
if state['used_bytes']>byte_cap and 'DEVICE_VRAM_LIMIT' not in monitor['reasons']:
|
||||
monitor['reasons'].append('DEVICE_VRAM_LIMIT')
|
||||
except Exception as exc:monitor['reasons'].append('GPU_MONITOR_FAILURE:'+str(exc))
|
||||
if monitor['reasons']:
|
||||
for p in psutil.Process().children(recursive=True):
|
||||
try:
|
||||
if p.cmdline()==command:stop_group(p.pid)
|
||||
except psutil.NoSuchProcess:pass
|
||||
done.wait(.5)
|
||||
thread=threading.Thread(target=watch,daemon=True);thread.start()
|
||||
try:
|
||||
receipt=supervise(command,guard_root,'worker',dict(total_cpu_seconds=cpu_limit,
|
||||
job_cpu_seconds=cpu_limit,job_wall_seconds=max(.1,wall_limit-3),tree_rss_bytes=24*2**30,
|
||||
disk_bytes=3*2**30,max_processes=1),poll=.1,cuda_device=uuid,numerical_threads=2)
|
||||
finally:
|
||||
done.set();thread.join(timeout=12)
|
||||
if thread.is_alive():monitor['reasons'].append('GPU_MONITOR_DID_NOT_STOP')
|
||||
try:scan()
|
||||
except Exception as exc:monitor['reasons'].append('FINAL_MEMORY_RECORD_READ_FAILED:'+str(exc))
|
||||
if receipt['status']=='PASS' and len(witnesses)!=2:monitor['reasons'].append('MISSING_FINAL_PEAKS')
|
||||
elapsed=time.monotonic()-start
|
||||
if elapsed>wall_limit:monitor['reasons'].append('TOTAL_GPU_WALL_LIMIT')
|
||||
receipt['underlying_worker_status']=receipt['status']
|
||||
if monitor['reasons']:receipt.update(status='STOPPED_OR_FAILED',stop_reason=monitor['reasons'][0])
|
||||
receipt.update(device_monitor=monitor,recorded_peaks=maxima,peak_witnesses=witnesses,
|
||||
total_gpu_wall_seconds=elapsed,peak_cap_bytes=byte_cap)
|
||||
return receipt
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Sequential E1 job guard: tree CPU/RSS, wall, process count and disk receipts.
|
||||
|
||||
Budget checks are sampled, not kernel-enforced RSS/directory quotas. Overshoot
|
||||
is measured and fails the job. Infrastructure recovery copy; worker command and numerics are unchanged.
|
||||
"""
|
||||
import fcntl
|
||||
import ctypes
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import psutil
|
||||
from s2_recovery_monitoring import disk_usage,Monitor
|
||||
|
||||
|
||||
def save(path, data):
|
||||
path = Path(path)
|
||||
tmp = path.with_suffix(path.suffix + '.tmp')
|
||||
tmp.write_text(json.dumps(data, indent=2, allow_nan=False) + '\n')
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def tree_usage(pid):
|
||||
try:
|
||||
root = psutil.Process(pid)
|
||||
processes = [root] + root.children(recursive=True)
|
||||
except psutil.NoSuchProcess:
|
||||
return 0., 0, 0
|
||||
cpu = 0.
|
||||
rss = count = 0
|
||||
for p in processes:
|
||||
try:
|
||||
t = p.cpu_times()
|
||||
# Linux child times include reaped descendants, not live children.
|
||||
cpu += t.user + t.system + t.children_user + t.children_system
|
||||
rss += p.memory_info().rss
|
||||
count += 1
|
||||
except psutil.NoSuchProcess:
|
||||
pass
|
||||
return cpu, rss, count
|
||||
|
||||
|
||||
def stop_group(pid):
|
||||
try:
|
||||
os.killpg(pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
|
||||
def supervise(command, root, name, limits, *, poll=.02, cuda_device='', numerical_threads=1):
|
||||
"""One locked attempt; no automatic retry/replacement; account failed CPU."""
|
||||
root = Path(root)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
monitor=Monitor(root/(name+'.monitor-errors.jsonl'))
|
||||
with (root/'execution.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
started_path = root/(name+'.started.json')
|
||||
receipt_path = root/(name+'.receipt.json')
|
||||
if started_path.exists() or receipt_path.exists():
|
||||
raise FileExistsError('Attempt already exists; no silent retry')
|
||||
if any(not math.isfinite(v) or v <= 0 for v in limits.values()):
|
||||
raise ValueError('Every limit must be finite and positive')
|
||||
ledger_path = root/'ledger.json'
|
||||
ledger = json.loads(ledger_path.read_text()) if ledger_path.exists() else {'cpu_seconds_charged':0.,'attempts':[]}
|
||||
if ledger['cpu_seconds_charged'] >= limits['total_cpu_seconds']:
|
||||
raise RuntimeError('Total CPU exhausted before spawn')
|
||||
if monitor.retry('initial_disk',lambda:disk_usage(root),poll) >= limits['disk_bytes']:
|
||||
raise RuntimeError('Disk exhausted before spawn')
|
||||
save(started_path, {'command':command,'limits':limits,'started_unix':time.time()})
|
||||
start = last = time.monotonic()
|
||||
cpu_peak = rss_peak = max_processes = max_gap = 0
|
||||
cpu=rss=count=used_disk=0
|
||||
reason = None
|
||||
exit_status = usage = None
|
||||
env = dict(os.environ, OMP_NUM_THREADS=str(numerical_threads), OPENBLAS_NUM_THREADS=str(numerical_threads),
|
||||
MKL_NUM_THREADS=str(numerical_threads), NUMEXPR_NUM_THREADS=str(numerical_threads),
|
||||
CUDA_VISIBLE_DEVICES=cuda_device)
|
||||
with (root/(name+'.log')).open('x') as log:
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
previous_subreaper = ctypes.c_int()
|
||||
if libc.prctl(37, ctypes.byref(previous_subreaper), 0, 0, 0) != 0 or libc.prctl(36, 1, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot enable descendant reaping')
|
||||
proc = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT,
|
||||
env=env, start_new_session=True)
|
||||
try:
|
||||
while True:
|
||||
now = time.monotonic()
|
||||
max_gap = max(max_gap, now-last)
|
||||
last = now
|
||||
(cpu,rss,count),_=monitor.sample('tree_usage',lambda:tree_usage(proc.pid),(cpu,rss,count))
|
||||
cpu_peak = max(cpu_peak,cpu)
|
||||
rss_peak = max(rss_peak,rss)
|
||||
max_processes = max(max_processes,count)
|
||||
used_disk,_=monitor.sample('live_disk',lambda:disk_usage(root),used_disk)
|
||||
if now-start >= limits['job_wall_seconds']:
|
||||
reason = 'wall_limit'
|
||||
elif rss > limits['tree_rss_bytes']:
|
||||
reason = 'tree_rss_limit'
|
||||
elif count > limits['max_processes']:
|
||||
reason = 'process_count_limit'
|
||||
elif used_disk >= limits['disk_bytes']:
|
||||
reason = 'disk_limit'
|
||||
elif cpu_peak >= limits['job_cpu_seconds']:
|
||||
reason = 'job_cpu_limit'
|
||||
elif ledger['cpu_seconds_charged']+cpu_peak >= limits['total_cpu_seconds']:
|
||||
reason = 'total_cpu_limit'
|
||||
if reason:
|
||||
stop_group(proc.pid)
|
||||
pid, status, child_usage = os.wait4(proc.pid, 0 if reason else os.WNOHANG)
|
||||
if pid:
|
||||
exit_status, usage = status, child_usage
|
||||
break
|
||||
time.sleep(poll)
|
||||
finally:
|
||||
# No detached work may outlive a supervised attempt.
|
||||
stop_group(proc.pid)
|
||||
if exit_status is None:
|
||||
_, exit_status, usage = os.wait4(proc.pid, 0)
|
||||
proc.returncode = os.waitstatus_to_exitcode(exit_status)
|
||||
orphan_cpu = 0.
|
||||
orphan_peak = 0
|
||||
# Killed descendants are adopted here rather than losing their
|
||||
# final CPU usage when the root worker cannot reap them itself.
|
||||
while True:
|
||||
try:
|
||||
_, _, orphan_usage = os.wait4(-proc.pid, 0)
|
||||
except ChildProcessError:
|
||||
break
|
||||
orphan_cpu += orphan_usage.ru_utime + orphan_usage.ru_stime
|
||||
orphan_peak = max(orphan_peak, orphan_usage.ru_maxrss*1024)
|
||||
if libc.prctl(36, previous_subreaper.value, 0, 0, 0) != 0:
|
||||
raise OSError(ctypes.get_errno(), 'Cannot restore subreaper state')
|
||||
elapsed = time.monotonic()-start
|
||||
charged = max(cpu_peak, usage.ru_utime+usage.ru_stime+orphan_cpu)
|
||||
rss_peak = max(rss_peak, usage.ru_maxrss*1024,orphan_peak)
|
||||
used_disk=monitor.retry('final_disk',lambda:disk_usage(root),poll)
|
||||
if reason is None:
|
||||
if elapsed > limits['job_wall_seconds']: reason='wall_limit_after_exit'
|
||||
elif rss_peak > limits['tree_rss_bytes']: reason='rss_limit_after_exit'
|
||||
elif charged > limits['job_cpu_seconds']: reason='cpu_limit_after_exit'
|
||||
elif ledger['cpu_seconds_charged']+charged > limits['total_cpu_seconds']: reason='total_cpu_limit_after_exit'
|
||||
elif used_disk >= limits['disk_bytes']: reason='disk_limit_after_exit'
|
||||
receipt = dict(status='PASS' if proc.returncode==0 and reason is None else 'STOPPED_OR_FAILED',
|
||||
command=command,returncode=proc.returncode,stop_reason=reason,
|
||||
cpu_seconds=charged,wall_seconds=elapsed,peak_tree_rss_bytes=rss_peak,
|
||||
peak_process_count=max_processes,output_disk_bytes_before_receipt=used_disk,
|
||||
max_poll_gap_seconds=max_gap,poll_target_seconds=poll,
|
||||
reaped_orphan_cpu_seconds=orphan_cpu,
|
||||
supervision='sampled process-tree monitoring; wait4 final CPU/HWM; overshoot retained as failure',
|
||||
gpu_visible=bool(cuda_device),cuda_device=cuda_device,
|
||||
numerical_threads=numerical_threads,retry=False,monitor_errors=monitor.errors,last_monitor_error=monitor.last_error)
|
||||
ledger['cpu_seconds_charged'] += charged
|
||||
ledger['attempts'].append({'name':name,'status':receipt['status'],'cpu_seconds':charged})
|
||||
monitor.retry('receipt_write',lambda:save(receipt_path,receipt),poll)
|
||||
monitor.retry('ledger_write',lambda:save(ledger_path,ledger),poll)
|
||||
return receipt
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Fixed Gre24 model choices; no test-driven selection or node subsampling."""
|
||||
import numpy as np
|
||||
|
||||
|
||||
def xgb_model(seed,threads=8):
|
||||
from xgboost import XGBRegressor
|
||||
return XGBRegressor(n_estimators=100,max_depth=8,learning_rate=.1,
|
||||
objective='reg:squarederror',random_state=seed,n_jobs=threads)
|
||||
|
||||
|
||||
def fnn_model():
|
||||
import torch
|
||||
layers=[];width=19
|
||||
for _ in range(4):
|
||||
layers.extend([torch.nn.Linear(width,100),torch.nn.ReLU()]);width=100
|
||||
layers.append(torch.nn.Linear(100,1))
|
||||
return torch.nn.Sequential(*layers)
|
||||
|
||||
|
||||
def eligible_roles(split,successful):
|
||||
successful=set(successful)
|
||||
roles={role:[c for c in split[role] if c in successful]
|
||||
for role in ['train','test','fnn_learning','fnn_validation']}
|
||||
if set(split['train'])&set(split['test']):raise ValueError('Train/test overlap')
|
||||
if set(split['fnn_learning'])&set(split['fnn_validation']):raise ValueError('FNN role overlap')
|
||||
if set(split['fnn_learning'])|set(split['fnn_validation'])!=set(split['train']):
|
||||
raise ValueError('FNN learning/validation do not partition train')
|
||||
missing={role:[c for c in split[role] if c not in successful] for role in roles}
|
||||
return roles,missing
|
||||
|
||||
|
||||
def require_training_role(case,split,role):
|
||||
if role not in ['train','fnn_learning','fnn_validation']:
|
||||
raise PermissionError('Only frozen training-side roles may read targets')
|
||||
if case in split['test'] or case not in split[role]:
|
||||
raise PermissionError('Case is not in the requested training role')
|
||||
|
||||
|
||||
def early_stop_update(best,stale,score,patience=5):
|
||||
if not np.isfinite(score):raise ValueError('Nonfinite validation MSE')
|
||||
improved=score<best
|
||||
return (score if improved else best),(0 if improved else stale+1),improved,(not improved and stale+1>=patience)
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Predictions frozen before test-target scoring, all successful tet nodes."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import numpy as np
|
||||
from gre24_prepare_v2 import sha
|
||||
from gre24_features_v2 import per_geometry_metrics
|
||||
from gre24_models import fnn_model
|
||||
from gre24_fit import contract
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
|
||||
def predict(root):
|
||||
c,prep,splits=contract(root)
|
||||
models=json.loads((root/'models-freeze.json').read_text())
|
||||
dest=root/'predictions';dest.mkdir(exist_ok=False)
|
||||
start=time.process_time();rows=[]
|
||||
for split in splits:
|
||||
available={'naive':None}
|
||||
for name in ['GBRT','FNN']:
|
||||
r=models['fits'][f"{split['index']}/{name}"]
|
||||
if r['status']!='PASS':continue
|
||||
if sha(r['checkpoint'])!=r['checkpoint_sha256']:raise ValueError('Checkpoint changed after freeze')
|
||||
if name=='GBRT':
|
||||
from xgboost import XGBRegressor
|
||||
m=XGBRegressor(n_jobs=4);m.load_model(r['checkpoint'])
|
||||
else:
|
||||
import torch
|
||||
torch.set_num_threads(4);m=fnn_model()
|
||||
m.load_state_dict(torch.load(r['checkpoint'],map_location='cpu',weights_only=True)['state_dict']);m.eval()
|
||||
available[name]=m
|
||||
for case in split['test']:
|
||||
if case not in prep['successful']:
|
||||
rows.append(dict(split=split['index'],case=case,status='PREPARATION_FAILED',predictions={}))
|
||||
continue
|
||||
r=prep['successful'][case];source=Path(r['output']);outputs={}
|
||||
for name in ['features.npy','Vf.npy','raw_ids.npy']:
|
||||
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Prediction geometry/feature changed')
|
||||
x=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
|
||||
# No seconds/target file is opened in this phase.
|
||||
for name,m in available.items():
|
||||
if name=='naive':pred=np.load(source/'Vf.npy',allow_pickle=False)
|
||||
elif name=='GBRT':pred=m.predict(np.asarray(x,dtype=np.float32))
|
||||
else:
|
||||
chunks=[]
|
||||
with torch.no_grad():
|
||||
for a in range(0,len(x),65536):
|
||||
chunks.append(m(torch.from_numpy(np.array(x[a:a+65536],dtype=np.float32))).squeeze(-1).numpy())
|
||||
pred=np.concatenate(chunks)
|
||||
if pred.shape!=(r['nodes'],) or not np.isfinite(pred).all():raise ValueError('Invalid full-node prediction')
|
||||
folder=dest/f"split-{split['index']}"/name;folder.mkdir(parents=True,exist_ok=True)
|
||||
path=folder/(case+'.npy');np.save(path,np.asarray(pred,dtype=np.float64)*r['t_scale_seconds'])
|
||||
outputs[name]=dict(path=str(path),sha256=sha(path),nodes=len(pred),no_clipping=True)
|
||||
rows.append(dict(split=split['index'],case=case,status='PREDICTED',predictions=outputs,
|
||||
raw_ids_sha256=r['output_sha256']['raw_ids.npy']))
|
||||
save(root/'prediction-progress.json',dict(test_case_visits=len(rows),worker_CPU_s=time.process_time()-start))
|
||||
save(root/'predictions-freeze.json',dict(models_freeze_sha256=sha(root/'models-freeze.json'),rows=rows,
|
||||
worker_CPU_s=time.process_time()-start,test_targets_read=False,device='CPU',prediction_units='seconds'))
|
||||
|
||||
|
||||
def score(root):
|
||||
c,prep,splits=contract(root);frozen=root/'predictions-freeze.json'
|
||||
if not frozen.exists():raise ValueError('Predictions must all be frozen before scoring')
|
||||
freeze=json.loads(frozen.read_text());rows=[];start=time.process_time()
|
||||
for r in freeze['rows']:
|
||||
if r['status']!='PREDICTED':
|
||||
rows.append(dict(split=r['split'],case=r['case'],status=r['status'],scores={}));continue
|
||||
p=prep['successful'][r['case']];source=Path(p['output'])
|
||||
if sha(source/'seconds.npy')!=p['output_sha256']['seconds.npy']:raise ValueError('Frozen test targets changed')
|
||||
y=np.load(source/'seconds.npy',allow_pickle=False);scores={}
|
||||
for name,value in r['predictions'].items():
|
||||
if sha(value['path'])!=value['sha256']:raise ValueError('Frozen predictions changed')
|
||||
pred=np.load(value['path'],allow_pickle=False);scores[name]=per_geometry_metrics(y,pred)
|
||||
rows.append(dict(split=r['split'],case=r['case'],status='SCORED',scores=scores))
|
||||
summary={}
|
||||
for name in ['naive','GBRT','FNN']:
|
||||
summary[name]={}
|
||||
for metric in ['rrmse_rms','rrmse_maximum','rrmse_mean']:
|
||||
per=[]
|
||||
for split in splits:
|
||||
a=[r['scores'][name][metric] for r in rows if r['split']==split['index'] and name in r['scores']
|
||||
and r['scores'][name][metric] is not None]
|
||||
per.append(dict(split=split['index'],requested_test_geometries=len(split['test']),scored=len(a),
|
||||
missing=len(split['test'])-len(a),min=float(np.min(a)) if a else None,
|
||||
median=float(np.median(a)) if a else None,max=float(np.max(a)) if a else None))
|
||||
# Do not average only whichever model splits happened to finish.
|
||||
avg={k:float(np.mean([r[k] for r in per])) if all(r[k] is not None for r in per) else None for k in ['min','median','max']}
|
||||
summary[name][metric]=dict(per_split=per,mean_of_five_split_summaries=avg,
|
||||
full_requested_coverage=all(r['missing']==0 for r in per))
|
||||
save(root/'scores.json',dict(rows=rows,predictions_freeze_sha256=sha(frozen)))
|
||||
save(root/'score-summary.json',dict(dataset_version=c['dataset_version'],protocol=c['protocol'],
|
||||
status='SCORED_WITH_EXPLICIT_COVERAGE',summary=summary,worker_CPU_s=time.process_time()-start,
|
||||
node_domain='All retained tetrahedral nodes including walls, not S2 queries; auxiliary nodes separately reported',
|
||||
outside_tet_node_counts={case:r.get('outside_tet_nodes',0) for case,r in prep['successful'].items() if r.get('outside_tet_nodes',0)},
|
||||
scores_sha256=sha(root/'scores.json'),paper_numbers_diagnostic_only=True,
|
||||
limitations=['v5 data instead of paper v4','R1 discretisation choices are explicit supplements',
|
||||
't_set substitutes unavailable applied Q','FNN fill-only and early stopping are declared choices',
|
||||
'Random geometry splits may leak near-duplicate geometry families','Failed preparations not replaced; coverage is explicit']))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('phase',choices=['predict','score'])
|
||||
a=p.parse_args();(predict if a.phase=='predict' else score)(a.root)
|
||||
@@ -0,0 +1,142 @@
|
||||
"""One sequential fixed-model chain after terminal preparation, within shared caps."""
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
import psutil
|
||||
from gre24_prepare_v2 import sha
|
||||
from gre24_job_guard import supervise,save
|
||||
from gre24_gpu_guard import gpu_state,gpu_fit
|
||||
|
||||
|
||||
def freeze_preparation(root):
|
||||
complete=json.loads((root/'complete.json').read_text());c=json.loads((root/'contract.json').read_text())
|
||||
if complete['status']!='COMPLETE' or not complete['accounting_complete'] or len(complete['records'])!=len(c['parts']):
|
||||
raise ValueError('Preparation globally stopped or missing terminal accounting')
|
||||
good={};fail=[]
|
||||
for r in complete['records']:
|
||||
if r['worker']['status']!='PASS':
|
||||
fail.append(dict(case=r['case'],guard_status=r['worker']['status'],reason=r['worker']['stop_reason']));continue
|
||||
p=Path(r['result_path'])
|
||||
if sha(p)!=r['result_sha256']:raise ValueError('Preparation result changed')
|
||||
result=json.loads(p.read_text())
|
||||
good[r['case']]=dict(output=str(p.parent),nodes=result['geometry']['tet_nodes'],
|
||||
raw_nodes=result['geometry'].get('raw_nodes',result['geometry']['tet_nodes']),
|
||||
t_scale_seconds=result['t_scale_seconds'],output_sha256=result['output_sha256'],result_sha256=sha(p),
|
||||
outside_tet_nodes=result['geometry']['outside_tet_nodes'])
|
||||
frozen=dict(successful=good,failed=fail,requested=len(c['parts']),
|
||||
preparation_complete_sha256=sha(root/'complete.json'),replacement=False,
|
||||
cumulative_initial_worker_cpu_seconds=complete['worker_cpu_seconds']+complete['initial_cpu_reserve_seconds'])
|
||||
save(root/'preparation-freeze.json',frozen);return frozen
|
||||
|
||||
|
||||
def run(root,plan_path):
|
||||
root=Path(root);plan_path=Path(plan_path)
|
||||
with (root/'model-chain.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
return run_locked(root,plan_path)
|
||||
|
||||
|
||||
def run_locked(root,plan_path):
|
||||
plan=json.loads(plan_path.read_text())
|
||||
for p,h in plan['code_sha256'].items():
|
||||
if sha(p)!=h:raise ValueError('Model plan source changed: '+p)
|
||||
started=root/'model-chain-started.json'
|
||||
if started.exists():raise FileExistsError('Never restart model chain')
|
||||
save(started,dict(pid=os.getpid(),unix=time.time(),plan_sha256=sha(plan_path),stage='WAIT_PREPARATION'))
|
||||
missing_since=None
|
||||
while not (root/'complete.json').exists():
|
||||
parent=json.loads((root/'queue-started.json').read_text())
|
||||
try:
|
||||
process=psutil.Process(parent['pid'])
|
||||
alive=process.create_time()<=parent['unix'] and process.status()!=psutil.STATUS_ZOMBIE and any('run_gre24_protocol_preparation.py' in a for a in process.cmdline())
|
||||
except psutil.NoSuchProcess:alive=False
|
||||
if not alive:
|
||||
missing_since=missing_since or time.monotonic()
|
||||
if time.monotonic()-missing_since>30:raise RuntimeError('Preparation exited without complete receipt; no restart')
|
||||
else:missing_since=None
|
||||
save(root/'model-chain-progress.json',dict(stage='WAIT_PREPARATION',unix=time.time()))
|
||||
time.sleep(10)
|
||||
frozen=freeze_preparation(root)
|
||||
split_path=root/'splits.json';splits=json.loads(split_path.read_text())
|
||||
c=dict(plan,preparation_freeze_sha256=sha(root/'preparation-freeze.json'),splits_sha256=sha(split_path))
|
||||
save(root/'training-contract.json',c)
|
||||
spent=frozen['cumulative_initial_worker_cpu_seconds'];fits={};gpu_started=None;gpu_uuid=None;gpu_lock=None
|
||||
gpu_previous=plan.get('initial_gpu_seconds',0.)
|
||||
os.sched_setaffinity(0,range(8))
|
||||
for name in ['GBRT','FNN']:
|
||||
for split in splits:
|
||||
index=split['index'];key=f'{index}/{name}';dest=root/'fits'/f'split-{index}'/name
|
||||
remain=plan['total_worker_cpu_seconds']-spent-plan['postfit_cpu_reserve_seconds']
|
||||
if remain<=5:
|
||||
fits[key]=dict(status='NOT_ATTEMPTED',reason='SHARED_CPU_CAP_RESERVE');continue
|
||||
command=[sys.executable,'tools/gre24_fit.py',str(root),str(index),name]
|
||||
if name=='GBRT':
|
||||
cap=min(plan['xgb_per_fit_cpu_seconds'],remain)
|
||||
receipt=supervise(command,dest,'worker',dict(total_cpu_seconds=cap,job_cpu_seconds=cap,
|
||||
job_wall_seconds=7200,tree_rss_bytes=24*2**30,disk_bytes=3*2**30,max_processes=1),
|
||||
poll=.1,numerical_threads=plan['xgb_threads'])
|
||||
else:
|
||||
if gpu_started is None:gpu_started=time.monotonic()
|
||||
remaining_gpu=plan['total_gpu_seconds']-gpu_previous-(time.monotonic()-gpu_started)
|
||||
if remaining_gpu<=20:
|
||||
fits[key]=dict(status='NOT_ATTEMPTED',reason='SHARED_GPU_CAP');continue
|
||||
# Fixed one device once selected, three idle polls; waiting counts toward 2 GPU-hours.
|
||||
idle=0
|
||||
while idle<3 and time.monotonic()-gpu_started+gpu_previous<plan['total_gpu_seconds']-20:
|
||||
states=gpu_state(gpu_uuid)
|
||||
if gpu_uuid is None:
|
||||
candidates=sorted((s for s in states if s['used_bytes']<=128*2**20 and s['utilization']==0),key=lambda s:s['uuid'])
|
||||
if candidates:
|
||||
gpu_uuid=candidates[0]['uuid']
|
||||
gpu_lock=Path('/tmp/meltmind-gre24-'+gpu_uuid+'.lock').open('a')
|
||||
fcntl.flock(gpu_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
states=[candidates[0]]
|
||||
clean=bool(gpu_uuid) and states[0]['used_bytes']<=128*2**20 and states[0]['utilization']==0
|
||||
idle=idle+1 if clean else 0
|
||||
save(root/'gpu-idle-progress.json',dict(uuid=gpu_uuid,consecutive_idle=idle,
|
||||
charged_gpu_seconds=gpu_previous+time.monotonic()-gpu_started,states=states))
|
||||
if idle<3:time.sleep(5)
|
||||
remaining_gpu=plan['total_gpu_seconds']-gpu_previous-(time.monotonic()-gpu_started)
|
||||
if idle<3 or remaining_gpu<=20:
|
||||
fits[key]=dict(status='NOT_ATTEMPTED',reason='NO_IDLE_GPU_WITHIN_SHARED_CAP');continue
|
||||
receipt=gpu_fit(command,dest,dest,gpu_uuid,remaining_gpu-2,
|
||||
min(remain,plan['fnn_per_fit_cpu_seconds']),plan['gpu_reserved_byte_cap'])
|
||||
spent+=receipt['cpu_seconds']
|
||||
save(dest/'final-guard-receipt.json',receipt)
|
||||
result_path=dest/'result.json';result=json.loads(result_path.read_text()) if result_path.exists() else None
|
||||
passed=receipt['status']=='PASS' and result and result['status']=='FIT_COMPLETE_NO_TEST_SCORES'
|
||||
if passed and sha(result['checkpoint'])!=result['checkpoint_sha256']:raise ValueError('Checkpoint changed before freeze')
|
||||
fits[key]=dict(status='PASS' if passed else 'FAILED',guard_receipt=str(dest/'final-guard-receipt.json'),
|
||||
guard_receipt_sha256=sha(dest/'final-guard-receipt.json'),result=str(result_path) if result else None,
|
||||
checkpoint=result['checkpoint'] if passed else None,checkpoint_sha256=result['checkpoint_sha256'] if passed else None,
|
||||
failure_checkpoint_retained=bool(result) and not passed)
|
||||
save(root/'model-chain-progress.json',dict(stage='FITTING',completed=len(fits),fits=fits,
|
||||
charged_worker_cpu_seconds=spent,charged_gpu_seconds=gpu_previous+(time.monotonic()-gpu_started if gpu_started else 0.)))
|
||||
gpu_seconds=gpu_previous+(time.monotonic()-gpu_started if gpu_started else 0.)
|
||||
if gpu_lock:gpu_lock.close()
|
||||
save(root/'models-freeze.json',dict(fits=fits,charged_worker_cpu_seconds=spent,charged_gpu_seconds=gpu_seconds,
|
||||
uuid=gpu_uuid,training_contract_sha256=sha(root/'training-contract.json'),all_fits_terminal=True,test_scores_read=False))
|
||||
postfit=[]
|
||||
for phase in ['predict','score']:
|
||||
remain=plan['total_worker_cpu_seconds']-spent
|
||||
if remain<=5:
|
||||
postfit.append(dict(phase=phase,status='NOT_ATTEMPTED_CPU_CAP'));break
|
||||
receipt=supervise([sys.executable,'tools/gre24_score.py',str(root),phase],root/'postfit-guards'/phase,'worker',
|
||||
dict(total_cpu_seconds=remain,job_cpu_seconds=remain,job_wall_seconds=7200,tree_rss_bytes=24*2**30,
|
||||
disk_bytes=3*2**30,max_processes=1),poll=.2,numerical_threads=4)
|
||||
spent+=receipt['cpu_seconds'];postfit.append(dict(phase=phase,receipt=receipt))
|
||||
if receipt['status']!='PASS':break
|
||||
save(root/'reproduction-complete.json',dict(status='SCORED' if (root/'score-summary.json').exists() else 'INCOMPLETE',
|
||||
fits=fits,postfit=postfit,charged_worker_cpu_seconds=spent,charged_gpu_seconds=gpu_seconds,
|
||||
total_worker_cpu_cap=plan['total_worker_cpu_seconds'],total_gpu_cap=plan['total_gpu_seconds'],
|
||||
dataset_version=plan['dataset_version'],paper_numbers_diagnostic_only=True))
|
||||
return 0 if (root/'score-summary.json').exists() else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('plan',type=Path)
|
||||
a=p.parse_args();raise SystemExit(run(a.root,a.plan))
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Exercise the real chain with exhausted fit allocation: no model/GPU launches."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
from gre24_prepare_v2 import sha
|
||||
from s2_recovery_supervisor import save
|
||||
|
||||
|
||||
def main():
|
||||
root=Path('artifacts/gre24-reproduction-2026-11-15/budget-chain-test');root.mkdir(exist_ok=False)
|
||||
reference=Path('artifacts/gre24-reproduction-2026-11-15/integration-03')
|
||||
good=json.loads((reference/'preparation-freeze.json').read_text())['successful'];records=[]
|
||||
for case,p in good.items():
|
||||
result=root/(case+'-prep-result.json')
|
||||
save(result,dict(status='PASS',case=case,geometry=dict(tet_nodes=p['nodes'],outside_tet_nodes=0),
|
||||
t_scale_seconds=2.,output_sha256=p['output_sha256']))
|
||||
# Result's parent defines input root; use separate existing arrays via copied tiny files.
|
||||
folder=root/case;folder.mkdir();result=folder/'result.json'
|
||||
for file in Path(p['output']).glob('*.npy'):(folder/file.name).write_bytes(file.read_bytes())
|
||||
save(result,dict(status='PASS',case=case,geometry=dict(tet_nodes=p['nodes'],outside_tet_nodes=0),
|
||||
t_scale_seconds=2.,output_sha256=p['output_sha256']))
|
||||
records.append(dict(case=case,worker={'status':'PASS'},result_path=str(result),result_sha256=sha(result)))
|
||||
save(root/'contract.json',dict(parts=list(good)))
|
||||
save(root/'complete.json',dict(status='COMPLETE',accounting_complete=True,records=records,
|
||||
worker_cpu_seconds=0.,initial_cpu_reserve_seconds=0.))
|
||||
(root/'splits.json').write_bytes((reference/'splits.json').read_bytes())
|
||||
paths=['tools/run_gre24_models.py','tools/gre24_fit.py','tools/gre24_score.py','tools/gre24_models.py','tools/gre24_gpu_guard.py','tools/gre24_job_guard.py']
|
||||
plan=dict(code_sha256={p:sha(p) for p in paths},total_worker_cpu_seconds=1000,postfit_cpu_reserve_seconds=998,
|
||||
total_gpu_seconds=60,initial_gpu_seconds=0,dataset_version='synthetic_budget_only',protocol='synthetic',
|
||||
xgb_threads=1,fnn_cpu_threads=1,xgb_per_fit_cpu_seconds=10,fnn_per_fit_cpu_seconds=10,gpu_reserved_byte_cap=15*2**30)
|
||||
save(root/'plan.json',plan)
|
||||
subprocess.run([sys.executable,'tools/run_gre24_models.py',str(root),str(root/'plan.json')],check=True)
|
||||
result=json.loads((root/'reproduction-complete.json').read_text())
|
||||
assert all(v['status']=='NOT_ATTEMPTED' for v in result['fits'].values())
|
||||
assert result['charged_gpu_seconds']==0
|
||||
assert not list((root/'fits').glob('**/started.json')) if (root/'fits').exists() else True
|
||||
assert result['charged_worker_cpu_seconds']<1000
|
||||
save(root/'test-acceptance.json',dict(status='PASS',checks=['CPU reserve prevents all fits','no GPU starts',
|
||||
'terminal model freeze exists before prediction','naive score survives missing models','actual postfit CPU charged'],result=result))
|
||||
print('PASS actual chain budget boundary, zero GPU fits')
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Actual fit/predict/score entry points on tiny synthetic data, one GPU only."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import sys
|
||||
import time
|
||||
import numpy as np
|
||||
from gre24_prepare_v2 import sha
|
||||
from gre24_job_guard import supervise,save
|
||||
from gre24_gpu_guard import gpu_state,gpu_fit,recorded_peaks
|
||||
|
||||
|
||||
def main():
|
||||
root=Path(sys.argv[1]);root.mkdir(exist_ok=False)
|
||||
os.sched_setaffinity(0,[0,1,2,3]);rng=np.random.default_rng(91);good={}
|
||||
for case in ['learn','validation','test']:
|
||||
d=root/'parts'/case;d.mkdir(parents=True)
|
||||
x=rng.uniform(0,1,(128,19));y=.2+.1*x[:,0]+.1*x[:,1]
|
||||
values={'features':x,'Vf':np.linspace(0,1,128,endpoint=False),'raw_ids':np.arange(128),'seconds':2*y}
|
||||
if case!='test':values['target']=y # Deliberately absent for test: no fit/predict may need it.
|
||||
for name,v in values.items():np.save(d/(name+'.npy'),v)
|
||||
good[case]=dict(output=str(d),nodes=len(y),t_scale_seconds=2.,
|
||||
output_sha256={p.name:sha(p) for p in d.glob('*.npy')})
|
||||
splits=[dict(index=i,seed=91+i,train=['learn','validation'],test=['test'],fnn_learning=['learn'],fnn_validation=['validation']) for i in range(5)]
|
||||
save(root/'splits.json',splits);save(root/'preparation-freeze.json',dict(successful=good,failed=[]))
|
||||
paths=['tools/gre24_models.py','tools/gre24_fit.py','tools/gre24_score.py','tools/gre24_gpu_guard.py','tools/gre24_job_guard.py']
|
||||
c=dict(code_sha256={p:sha(p) for p in paths},preparation_freeze_sha256=sha(root/'preparation-freeze.json'),
|
||||
splits_sha256=sha(root/'splits.json'),dataset_version='synthetic_only',protocol='synthetic smoke, not scientific data',
|
||||
xgb_threads=1,fnn_cpu_threads=2,gpu_reserved_byte_cap=15*2**30)
|
||||
save(root/'training-contract.json',c)
|
||||
receipts={};models={};gpu_seconds=0
|
||||
for name in ['GBRT','FNN']:
|
||||
dest=root/'fits'/'split-0'/name;cmd=[sys.executable,'tools/gre24_fit.py',str(root),'0',name]
|
||||
if name=='GBRT':
|
||||
r=supervise(cmd,dest,'worker',dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120,
|
||||
tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1)
|
||||
else:
|
||||
begin=time.monotonic();available=sorted((s for s in gpu_state() if s['used_bytes']<128*2**20 and s['utilization']==0),key=lambda s:s['uuid'])
|
||||
if not available:raise RuntimeError('No idle GPU for bounded synthetic acceptance')
|
||||
uuid=available[0]['uuid']
|
||||
r=gpu_fit(cmd,dest,dest,uuid,60.,120.,15*2**30);gpu_seconds=time.monotonic()-begin
|
||||
receipts[name]=r;save(root/(name+'-guard.json'),r)
|
||||
if r['status']!='PASS':
|
||||
save(root/'acceptance.json',dict(status='FAILED',receipts=receipts,GPU_seconds=gpu_seconds));raise RuntimeError(name+' integration failed')
|
||||
result=json.loads((dest/'result.json').read_text())
|
||||
access=json.loads((dest/'training-target-access.json').read_text())
|
||||
assert all(a['case']!='test' for a in access)
|
||||
models['0/'+name]=dict(status='PASS',checkpoint=result['checkpoint'],checkpoint_sha256=result['checkpoint_sha256'])
|
||||
for i in range(1,5):
|
||||
for name in ['GBRT','FNN']:models[f'{i}/{name}']=dict(status='NOT_ATTEMPTED_SYNTHETIC')
|
||||
save(root/'models-freeze.json',dict(fits=models))
|
||||
test_seconds=root/'parts/test/seconds.npy';hidden=root/'parts/test/seconds.unavailable'
|
||||
test_seconds.rename(hidden)
|
||||
try:
|
||||
r=supervise([sys.executable,'tools/gre24_score.py',str(root),'predict'],root/'predict-guard','worker',
|
||||
dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120,tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1)
|
||||
receipts['predict']=r
|
||||
if r['status']!='PASS':raise RuntimeError('Prediction failed with test seconds unavailable')
|
||||
finally:hidden.rename(test_seconds)
|
||||
r=supervise([sys.executable,'tools/gre24_score.py',str(root),'score'],root/'score-guard','worker',
|
||||
dict(total_cpu_seconds=120,job_cpu_seconds=120,job_wall_seconds=120,tree_rss_bytes=4*2**30,disk_bytes=2**28,max_processes=1),poll=.1)
|
||||
receipts['score']=r;assert r['status']=='PASS'
|
||||
summary=json.loads((root/'score-summary.json').read_text())['summary']
|
||||
assert summary['naive']['rrmse_rms']['full_requested_coverage']
|
||||
assert summary['FNN']['rrmse_rms']['mean_of_five_split_summaries']['median'] is None
|
||||
# Failed-result records still contribute their peaks, independently of live memory.
|
||||
fake=root/'peak-check';fake.mkdir();save(fake/'failure.json',dict(peak_reserved_bytes=16*2**30))
|
||||
peaks,witness=recorded_peaks(fake);assert peaks['peak_reserved_bytes']==16*2**30
|
||||
save(root/'acceptance.json',dict(status='PASS',receipts=receipts,GPU_seconds=gpu_seconds,
|
||||
worker_CPU_seconds=sum(r['cpu_seconds'] for r in receipts.values()),
|
||||
checks=['real GBRT and FNN entrypoints','test target absent during fits','test seconds unavailable during prediction',
|
||||
'all predictions frozen before scoring','missing splits not silently averaged','failure VRAM peaks retained'],
|
||||
scientific_data_read=False,coordinator_CPU_seconds=sum(resource.getrusage(resource.RUSAGE_SELF)[:2])))
|
||||
print(json.dumps(dict(status='PASS',GPU_seconds=gpu_seconds,worker_CPU_seconds=sum(r['cpu_seconds'] for r in receipts.values()))))
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Role isolation, full-row learning, early stopping and real guard receipts."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import numpy as np
|
||||
from gre24_models import xgb_model,fnn_model,eligible_roles,require_training_role,early_stop_update
|
||||
from gre24_job_guard import supervise
|
||||
|
||||
|
||||
class ModelChecks(unittest.TestCase):
|
||||
def test_role_isolation_and_missing_no_replacement(self):
|
||||
s=dict(train=['a','b'],test=['c','d'],fnn_learning=['a'],fnn_validation=['b'])
|
||||
roles,missing=eligible_roles(s,['a','c','d'])
|
||||
self.assertEqual(roles['train'],['a']);self.assertEqual(missing['train'],['b'])
|
||||
with self.assertRaises(PermissionError):require_training_role('c',s,'train')
|
||||
with self.assertRaises(PermissionError):require_training_role('a',s,'test')
|
||||
|
||||
def test_earlystop_strict_improvement_patience(self):
|
||||
best=1.;stale=0
|
||||
for i in range(5):
|
||||
best,stale,improved,stop=early_stop_update(best,stale,1.)
|
||||
self.assertFalse(improved);self.assertEqual(stop,i==4)
|
||||
best,stale,improved,stop=early_stop_update(best,stale,.9)
|
||||
self.assertTrue(improved);self.assertEqual(stale,0);self.assertFalse(stop)
|
||||
with self.assertRaises(ValueError):early_stop_update(best,stale,float('nan'))
|
||||
|
||||
def test_xgboost_fixed_recipe_learns_all_rows(self):
|
||||
rng=np.random.default_rng(42);x=rng.normal(size=(512,19)).astype(np.float32);y=x[:,0]+.5*x[:,1]
|
||||
m=xgb_model(42,threads=1);m.fit(x,y)
|
||||
self.assertLess(np.mean((m.predict(x)-y)**2),.02)
|
||||
self.assertEqual(m.n_estimators,100);self.assertEqual(m.max_depth,8)
|
||||
|
||||
def test_fnn_architecture_gradients_and_checkpoint_restore(self):
|
||||
import torch
|
||||
torch.set_num_threads(1);torch.manual_seed(42);m=fnn_model()
|
||||
linear=[a for a in m if isinstance(a,torch.nn.Linear)]
|
||||
self.assertEqual([(a.in_features,a.out_features) for a in linear],[(19,100),(100,100),(100,100),(100,100),(100,1)])
|
||||
x=torch.randn(64,19);y=torch.ones(64,1);o=torch.optim.Adam(m.parameters(),lr=.001)
|
||||
before=float(((m(x)-y)**2).mean())
|
||||
for _ in range(15):
|
||||
o.zero_grad();loss=((m(x)-y)**2).mean();loss.backward();o.step()
|
||||
self.assertLess(float(((m(x)-y)**2).mean()),before)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
p=Path(tmp)/'m.pt';torch.save(m.state_dict(),p);n=fnn_model();n.load_state_dict(torch.load(p,weights_only=True))
|
||||
torch.testing.assert_close(m(x),n(x))
|
||||
|
||||
def test_guard_records_failure_cpu_and_refuses_repeat(self):
|
||||
limits=dict(total_cpu_seconds=10,job_cpu_seconds=5,job_wall_seconds=5,tree_rss_bytes=256*2**20,disk_bytes=2**20,max_processes=1)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
r=supervise([sys.executable,'-c','raise SystemExit(7)'],Path(tmp),'fail',limits,poll=.02)
|
||||
self.assertEqual(r['returncode'],7);self.assertGreater(r['cpu_seconds'],0)
|
||||
self.assertEqual(r['status'],'STOPPED_OR_FAILED')
|
||||
with self.assertRaises(FileExistsError):supervise([sys.executable,'-c','pass'],Path(tmp),'fail',limits)
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
result=unittest.TextTestRunner(verbosity=2).run(unittest.defaultTestLoader.loadTestsFromTestCase(ModelChecks))
|
||||
u=resource.getrusage(resource.RUSAGE_SELF);ch=resource.getrusage(resource.RUSAGE_CHILDREN)
|
||||
Path('artifacts/gre24-reproduction-2026-11-15/model-synthetic-receipt.json').write_text(json.dumps(dict(
|
||||
tests=result.testsRun,passed=result.wasSuccessful(),failures=len(result.failures),errors=len(result.errors),
|
||||
CPU_s=u.ru_utime+u.ru_stime+ch.ru_utime+ch.ru_stime,GPU_used=False),indent=2))
|
||||
raise SystemExit(0 if result.wasSuccessful() else 1)
|
||||
Reference in new issue
Block a user