Files
MeltMind/tools/gre24_fit.py

140 lines
8.7 KiB
Python

"""One fixed split/model fit; reads training-side targets only, no test scores."""
import argparse
import json
import os
from pathlib import Path
import time
import traceback
import numpy as np
from gre24_prepare_v2 import sha
from gre24_models import xgb_model,fnn_model,eligible_roles,require_training_role,early_stop_update
from s2_recovery_supervisor import save
def contract(root):
c=json.loads((root/'training-contract.json').read_text())
for p,h in c['code_sha256'].items():
if sha(p)!=h:raise ValueError('Pinned training source changed: '+p)
if sha(root/'preparation-freeze.json')!=c['preparation_freeze_sha256']:
raise ValueError('Preparation freeze changed')
if sha(root/'splits.json')!=c['splits_sha256']:raise ValueError('Splits changed')
return c,json.loads((root/'preparation-freeze.json').read_text()),json.loads((root/'splits.json').read_text())
def training_arrays(root,freeze,split,role,read_log):
selected=[case for case in split[role] if case in freeze['successful']]
n=sum(freeze['successful'][case]['nodes'] for case in selected)
if not n:raise ValueError('No successful training rows')
x=np.empty((n,19),dtype=np.float32);y=np.empty(n,dtype=np.float32);cursor=0
for case in selected:
require_training_role(case,split,role)
r=freeze['successful'][case];source=Path(r['output'])
for name in ['features.npy','target.npy']:
if sha(source/name)!=r['output_sha256'][name]:raise ValueError('Frozen preparation changed')
a=np.load(source/'features.npy',mmap_mode='r',allow_pickle=False)
b=np.load(source/'target.npy',mmap_mode='r',allow_pickle=False)
count=len(b)
if a.shape!=(count,19) or count!=r['nodes']:raise ValueError('Training row identity mismatch')
x[cursor:cursor+count]=a;y[cursor:cursor+count]=b;cursor+=count
read_log.append(dict(case=case,role=role,nodes=count,source=str(source),
feature_sha256=r['output_sha256']['features.npy'],target_sha256=r['output_sha256']['target.npy']))
if not np.isfinite(x).all() or not np.isfinite(y).all():raise ValueError('Nonfinite training input')
return x,y
def fit(root,index,model_name):
c,f,splits=contract(root);split=splits[index]
dest=root/'fits'/f'split-{index}'/model_name;dest.mkdir(parents=True,exist_ok=True)
if (dest/'started.json').exists():raise FileExistsError('No fit restart or overwrite')
start=time.monotonic();cpu=time.process_time();read_log=[]
def progress(stage,**kw):
record=dict(stage=stage,pid=os.getpid(),unix=time.time(),split=index,model=model_name,
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start)
record.update(kw);save(dest/'progress.json',record)
save(dest/'started.json',dict(pid=os.getpid(),unix=time.time(),split=index,model=model_name,seed=split['seed'],
training_contract_sha256=sha(root/'training-contract.json')))
roles,missing=eligible_roles(split,f['successful'])
try:
if model_name=='GBRT':
progress('LOAD_TRAIN_ONLY')
x,y=training_arrays(root,f,split,'train',read_log)
save(dest/'training-target-access.json',read_log)
model=xgb_model(split['seed'],c['xgb_threads'])
progress('FIT',nodes=len(y),geometries=len(roles['train']))
model.fit(x,y,verbose=False)
checkpoint=dest/'model.ubj';model.save_model(checkpoint)
(dest/'xgboost-config.json').write_text(model.get_booster().save_config())
params=model.get_params()
# Preserve XGBoost's default missing=NaN as an explicit textual parameter;
# this is not a nonfinite observation or a change to the fitted estimator.
params={k:('NaN (library default missing marker)' if isinstance(v,float) and np.isnan(v) else v)
for k,v in params.items()}
extra=dict(train_nodes=len(y),train_geometries=len(roles['train']),parameters=params)
elif model_name=='FNN':
import torch
torch.set_num_threads(c['fnn_cpu_threads']);torch.manual_seed(split['seed'])
if not torch.cuda.is_available():raise RuntimeError('Selected CUDA GPU unavailable')
torch.cuda.set_device(0) # Initialise the selected device before resetting its allocator counters.
device=torch.device('cuda:0');torch.cuda.reset_peak_memory_stats(device)
torch.backends.cuda.matmul.allow_tf32=False
progress('LOAD_FNN_TRAIN_VALIDATION_ONLY')
x,y=training_arrays(root,f,split,'fnn_learning',read_log)
vx,vy=training_arrays(root,f,split,'fnn_validation',read_log)
save(dest/'training-target-access.json',read_log)
x=torch.as_tensor(x,device=device);y=torch.as_tensor(y,device=device)
vx=torch.as_tensor(vx,device=device);vy=torch.as_tensor(vy,device=device)
model=fnn_model().to(device);optimizer=torch.optim.Adam(model.parameters(),lr=.001)
best=float('inf');stale=0;best_epoch=None;history=[];checkpoint=dest/'model.pt'
for epoch in range(1,31):
model.train();order=torch.randperm(len(y),device=device);total=torch.zeros((),device=device)
for begin in range(0,len(y),1024):
ids=order[begin:begin+1024];optimizer.zero_grad(set_to_none=True)
prediction=model(x[ids]).squeeze(-1);loss=torch.mean((prediction-y[ids])**2)
if not torch.isfinite(loss):raise ValueError('Nonfinite training loss')
loss.backward();optimizer.step();total+=loss.detach()*len(ids)
model.eval();sse=torch.zeros((),device=device)
with torch.no_grad():
for begin in range(0,len(vy),65536):
sse+=torch.sum((model(vx[begin:begin+65536]).squeeze(-1)-vy[begin:begin+65536])**2)
validation=float((sse/len(vy)).item());train=float((total/len(y)).item())
best,stale,improved,stop=early_stop_update(best,stale,validation)
if improved:
torch.save({'state_dict':{k:v.detach().cpu() for k,v in model.state_dict().items()},
'epoch':epoch,'validation_mse':validation,'seed':split['seed']},checkpoint)
best_epoch=epoch
row=dict(epoch=epoch,training_mse=train,validation_mse=validation,best_epoch=best_epoch,
stale_epochs=stale,peak_allocated_bytes=torch.cuda.max_memory_allocated(device),
peak_reserved_bytes=torch.cuda.max_memory_reserved(device),wall_s=time.monotonic()-start)
history.append(row);save(dest/'history.json',history);progress('EPOCH_COMPLETE',**row)
if row['peak_reserved_bytes']>c['gpu_reserved_byte_cap']:raise RuntimeError('Recorded VRAM peak exceeds cap')
if stop:break
restored=torch.load(checkpoint,map_location='cpu',weights_only=True)
model.load_state_dict(restored['state_dict'])
extra=dict(epochs=len(history),best_epoch=best_epoch,best_validation_mse=best,
train_nodes=len(y),validation_nodes=len(vy),train_geometries=len(roles['fnn_learning']),
validation_geometries=len(roles['fnn_validation']),restored_best=True,amp=False,
input_standardization='none; use eq12 columns',peak_allocated_bytes=torch.cuda.max_memory_allocated(device),
peak_reserved_bytes=torch.cuda.max_memory_reserved(device))
else:raise ValueError('Unknown model')
save(dest/'result.json',dict(status='FIT_COMPLETE_NO_TEST_SCORES',split=index,model=model_name,
checkpoint=str(checkpoint),checkpoint_sha256=sha(checkpoint),missing_preparation=missing,
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,**extra))
progress('FIT_COMPLETE_NO_TEST_SCORES');return 0
except Exception as exc:
peaks={}
if model_name=='FNN':
try:
import torch
if torch.cuda.is_initialized():
peaks=dict(peak_allocated_bytes=torch.cuda.max_memory_allocated(),
peak_reserved_bytes=torch.cuda.max_memory_reserved())
except Exception as peak_error:peaks={'peak_read_error':str(peak_error)}
save(dest/'failure.json',dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),
CPU_s=time.process_time()-cpu,wall_s=time.monotonic()-start,automatic_retry=False,**peaks))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('split',type=int);p.add_argument('model',choices=['GBRT','FNN'])
a=p.parse_args();raise SystemExit(fit(a.root,a.split,a.model))