Files
MeltMind/tools/run_s2_pilot_training.py
T

127 lines
8.9 KiB
Python

"""One fixed pilot arm/width. Must be launched inside the GPU resource guard."""
import argparse
from collections import Counter
from collections.abc import Mapping
import json
from pathlib import Path
import random
import time
import traceback
import numpy as np
import torch
from freeze_g1_components import sha
from s2_e1_supervisor import save
from s2_models.network import QueryModel,TRAINING_CONTRACT
from s2_models.normalization import BaseStandardizer,fit_base_standardizer
from s2_models.objectives import relative_l2
from s2_pilot_data import permitted_role
from s2_pilot_train import part_weights,train_part,component_score
class LearningBases(Mapping):
def __init__(self,records,ids,roles):self.records=records;self.ids=tuple(ids);self.roles=roles
def __iter__(self):return iter(self.ids)
def __len__(self):return len(self.ids)
def __getitem__(self,case):
if case not in self.ids or permitted_role(self.roles,case)!='learning':raise PermissionError('Learning-only normalization')
return np.load(Path(self.records[case]['preparation_root'])/'base.npy',allow_pickle=False,mmap_mode='r')
def load_part(record,roles,device):
case=record['case'];permitted_role(roles,case)
root=Path(record['preparation_root']);label=Path(record['label_path'])
assert sha(root/'base.npy')==record['input_sha256']['base.npy']
assert sha(root/'queries.npz')==record['input_sha256']['queries.npz']
assert sha(label)==record['input_sha256']['labels.npz']
with np.load(root/'queries.npz',allow_pickle=False) as q:
mask=q['eligible_s2'];ids=q['anchor_ids'][mask];g=q['g'][mask];edge=q['edge_summary'][mask];counts=q['multiplicity'][mask]
with np.load(label,allow_pickle=False) as z:
assert np.array_equal(ids,z['anchor_ids']) and np.array_equal(counts,z['multiplicity'])
target=z['target_t_star'];seconds=z['seconds']
base=np.load(root/'base.npy',allow_pickle=False)
if len(base)!=record['nodes']:raise ValueError('Full-node base size mismatch')
tensor=lambda a,dtype=torch.float32:torch.as_tensor(np.array(a,copy=True),dtype=dtype,device=device)
return dict(base=tensor(base),ids=tensor(ids,torch.long),gaps=tensor(g),edge=tensor(edge),target=tensor(target),
counts=counts,truth_seconds=seconds)
def run(root,index):
started=time.monotonic();c=json.loads((root/'training-contract.json').read_text());run=c['runs'][index]
dest=root/run['name'];dest.mkdir(exist_ok=False);save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
try:
for p,h in c['code_sha256'].items():assert sha(p)==h,p
source=Path(c['success_manifest']);assert sha(source)==c['success_manifest_sha256']
manifest=json.loads(source.read_text());records={r['case']:r for r in manifest['ready_parts']}
roles=json.loads(Path(c['role_manifest']).read_text());assert sha(c['role_manifest'])==c['role_manifest_sha256']
learning=sorted(p for p in records if permitted_role(roles,p)=='learning')
validation=sorted(p for p in records if permitted_role(roles,p)=='validation')
if not learning or not validation:raise ValueError('Empty successful learning/validation roster')
if any(p in roles['sealed_test_parts'] for p in records):raise PermissionError('Test case present in ready manifest')
assert run['seed']==0 and c['fold']==0 and c['pool']==100
torch.set_num_threads(1);random.seed(run['seed']);np.random.seed(run['seed']);torch.manual_seed(run['seed'])
if not torch.cuda.is_available() or torch.cuda.device_count()!=1:raise RuntimeError('Exactly one visible CUDA device required')
device=torch.device('cuda:0');torch.cuda.manual_seed_all(run['seed']);torch.cuda.reset_peak_memory_stats()
torch.backends.cuda.matmul.allow_tf32=False
if 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Frozen pilot requires one V100')
# Every base statistics calculation is learning-only and inside this run's wall guard.
stats=fit_base_standardizer(LearningBases(records,learning,roles),learning_part_ids=learning,fold='0',pool='100')
save(dest/'standardizer.json',dict(**stats.as_dict(),fingerprint=stats.fingerprint))
shared=root/'shared-standardizer-fingerprint.json'
if shared.exists():assert json.loads(shared.read_text())['fingerprint']==stats.fingerprint
else:save(shared,dict(fingerprint=stats.fingerprint,learning_parts=learning))
model=QueryModel(run['arm'],run['width'],standardizer=stats).to(device)
optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5)
scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=200)
scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(run['seed']))
group_counts=Counter(records[p]['component_id'] for p in learning);group_number=len(group_counts)
val_groups={p:records[p]['component_id'] for p in validation}
best=float('inf');best_epoch=None;stale=0;history=[];updates=0
for epoch in range(200):
model.train();loss_sum=0.
order=[learning[int(j)] for j in rng.permutation(len(learning))]
for j,case in enumerate(order):
record=records[case];data=load_part(record,roles,device)
weights=torch.tensor(part_weights(data['counts'],group_counts[record['component_id']],group_number,len(learning)),dtype=torch.float32,device=device)
loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True)
loss_sum+=loss/len(learning);updates+=1
del data,weights
if j%16==0:
torch.cuda.synchronize();save(dest/'progress.json',dict(stage='training',epoch=epoch,parts_done=j+1,parts_total=len(learning),updates=updates,elapsed_seconds=time.monotonic()-started,
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()))
model.eval();errors={}
with torch.no_grad():
for case in validation:
data=load_part(records[case],roles,device)
with torch.autocast(device_type='cuda',dtype=torch.float16):
pred=model(data['base'],data['ids'],data['gaps'],data['edge'] if run['arm']=='E' else None)
predicted_seconds=pred.float().cpu().numpy().astype(np.float64)*records[case]['t_set']
score=relative_l2(predicted_seconds,data['truth_seconds'],data['counts'],np.ones(len(data['counts']),bool))
errors[case]=score['value'];del data,pred
value=component_score(errors,val_groups);torch.cuda.synchronize()
row=dict(epoch=epoch,training_global_weighted_mse=loss_sum,validation_component_equal_relative_l2=value,updates=updates,elapsed_seconds=time.monotonic()-started,
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())
history.append(row);save(dest/'history.json',history)
if value<best:
best=value;best_epoch=epoch;stale=0
torch.save(dict(state_dict={k:v.detach().cpu() for k,v in model.state_dict().items()},arm=run['arm'],width=run['width'],seed=run['seed'],epoch=epoch,
standardizer=stats.as_dict(),standardizer_fingerprint=stats.fingerprint),dest/'best.pt')
save(dest/'best-validation-parts.json',errors)
else:stale+=1
scheduler.step()
save(dest/'progress.json',dict(stage='epoch_complete',**row,patience_count=stale))
if stale>=20:break
torch.cuda.synchronize()
result=dict(status='COMPLETE',run=run,epochs=len(history),updates=updates,best_epoch=best_epoch,best_validation_score=best,
learning_parts=len(learning),validation_parts=len(validation),learning_components=group_number,validation_components=len(set(val_groups.values())),
failed_preparation_parts_dropped=manifest['failed_parts'],standardizer_fingerprint=stats.fingerprint,
wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(),
test_labels_read=False,test_predictions=False,capacity_selected=False,checkpoint_sha256=sha(dest/'best.pt'))
save(dest/'result.json',result);print(json.dumps(result),flush=True);return 0
except Exception as exc:
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-started))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))