Files
MeltMind/tools/run_s2_pilot_training_v2.py

151 lines
11 KiB
Python

"""One fixed pilot arm/width. Must be launched inside the GPU resource guard."""
import argparse
from collections import Counter
from collections.abc import Mapping
import json
from pathlib import Path
import random
import time
import traceback
import numpy as np
import torch
from freeze_g1_components import sha
from s2_e1_supervisor import save
from s2_models_v2.network import QueryModel,TRAINING_CONTRACT
from s2_models.normalization import BaseStandardizer,fit_base_standardizer
from s2_models.objectives import relative_l2
from s2_pilot_data import permitted_role
from s2_pilot_train import part_weights,train_part,component_score,SkipPolicy
class LearningBases(Mapping):
def __init__(self,records,ids,roles,cache=None):self.records=records;self.ids=tuple(ids);self.roles=roles;self.cache=cache
def __iter__(self):return iter(self.ids)
def __len__(self):return len(self.ids)
def __getitem__(self,case):
if case not in self.ids or permitted_role(self.roles,case)!='learning':raise PermissionError('Learning-only normalization')
if self.cache is not None:return self.cache[case]['base']
assert sha(Path(self.records[case]['preparation_root'])/'base.npy')==self.records[case]['input_sha256']['base.npy']
return np.load(Path(self.records[case]['preparation_root'])/'base.npy',allow_pickle=False,mmap_mode='r')
def read_part_arrays(record,roles,*,synthetic_targets=False):
case=record['case'];role=permitted_role(roles,case)
if synthetic_targets and role!='learning':raise PermissionError('Timing probe accepts only learning parts')
root=Path(record['preparation_root']);label=Path(record['label_path'])
assert sha(root/'base.npy')==record['input_sha256']['base.npy']
assert sha(root/'queries.npz')==record['input_sha256']['queries.npz']
with np.load(root/'queries.npz',allow_pickle=False) as q:
mask=q['eligible_s2'];ids=q['anchor_ids'][mask];g=q['g'][mask];edge=q['edge_summary'][mask];counts=q['multiplicity'][mask]
if synthetic_targets:
target=np.zeros(len(ids));seconds=None
else:
assert sha(label)==record['input_sha256']['labels.npz']
with np.load(label,allow_pickle=False) as z:
assert np.array_equal(ids,z['anchor_ids']) and np.array_equal(counts,z['multiplicity'])
target=z['target_t_star'];seconds=z['seconds']
base=np.load(root/'base.npy',allow_pickle=False)
if len(base)!=record['nodes']:raise ValueError('Full-node base size mismatch')
return dict(base=base,ids=ids,gaps=g,edge=edge,target=target,counts=counts,truth_seconds=seconds)
def to_device(data,device):
result={k:v for k,v in data.items() if k not in ['base','ids','gaps','edge','target']}
for key in ['base','ids','gaps','edge','target']:
result[key]=torch.as_tensor(np.array(data[key],copy=True),dtype=torch.long if key=='ids' else torch.float32,device=device)
return result
def load_part(record,roles,device):
return to_device(read_part_arrays(record,roles),device)
def run(root,index):
started=time.monotonic();c=json.loads((root/'training-contract.json').read_text());run=c['runs'][index]
dest=root/run['name'];dest.mkdir(exist_ok=False);save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
try:
for p,h in c['code_sha256'].items():assert sha(p)==h,p
source=Path(c['success_manifest']);assert sha(source)==c['success_manifest_sha256']
manifest=json.loads(source.read_text());records={r['case']:r for r in manifest['ready_parts']}
roles=json.loads(Path(c['role_manifest']).read_text());assert sha(c['role_manifest'])==c['role_manifest_sha256']
learning=sorted(p for p in records if permitted_role(roles,p)=='learning')
validation=sorted(p for p in records if permitted_role(roles,p)=='validation')
if not learning or not validation:raise ValueError('Empty successful learning/validation roster')
if any(p in roles['sealed_test_parts'] for p in records):raise PermissionError('Test case present in ready manifest')
assert run['seed']==0 and c['fold']==0 and c['pool']==100
torch.set_num_threads(1);random.seed(run['seed']);np.random.seed(run['seed']);torch.manual_seed(run['seed'])
if not torch.cuda.is_available() or torch.cuda.device_count()!=1:raise RuntimeError('Exactly one visible CUDA device required')
device=torch.device('cuda:0');torch.cuda.manual_seed_all(run['seed']);torch.cuda.reset_peak_memory_stats()
torch.backends.cuda.matmul.allow_tf32=False
if 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Frozen pilot requires one V100')
# Check hashes once, cache raw FP64 arrays, and fit only the learning subset.
cache={case:read_part_arrays(records[case],roles) for case in sorted(records)}
stats=fit_base_standardizer(LearningBases(records,learning,roles,cache),learning_part_ids=learning,fold='0',pool='100')
assert stats.fingerprint==c['standardizer_fingerprint'],'Probe and actual runs must share learning statistics'
save(dest/'standardizer.json',dict(**stats.as_dict(),fingerprint=stats.fingerprint))
shared=root/'shared-standardizer-fingerprint.json'
if shared.exists():assert json.loads(shared.read_text())['fingerprint']==stats.fingerprint
else:save(shared,dict(fingerprint=stats.fingerprint,learning_parts=learning))
model=QueryModel(run['arm'],run['width'],standardizer=stats).to(device)
optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5)
epoch_limit=int(c['E_max']);assert 1<=epoch_limit<=200 and c['cosine_T_max']==epoch_limit
assert sha(c['timing_probe_receipt'])==c['timing_probe_sha256']
scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=epoch_limit)
scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(run['seed']))
group_counts=Counter(records[p]['component_id'] for p in learning);group_number=len(group_counts)
val_groups={p:records[p]['component_id'] for p in validation}
best=float('inf');best_epoch=None;stale=0;history=[];updates=0;attempted_updates=0;skip_policy=SkipPolicy()
for epoch in range(epoch_limit):
model.train();loss_sum=0.;skip_policy.start_epoch()
order=[learning[int(j)] for j in rng.permutation(len(learning))]
for j,case in enumerate(order):
record=records[case];data=to_device(cache[case],device)
weights=torch.tensor(part_weights(data['counts'],group_counts[record['component_id']],group_number,len(learning)),dtype=torch.float32,device=device)
step_info={}
loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=step_info)
loss_sum+=loss/len(learning);attempted_updates+=1;updates+=int(not step_info['step_skipped'])
skip_policy.observe(step_info['step_skipped'])
del data,weights
if j%16==0:
torch.cuda.synchronize();save(dest/'progress.json',dict(stage='training',epoch=epoch,parts_done=j+1,parts_total=len(learning),updates=updates,elapsed_seconds=time.monotonic()-started,
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()))
amp_summary=skip_policy.summary();save(dest/f'epoch-{epoch:03d}-amp.json',amp_summary)
skip_policy.finish_epoch(epoch)
model.eval();errors={}
with torch.no_grad():
for case in validation:
data=to_device(cache[case],device)
with torch.autocast(device_type='cuda',enabled=False):
pred=model(data['base'],data['ids'],data['gaps'],data['edge'] if run['arm']=='E' else None)
predicted_seconds=pred.float().cpu().numpy().astype(np.float64)*records[case]['t_set']
score=relative_l2(predicted_seconds,data['truth_seconds'],data['counts'],np.ones(len(data['counts']),bool))
errors[case]=score['value'];del data,pred
value=component_score(errors,val_groups);torch.cuda.synchronize()
row=dict(epoch=epoch,amp=amp_summary,attempted_updates=attempted_updates,training_global_weighted_mse=loss_sum,validation_component_equal_relative_l2=value,updates=updates,elapsed_seconds=time.monotonic()-started,
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())
history.append(row);save(dest/'history.json',history)
if value<best:
best=value;best_epoch=epoch;stale=0
torch.save(dict(state_dict={k:v.detach().cpu() for k,v in model.state_dict().items()},arm=run['arm'],width=run['width'],seed=run['seed'],epoch=epoch,
standardizer=stats.as_dict(),standardizer_fingerprint=stats.fingerprint),dest/'best.pt')
save(dest/'best-validation-parts.json',errors)
else:stale+=1
scheduler.step()
save(dest/'progress.json',dict(stage='epoch_complete',**row,patience_count=stale))
if stale>=20:break
torch.cuda.synchronize()
result=dict(status='COMPLETE',run=run,E_max=epoch_limit,cosine_T_max=epoch_limit,epochs=len(history),attempted_updates=attempted_updates,updates=updates,best_epoch=best_epoch,best_validation_score=best,
learning_parts=len(learning),validation_parts=len(validation),learning_components=group_number,validation_components=len(set(val_groups.values())),
failed_preparation_parts_dropped=manifest['failed_parts'],standardizer_fingerprint=stats.fingerprint,
wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(),
test_labels_read=False,test_predictions=False,capacity_selected=False,checkpoint_sha256=sha(dest/'best.pt'))
save(dest/'result.json',result);print(json.dumps(result),flush=True);return 0
except Exception as exc:
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-started,
epoch=locals().get('epoch'),amp=skip_policy.summary() if 'skip_policy' in locals() else None))
raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))