149 lines
11 KiB
Python
149 lines
11 KiB
Python
"""One fresh frozen matrix cell; unchanged v2 numerical training loop inside the GPU guard."""
|
|
import argparse
|
|
from collections import Counter
|
|
from collections.abc import Mapping
|
|
import json
|
|
from pathlib import Path
|
|
import random
|
|
import time
|
|
import traceback
|
|
import numpy as np
|
|
import torch
|
|
from freeze_g1_components import sha
|
|
from s2_e1_supervisor import save
|
|
from s2_models_v2.network import QueryModel,TRAINING_CONTRACT
|
|
from s2_models.normalization import BaseStandardizer,fit_base_standardizer
|
|
from s2_models.objectives import relative_l2
|
|
from s2_pilot_data import permitted_role
|
|
from s2_main_data import cell_records
|
|
from s2_pilot_train import part_weights,train_part,component_score,SkipPolicy
|
|
|
|
|
|
class LearningBases(Mapping):
|
|
def __init__(self,records,ids,roles,cache=None):self.records=records;self.ids=tuple(ids);self.roles=roles;self.cache=cache
|
|
def __iter__(self):return iter(self.ids)
|
|
def __len__(self):return len(self.ids)
|
|
def __getitem__(self,case):
|
|
if case not in self.ids or permitted_role(self.roles,case)!='learning':raise PermissionError('Learning-only normalization')
|
|
if self.cache is not None:return self.cache[case]['base']
|
|
assert sha(Path(self.records[case]['preparation_root'])/'base.npy')==self.records[case]['input_sha256']['base.npy']
|
|
return np.load(Path(self.records[case]['preparation_root'])/'base.npy',allow_pickle=False,mmap_mode='r')
|
|
|
|
|
|
def read_part_arrays(record,roles,*,synthetic_targets=False):
|
|
case=record['case'];role=permitted_role(roles,case)
|
|
if synthetic_targets and role!='learning':raise PermissionError('Timing probe accepts only learning parts')
|
|
root=Path(record['preparation_root']);label=Path(record['label_path'])
|
|
assert sha(root/'base.npy')==record['input_sha256']['base.npy']
|
|
assert sha(root/'queries.npz')==record['input_sha256']['queries.npz']
|
|
with np.load(root/'queries.npz',allow_pickle=False) as q:
|
|
mask=q['eligible_s2'];ids=q['anchor_ids'][mask];g=q['g'][mask];edge=q['edge_summary'][mask];counts=q['multiplicity'][mask]
|
|
if synthetic_targets:
|
|
target=np.zeros(len(ids));seconds=None
|
|
else:
|
|
assert sha(label)==record['input_sha256']['labels.npz']
|
|
with np.load(label,allow_pickle=False) as z:
|
|
assert np.array_equal(ids,z['anchor_ids']) and np.array_equal(counts,z['multiplicity'])
|
|
target=z['target_t_star'];seconds=z['seconds']
|
|
base=np.load(root/'base.npy',allow_pickle=False)
|
|
if len(base)!=record['nodes']:raise ValueError('Full-node base size mismatch')
|
|
return dict(base=base,ids=ids,gaps=g,edge=edge,target=target,counts=counts,truth_seconds=seconds)
|
|
|
|
|
|
def to_device(data,device):
|
|
result={k:v for k,v in data.items() if k not in ['base','ids','gaps','edge','target']}
|
|
for key in ['base','ids','gaps','edge','target']:
|
|
result[key]=torch.as_tensor(np.array(data[key],copy=True),dtype=torch.long if key=='ids' else torch.float32,device=device)
|
|
return result
|
|
|
|
|
|
def load_part(record,roles,device):
|
|
return to_device(read_part_arrays(record,roles),device)
|
|
|
|
|
|
def run(root,index):
|
|
started=time.monotonic();c=json.loads((root/'training-contract.json').read_text());run=c['runs'][index]
|
|
dest=root/'cells'/run['name']/run['name'];dest.mkdir(exist_ok=False);save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
|
|
try:
|
|
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
|
records,roles,manifest,target_audit=cell_records(root,c,run)
|
|
save(dest/'target-access.json',dict(fold=run['fold'],pool=run['pool'],roster_sha256=c['roster_sha256'],access=target_audit,outer_fold_test_labels_read=False))
|
|
learning=sorted(p for p in records if permitted_role(roles,p)=='learning')
|
|
validation=sorted(p for p in records if permitted_role(roles,p)=='validation')
|
|
if not learning or not validation:raise ValueError('Empty successful learning/validation roster')
|
|
if any(p in roles['sealed_test_parts'] for p in records):raise PermissionError('Test case present in ready manifest')
|
|
assert run['seed'] in [0,1,2] and run['fold'] in range(5) and run['pool'] in [25,50,100]
|
|
assert run['width']==128 and c['E_max']==42 and run['initialization']=='fresh_seeded'
|
|
torch.set_num_threads(1);random.seed(run['seed']);np.random.seed(run['seed']);torch.manual_seed(run['seed'])
|
|
if not torch.cuda.is_available() or torch.cuda.device_count()!=1:raise RuntimeError('Exactly one visible CUDA device required')
|
|
device=torch.device('cuda:0');torch.cuda.manual_seed_all(run['seed']);torch.cuda.reset_peak_memory_stats()
|
|
torch.backends.cuda.matmul.allow_tf32=False
|
|
if 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Frozen pilot requires one V100')
|
|
# Check hashes once, cache raw FP64 arrays, and fit only the learning subset.
|
|
cache={case:read_part_arrays(records[case],roles) for case in sorted(records)}
|
|
stats=fit_base_standardizer(LearningBases(records,learning,roles,cache),learning_part_ids=learning,fold=str(run['fold']),pool=str(run['pool']))
|
|
save(dest/'standardizer.json',dict(**stats.as_dict(),fingerprint=stats.fingerprint))
|
|
model=QueryModel(run['arm'],run['width'],standardizer=stats).to(device)
|
|
optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5)
|
|
epoch_limit=int(c['E_max']);assert 1<=epoch_limit<=200 and c['cosine_T_max']==epoch_limit
|
|
scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=epoch_limit)
|
|
scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(run['seed']))
|
|
group_counts=Counter(records[p]['component_id'] for p in learning);group_number=len(group_counts)
|
|
val_groups={p:records[p]['component_id'] for p in validation}
|
|
best=float('inf');best_epoch=None;stale=0;history=[];updates=0;attempted_updates=0;skip_policy=SkipPolicy()
|
|
for epoch in range(epoch_limit):
|
|
model.train();loss_sum=0.;skip_policy.start_epoch()
|
|
order=[learning[int(j)] for j in rng.permutation(len(learning))]
|
|
for j,case in enumerate(order):
|
|
record=records[case];data=to_device(cache[case],device)
|
|
weights=torch.tensor(part_weights(data['counts'],group_counts[record['component_id']],group_number,len(learning)),dtype=torch.float32,device=device)
|
|
step_info={}
|
|
loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=step_info)
|
|
loss_sum+=loss/len(learning);attempted_updates+=1;updates+=int(not step_info['step_skipped'])
|
|
skip_policy.observe(step_info['step_skipped'])
|
|
del data,weights
|
|
if j%16==0:
|
|
torch.cuda.synchronize();save(dest/'progress.json',dict(stage='training',epoch=epoch,parts_done=j+1,parts_total=len(learning),updates=updates,elapsed_seconds=time.monotonic()-started,
|
|
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()))
|
|
amp_summary=skip_policy.summary();save(dest/f'epoch-{epoch:03d}-amp.json',amp_summary)
|
|
skip_policy.finish_epoch(epoch)
|
|
model.eval();errors={}
|
|
with torch.no_grad():
|
|
for case in validation:
|
|
data=to_device(cache[case],device)
|
|
with torch.autocast(device_type='cuda',enabled=False):
|
|
pred=model(data['base'],data['ids'],data['gaps'],data['edge'] if run['arm']=='E' else None)
|
|
predicted_seconds=pred.float().cpu().numpy().astype(np.float64)*records[case]['t_set']
|
|
score=relative_l2(predicted_seconds,data['truth_seconds'],data['counts'],np.ones(len(data['counts']),bool))
|
|
errors[case]=score['value'];del data,pred
|
|
value=component_score(errors,val_groups);torch.cuda.synchronize()
|
|
row=dict(epoch=epoch,amp=amp_summary,attempted_updates=attempted_updates,training_global_weighted_mse=loss_sum,validation_component_equal_relative_l2=value,updates=updates,elapsed_seconds=time.monotonic()-started,
|
|
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())
|
|
history.append(row);save(dest/'history.json',history)
|
|
if value<best:
|
|
best=value;best_epoch=epoch;stale=0
|
|
torch.save(dict(state_dict={k:v.detach().cpu() for k,v in model.state_dict().items()},arm=run['arm'],width=run['width'],seed=run['seed'],epoch=epoch,
|
|
standardizer=stats.as_dict(),standardizer_fingerprint=stats.fingerprint),dest/'best.pt')
|
|
save(dest/'best-validation-parts.json',errors)
|
|
else:stale+=1
|
|
scheduler.step()
|
|
save(dest/'progress.json',dict(stage='epoch_complete',**row,patience_count=stale))
|
|
if stale>=20:break
|
|
torch.cuda.synchronize()
|
|
result=dict(status='COMPLETE',run=run,E_max=epoch_limit,cosine_T_max=epoch_limit,epochs=len(history),attempted_updates=attempted_updates,updates=updates,best_epoch=best_epoch,best_validation_score=best,
|
|
learning_parts=len(learning),validation_parts=len(validation),learning_components=group_number,validation_components=len(set(val_groups.values())),
|
|
roster_sha256=c['roster_sha256'],fold=run['fold'],pool=run['pool'],fresh_initialization=True,failed_preparation_parts_dropped=manifest['failed_parts'],standardizer_fingerprint=stats.fingerprint,
|
|
wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(),
|
|
test_labels_read=False,test_predictions=False,capacity_selected=False,checkpoint_sha256=sha(dest/'best.pt'))
|
|
save(dest/'result.json',result);print(json.dumps(result),flush=True);return 0
|
|
except Exception as exc:
|
|
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-started,
|
|
peak_allocated_bytes=torch.cuda.max_memory_allocated() if torch.cuda.is_initialized() else 0,
|
|
peak_reserved_bytes=torch.cuda.max_memory_reserved() if torch.cuda.is_initialized() else 0,
|
|
epoch=locals().get('epoch'),amp=skip_policy.summary() if 'skip_policy' in locals() else None))
|
|
raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))
|