127 lines
8.9 KiB
Python
127 lines
8.9 KiB
Python
"""One fixed pilot arm/width. Must be launched inside the GPU resource guard."""
|
|
import argparse
|
|
from collections import Counter
|
|
from collections.abc import Mapping
|
|
import json
|
|
from pathlib import Path
|
|
import random
|
|
import time
|
|
import traceback
|
|
import numpy as np
|
|
import torch
|
|
from freeze_g1_components import sha
|
|
from s2_e1_supervisor import save
|
|
from s2_models.network import QueryModel,TRAINING_CONTRACT
|
|
from s2_models.normalization import BaseStandardizer,fit_base_standardizer
|
|
from s2_models.objectives import relative_l2
|
|
from s2_pilot_data import permitted_role
|
|
from s2_pilot_train import part_weights,train_part,component_score
|
|
|
|
|
|
class LearningBases(Mapping):
|
|
def __init__(self,records,ids,roles):self.records=records;self.ids=tuple(ids);self.roles=roles
|
|
def __iter__(self):return iter(self.ids)
|
|
def __len__(self):return len(self.ids)
|
|
def __getitem__(self,case):
|
|
if case not in self.ids or permitted_role(self.roles,case)!='learning':raise PermissionError('Learning-only normalization')
|
|
return np.load(Path(self.records[case]['preparation_root'])/'base.npy',allow_pickle=False,mmap_mode='r')
|
|
|
|
|
|
def load_part(record,roles,device):
|
|
case=record['case'];permitted_role(roles,case)
|
|
root=Path(record['preparation_root']);label=Path(record['label_path'])
|
|
assert sha(root/'base.npy')==record['input_sha256']['base.npy']
|
|
assert sha(root/'queries.npz')==record['input_sha256']['queries.npz']
|
|
assert sha(label)==record['input_sha256']['labels.npz']
|
|
with np.load(root/'queries.npz',allow_pickle=False) as q:
|
|
mask=q['eligible_s2'];ids=q['anchor_ids'][mask];g=q['g'][mask];edge=q['edge_summary'][mask];counts=q['multiplicity'][mask]
|
|
with np.load(label,allow_pickle=False) as z:
|
|
assert np.array_equal(ids,z['anchor_ids']) and np.array_equal(counts,z['multiplicity'])
|
|
target=z['target_t_star'];seconds=z['seconds']
|
|
base=np.load(root/'base.npy',allow_pickle=False)
|
|
if len(base)!=record['nodes']:raise ValueError('Full-node base size mismatch')
|
|
tensor=lambda a,dtype=torch.float32:torch.as_tensor(np.array(a,copy=True),dtype=dtype,device=device)
|
|
return dict(base=tensor(base),ids=tensor(ids,torch.long),gaps=tensor(g),edge=tensor(edge),target=tensor(target),
|
|
counts=counts,truth_seconds=seconds)
|
|
|
|
|
|
def run(root,index):
|
|
started=time.monotonic();c=json.loads((root/'training-contract.json').read_text());run=c['runs'][index]
|
|
dest=root/run['name'];dest.mkdir(exist_ok=False);save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'training-contract.json')))
|
|
try:
|
|
for p,h in c['code_sha256'].items():assert sha(p)==h,p
|
|
source=Path(c['success_manifest']);assert sha(source)==c['success_manifest_sha256']
|
|
manifest=json.loads(source.read_text());records={r['case']:r for r in manifest['ready_parts']}
|
|
roles=json.loads(Path(c['role_manifest']).read_text());assert sha(c['role_manifest'])==c['role_manifest_sha256']
|
|
learning=sorted(p for p in records if permitted_role(roles,p)=='learning')
|
|
validation=sorted(p for p in records if permitted_role(roles,p)=='validation')
|
|
if not learning or not validation:raise ValueError('Empty successful learning/validation roster')
|
|
if any(p in roles['sealed_test_parts'] for p in records):raise PermissionError('Test case present in ready manifest')
|
|
assert run['seed']==0 and c['fold']==0 and c['pool']==100
|
|
torch.set_num_threads(1);random.seed(run['seed']);np.random.seed(run['seed']);torch.manual_seed(run['seed'])
|
|
if not torch.cuda.is_available() or torch.cuda.device_count()!=1:raise RuntimeError('Exactly one visible CUDA device required')
|
|
device=torch.device('cuda:0');torch.cuda.manual_seed_all(run['seed']);torch.cuda.reset_peak_memory_stats()
|
|
torch.backends.cuda.matmul.allow_tf32=False
|
|
if 'V100' not in torch.cuda.get_device_name():raise RuntimeError('Frozen pilot requires one V100')
|
|
# Every base statistics calculation is learning-only and inside this run's wall guard.
|
|
stats=fit_base_standardizer(LearningBases(records,learning,roles),learning_part_ids=learning,fold='0',pool='100')
|
|
save(dest/'standardizer.json',dict(**stats.as_dict(),fingerprint=stats.fingerprint))
|
|
shared=root/'shared-standardizer-fingerprint.json'
|
|
if shared.exists():assert json.loads(shared.read_text())['fingerprint']==stats.fingerprint
|
|
else:save(shared,dict(fingerprint=stats.fingerprint,learning_parts=learning))
|
|
model=QueryModel(run['arm'],run['width'],standardizer=stats).to(device)
|
|
optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5)
|
|
scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=200)
|
|
scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(run['seed']))
|
|
group_counts=Counter(records[p]['component_id'] for p in learning);group_number=len(group_counts)
|
|
val_groups={p:records[p]['component_id'] for p in validation}
|
|
best=float('inf');best_epoch=None;stale=0;history=[];updates=0
|
|
for epoch in range(200):
|
|
model.train();loss_sum=0.
|
|
order=[learning[int(j)] for j in rng.permutation(len(learning))]
|
|
for j,case in enumerate(order):
|
|
record=records[case];data=load_part(record,roles,device)
|
|
weights=torch.tensor(part_weights(data['counts'],group_counts[record['component_id']],group_number,len(learning)),dtype=torch.float32,device=device)
|
|
loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True)
|
|
loss_sum+=loss/len(learning);updates+=1
|
|
del data,weights
|
|
if j%16==0:
|
|
torch.cuda.synchronize();save(dest/'progress.json',dict(stage='training',epoch=epoch,parts_done=j+1,parts_total=len(learning),updates=updates,elapsed_seconds=time.monotonic()-started,
|
|
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved()))
|
|
model.eval();errors={}
|
|
with torch.no_grad():
|
|
for case in validation:
|
|
data=load_part(records[case],roles,device)
|
|
with torch.autocast(device_type='cuda',dtype=torch.float16):
|
|
pred=model(data['base'],data['ids'],data['gaps'],data['edge'] if run['arm']=='E' else None)
|
|
predicted_seconds=pred.float().cpu().numpy().astype(np.float64)*records[case]['t_set']
|
|
score=relative_l2(predicted_seconds,data['truth_seconds'],data['counts'],np.ones(len(data['counts']),bool))
|
|
errors[case]=score['value'];del data,pred
|
|
value=component_score(errors,val_groups);torch.cuda.synchronize()
|
|
row=dict(epoch=epoch,training_global_weighted_mse=loss_sum,validation_component_equal_relative_l2=value,updates=updates,elapsed_seconds=time.monotonic()-started,
|
|
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved())
|
|
history.append(row);save(dest/'history.json',history)
|
|
if value<best:
|
|
best=value;best_epoch=epoch;stale=0
|
|
torch.save(dict(state_dict={k:v.detach().cpu() for k,v in model.state_dict().items()},arm=run['arm'],width=run['width'],seed=run['seed'],epoch=epoch,
|
|
standardizer=stats.as_dict(),standardizer_fingerprint=stats.fingerprint),dest/'best.pt')
|
|
save(dest/'best-validation-parts.json',errors)
|
|
else:stale+=1
|
|
scheduler.step()
|
|
save(dest/'progress.json',dict(stage='epoch_complete',**row,patience_count=stale))
|
|
if stale>=20:break
|
|
torch.cuda.synchronize()
|
|
result=dict(status='COMPLETE',run=run,epochs=len(history),updates=updates,best_epoch=best_epoch,best_validation_score=best,
|
|
learning_parts=len(learning),validation_parts=len(validation),learning_components=group_number,validation_components=len(set(val_groups.values())),
|
|
failed_preparation_parts_dropped=manifest['failed_parts'],standardizer_fingerprint=stats.fingerprint,
|
|
wall_seconds=time.monotonic()-started,peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(),
|
|
test_labels_read=False,test_predictions=False,capacity_selected=False,checkpoint_sha256=sha(dest/'best.pt'))
|
|
save(dest/'result.json',result);print(json.dumps(result),flush=True);return 0
|
|
except Exception as exc:
|
|
save(dest/'failure.json',dict(status='FAILED_NO_RETRY',error_type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-started))
|
|
raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))
|