Files
MeltMind/tools/s2_pilot_train.py
T

80 lines
4.7 KiB
Python

"""Bounded pilot training primitives; corpus runs require a frozen ready manifest."""
import math
from pathlib import Path
import numpy as np
import torch
from s2_models.network import weighted_mse
def part_weights(counts, parts_in_component, component_count, learning_parts):
c=np.asarray(counts,dtype=np.float64)
if c.ndim!=1 or not len(c) or np.any(c<=0) or not np.isfinite(c).all():raise ValueError('Invalid common query counts')
if min(parts_in_component,component_count,learning_parts)<=0:raise ValueError('Empty training hierarchy')
# Uniform shuffled part steps estimate the full component-balanced objective.
return c/c.sum()/parts_in_component/component_count*learning_parts
def train_part(model,optimizer,base,ids,gaps,edge,target,weights,*,scaler=None,amp=False,step_info=None):
optimizer.zero_grad(set_to_none=True)
with torch.autocast(device_type=base.device.type,dtype=torch.float16,enabled=amp):
prediction=model(base,ids,gaps,edge if model.arm=='E' else None)
loss=weighted_mse(prediction,target,weights)
if not torch.isfinite(loss):raise FloatingPointError('Nonfinite training loss')
scale_before=float(scaler.get_scale()) if scaler is not None else None
if scaler is None:
loss.backward()
else:
scaler.scale(loss).backward();scaler.unscale_(optimizer)
if scaler is None and any(p.grad is not None and not torch.isfinite(p.grad).all() for p in model.parameters()):
raise FloatingPointError('Nonfinite gradient; no silent retry or precision fallback')
if scaler is None:optimizer.step()
else:scaler.step(optimizer);scaler.update()
scale_after=float(scaler.get_scale()) if scaler is not None else None
skipped=scaler is not None and scale_after<scale_before
if step_info is not None:
step_info.update(step_skipped=skipped,scale_before=scale_before,scale_after=scale_after)
if any(not torch.isfinite(p).all() for p in model.parameters()):raise FloatingPointError('Nonfinite updated parameter')
return float(loss.detach())
class SkipPolicy:
"""Claude 1794681576: epoch 0 exempt from fraction, never from streak cap."""
def __init__(self):self.consecutive=0;self.start_epoch()
def start_epoch(self):self.attempted=0;self.skipped=0
def observe(self,skipped):
self.attempted+=1;self.skipped+=int(skipped)
self.consecutive=self.consecutive+1 if skipped else 0
if self.consecutive>20:raise FloatingPointError('More than 20 consecutive AMP skipped steps')
def summary(self):
return dict(attempted_steps=self.attempted,skipped_steps=self.skipped,successful_updates=self.attempted-self.skipped,
skipped_fraction=self.skipped/self.attempted if self.attempted else 0.,consecutive_skipped=self.consecutive)
def finish_epoch(self,epoch):
if epoch>0 and self.skipped>self.attempted*.01:raise FloatingPointError('AMP skipped steps exceed 1 percent after epoch 0')
def common_epoch_limit(training_seconds_per_node,inference_seconds_per_node,learning_nodes,validation_nodes):
if min(training_seconds_per_node,inference_seconds_per_node,learning_nodes,validation_nodes)<=0:raise ValueError('Positive per-node timing and node counts required')
epoch=training_seconds_per_node*learning_nodes+inference_seconds_per_node*validation_nodes
if not math.isfinite(epoch):raise ValueError('Nonfinite epoch time')
limit=min(200,math.floor(25*60/epoch))
if limit<1:raise ValueError('One projected epoch exceeds the 25-minute run reserve')
return dict(E_max=limit,cosine_T_max=limit,projected_epoch_seconds=epoch,projected_epochs_seconds=limit*epoch,
training_seconds_per_node=training_seconds_per_node,inference_seconds_per_node=inference_seconds_per_node,
learning_nodes=learning_nodes,validation_nodes=validation_nodes)
def component_score(part_errors,component_ids):
if not part_errors or set(part_errors)!=set(component_ids):raise ValueError('Incomplete validation roster')
groups={}
for part,value in part_errors.items():
if value is None or not math.isfinite(value):raise ValueError('Invalid validation part score')
groups.setdefault(component_ids[part],[]).append(value)
return float(np.mean([np.mean(values) for values in groups.values()]))
def select_capacity(scores):
# Exactly four run scores, no choice from an incomplete/failed run matrix.
if set(scores)!={(arm,width) for arm in ['N','E'] for width in [128,256]}:raise ValueError('All four pilot runs required')
if any(v is None or not math.isfinite(v) for v in scores.values()):raise ValueError('All four pilot validation scores must be finite')
return min([128,256],key=lambda w:((scores['N',w]+scores['E',w])/2,w))