80 lines
4.7 KiB
Python
80 lines
4.7 KiB
Python
"""Bounded pilot training primitives; corpus runs require a frozen ready manifest."""
|
|
import math
|
|
from pathlib import Path
|
|
import numpy as np
|
|
import torch
|
|
from s2_models.network import weighted_mse
|
|
|
|
|
|
def part_weights(counts, parts_in_component, component_count, learning_parts):
|
|
c=np.asarray(counts,dtype=np.float64)
|
|
if c.ndim!=1 or not len(c) or np.any(c<=0) or not np.isfinite(c).all():raise ValueError('Invalid common query counts')
|
|
if min(parts_in_component,component_count,learning_parts)<=0:raise ValueError('Empty training hierarchy')
|
|
# Uniform shuffled part steps estimate the full component-balanced objective.
|
|
return c/c.sum()/parts_in_component/component_count*learning_parts
|
|
|
|
|
|
def train_part(model,optimizer,base,ids,gaps,edge,target,weights,*,scaler=None,amp=False,step_info=None):
|
|
optimizer.zero_grad(set_to_none=True)
|
|
with torch.autocast(device_type=base.device.type,dtype=torch.float16,enabled=amp):
|
|
prediction=model(base,ids,gaps,edge if model.arm=='E' else None)
|
|
loss=weighted_mse(prediction,target,weights)
|
|
if not torch.isfinite(loss):raise FloatingPointError('Nonfinite training loss')
|
|
scale_before=float(scaler.get_scale()) if scaler is not None else None
|
|
if scaler is None:
|
|
loss.backward()
|
|
else:
|
|
scaler.scale(loss).backward();scaler.unscale_(optimizer)
|
|
if scaler is None and any(p.grad is not None and not torch.isfinite(p.grad).all() for p in model.parameters()):
|
|
raise FloatingPointError('Nonfinite gradient; no silent retry or precision fallback')
|
|
if scaler is None:optimizer.step()
|
|
else:scaler.step(optimizer);scaler.update()
|
|
scale_after=float(scaler.get_scale()) if scaler is not None else None
|
|
skipped=scaler is not None and scale_after<scale_before
|
|
if step_info is not None:
|
|
step_info.update(step_skipped=skipped,scale_before=scale_before,scale_after=scale_after)
|
|
if any(not torch.isfinite(p).all() for p in model.parameters()):raise FloatingPointError('Nonfinite updated parameter')
|
|
return float(loss.detach())
|
|
|
|
|
|
class SkipPolicy:
|
|
"""Claude 1794681576: epoch 0 exempt from fraction, never from streak cap."""
|
|
def __init__(self):self.consecutive=0;self.start_epoch()
|
|
def start_epoch(self):self.attempted=0;self.skipped=0
|
|
def observe(self,skipped):
|
|
self.attempted+=1;self.skipped+=int(skipped)
|
|
self.consecutive=self.consecutive+1 if skipped else 0
|
|
if self.consecutive>20:raise FloatingPointError('More than 20 consecutive AMP skipped steps')
|
|
def summary(self):
|
|
return dict(attempted_steps=self.attempted,skipped_steps=self.skipped,successful_updates=self.attempted-self.skipped,
|
|
skipped_fraction=self.skipped/self.attempted if self.attempted else 0.,consecutive_skipped=self.consecutive)
|
|
def finish_epoch(self,epoch):
|
|
if epoch>0 and self.skipped>self.attempted*.01:raise FloatingPointError('AMP skipped steps exceed 1 percent after epoch 0')
|
|
|
|
|
|
def common_epoch_limit(training_seconds_per_node,inference_seconds_per_node,learning_nodes,validation_nodes):
|
|
if min(training_seconds_per_node,inference_seconds_per_node,learning_nodes,validation_nodes)<=0:raise ValueError('Positive per-node timing and node counts required')
|
|
epoch=training_seconds_per_node*learning_nodes+inference_seconds_per_node*validation_nodes
|
|
if not math.isfinite(epoch):raise ValueError('Nonfinite epoch time')
|
|
limit=min(200,math.floor(25*60/epoch))
|
|
if limit<1:raise ValueError('One projected epoch exceeds the 25-minute run reserve')
|
|
return dict(E_max=limit,cosine_T_max=limit,projected_epoch_seconds=epoch,projected_epochs_seconds=limit*epoch,
|
|
training_seconds_per_node=training_seconds_per_node,inference_seconds_per_node=inference_seconds_per_node,
|
|
learning_nodes=learning_nodes,validation_nodes=validation_nodes)
|
|
|
|
|
|
def component_score(part_errors,component_ids):
|
|
if not part_errors or set(part_errors)!=set(component_ids):raise ValueError('Incomplete validation roster')
|
|
groups={}
|
|
for part,value in part_errors.items():
|
|
if value is None or not math.isfinite(value):raise ValueError('Invalid validation part score')
|
|
groups.setdefault(component_ids[part],[]).append(value)
|
|
return float(np.mean([np.mean(values) for values in groups.values()]))
|
|
|
|
|
|
def select_capacity(scores):
|
|
# Exactly four run scores, no choice from an incomplete/failed run matrix.
|
|
if set(scores)!={(arm,width) for arm in ['N','E'] for width in [128,256]}:raise ValueError('All four pilot runs required')
|
|
if any(v is None or not math.isfinite(v) for v in scores.values()):raise ValueError('All four pilot validation scores must be finite')
|
|
return min([128,256],key=lambda w:((scores['N',w]+scores['E',w])/2,w))
|