"""Bounded pilot training primitives; corpus runs require a frozen ready manifest.""" import math from pathlib import Path import numpy as np import torch from s2_models.network import weighted_mse def part_weights(counts, parts_in_component, component_count, learning_parts): c=np.asarray(counts,dtype=np.float64) if c.ndim!=1 or not len(c) or np.any(c<=0) or not np.isfinite(c).all():raise ValueError('Invalid common query counts') if min(parts_in_component,component_count,learning_parts)<=0:raise ValueError('Empty training hierarchy') # Uniform shuffled part steps estimate the full component-balanced objective. return c/c.sum()/parts_in_component/component_count*learning_parts def train_part(model,optimizer,base,ids,gaps,edge,target,weights,*,scaler=None,amp=False,step_info=None): optimizer.zero_grad(set_to_none=True) with torch.autocast(device_type=base.device.type,dtype=torch.float16,enabled=amp): prediction=model(base,ids,gaps,edge if model.arm=='E' else None) loss=weighted_mse(prediction,target,weights) if not torch.isfinite(loss):raise FloatingPointError('Nonfinite training loss') scale_before=float(scaler.get_scale()) if scaler is not None else None if scaler is None: loss.backward() else: scaler.scale(loss).backward();scaler.unscale_(optimizer) if scaler is None and any(p.grad is not None and not torch.isfinite(p.grad).all() for p in model.parameters()): raise FloatingPointError('Nonfinite gradient; no silent retry or precision fallback') if scaler is None:optimizer.step() else:scaler.step(optimizer);scaler.update() scale_after=float(scaler.get_scale()) if scaler is not None else None skipped=scaler is not None and scale_after20:raise FloatingPointError('More than 20 consecutive AMP skipped steps') def summary(self): return dict(attempted_steps=self.attempted,skipped_steps=self.skipped,successful_updates=self.attempted-self.skipped, skipped_fraction=self.skipped/self.attempted if self.attempted else 0.,consecutive_skipped=self.consecutive) def finish_epoch(self,epoch): if epoch>0 and self.skipped>self.attempted*.01:raise FloatingPointError('AMP skipped steps exceed 1 percent after epoch 0') def common_epoch_limit(training_seconds_per_node,inference_seconds_per_node,learning_nodes,validation_nodes): if min(training_seconds_per_node,inference_seconds_per_node,learning_nodes,validation_nodes)<=0:raise ValueError('Positive per-node timing and node counts required') epoch=training_seconds_per_node*learning_nodes+inference_seconds_per_node*validation_nodes if not math.isfinite(epoch):raise ValueError('Nonfinite epoch time') limit=min(200,math.floor(25*60/epoch)) if limit<1:raise ValueError('One projected epoch exceeds the 25-minute run reserve') return dict(E_max=limit,cosine_T_max=limit,projected_epoch_seconds=epoch,projected_epochs_seconds=limit*epoch, training_seconds_per_node=training_seconds_per_node,inference_seconds_per_node=inference_seconds_per_node, learning_nodes=learning_nodes,validation_nodes=validation_nodes) def component_score(part_errors,component_ids): if not part_errors or set(part_errors)!=set(component_ids):raise ValueError('Incomplete validation roster') groups={} for part,value in part_errors.items(): if value is None or not math.isfinite(value):raise ValueError('Invalid validation part score') groups.setdefault(component_ids[part],[]).append(value) return float(np.mean([np.mean(values) for values in groups.values()])) def select_capacity(scores): # Exactly four run scores, no choice from an incomplete/failed run matrix. if set(scores)!={(arm,width) for arm in ['N','E'] for width in [128,256]}:raise ValueError('All four pilot runs required') if any(v is None or not math.isfinite(v) for v in scores.values()):raise ValueError('All four pilot validation scores must be finite') return min([128,256],key=lambda w:((scores['N',w]+scores['E',w])/2,w))