"""One separately budgeted replay of the preserved N128 forward failure. No validation arrays, validation scoring, test arrays, or pilot continuation. Original pinned numerical modules are imported without editing them. """ import argparse from collections import Counter import json import os from pathlib import Path import random import sys import time import traceback import types import numpy as np import psutil import torch from freeze_g1_components import sha from s2_e1_supervisor import save from s2_models.network import QueryModel from s2_models.normalization import fit_base_standardizer from s2_pilot_data import permitted_role from s2_pilot_train import part_weights, train_part, SkipPolicy from run_s2_pilot_training import LearningBases, read_part_arrays, to_device def describe(x): x=x.detach(); finite=torch.isfinite(x); n=int(finite.sum()) return dict(shape=list(x.shape),dtype=str(x.dtype),finite=n==x.numel(), nonfinite=x.numel()-n,nan=int(torch.isnan(x).sum()), inf=int(torch.isinf(x).sum()), max_abs_finite=float(x[finite].abs().max()) if n else None) def traced_attention(self,x,*,trace,prefix): def record(name,value): trace[prefix+'.'+name]=describe(value);return value B,N,C=x.shape fx_mid=record('fx_mid',self.in_project_fx(x).reshape(B,N,self.heads,self.dim_head).permute(0,2,1,3).contiguous()) x_mid=record('x_mid',self.in_project_x(x).reshape(B,N,self.heads,self.dim_head).permute(0,2,1,3).contiguous()) weights=record('slice_weights',self.softmax(self.in_project_slice(x_mid)/self.temperature)) norm=record('slice_denominator',weights.sum(2)) numerator=record('slice_numerator',torch.einsum('bhnc,bhng->bhgc',fx_mid,weights)) token=record('slice_normalized',numerator/((norm+1e-5)[:,:,:,None].repeat(1,1,1,self.dim_head))) q=record('q',self.to_q(token));k=record('k',self.to_k(token));v=record('v',self.to_v(token)) dots=record('attention_logits',torch.matmul(q,k.transpose(-1,-2))*self.scale) attn=self.dropout(self.softmax(dots)) out=record('out_slice_token',torch.matmul(attn,v)) out=record('deslice',torch.einsum('bhgc,bhng->bhnc',out,weights)) return self.to_out(out.permute(0,2,1,3).reshape(B,N,-1)) def trace_forward(model,data,amp): trace={};handles=[];original=[] for name,module in model.named_modules(): if name: def hook(_module,_args,output,name=name): if isinstance(output,torch.Tensor):trace[name+'.output']=describe(output) handles.append(module.register_forward_hook(hook)) if name.endswith('.Attn'): original.append((module,module.forward)) def forward(self,x,name=name):return traced_attention(self,x,trace=trace,prefix=name) module.forward=types.MethodType(forward,module) try: with torch.no_grad(),torch.autocast(device_type=data['base'].device.type,dtype=torch.float16,enabled=amp): output=model(data['base'],data['ids'],data['gaps'],None) result=dict(finite=bool(torch.isfinite(output).all()),output=describe(output),error=None) except Exception as exc: result=dict(finite=False,error=str(exc),error_type=type(exc).__name__) finally: for h in handles:h.remove() for module,forward in original:module.forward=forward return dict(**result,trace=trace) def worker(root): begin=time.monotonic();dest=root/'replay';dest.mkdir(exist_ok=False) contract=json.loads((root/'contract.json').read_text());source=Path(contract['source_pilot']) for path,digest in contract['code_sha256'].items():assert sha(path)==digest,path for path,digest in contract['input_sha256'].items():assert sha(path)==digest,path c=json.loads((source/'training-contract.json').read_text()) roles=json.loads(Path(c['role_manifest']).read_text());manifest=json.loads(Path(c['success_manifest']).read_text()) records={r['case']:r for r in manifest['ready_parts'] if permitted_role(roles,r['case'])=='learning'} learning=sorted(records);assert len(learning)==382 and not set(learning)&set(roles['sealed_test_parts']) torch.set_num_threads(1);random.seed(0);np.random.seed(0);torch.manual_seed(0) assert torch.cuda.is_available() and torch.cuda.device_count()==1 device=torch.device('cuda:0');torch.cuda.manual_seed_all(0);torch.cuda.reset_peak_memory_stats() torch.backends.cuda.matmul.allow_tf32=False assert 'V100' in torch.cuda.get_device_name() cache={case:read_part_arrays(records[case],roles) for case in learning} stats=fit_base_standardizer(LearningBases(records,learning,roles,cache),learning_part_ids=learning,fold='0',pool='100') assert stats.fingerprint==c['standardizer_fingerprint'] model=QueryModel('N',128,standardizer=stats).to(device) optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5) scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=c['E_max']) scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(0)) groups=Counter(records[p]['component_id'] for p in learning);policy=SkipPolicy();rows=[] previous=json.loads((source/'N128-seed0/history.json').read_text()) failure=json.loads((source/'N128-seed0/failure.json').read_text()) target_epoch=failure['epoch'];target_step=failure['amp']['attempted_steps'] expected_rng=np.random.Generator(np.random.PCG64(0)) for _ in range(target_epoch+1):expected_order=expected_rng.permutation(len(learning)) expected_case=learning[int(expected_order[target_step])] save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'contract.json'), expected_case=expected_case,expected_epoch=target_epoch,expected_step_zero_based=target_step, learning_parts=len(learning),validation_arrays_read=False,test_data_read=False)) for epoch in range(target_epoch+1): model.train();policy.start_epoch();loss_sum=0. order=[learning[int(j)] for j in rng.permutation(len(learning))] for step,case in enumerate(order): data=to_device(cache[case],device) weights=torch.tensor(part_weights(data['counts'],groups[records[case]['component_id']],len(groups),len(learning)),dtype=torch.float32,device=device) error=None;info={} try: loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=info) except Exception as exc: error=dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc()) if error is not None or (epoch==target_epoch and step==target_step): # Save the actual diagnostic state, never present best.pt as a failure state. optimizer.zero_grad(set_to_none=True) torch.save(dict(state_dict=model.state_dict(),optimizer=optimizer.state_dict(),scaler=scaler.state_dict(), scheduler=scheduler.state_dict(),epoch=epoch,step_zero_based=step,case=case, standardizer=stats.as_dict(),numpy_order_rng=rng.bit_generator.state, torch_rng=torch.get_rng_state(),cuda_rng=torch.cuda.get_rng_state()),dest/'failure-state.pt') parameters={name:describe(p) for name,p in model.named_parameters()} half=trace_forward(model,data,True);single=trace_forward(model,data,False) prior_match=all(row['matches_original_training_summary'] for row in rows) reproduced=error is not None and error['message']==failure['message'] and epoch==target_epoch and step==target_step and case==expected_case and prior_match decision='FP32_FINITE_FP16_NONFINITE' if reproduced and single['finite'] and not half['finite'] else 'FP32_NONFINITE' if reproduced and not single['finite'] else 'REPLAY_NOT_CONFIRMED' result=dict(status='COMPLETE',diagnostic_only=True,decision=decision,reproduced_original_failure=reproduced, case=case,nodes=records[case]['nodes'],epoch=epoch,step_zero_based=step,step_one_based=step+1, original_step_error=error,prior_epoch_training_summaries=rows,parameters=parameters, fp16=half,fp32=single,validation_arrays_read=False,validation_scoring=False,test_data_read=False, pilot_run=False,standardizer_fingerprint=stats.fingerprint,wall_seconds=time.monotonic()-begin, peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(), failure_state_sha256=sha(dest/'failure-state.pt')) save(dest/'result.json',result);print(json.dumps({k:v for k,v in result.items() if k not in ['fp16','fp32','parameters','prior_epoch_training_summaries','original_step_error']}),flush=True) return policy.observe(info['step_skipped']);loss_sum+=loss/len(learning) del data,weights if step%32==0:save(dest/'progress.json',dict(epoch=epoch,step_zero_based=step,case=case,wall_seconds=time.monotonic()-begin,amp=policy.summary())) policy.finish_epoch(epoch) reference=previous[epoch] row=dict(epoch=epoch,training_global_weighted_mse=loss_sum,amp=policy.summary(), original_training_global_weighted_mse=reference['training_global_weighted_mse'], matches_original_training_summary=abs(loss_sum-reference['training_global_weighted_mse'])<=1e-10 and policy.summary()==reference['amp']) rows.append(row);save(dest/'replay-history.json',rows);scheduler.step() raise RuntimeError('Diagnostic ended without reaching the specified step') def run(root): import s2_recovered_pilot_gpu_guard as guard contract=json.loads((root/'contract.json').read_text()) assert not (root/'started.json').exists(),'No automatic diagnostic retry' def diagnostic_pids(parent,_root,_index): result=[] for p in psutil.Process(parent).children(recursive=True): try: cmd=p.cmdline() if 'tools/run_s2_encoder_diagnostic.py' in cmd and '--worker' in cmd and str(root) in cmd:result.append(p.pid) except psutil.NoSuchProcess:pass return result guard.active_training_pids=diagnostic_pids start=time.monotonic();save(root/'started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json'))) try: receipt,result=guard.guarded_worker(root,dict(name='replay'),0,contract['gpu_uuid'],900, [sys.executable,'tools/run_s2_encoder_diagnostic.py',str(root),'--worker']) save(root/'complete.json',dict(status='COMPLETE' if receipt['status']=='PASS' and result else 'DIAGNOSTIC_INCOMPLETE', decision=result['decision'] if result else None,total_wall_seconds=time.monotonic()-start, limit_seconds=900,separate_from_pilot=True,validation_scoring=False,test_data_read=False)) except Exception as exc: save(root/'failure.json',dict(error=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-start));raise if __name__=='__main__': p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--worker',action='store_true');a=p.parse_args() worker(a.root) if a.worker else run(a.root)