diff --git a/artifacts/s2-encoder-diagnostic-2026-11-15/contract.json b/artifacts/s2-encoder-diagnostic-2026-11-15/contract.json new file mode 100644 index 000000000..90746ed07 --- /dev/null +++ b/artifacts/s2-encoder-diagnostic-2026-11-15/contract.json @@ -0,0 +1,57 @@ +{ + "version": "S2-N128-forward-diagnostic-v1", + "authorizing_ruling": "1794694544.819729", + "source_pilot": "artifacts/s2-pilot-training-2026-11-15", + "source_stage_commit": "5744cc119f638b46c50edf38579575ed5555318f", + "gpu_uuid": "GPU-e8853ae7-a302-6a17-bcfa-aa767e9ed20c", + "limits": { + "wall_seconds": 900, + "vram_bytes": 16106127360, + "host_rss_bytes": 17179869184, + "disk_bytes": 5368709120 + }, + "code_sha256": { + "tools/prepare_s2_recovered_pilot_training.py": "ab5a4274363598ff933ff873df2b00497f069b1fb45756a1d1dba8d723294f68", + "tools/run_s2_recovered_pilot.py": "56962009bf5eaaee599ffea6a3d454372e0aea2842fa79eac609fd4a24ecbb39", + "tools/run_s2_pilot_probe.py": "2360ab7b44b179144a5ecd2d691e33cd36a6ec4bfeac71452bf062e7f42f81eb", + "tools/run_s2_pilot_training.py": "d5e543478543243fe1db3475a4d4cd135fbdcf631d3bce48c4b5af8b2edafb22", + "tools/s2_pilot_train.py": "e9cceead51a5cc092aad23409ad98909c84a62e7e10571e750d1e9be1ad3b948", + "tools/s2_recovered_pilot_gpu_guard.py": "9a27f47918c865ce37e46b0be54144dc3f9ab1058e4eddc9f6fbc771f4959eb0", + "tools/s2_recovery_gpu_supervisor.py": "689c0f3179e4fcd362e546d90214829a9e4f8739abdbdf74b560763e5a392f15", + "tools/s2_e1_supervisor.py": "6753c9ceb0b932047ba505bd15681f109e9407bdb58fd96c55e1431a3f1b4a02", + "tools/s2_pilot_data.py": "82c1fd18ec51757479c8fd7dda2db4d5f38c366d77fedcca3216a08998214c97", + "tools/s2_incident_recovery_manifest.py": "8cbb2e1067c2cc22762d8ae3cf192aa2c5fa7c7a199a22e7a39a25570ee1c3b1", + "tools/freeze_g1_components.py": "d0cee60d95243ac2c59ef3f2244ba818315718c4e3d1895f2f9f064f1f7f5ecd", + "tools/s2_recovery_monitoring.py": "442c2a9c75e8c00b30a8f2e91cb52d87c466b19a80aea9128b10c021e61b922e", + "tools/prepare_s2_incident_recovery.py": "4f772573dedc2b7921986e25eaedb14f91d98b235421829c284f3b40b85710aa", + "tools/run_s2_recovery_chain.py": "76d0808bdb2d624cfbaadf5a3c5e7e9a0c2be5d5ac41d4ac3fd9ccd48743933c", + "tools/s2_models/__init__.py": "a0fa1323d3553172da66c5d442c5fc4fcb0d5055c886c776928604550ea97609", + "tools/s2_models/attention.py": "7714adfccf6a0fc6b52a44aee3a50321a88898bf3d743680a1108c01e1ff8a61", + "tools/s2_models/blocks.py": "b13e986cea5f1d7f325a1638c1505197b4a105baacff5a9a515f23fcaf28ea75", + "tools/s2_models/features.py": "3b34ae61cf8012ebd58b7505a73d7aa43d8ee1890ac54f98d7917b67887a45a4", + "tools/s2_models/network.py": "5137e845d81d1223afc13bf87e03ccba1132a3baa75d5189a8f320b2752be071", + "tools/s2_models/normalization.py": "7180696d3845799e28d8c31ddef957d075de6a2e46d96a3a0f0df16fc05234a0", + "tools/s2_models/objectives.py": "5b6527a45f08690055d572335470ce5979e8bb86e1cab00de594f79ae21316b1", + "tools/run_s2_encoder_diagnostic.py": "8f31d312fb53bfdb9b1804af064e463d3989f1b33c4aa7279a967ca82dd2c0a4", + "tools/s2_recovery_supervisor.py": "f563ca3c2665f5baf0c04f296616e4640423a59b815fbb00479af0f29e57a919" + }, + "input_sha256": { + "artifacts/s2-pilot-training-2026-11-15/training-contract.json": "f6987431f478275485d18f62240144c2626cab3fbeb75c3e130b68b5d57a1fd2", + "artifacts/s2-pilot-training-2026-11-15/success-manifest.json": "678ca725214c8b7469c55990ded3a16826fc62397ff1a3e5a6590b6ba309c2f3", + "artifacts/s2-pilot-training-2026-11-15/N128-seed0/history.json": "454537d13f209fb252f40840f81ae87e55965a91a759cf7663c7d7a0c7e38e73", + "artifacts/s2-pilot-training-2026-11-15/N128-seed0/failure.json": "a52ba16974e0d5d89334268af35a982f607bf0e225920b70b67339e13286ff17", + "artifacts/s2-pilot-training-2026-11-15/N128-seed0/standardizer.json": "d12d9cb1ba4fc4b9cb00b289ea71e630b05e776a9976c7998351be4244dcdc1c", + "artifacts/s2-pilot-inputs-2026-11-15/role-manifest.json": "a6dce3ebf875dbbe2df349f06e728960140f131d4d55ebd00af03945a46ac35c" + }, + "learning_only": true, + "validation_scoring": false, + "test_data": false, + "pilot_run": false, + "replay_seed": 0, + "replay_arm": "N", + "replay_width": 128, + "cosine_T_max": 38, + "no_automatic_retry": true, + "trace": "same pre-update failure state fp16 autocast and FP32, layer maxima including slice numerator/denominator", + "precision_fix_gate": "Original failure reproduced with matching previous training summaries, FP32 finite and fp16 nonfinite; otherwise report without changing model" +} diff --git a/artifacts/s2-encoder-diagnostic-2026-11-15/synthetic-check.json b/artifacts/s2-encoder-diagnostic-2026-11-15/synthetic-check.json new file mode 100644 index 000000000..e011cbe18 --- /dev/null +++ b/artifacts/s2-encoder-diagnostic-2026-11-15/synthetic-check.json @@ -0,0 +1,10 @@ +{ + "status": "PASS", + "CUDA_VISIBLE_DEVICES": "", + "checks": [ + "trace leaves output bitwise unchanged on synthetic CPU FP32 model", + "all four numerator and denominator traces present", + "all hooks removed after trace" + ], + "production_code_changed": false +} diff --git a/tools/run_s2_encoder_diagnostic.py b/tools/run_s2_encoder_diagnostic.py new file mode 100644 index 000000000..90f8c8701 --- /dev/null +++ b/tools/run_s2_encoder_diagnostic.py @@ -0,0 +1,179 @@ +"""One separately budgeted replay of the preserved N128 forward failure. + +No validation arrays, validation scoring, test arrays, or pilot continuation. +Original pinned numerical modules are imported without editing them. +""" +import argparse +from collections import Counter +import json +import os +from pathlib import Path +import random +import sys +import time +import traceback +import types + +import numpy as np +import psutil +import torch + +from freeze_g1_components import sha +from s2_e1_supervisor import save +from s2_models.network import QueryModel +from s2_models.normalization import fit_base_standardizer +from s2_pilot_data import permitted_role +from s2_pilot_train import part_weights, train_part, SkipPolicy +from run_s2_pilot_training import LearningBases, read_part_arrays, to_device + + +def describe(x): + x=x.detach(); finite=torch.isfinite(x); n=int(finite.sum()) + return dict(shape=list(x.shape),dtype=str(x.dtype),finite=n==x.numel(), + nonfinite=x.numel()-n,nan=int(torch.isnan(x).sum()), + inf=int(torch.isinf(x).sum()), + max_abs_finite=float(x[finite].abs().max()) if n else None) + + +def traced_attention(self,x,*,trace,prefix): + def record(name,value): trace[prefix+'.'+name]=describe(value);return value + B,N,C=x.shape + fx_mid=record('fx_mid',self.in_project_fx(x).reshape(B,N,self.heads,self.dim_head).permute(0,2,1,3).contiguous()) + x_mid=record('x_mid',self.in_project_x(x).reshape(B,N,self.heads,self.dim_head).permute(0,2,1,3).contiguous()) + weights=record('slice_weights',self.softmax(self.in_project_slice(x_mid)/self.temperature)) + norm=record('slice_denominator',weights.sum(2)) + numerator=record('slice_numerator',torch.einsum('bhnc,bhng->bhgc',fx_mid,weights)) + token=record('slice_normalized',numerator/((norm+1e-5)[:,:,:,None].repeat(1,1,1,self.dim_head))) + q=record('q',self.to_q(token));k=record('k',self.to_k(token));v=record('v',self.to_v(token)) + dots=record('attention_logits',torch.matmul(q,k.transpose(-1,-2))*self.scale) + attn=self.dropout(self.softmax(dots)) + out=record('out_slice_token',torch.matmul(attn,v)) + out=record('deslice',torch.einsum('bhgc,bhng->bhnc',out,weights)) + return self.to_out(out.permute(0,2,1,3).reshape(B,N,-1)) + + +def trace_forward(model,data,amp): + trace={};handles=[];original=[] + for name,module in model.named_modules(): + if name: + def hook(_module,_args,output,name=name): + if isinstance(output,torch.Tensor):trace[name+'.output']=describe(output) + handles.append(module.register_forward_hook(hook)) + if name.endswith('.Attn'): + original.append((module,module.forward)) + def forward(self,x,name=name):return traced_attention(self,x,trace=trace,prefix=name) + module.forward=types.MethodType(forward,module) + try: + with torch.no_grad(),torch.autocast(device_type=data['base'].device.type,dtype=torch.float16,enabled=amp): + output=model(data['base'],data['ids'],data['gaps'],None) + result=dict(finite=bool(torch.isfinite(output).all()),output=describe(output),error=None) + except Exception as exc: + result=dict(finite=False,error=str(exc),error_type=type(exc).__name__) + finally: + for h in handles:h.remove() + for module,forward in original:module.forward=forward + return dict(**result,trace=trace) + + +def worker(root): + begin=time.monotonic();dest=root/'replay';dest.mkdir(exist_ok=False) + contract=json.loads((root/'contract.json').read_text());source=Path(contract['source_pilot']) + for path,digest in contract['code_sha256'].items():assert sha(path)==digest,path + for path,digest in contract['input_sha256'].items():assert sha(path)==digest,path + c=json.loads((source/'training-contract.json').read_text()) + roles=json.loads(Path(c['role_manifest']).read_text());manifest=json.loads(Path(c['success_manifest']).read_text()) + records={r['case']:r for r in manifest['ready_parts'] if permitted_role(roles,r['case'])=='learning'} + learning=sorted(records);assert len(learning)==382 and not set(learning)&set(roles['sealed_test_parts']) + torch.set_num_threads(1);random.seed(0);np.random.seed(0);torch.manual_seed(0) + assert torch.cuda.is_available() and torch.cuda.device_count()==1 + device=torch.device('cuda:0');torch.cuda.manual_seed_all(0);torch.cuda.reset_peak_memory_stats() + torch.backends.cuda.matmul.allow_tf32=False + assert 'V100' in torch.cuda.get_device_name() + cache={case:read_part_arrays(records[case],roles) for case in learning} + stats=fit_base_standardizer(LearningBases(records,learning,roles,cache),learning_part_ids=learning,fold='0',pool='100') + assert stats.fingerprint==c['standardizer_fingerprint'] + model=QueryModel('N',128,standardizer=stats).to(device) + optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5) + scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=c['E_max']) + scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(0)) + groups=Counter(records[p]['component_id'] for p in learning);policy=SkipPolicy();rows=[] + previous=json.loads((source/'N128-seed0/history.json').read_text()) + failure=json.loads((source/'N128-seed0/failure.json').read_text()) + target_epoch=failure['epoch'];target_step=failure['amp']['attempted_steps'] + expected_rng=np.random.Generator(np.random.PCG64(0)) + for _ in range(target_epoch+1):expected_order=expected_rng.permutation(len(learning)) + expected_case=learning[int(expected_order[target_step])] + save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'contract.json'), + expected_case=expected_case,expected_epoch=target_epoch,expected_step_zero_based=target_step, + learning_parts=len(learning),validation_arrays_read=False,test_data_read=False)) + for epoch in range(target_epoch+1): + model.train();policy.start_epoch();loss_sum=0. + order=[learning[int(j)] for j in rng.permutation(len(learning))] + for step,case in enumerate(order): + data=to_device(cache[case],device) + weights=torch.tensor(part_weights(data['counts'],groups[records[case]['component_id']],len(groups),len(learning)),dtype=torch.float32,device=device) + error=None;info={} + try: + loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=info) + except Exception as exc: + error=dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc()) + if error is not None or (epoch==target_epoch and step==target_step): + # Save the actual diagnostic state, never present best.pt as a failure state. + optimizer.zero_grad(set_to_none=True) + torch.save(dict(state_dict=model.state_dict(),optimizer=optimizer.state_dict(),scaler=scaler.state_dict(), + scheduler=scheduler.state_dict(),epoch=epoch,step_zero_based=step,case=case, + standardizer=stats.as_dict(),numpy_order_rng=rng.bit_generator.state, + torch_rng=torch.get_rng_state(),cuda_rng=torch.cuda.get_rng_state()),dest/'failure-state.pt') + parameters={name:describe(p) for name,p in model.named_parameters()} + half=trace_forward(model,data,True);single=trace_forward(model,data,False) + prior_match=all(row['matches_original_training_summary'] for row in rows) + reproduced=error is not None and error['message']==failure['message'] and epoch==target_epoch and step==target_step and case==expected_case and prior_match + decision='FP32_FINITE_FP16_NONFINITE' if reproduced and single['finite'] and not half['finite'] else 'FP32_NONFINITE' if reproduced and not single['finite'] else 'REPLAY_NOT_CONFIRMED' + result=dict(status='COMPLETE',diagnostic_only=True,decision=decision,reproduced_original_failure=reproduced, + case=case,nodes=records[case]['nodes'],epoch=epoch,step_zero_based=step,step_one_based=step+1, + original_step_error=error,prior_epoch_training_summaries=rows,parameters=parameters, + fp16=half,fp32=single,validation_arrays_read=False,validation_scoring=False,test_data_read=False, + pilot_run=False,standardizer_fingerprint=stats.fingerprint,wall_seconds=time.monotonic()-begin, + peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(), + failure_state_sha256=sha(dest/'failure-state.pt')) + save(dest/'result.json',result);print(json.dumps({k:v for k,v in result.items() if k not in ['fp16','fp32','parameters','prior_epoch_training_summaries','original_step_error']}),flush=True) + return + policy.observe(info['step_skipped']);loss_sum+=loss/len(learning) + del data,weights + if step%32==0:save(dest/'progress.json',dict(epoch=epoch,step_zero_based=step,case=case,wall_seconds=time.monotonic()-begin,amp=policy.summary())) + policy.finish_epoch(epoch) + reference=previous[epoch] + row=dict(epoch=epoch,training_global_weighted_mse=loss_sum,amp=policy.summary(), + original_training_global_weighted_mse=reference['training_global_weighted_mse'], + matches_original_training_summary=abs(loss_sum-reference['training_global_weighted_mse'])<=1e-10 and policy.summary()==reference['amp']) + rows.append(row);save(dest/'replay-history.json',rows);scheduler.step() + raise RuntimeError('Diagnostic ended without reaching the specified step') + + +def run(root): + import s2_recovered_pilot_gpu_guard as guard + contract=json.loads((root/'contract.json').read_text()) + assert not (root/'started.json').exists(),'No automatic diagnostic retry' + def diagnostic_pids(parent,_root,_index): + result=[] + for p in psutil.Process(parent).children(recursive=True): + try: + cmd=p.cmdline() + if 'tools/run_s2_encoder_diagnostic.py' in cmd and '--worker' in cmd and str(root) in cmd:result.append(p.pid) + except psutil.NoSuchProcess:pass + return result + guard.active_training_pids=diagnostic_pids + start=time.monotonic();save(root/'started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json'))) + try: + receipt,result=guard.guarded_worker(root,dict(name='replay'),0,contract['gpu_uuid'],900, + [sys.executable,'tools/run_s2_encoder_diagnostic.py',str(root),'--worker']) + save(root/'complete.json',dict(status='COMPLETE' if receipt['status']=='PASS' and result else 'DIAGNOSTIC_INCOMPLETE', + decision=result['decision'] if result else None,total_wall_seconds=time.monotonic()-start, + limit_seconds=900,separate_from_pilot=True,validation_scoring=False,test_data_read=False)) + except Exception as exc: + save(root/'failure.json',dict(error=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-start));raise + + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--worker',action='store_true');a=p.parse_args() + worker(a.root) if a.worker else run(a.root)