Files
MeltMind/tools/run_s2_encoder_diagnostic.py

180 lines
11 KiB
Python

"""One separately budgeted replay of the preserved N128 forward failure.
No validation arrays, validation scoring, test arrays, or pilot continuation.
Original pinned numerical modules are imported without editing them.
"""
import argparse
from collections import Counter
import json
import os
from pathlib import Path
import random
import sys
import time
import traceback
import types
import numpy as np
import psutil
import torch
from freeze_g1_components import sha
from s2_e1_supervisor import save
from s2_models.network import QueryModel
from s2_models.normalization import fit_base_standardizer
from s2_pilot_data import permitted_role
from s2_pilot_train import part_weights, train_part, SkipPolicy
from run_s2_pilot_training import LearningBases, read_part_arrays, to_device
def describe(x):
x=x.detach(); finite=torch.isfinite(x); n=int(finite.sum())
return dict(shape=list(x.shape),dtype=str(x.dtype),finite=n==x.numel(),
nonfinite=x.numel()-n,nan=int(torch.isnan(x).sum()),
inf=int(torch.isinf(x).sum()),
max_abs_finite=float(x[finite].abs().max()) if n else None)
def traced_attention(self,x,*,trace,prefix):
def record(name,value): trace[prefix+'.'+name]=describe(value);return value
B,N,C=x.shape
fx_mid=record('fx_mid',self.in_project_fx(x).reshape(B,N,self.heads,self.dim_head).permute(0,2,1,3).contiguous())
x_mid=record('x_mid',self.in_project_x(x).reshape(B,N,self.heads,self.dim_head).permute(0,2,1,3).contiguous())
weights=record('slice_weights',self.softmax(self.in_project_slice(x_mid)/self.temperature))
norm=record('slice_denominator',weights.sum(2))
numerator=record('slice_numerator',torch.einsum('bhnc,bhng->bhgc',fx_mid,weights))
token=record('slice_normalized',numerator/((norm+1e-5)[:,:,:,None].repeat(1,1,1,self.dim_head)))
q=record('q',self.to_q(token));k=record('k',self.to_k(token));v=record('v',self.to_v(token))
dots=record('attention_logits',torch.matmul(q,k.transpose(-1,-2))*self.scale)
attn=self.dropout(self.softmax(dots))
out=record('out_slice_token',torch.matmul(attn,v))
out=record('deslice',torch.einsum('bhgc,bhng->bhnc',out,weights))
return self.to_out(out.permute(0,2,1,3).reshape(B,N,-1))
def trace_forward(model,data,amp):
trace={};handles=[];original=[]
for name,module in model.named_modules():
if name:
def hook(_module,_args,output,name=name):
if isinstance(output,torch.Tensor):trace[name+'.output']=describe(output)
handles.append(module.register_forward_hook(hook))
if name.endswith('.Attn'):
original.append((module,module.forward))
def forward(self,x,name=name):return traced_attention(self,x,trace=trace,prefix=name)
module.forward=types.MethodType(forward,module)
try:
with torch.no_grad(),torch.autocast(device_type=data['base'].device.type,dtype=torch.float16,enabled=amp):
output=model(data['base'],data['ids'],data['gaps'],None)
result=dict(finite=bool(torch.isfinite(output).all()),output=describe(output),error=None)
except Exception as exc:
result=dict(finite=False,error=str(exc),error_type=type(exc).__name__)
finally:
for h in handles:h.remove()
for module,forward in original:module.forward=forward
return dict(**result,trace=trace)
def worker(root):
begin=time.monotonic();dest=root/'replay';dest.mkdir(exist_ok=False)
contract=json.loads((root/'contract.json').read_text());source=Path(contract['source_pilot'])
for path,digest in contract['code_sha256'].items():assert sha(path)==digest,path
for path,digest in contract['input_sha256'].items():assert sha(path)==digest,path
c=json.loads((source/'training-contract.json').read_text())
roles=json.loads(Path(c['role_manifest']).read_text());manifest=json.loads(Path(c['success_manifest']).read_text())
records={r['case']:r for r in manifest['ready_parts'] if permitted_role(roles,r['case'])=='learning'}
learning=sorted(records);assert len(learning)==382 and not set(learning)&set(roles['sealed_test_parts'])
torch.set_num_threads(1);random.seed(0);np.random.seed(0);torch.manual_seed(0)
assert torch.cuda.is_available() and torch.cuda.device_count()==1
device=torch.device('cuda:0');torch.cuda.manual_seed_all(0);torch.cuda.reset_peak_memory_stats()
torch.backends.cuda.matmul.allow_tf32=False
assert 'V100' in torch.cuda.get_device_name()
cache={case:read_part_arrays(records[case],roles) for case in learning}
stats=fit_base_standardizer(LearningBases(records,learning,roles,cache),learning_part_ids=learning,fold='0',pool='100')
assert stats.fingerprint==c['standardizer_fingerprint']
model=QueryModel('N',128,standardizer=stats).to(device)
optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5)
scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=c['E_max'])
scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(0))
groups=Counter(records[p]['component_id'] for p in learning);policy=SkipPolicy();rows=[]
previous=json.loads((source/'N128-seed0/history.json').read_text())
failure=json.loads((source/'N128-seed0/failure.json').read_text())
target_epoch=failure['epoch'];target_step=failure['amp']['attempted_steps']
expected_rng=np.random.Generator(np.random.PCG64(0))
for _ in range(target_epoch+1):expected_order=expected_rng.permutation(len(learning))
expected_case=learning[int(expected_order[target_step])]
save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'contract.json'),
expected_case=expected_case,expected_epoch=target_epoch,expected_step_zero_based=target_step,
learning_parts=len(learning),validation_arrays_read=False,test_data_read=False))
for epoch in range(target_epoch+1):
model.train();policy.start_epoch();loss_sum=0.
order=[learning[int(j)] for j in rng.permutation(len(learning))]
for step,case in enumerate(order):
data=to_device(cache[case],device)
weights=torch.tensor(part_weights(data['counts'],groups[records[case]['component_id']],len(groups),len(learning)),dtype=torch.float32,device=device)
error=None;info={}
try:
loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=info)
except Exception as exc:
error=dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc())
if error is not None or (epoch==target_epoch and step==target_step):
# Save the actual diagnostic state, never present best.pt as a failure state.
optimizer.zero_grad(set_to_none=True)
torch.save(dict(state_dict=model.state_dict(),optimizer=optimizer.state_dict(),scaler=scaler.state_dict(),
scheduler=scheduler.state_dict(),epoch=epoch,step_zero_based=step,case=case,
standardizer=stats.as_dict(),numpy_order_rng=rng.bit_generator.state,
torch_rng=torch.get_rng_state(),cuda_rng=torch.cuda.get_rng_state()),dest/'failure-state.pt')
parameters={name:describe(p) for name,p in model.named_parameters()}
half=trace_forward(model,data,True);single=trace_forward(model,data,False)
prior_match=all(row['matches_original_training_summary'] for row in rows)
reproduced=error is not None and error['message']==failure['message'] and epoch==target_epoch and step==target_step and case==expected_case and prior_match
decision='FP32_FINITE_FP16_NONFINITE' if reproduced and single['finite'] and not half['finite'] else 'FP32_NONFINITE' if reproduced and not single['finite'] else 'REPLAY_NOT_CONFIRMED'
result=dict(status='COMPLETE',diagnostic_only=True,decision=decision,reproduced_original_failure=reproduced,
case=case,nodes=records[case]['nodes'],epoch=epoch,step_zero_based=step,step_one_based=step+1,
original_step_error=error,prior_epoch_training_summaries=rows,parameters=parameters,
fp16=half,fp32=single,validation_arrays_read=False,validation_scoring=False,test_data_read=False,
pilot_run=False,standardizer_fingerprint=stats.fingerprint,wall_seconds=time.monotonic()-begin,
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(),
failure_state_sha256=sha(dest/'failure-state.pt'))
save(dest/'result.json',result);print(json.dumps({k:v for k,v in result.items() if k not in ['fp16','fp32','parameters','prior_epoch_training_summaries','original_step_error']}),flush=True)
return
policy.observe(info['step_skipped']);loss_sum+=loss/len(learning)
del data,weights
if step%32==0:save(dest/'progress.json',dict(epoch=epoch,step_zero_based=step,case=case,wall_seconds=time.monotonic()-begin,amp=policy.summary()))
policy.finish_epoch(epoch)
reference=previous[epoch]
row=dict(epoch=epoch,training_global_weighted_mse=loss_sum,amp=policy.summary(),
original_training_global_weighted_mse=reference['training_global_weighted_mse'],
matches_original_training_summary=abs(loss_sum-reference['training_global_weighted_mse'])<=1e-10 and policy.summary()==reference['amp'])
rows.append(row);save(dest/'replay-history.json',rows);scheduler.step()
raise RuntimeError('Diagnostic ended without reaching the specified step')
def run(root):
import s2_recovered_pilot_gpu_guard as guard
contract=json.loads((root/'contract.json').read_text())
assert not (root/'started.json').exists(),'No automatic diagnostic retry'
def diagnostic_pids(parent,_root,_index):
result=[]
for p in psutil.Process(parent).children(recursive=True):
try:
cmd=p.cmdline()
if 'tools/run_s2_encoder_diagnostic.py' in cmd and '--worker' in cmd and str(root) in cmd:result.append(p.pid)
except psutil.NoSuchProcess:pass
return result
guard.active_training_pids=diagnostic_pids
start=time.monotonic();save(root/'started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
try:
receipt,result=guard.guarded_worker(root,dict(name='replay'),0,contract['gpu_uuid'],900,
[sys.executable,'tools/run_s2_encoder_diagnostic.py',str(root),'--worker'])
save(root/'complete.json',dict(status='COMPLETE' if receipt['status']=='PASS' and result else 'DIAGNOSTIC_INCOMPLETE',
decision=result['decision'] if result else None,total_wall_seconds=time.monotonic()-start,
limit_seconds=900,separate_from_pilot=True,validation_scoring=False,test_data_read=False))
except Exception as exc:
save(root/'failure.json',dict(error=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-start));raise
if __name__=='__main__':
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--worker',action='store_true');a=p.parse_args()
worker(a.root) if a.worker else run(a.root)