180 lines
11 KiB
Python
180 lines
11 KiB
Python
"""One separately budgeted replay of the preserved N128 forward failure.
|
|
|
|
No validation arrays, validation scoring, test arrays, or pilot continuation.
|
|
Original pinned numerical modules are imported without editing them.
|
|
"""
|
|
import argparse
|
|
from collections import Counter
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import types
|
|
|
|
import numpy as np
|
|
import psutil
|
|
import torch
|
|
|
|
from freeze_g1_components import sha
|
|
from s2_e1_supervisor import save
|
|
from s2_models.network import QueryModel
|
|
from s2_models.normalization import fit_base_standardizer
|
|
from s2_pilot_data import permitted_role
|
|
from s2_pilot_train import part_weights, train_part, SkipPolicy
|
|
from run_s2_pilot_training import LearningBases, read_part_arrays, to_device
|
|
|
|
|
|
def describe(x):
|
|
x=x.detach(); finite=torch.isfinite(x); n=int(finite.sum())
|
|
return dict(shape=list(x.shape),dtype=str(x.dtype),finite=n==x.numel(),
|
|
nonfinite=x.numel()-n,nan=int(torch.isnan(x).sum()),
|
|
inf=int(torch.isinf(x).sum()),
|
|
max_abs_finite=float(x[finite].abs().max()) if n else None)
|
|
|
|
|
|
def traced_attention(self,x,*,trace,prefix):
|
|
def record(name,value): trace[prefix+'.'+name]=describe(value);return value
|
|
B,N,C=x.shape
|
|
fx_mid=record('fx_mid',self.in_project_fx(x).reshape(B,N,self.heads,self.dim_head).permute(0,2,1,3).contiguous())
|
|
x_mid=record('x_mid',self.in_project_x(x).reshape(B,N,self.heads,self.dim_head).permute(0,2,1,3).contiguous())
|
|
weights=record('slice_weights',self.softmax(self.in_project_slice(x_mid)/self.temperature))
|
|
norm=record('slice_denominator',weights.sum(2))
|
|
numerator=record('slice_numerator',torch.einsum('bhnc,bhng->bhgc',fx_mid,weights))
|
|
token=record('slice_normalized',numerator/((norm+1e-5)[:,:,:,None].repeat(1,1,1,self.dim_head)))
|
|
q=record('q',self.to_q(token));k=record('k',self.to_k(token));v=record('v',self.to_v(token))
|
|
dots=record('attention_logits',torch.matmul(q,k.transpose(-1,-2))*self.scale)
|
|
attn=self.dropout(self.softmax(dots))
|
|
out=record('out_slice_token',torch.matmul(attn,v))
|
|
out=record('deslice',torch.einsum('bhgc,bhng->bhnc',out,weights))
|
|
return self.to_out(out.permute(0,2,1,3).reshape(B,N,-1))
|
|
|
|
|
|
def trace_forward(model,data,amp):
|
|
trace={};handles=[];original=[]
|
|
for name,module in model.named_modules():
|
|
if name:
|
|
def hook(_module,_args,output,name=name):
|
|
if isinstance(output,torch.Tensor):trace[name+'.output']=describe(output)
|
|
handles.append(module.register_forward_hook(hook))
|
|
if name.endswith('.Attn'):
|
|
original.append((module,module.forward))
|
|
def forward(self,x,name=name):return traced_attention(self,x,trace=trace,prefix=name)
|
|
module.forward=types.MethodType(forward,module)
|
|
try:
|
|
with torch.no_grad(),torch.autocast(device_type=data['base'].device.type,dtype=torch.float16,enabled=amp):
|
|
output=model(data['base'],data['ids'],data['gaps'],None)
|
|
result=dict(finite=bool(torch.isfinite(output).all()),output=describe(output),error=None)
|
|
except Exception as exc:
|
|
result=dict(finite=False,error=str(exc),error_type=type(exc).__name__)
|
|
finally:
|
|
for h in handles:h.remove()
|
|
for module,forward in original:module.forward=forward
|
|
return dict(**result,trace=trace)
|
|
|
|
|
|
def worker(root):
|
|
begin=time.monotonic();dest=root/'replay';dest.mkdir(exist_ok=False)
|
|
contract=json.loads((root/'contract.json').read_text());source=Path(contract['source_pilot'])
|
|
for path,digest in contract['code_sha256'].items():assert sha(path)==digest,path
|
|
for path,digest in contract['input_sha256'].items():assert sha(path)==digest,path
|
|
c=json.loads((source/'training-contract.json').read_text())
|
|
roles=json.loads(Path(c['role_manifest']).read_text());manifest=json.loads(Path(c['success_manifest']).read_text())
|
|
records={r['case']:r for r in manifest['ready_parts'] if permitted_role(roles,r['case'])=='learning'}
|
|
learning=sorted(records);assert len(learning)==382 and not set(learning)&set(roles['sealed_test_parts'])
|
|
torch.set_num_threads(1);random.seed(0);np.random.seed(0);torch.manual_seed(0)
|
|
assert torch.cuda.is_available() and torch.cuda.device_count()==1
|
|
device=torch.device('cuda:0');torch.cuda.manual_seed_all(0);torch.cuda.reset_peak_memory_stats()
|
|
torch.backends.cuda.matmul.allow_tf32=False
|
|
assert 'V100' in torch.cuda.get_device_name()
|
|
cache={case:read_part_arrays(records[case],roles) for case in learning}
|
|
stats=fit_base_standardizer(LearningBases(records,learning,roles,cache),learning_part_ids=learning,fold='0',pool='100')
|
|
assert stats.fingerprint==c['standardizer_fingerprint']
|
|
model=QueryModel('N',128,standardizer=stats).to(device)
|
|
optimizer=torch.optim.AdamW(model.parameters(),lr=1e-3,weight_decay=1e-5)
|
|
scheduler=torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=c['E_max'])
|
|
scaler=torch.amp.GradScaler('cuda');rng=np.random.Generator(np.random.PCG64(0))
|
|
groups=Counter(records[p]['component_id'] for p in learning);policy=SkipPolicy();rows=[]
|
|
previous=json.loads((source/'N128-seed0/history.json').read_text())
|
|
failure=json.loads((source/'N128-seed0/failure.json').read_text())
|
|
target_epoch=failure['epoch'];target_step=failure['amp']['attempted_steps']
|
|
expected_rng=np.random.Generator(np.random.PCG64(0))
|
|
for _ in range(target_epoch+1):expected_order=expected_rng.permutation(len(learning))
|
|
expected_case=learning[int(expected_order[target_step])]
|
|
save(dest/'started.json',dict(started_unix=time.time(),contract_sha256=sha(root/'contract.json'),
|
|
expected_case=expected_case,expected_epoch=target_epoch,expected_step_zero_based=target_step,
|
|
learning_parts=len(learning),validation_arrays_read=False,test_data_read=False))
|
|
for epoch in range(target_epoch+1):
|
|
model.train();policy.start_epoch();loss_sum=0.
|
|
order=[learning[int(j)] for j in rng.permutation(len(learning))]
|
|
for step,case in enumerate(order):
|
|
data=to_device(cache[case],device)
|
|
weights=torch.tensor(part_weights(data['counts'],groups[records[case]['component_id']],len(groups),len(learning)),dtype=torch.float32,device=device)
|
|
error=None;info={}
|
|
try:
|
|
loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=info)
|
|
except Exception as exc:
|
|
error=dict(type=type(exc).__name__,message=str(exc),traceback=traceback.format_exc())
|
|
if error is not None or (epoch==target_epoch and step==target_step):
|
|
# Save the actual diagnostic state, never present best.pt as a failure state.
|
|
optimizer.zero_grad(set_to_none=True)
|
|
torch.save(dict(state_dict=model.state_dict(),optimizer=optimizer.state_dict(),scaler=scaler.state_dict(),
|
|
scheduler=scheduler.state_dict(),epoch=epoch,step_zero_based=step,case=case,
|
|
standardizer=stats.as_dict(),numpy_order_rng=rng.bit_generator.state,
|
|
torch_rng=torch.get_rng_state(),cuda_rng=torch.cuda.get_rng_state()),dest/'failure-state.pt')
|
|
parameters={name:describe(p) for name,p in model.named_parameters()}
|
|
half=trace_forward(model,data,True);single=trace_forward(model,data,False)
|
|
prior_match=all(row['matches_original_training_summary'] for row in rows)
|
|
reproduced=error is not None and error['message']==failure['message'] and epoch==target_epoch and step==target_step and case==expected_case and prior_match
|
|
decision='FP32_FINITE_FP16_NONFINITE' if reproduced and single['finite'] and not half['finite'] else 'FP32_NONFINITE' if reproduced and not single['finite'] else 'REPLAY_NOT_CONFIRMED'
|
|
result=dict(status='COMPLETE',diagnostic_only=True,decision=decision,reproduced_original_failure=reproduced,
|
|
case=case,nodes=records[case]['nodes'],epoch=epoch,step_zero_based=step,step_one_based=step+1,
|
|
original_step_error=error,prior_epoch_training_summaries=rows,parameters=parameters,
|
|
fp16=half,fp32=single,validation_arrays_read=False,validation_scoring=False,test_data_read=False,
|
|
pilot_run=False,standardizer_fingerprint=stats.fingerprint,wall_seconds=time.monotonic()-begin,
|
|
peak_allocated_bytes=torch.cuda.max_memory_allocated(),peak_reserved_bytes=torch.cuda.max_memory_reserved(),
|
|
failure_state_sha256=sha(dest/'failure-state.pt'))
|
|
save(dest/'result.json',result);print(json.dumps({k:v for k,v in result.items() if k not in ['fp16','fp32','parameters','prior_epoch_training_summaries','original_step_error']}),flush=True)
|
|
return
|
|
policy.observe(info['step_skipped']);loss_sum+=loss/len(learning)
|
|
del data,weights
|
|
if step%32==0:save(dest/'progress.json',dict(epoch=epoch,step_zero_based=step,case=case,wall_seconds=time.monotonic()-begin,amp=policy.summary()))
|
|
policy.finish_epoch(epoch)
|
|
reference=previous[epoch]
|
|
row=dict(epoch=epoch,training_global_weighted_mse=loss_sum,amp=policy.summary(),
|
|
original_training_global_weighted_mse=reference['training_global_weighted_mse'],
|
|
matches_original_training_summary=abs(loss_sum-reference['training_global_weighted_mse'])<=1e-10 and policy.summary()==reference['amp'])
|
|
rows.append(row);save(dest/'replay-history.json',rows);scheduler.step()
|
|
raise RuntimeError('Diagnostic ended without reaching the specified step')
|
|
|
|
|
|
def run(root):
|
|
import s2_recovered_pilot_gpu_guard as guard
|
|
contract=json.loads((root/'contract.json').read_text())
|
|
assert not (root/'started.json').exists(),'No automatic diagnostic retry'
|
|
def diagnostic_pids(parent,_root,_index):
|
|
result=[]
|
|
for p in psutil.Process(parent).children(recursive=True):
|
|
try:
|
|
cmd=p.cmdline()
|
|
if 'tools/run_s2_encoder_diagnostic.py' in cmd and '--worker' in cmd and str(root) in cmd:result.append(p.pid)
|
|
except psutil.NoSuchProcess:pass
|
|
return result
|
|
guard.active_training_pids=diagnostic_pids
|
|
start=time.monotonic();save(root/'started.json',dict(started_unix=time.time(),pid=os.getpid(),contract_sha256=sha(root/'contract.json')))
|
|
try:
|
|
receipt,result=guard.guarded_worker(root,dict(name='replay'),0,contract['gpu_uuid'],900,
|
|
[sys.executable,'tools/run_s2_encoder_diagnostic.py',str(root),'--worker'])
|
|
save(root/'complete.json',dict(status='COMPLETE' if receipt['status']=='PASS' and result else 'DIAGNOSTIC_INCOMPLETE',
|
|
decision=result['decision'] if result else None,total_wall_seconds=time.monotonic()-start,
|
|
limit_seconds=900,separate_from_pilot=True,validation_scoring=False,test_data_read=False))
|
|
except Exception as exc:
|
|
save(root/'failure.json',dict(error=str(exc),traceback=traceback.format_exc(),wall_seconds=time.monotonic()-start));raise
|
|
|
|
|
|
if __name__=='__main__':
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('--worker',action='store_true');a=p.parse_args()
|
|
worker(a.root) if a.worker else run(a.root)
|