21 lines
1.7 KiB
Plaintext
21 lines
1.7 KiB
Plaintext
Traceback (most recent call last):
|
|
File "/home/frank14f/optane/MeltMind/tools/run_s2_pilot_training.py", line 150, in <module>
|
|
p=argparse.ArgumentParser();p.add_argument('root',type=Path);p.add_argument('index',type=int);a=p.parse_args();raise SystemExit(run(a.root,a.index))
|
|
File "/home/frank14f/optane/MeltMind/tools/run_s2_pilot_training.py", line 105, in run
|
|
loss=train_part(model,optimizer,data['base'],data['ids'],data['gaps'],data['edge'],data['target'],weights,scaler=scaler,amp=True,step_info=step_info)
|
|
File "/home/frank14f/optane/MeltMind/tools/s2_pilot_train.py", line 20, in train_part
|
|
prediction=model(base,ids,gaps,edge if model.arm=='E' else None)
|
|
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
|
return self._call_impl(*args, **kwargs)
|
|
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
|
return forward_call(*args, **kwargs)
|
|
File "/home/frank14f/optane/MeltMind/tools/s2_models/network.py", line 94, in forward
|
|
embedding = self.encoder(base)[query_ids]
|
|
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
|
|
return self._call_impl(*args, **kwargs)
|
|
File "/home/frank14f/anaconda3/envs/pycuda_3_10/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
|
|
return forward_call(*args, **kwargs)
|
|
File "/home/frank14f/optane/MeltMind/tools/s2_models/network.py", line 63, in forward
|
|
raise ValueError("nonfinite encoder output")
|
|
ValueError: nonfinite encoder output
|