I'm working on a highly imbalanced dataset for a tabular binary classification task. I'm using pytorch and skorch to build my DNN with output_dim = 1. Assigning class weights is better than upsampling based on the nature of my dataset. However, when I run:
from sklearn.utils.class_weight import compute_class_weight
class_weight = compute_class_weight(class_weight = 'balanced', classes = np.unique(y_train), y = y_train)
net = NeuralNetBinaryClassifier(
DNN,
lr = 0.001,
optimizer = torch.optim.SGD,
criterion = torch.nn.BCEWithLogitsLoss(),
train_split = predefined_split(val_data),
max_epochs = 1000,
batch_size = 2048,
criterion__weight = class_weight,
callbacks=[
(skorch.callbacks.EarlyStopping(patience = 5, threshold = 1e-3))
]
)
net.fit(X_train, y_train)
a RuntimeError pops up:
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
Cell In[27], line 1
----> 1 net.fit(X_train, y_train)
File ~\anaconda3\lib\site-packages\skorch\classifier.py:348, in NeuralNetBinaryClassifier.fit(self, X, y, **fit_params)
337 """See ``NeuralNet.fit``.
338
339 In contrast to ``NeuralNet.fit``, ``y`` is non-optional to
(...)
343
344 """
345 # pylint: disable=useless-super-delegation
346 # this is actually a pylint bug:
347 # https://github.com/PyCQA/pylint/issues/1085
--> 348 return super().fit(X, y, **fit_params)
File ~\anaconda3\lib\site-packages\skorch\net.py:1319, in NeuralNet.fit(self, X, y, **fit_params)
1316 if not self.warm_start or not self.initialized_:
1317 self.initialize()
-> 1319 self.partial_fit(X, y, **fit_params)
1320 return self
File ~\anaconda3\lib\site-packages\skorch\net.py:1278, in NeuralNet.partial_fit(self, X, y, classes, **fit_params)
1276 self.notify('on_train_begin', X=X, y=y)
1277 try:
-> 1278 self.fit_loop(X, y, **fit_params)
1279 except KeyboardInterrupt:
1280 pass
File ~\anaconda3\lib\site-packages\skorch\net.py:1190, in NeuralNet.fit_loop(self, X, y, epochs, **fit_params)
1187 for _ in range(epochs):
1188 self.notify('on_epoch_begin', **on_epoch_kwargs)
-> 1190 self.run_single_epoch(iterator_train, training=True, prefix="train",
1191 step_fn=self.train_step, **fit_params)
1193 self.run_single_epoch(iterator_valid, training=False, prefix="valid",
1194 step_fn=self.validation_step, **fit_params)
1196 self.notify("on_epoch_end", **on_epoch_kwargs)
File ~\anaconda3\lib\site-packages\skorch\net.py:1226, in NeuralNet.run_single_epoch(self, iterator, training, prefix, step_fn, **fit_params)
1224 for batch in iterator:
1225 self.notify("on_batch_begin", batch=batch, training=training)
-> 1226 step = step_fn(batch, **fit_params)
1227 self.history.record_batch(prefix + "_loss", step["loss"].item())
1228 batch_size = (get_len(batch[0]) if isinstance(batch, (tuple, list))
1229 else get_len(batch))
File ~\anaconda3\lib\site-packages\skorch\net.py:1105, in NeuralNet.train_step(self, batch, **fit_params)
1097 self.notify(
1098 'on_grad_computed',
1099 named_parameters=TeeGenerator(self.get_all_learnable_params()),
1100 batch=batch,
1101 training=True,
1102 )
1103 return step['loss']
-> 1105 self._step_optimizer(step_fn)
1106 return step_accumulator.get_step()
File ~\anaconda3\lib\site-packages\skorch\net.py:1060, in NeuralNet._step_optimizer(self, step_fn)
1058 optimizer.step()
1059 else:
-> 1060 optimizer.step(step_fn)
File ~\anaconda3\lib\site-packages\torch\optim\optimizer.py:113, in Optimizer._hook_for_profile.<locals>.profile_hook_step.<locals>.wrapper(*args, **kwargs)
111 profile_name = "Optimizer.step#{}.step".format(obj.__class__.__name__)
112 with torch.autograd.profiler.record_function(profile_name):
--> 113 return func(*args, **kwargs)
File ~\anaconda3\lib\site-packages\torch\autograd\grad_mode.py:27, in _DecoratorContextManager.__call__.<locals>.decorate_context(*args, **kwargs)
24 @functools.wraps(func)
25 def decorate_context(*args, **kwargs):
26 with self.clone():
---> 27 return func(*args, **kwargs)
File ~\anaconda3\lib\site-packages\torch\optim\sgd.py:125, in SGD.step(self, closure)
123 if closure is not None:
124 with torch.enable_grad():
--> 125 loss = closure()
127 for group in self.param_groups:
128 params_with_grad = []
File ~\anaconda3\lib\site-packages\skorch\net.py:1094, in NeuralNet.train_step.<locals>.step_fn()
1092 def step_fn():
1093 self._zero_grad_optimizer()
-> 1094 step = self.train_step_single(batch, **fit_params)
1095 step_accumulator.store_step(step)
1097 self.notify(
1098 'on_grad_computed',
1099 named_parameters=TeeGenerator(self.get_all_learnable_params()),
1100 batch=batch,
1101 training=True,
1102 )
File ~\anaconda3\lib\site-packages\skorch\net.py:994, in NeuralNet.train_step_single(self, batch, **fit_params)
992 Xi, yi = unpack_data(batch)
993 y_pred = self.infer(Xi, **fit_params)
--> 994 loss = self.get_loss(y_pred, yi, X=Xi, training=True)
995 loss.backward()
996 return {
997 'loss': loss,
998 'y_pred': y_pred,
999 }
File ~\anaconda3\lib\site-packages\skorch\net.py:1665, in NeuralNet.get_loss(self, y_pred, y_true, X, training)
1636 """Return the loss for this batch.
1637
1638 Parameters
(...)
1662
1663 """
1664 y_true = to_tensor(y_true, device=self.device)
-> 1665 return self.criterion_(y_pred, y_true)
File ~\anaconda3\lib\site-packages\torch\nn\modules\module.py:1130, in Module._call_impl(self, *input, **kwargs)
1126 # If we don't have any hooks, we want to skip the rest of the logic in
1127 # this function, and just call forward.
1128 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1129 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1130 return forward_call(*input, **kwargs)
1131 # Do not call functions when jit is used
1132 full_backward_hooks, non_full_backward_hooks = [], []
File ~\anaconda3\lib\site-packages\torch\nn\modules\loss.py:714, in BCEWithLogitsLoss.forward(self, input, target)
713 def forward(self, input: Tensor, target: Tensor) -> Tensor:
--> 714 return F.binary_cross_entropy_with_logits(input, target,
715 self.weight,
716 pos_weight=self.pos_weight,
717 reduction=self.reduction)
File ~\anaconda3\lib\site-packages\torch\nn\functional.py:3150, in binary_cross_entropy_with_logits(input, target, weight, size_average, reduce, reduction, pos_weight)
3147 if not (target.size() == input.size()):
3148 raise ValueError("Target size ({}) must be the same as input size ({})".format(target.size(), input.size()))
-> 3150 return torch.binary_cross_entropy_with_logits(input, target, weight, pos_weight, reduction_enum)
RuntimeError: The size of tensor a (2048) must match the size of tensor b (2) at non-singleton dimension 0
I guess it has something to do with my loss fn. Tensor 'a' must be a batch, but I don't know where tensor 'b' comes from. How do I check or what am I doing wrong?
Also, I tried running the network without class weights and this error didn't show up.