Instruction: Making transformers BertForSequenceClassification initial layers non-trainable for pytorch training

Input:
I'm trying to do a transfer learning with BertForSequenceClassification https://huggingface.co/transformers/model_doc/bert.html#bertforsequenceclassification

This is my simple NN model for classification.

from transformers import BertTokenizer, BertForSequenceClassification
class NN(nn.Module):
    def __init__(self):
        super(NN, self).__init__()
        self.bert = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels = 17)
    def forward(self, x):
        return self.bert(x)

a = NN()

Once I print my model I get this:

NN(
  (bert): BertForSequenceClassification(
    (bert): BertModel(
      (embeddings): BertEmbeddings(
        (word_embeddings): Embedding(30522, 768, padding_idx=0)
        (position_embeddings): Embedding(512, 768)
        (token_type_embeddings): Embedding(2, 768)
        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (encoder): BertEncoder(
        (layer): ModuleList(
          (0): BertLayer(
            (attention): BertAttention(
              (self): BertSelfAttention(
                (query): Linear(in_features=768, out_features=768, bias=True)
                (key): Linear(in_features=768, out_features=768, bias=True)
                (value): Linear(in_features=768, out_features=768, bias=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (output): BertSelfOutput(
                (dense): Linear(in_features=768, out_features=768, bias=True)
                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
            (intermediate): BertIntermediate(
              (dense): Linear(in_features=768, out_features=3072, bias=True)
            )
            (output): BertOutput(
              (dense): Linear(in_features=3072, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (1): BertLayer(
            (attention): BertAttention(
              (self): BertSelfAttention(
                (query): Linear(in_features=768, out_features=768, bias=True)
                (key): Linear(in_features=768, out_features=768, bias=True)
                (value): Linear(in_features=768, out_features=768, bias=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (output): BertSelfOutput(
                (dense): Linear(in_features=768, out_features=768, bias=True)
                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
            (intermediate): BertIntermediate(
              (dense): Linear(in_features=768, out_features=3072, bias=True)
            )
            (output): BertOutput(
              (dense): Linear(in_features=3072, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (2): BertLayer(
            (attention): BertAttention(
              (self): BertSelfAttention(
                (query): Linear(in_features=768, out_features=768, bias=True)
                (key): Linear(in_features=768, out_features=768, bias=True)
                (value): Linear(in_features=768, out_features=768, bias=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (output): BertSelfOutput(
                (dense): Linear(in_features=768, out_features=768, bias=True)
                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
            (intermediate): BertIntermediate(
              (dense): Linear(in_features=768, out_features=3072, bias=True)
            )
            (output): BertOutput(
              (dense): Linear(in_features=3072, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (3): BertLayer(
            (attention): BertAttention(
              (self): BertSelfAttention(
                (query): Linear(in_features=768, out_features=768, bias=True)
                (key): Linear(in_features=768, out_features=768, bias=True)
                (value): Linear(in_features=768, out_features=768, bias=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (output): BertSelfOutput(
                (dense): Linear(in_features=768, out_features=768, bias=True)
                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
            (intermediate): BertIntermediate(
              (dense): Linear(in_features=768, out_features=3072, bias=True)
            )
            (output): BertOutput(
              (dense): Linear(in_features=3072, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (4): BertLayer(
            (attention): BertAttention(
              (self): BertSelfAttention(
                (query): Linear(in_features=768, out_features=768, bias=True)
                (key): Linear(in_features=768, out_features=768, bias=True)
                (value): Linear(in_features=768, out_features=768, bias=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (output): BertSelfOutput(
                (dense): Linear(in_features=768, out_features=768, bias=True)
                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
            (intermediate): BertIntermediate(
              (dense): Linear(in_features=768, out_features=3072, bias=True)
            )
            (output): BertOutput(
              (dense): Linear(in_features=3072, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (5): BertLayer(
            (attention): BertAttention(
              (self): BertSelfAttention(
                (query): Linear(in_features=768, out_features=768, bias=True)
                (key): Linear(in_features=768, out_features=768, bias=True)
                (value): Linear(in_features=768, out_features=768, bias=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (output): BertSelfOutput(
                (dense): Linear(in_features=768, out_features=768, bias=True)
                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
            (intermediate): BertIntermediate(
              (dense): Linear(in_features=768, out_features=3072, bias=True)
            )
            (output): BertOutput(
              (dense): Linear(in_features=3072, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (6): BertLayer(
            (attention): BertAttention(
              (self): BertSelfAttention(
                (query): Linear(in_features=768, out_features=768, bias=True)
                (key): Linear(in_features=768, out_features=768, bias=True)
                (value): Linear(in_features=768, out_features=768, bias=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (output): BertSelfOutput(
                (dense): Linear(in_features=768, out_features=768, bias=True)
                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
            (intermediate): BertIntermediate(
              (dense): Linear(in_features=768, out_features=3072, bias=True)
            )
            (output): BertOutput(
              (dense): Linear(in_features=3072, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (7): BertLayer(
            (attention): BertAttention(
              (self): BertSelfAttention(
                (query): Linear(in_features=768, out_features=768, bias=True)
                (key): Linear(in_features=768, out_features=768, bias=True)
                (value): Linear(in_features=768, out_features=768, bias=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (output): BertSelfOutput(
                (dense): Linear(in_features=768, out_features=768, bias=True)
                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
            (intermediate): BertIntermediate(
              (dense): Linear(in_features=768, out_features=3072, bias=True)
            )
            (output): BertOutput(
              (dense): Linear(in_features=3072, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (8): BertLayer(
            (attention): BertAttention(
              (self): BertSelfAttention(
                (query): Linear(in_features=768, out_features=768, bias=True)
                (key): Linear(in_features=768, out_features=768, bias=True)
                (value): Linear(in_features=768, out_features=768, bias=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (output): BertSelfOutput(
                (dense): Linear(in_features=768, out_features=768, bias=True)
                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
            (intermediate): BertIntermediate(
              (dense): Linear(in_features=768, out_features=3072, bias=True)
            )
            (output): BertOutput(
              (dense): Linear(in_features=3072, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (9): BertLayer(
            (attention): BertAttention(
              (self): BertSelfAttention(
                (query): Linear(in_features=768, out_features=768, bias=True)
                (key): Linear(in_features=768, out_features=768, bias=True)
                (value): Linear(in_features=768, out_features=768, bias=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (output): BertSelfOutput(
                (dense): Linear(in_features=768, out_features=768, bias=True)
                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
            (intermediate): BertIntermediate(
              (dense): Linear(in_features=768, out_features=3072, bias=True)
            )
            (output): BertOutput(
              (dense): Linear(in_features=3072, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (10): BertLayer(
            (attention): BertAttention(
              (self): BertSelfAttention(
                (query): Linear(in_features=768, out_features=768, bias=True)
                (key): Linear(in_features=768, out_features=768, bias=True)
                (value): Linear(in_features=768, out_features=768, bias=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (output): BertSelfOutput(
                (dense): Linear(in_features=768, out_features=768, bias=True)
                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
            (intermediate): BertIntermediate(
              (dense): Linear(in_features=768, out_features=3072, bias=True)
            )
            (output): BertOutput(
              (dense): Linear(in_features=3072, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (11): BertLayer(
            (attention): BertAttention(
              (self): BertSelfAttention(
                (query): Linear(in_features=768, out_features=768, bias=True)
                (key): Linear(in_features=768, out_features=768, bias=True)
                (value): Linear(in_features=768, out_features=768, bias=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (output): BertSelfOutput(
                (dense): Linear(in_features=768, out_features=768, bias=True)
                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
            (intermediate): BertIntermediate(
              (dense): Linear(in_features=768, out_features=3072, bias=True)
            )
            (output): BertOutput(
              (dense): Linear(in_features=3072, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
        )
      )
      (pooler): BertPooler(
        (dense): Linear(in_features=768, out_features=768, bias=True)
        (activation): Tanh()
      )
    )
    (dropout): Dropout(p=0.1, inplace=False)
    (classifier): Linear(in_features=768, out_features=17, bias=True)
  )
)

I want to make only the last Linear layer trainable but I can't access the layers of my model. I have tried iterating model as a list and features but both gives error that there is no such attribute.

a.features

How can make all my layers frozen except the last linear layer?

Output:
As the documentation says,

  class transformers.BertForSequenceClassification(config)[source]

Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. This model is a PyTorch torch.nn.Module sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matters related to general usage and behavior.

So, you can treat your bert layer as an nn.Module container and use these attributes and functions https://pytorch.org/docs/stable/nn.html

You can use list(a.parameters()) to get the layers and use requires_grad to make layers trainable or non-trainable.

for pp in a.parameters():
    pp.requires_grad = False

list(a.parameters())[-1].requires_grad = True


Instruction: How to initialize empty tensor with certain dimension and append to it through a loop without CUDA out of memory?

Input:
I am trying to append tensors (t) generated in a for-loop to a list [T] that accumulates all these tensors. Next, the list [T] requires to be converted into a tensor and needs to be loaded onto GPU.

        b_output = []

        for eachInputId, eachMask in zip(b_input_ids, b_input_mask):
            # unrolled into each individual document
            # print(eachInputId.size()) # individual document here

            outputs = model(eachInputId,
                    token_type_ids=None,
                    attention_mask=eachMask)

            # combine the [CLS] output layer to form the document
            doc_output = torch.mean(outputs[1], dim=0) # size = [1, ncol]

            b_output.append( doc_output )

        t_b_output = torch.tensor( b_output )

Another method that I tried was initializing a tensor {T} with fixed dimensions and appending the tensors (t) to it from the for-loop.

        b_output = torch.zeros(batch_size, hidden_units)
        b_output.to(device) # cuda device

        for index, (eachInputId, eachMask) in enumerate(zip(b_input_ids, b_input_mask)):
            # unrolled into each individual document
            # print(eachInputId.size()) # individual document here

            outputs = model(eachInputId,
                    token_type_ids=None,
                    attention_mask=eachMask)

            # combine the [CLS] output layer to form the document
            doc_output = torch.mean(outputs[1], dim=0) # size = [1, ncol]

            b_output[index] = doc_output

Doing either of this produces this error:

RuntimeError: CUDA out of memory. Tried to allocate 20.00 MiB (GPU 0; 11.17 GiB total capacity; 10.65 GiB already allocated; 2.81 MiB free; 10.86 GiB reserved in total by PyTorch)

I assume this is because of appending the tensors (that are on the GPU) to a list (of course not on the GPU) and then trying to convert the list into a tensor (thats not on the GPU).

What could be done to append those tensors to another tensor and then load the tensor to GPU for further processing?

I will be grateful for any hint or information.

Output:
Try using torch.cat instead of torch.tensor. You are currently trying to allocate memory for you new tensor while all the other tensors are still stored, which might be the cause of the out of memory error. Change :

t_b_output = torch.tensor( b_output )

with:

t_b_output = torch.cat( b_output )

Hope this help


Instruction: Libtorch:how to create tensor from tensorRT fp16 half type pointer?

Input:
how to create tensor from tensorRT fp16 half type pointer in libtorch?
I am working on a detection model. I change the backbone of it to tensorRT to do FP16 inference, and the detection code such as decode boxes and nms is done in libtorch and torchvisoin, so how to create fp16 tensor from tensorRT half type pointers?
The important code is to illustrate the issue:

// tensorRT code to get half type outpus
half_float::half* outputs[18];
doInference(*engine, data, outputs, 1);
// to get the final outputs with libtorch
vector<torch::Tensor> output;
//???? how to feed the date in outpus to output????
// get the result with libtorch method detect_trt->forward
 auto res = detect_trt->forward(output);

Thanks in advance.

Output:
I have to do backbone inference in TensorRT, but the post process is using libtorch for convenience.And now I figure it out by using the following code:

out = torch::from_blob(outputs[i], {1, num, dim, dim}, torch::kFloat16).to(device_used);


Instruction: Pytorch: Weight in cross entropy loss

Input:
I was trying to understand how weight is in CrossEntropyLoss works by a practical example. So I first run as standard PyTorch code and then manually both. But the losses are not the same.

from torch import nn
import torch
softmax=nn.Softmax()
sc=torch.tensor([0.4,0.36])
loss = nn.CrossEntropyLoss(weight=sc)
input = torch.tensor([[3.0,4.0],[6.0,9.0]])
target = torch.tensor([1,0])
output = loss(input, target)
print(output)
>>1.7529

Now for manual Calculation, first softmax the input:

print(softmax(input))
>>
tensor([[0.2689, 0.7311],
        [0.0474, 0.9526]])

and then negetive log of the correct class probality and multiply with the respective weight:

((-math.log(0.7311)*0.36) - (math.log(0.0474)*0.4))/2
>>
0.6662

What I am missing here?

Output:
To compute class weight of your classes use sklearn.utils.class_weight.compute_class_weight(class_weight, *, classes, y) read it here
This will return you an array i.e  weight.
eg .
x = torch.randn(20, 5)
y = torch.randint(0, 5, (20,)) # classes
class_weights=class_weight.compute_class_weight('balanced',np.unique(y),y.numpy())
class_weights=torch.tensor(class_weights,dtype=torch.float)

print(class_weights) #([1.0000, 1.0000, 4.0000, 1.0000, 0.5714])

Then pass it to nn.CrossEntropyLoss's weight variable
criterion = nn.CrossEntropyLoss(weight=class_weights,reduction='mean')

loss = criterion(...)


Instruction: Installing Pytorch Transformers in AWS Sagemaker

Input:
I'm trying to install the pytorch transformers package for my AWS Sagemaker notebook instance. However, it keeps giving me error of "No Module Found" for the package when i run my entry point script.

I saw in an example for TensorFlowModel which requires to set up env but for Pytorch it is not the case (How do I load python modules which are not available in Sagemaker?). Anyway, below is my code :

estimator = PyTorch(entry_point='model.py',
                role=role,
                framework_version='1.4.0',
                train_instance_count=2,
                train_instance_type='ml.c4.xlarge',
                source_dir = 'src',
                hyperparameters={
                    'train_path': 's3://bucket-train',
                    'validation_path': 's3://bucket-val',
                    'epochs': 3,
                    'backend': 'gloo'
                })

Output: although you may be running that command from a SageMaker notebook, the training job you launch with the PyTorch estimator does not run on the notebook. It runs on remote, ephemeral infrastructure. You need to install your package on that remote machine. You need to add in the srcsource directory a requirements.txt file that contains the list of extra packages you want to install, such as the transformers package


Instruction: ResNet model of pytorch and tensorflow give different results when stride=2

Input:
class BasicBlock(nn.Module):
    def __init__(self, in_planes, out_planes, stride, dropRate=0.0):
        super(BasicBlock, self).__init__()
        self.bn1 = nn.BatchNorm2d(in_planes)
        self.relu1 = nn.ReLU(inplace=True)
        self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
                               padding=1, bias=False) # 1
        self.bn2 = nn.BatchNorm2d(out_planes)
        self.relu2 = nn.ReLU(inplace=True)
        self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1,
                               padding=1, bias=False)
        self.droprate = dropRate
        self.equalInOut = (in_planes == out_planes)
        self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride,
                                                                padding=0, bias=False) or None
    def forward(self, x):
        if not self.equalInOut:
            x = self.relu1(self.bn1(x))
        else:
            out = self.relu1(self.bn1(x))
        out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x)))
        if self.droprate > 0:
            out = F.dropout(out, p=self.droprate, training=self.training)
        out = self.conv2(out)
        if self.convShortcut is not None:
        return torch.add(x if self.equalInOut else self.convShortcut(x), out)

class NetworkBlock(nn.Module):
    def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0):
        super(NetworkBlock, self).__init__()
        self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate)

    def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate):
        layers = []
        for i in range(int(nb_layers)):
            layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate))
        return nn.Sequential(*layers)

    def forward(self, x):
        return self.layer(x)

class WideResNet(nn.Module):
    def __init__(self, depth=34, num_classes=10, widen_factor=10, dropRate=0.0):
        super(WideResNet, self).__init__()
        nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor]
        assert ((depth - 4) % 6 == 0)
        n = (depth - 4) / 6
        block = BasicBlock
        # 1st conv before any network block
        self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1,
                               padding=1, bias=False)
        # 1st block
        self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate)
        # 1st sub-block
        self.sub_block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate)
        # 2nd block
        self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) # 2
        # 3rd block
        self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) # 2
        # global average pooling and classifier
        self.bn1 = nn.BatchNorm2d(nChannels[3])
        self.relu = nn.ReLU(inplace=True)
        self.fc = nn.Linear(nChannels[3], num_classes)
        self.nChannels = nChannels[3]

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                m.bias.data.zero_()

    def forward(self, x):
        out = self.conv1(x)
        out = self.block1(out)
        out = self.block2(out)
        out = self.block3(out)
        out = self.relu(self.bn1(out))
        out = F.avg_pool2d(out, 8)
        out = out.view(-1, self.nChannels)
        return self.fc(out)

  def _conv(self, name, x, filter_size, in_filters, out_filters, strides, padding='SAME'):
    """Convolution."""
    with tf.variable_scope(name):
      n = filter_size * filter_size * out_filters
      kernel = tf.get_variable(
          'DW', [filter_size, filter_size, in_filters, out_filters],
          tf.float32, initializer=tf.random_normal_initializer(
              stddev=np.sqrt(2.0/n)))
      return tf.nn.conv2d(x, kernel, strides, padding=padding)

  def _residual(self, x, in_filter, out_filter, stride,
                activate_before_residual=False, is_log=False):
    """Residual unit with 2 sub layers."""
    if activate_before_residual:
        x = self._batch_norm('bn1', x)
        x = self._relu(x)
        orig_x = x
    else:
        orig_x = x
        x = self._batch_norm('bn1', x)
        x = self._relu(x)

    x = self._conv('conv1', x, 3, in_filter, out_filter, stride)
    x = self._batch_norm('bn2', x)
    x = self._relu(x)
    x = self._conv('conv2', x, 3, out_filter, out_filter, [1, 1, 1, 1])
    if in_filter != out_filter:
        orig_x = self._conv('shortcut_conv', orig_x, filter_size=1, in_filters=in_filter, out_filters=out_filter,
                            strides=stride, padding="VALID")
    x += orig_x
    return x

  def _build_model(self):
    assert self.mode == 'train' or self.mode == 'eval'
    with tf.variable_scope('input'):

      self.x_input = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
      self.y_input = tf.placeholder(tf.float32, shape=[None, 10])
      self.is_training = tf.placeholder(tf.bool, shape=None)
    x = self._conv('conv1.weight', self.x_input, 3, 3, 16, self._stride_arr(1))

    strides = [1, 2, 2]
    activate_before_residual = [True, True, True]
    res_func = self._residual
    # wide residual network (https://arxiv.org/abs/1605.07146v1)
    filters = [16, 160, 320, 640]

    with tf.variable_scope('block1.layer.0'):
      x = res_func(x, filters[0], filters[1], self._stride_arr(strides[0]),
                   activate_before_residual[0])
    for i in range(1, 5):
      with tf.variable_scope('block1.layer.%d' % i):
        x = res_func(x, filters[1], filters[1], self._stride_arr(1), False)

    with tf.variable_scope('block2.layer.0'):
      x = res_func(x, filters[1], filters[2], self._stride_arr(strides[1]),
                   activate_before_residual[1], is_log=True)
    for i in range(1, 5):
      with tf.variable_scope('block2.layer.%d' % i):
        x = res_func(x, filters[2], filters[2], self._stride_arr(1), False)

    with tf.variable_scope('block3.layer.0'):
      x = res_func(x, filters[2], filters[3], self._stride_arr(strides[2]),
                   activate_before_residual[2])
    for i in range(1, 5):
      with tf.variable_scope('block3.layer.%d' % i):
        x = res_func(x, filters[3], filters[3], self._stride_arr(1), False)

    x = self._batch_norm('bn1', x)
    x = self._relu(x)
    x = self._global_avg_pool(x)

    with tf.variable_scope('fc'):
      self.pre_softmax = self._fully_connected(x, 10)

I'm doing experiment on "adversarial defense", and I checked that the performances of pytorch and tensorflow is different with same weights (I exported it as numpy and loaded to pytorch and tensorflow) I printed out each result of WideResNet34 and calculate the difference of each output, then, the above output of below image comes out

The results start to be different from block2. Then, I only change the stride of each block to all 1 (stride of block 2 and 3),  the below output of above image comes out

The differences are negligible at all layers, so I think the difference appear only when stride=2. I don't know why there is no difference when stride=1 but different when stride=2... Who knows about this thing?

Output:
I finally found that the problem was the "padding". Tensorflow's "SAME" padding zero-pads assymmetrically (left=0, right=1, top=0, bottom=1) when symmetric padding results in odd number... While, pytorch do not support assymmetric padding in nn.conv2d, so it zero-pads symmetrically (left=1, right=1, top=1, bottom=1)..

So, I think that when input size=8, filter size=3, and stride=2, index of left-top of filter in tensorflow would be 0,2,4,6 but in pytorch it would be -1(zero-pad), 1, 3, 5... I checked that when I zero-pads assymetrically using nn.Zero-pad2d , it gives almost same results (2-norm diff < 1e-2)

class BasicBlock(nn.Module):
    def __init__(self, in_planes, out_planes, stride, dropRate=0.0):
        super(BasicBlock, self).__init__()
        self.bn1 = nn.BatchNorm2d(in_planes)
        self.relu1 = nn.ReLU(inplace=True)
        if stride==1:
            self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False) # 1
        else:
            self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=0, bias=False)  # 1
        self.pad1 = nn.ZeroPad2d((0,1,0,1)) # 0,1,0,1
        self.stride = stride
        self.bn2 = nn.BatchNorm2d(out_planes)
        self.relu2 = nn.ReLU(inplace=True)
        self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1,
                               padding=1, bias=False)
        self.droprate = dropRate
        self.equalInOut = (in_planes == out_planes)
        self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride,
                                                                padding=0, bias=False) or None

    def forward(self, x):
        if not self.equalInOut:
            x = self.relu1(self.bn1(x))
        else:
            out = self.relu1(self.bn1(x))
        if self.stride==1:
            out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x)))
        else:
            out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else self.pad1(x))))
        if self.droprate > 0:
            out = F.dropout(out, p=self.droprate, training=self.training)
        out = self.conv2(out)
        return torch.add(x if self.equalInOut else self.convShortcut(x), out)


Instruction: Loss not Converging for CNN Model

Input:
Image Transformation and Batch

transform = transforms.Compose([

                                transforms.Resize((100,100)),

                                transforms.ToTensor(),

                                transforms.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225])

                                ])

data_set = datasets.ImageFolder(root="/content/drive/My Drive/models/pokemon/dataset",transform=transform)

train_loader = DataLoader(data_set,batch_size=10,shuffle=True,num_workers=6)

Below is my Model

class pokimonClassifier(nn.Module):

  def __init__(self):

    super().__init__()

    self.conv1 = nn.Conv2d(3,6,3,1)

    self.conv2 = nn.Conv2d(6,18,3,1)

    self.fc1 = nn.Linear(23*23*18,520)

    self.fc2 = nn.Linear(520,400)

    self.fc3 = nn.Linear(400,320)

    self.fc4 = nn.Linear(320,149)

  def forward(self,x):

    x = F.relu(self.conv1(x))

    x = F.max_pool2d(x,2,2)

    x = F.relu(self.conv2(x))

    x = F.max_pool2d(x,2,2)

    x = x.view(-1,23*23*18)

    x = F.relu(self.fc1(x))

    x = F.relu(self.fc2(x))

    x = F.relu(self.fc3(x))

    x = F.log_softmax(self.fc4(x), dim=1)

    return x

Creating Instance of model, Use GPU, Set Criterion and optimizer
Here is firsr set lr = 0.001 then later changed to 0.0001

model = pokimonClassifier()
model.to('cuda')
criterion = nn.CrossEntropyLoss()

optimizer = torch.optim.Adam(model.parameters(),lr = 0.0001)

Training Dataset

for e in range(epochs):

  train_crt = 0

  for b,(train_x,train_y) in enumerate(train_loader):

    b+=1

    train_x, train_y = train_x.to('cuda'), train_y.to('cuda')

    # train model

    y_preds = model(train_x)

    loss = criterion(y_preds,train_y)

    # analysis model

    predicted = torch.max(y_preds,1)[1]

    correct = (predicted == train_y).sum()

    train_crt += correct

    # print loss and accuracy

    if b%50 == 0:

        print(f'Epoch {e} batch{b} loss:{loss.item()} ')

    # updating weights and bais

    optimizer.zero_grad()

    loss.backward()

    optimizer.step()

  train_loss.append(loss)

  train_correct.append(train_crt)

My loss value remains between 4 - 3  and its not converging to 0.
I am super new to deep learning and  I don't know much about it.

The dataset I am using is here: https://www.kaggle.com/thedagger/pokemon-generation-one

A help will be much appreciated.
Thank You

Output:
The problem with your network is that you are applying softmax() twice - once at fc4() layer and once more while using nn.CrossEntropyLoss().

According to the official documentation, Pytorch takes care of softmax() while applying nn.CrossEntropyLoss().

So in your code, please change this line

x = F.log_softmax(self.fc4(x), dim=1)

to

x = self.fc4(x)


Instruction: XLNetForSequenceClassification Pretrained model unable to load

Input:
I tried loading the XLNet pretrained but this occurred. I've tried this before and it worked, however, now it doesn't. Any suggestion on how to fix this problem?

model = XLNetForSequenceClassification.from_pretrained("xlnet-large-cased", num_labels = 2)
model.to(device)

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-55-d6f698a3714b> in <module>()
----> 1 model = XLNetForSequenceClassification.from_pretrained("xlnet-large-cased", num_labels = 2)
      2 model.to(device)

3 frames
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/sparse.py in __init__(self, num_embeddings, embedding_dim, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse, _weight)
     95         self.scale_grad_by_freq = scale_grad_by_freq
     96         if _weight is None:
---> 97             self.weight = Parameter(torch.Tensor(num_embeddings, embedding_dim))
     98             self.reset_parameters()
     99         else:

RuntimeError: Trying to create tensor with negative dimension -1: [-1, 1024]

Output:
You should import XLNetForSequenceClassification from transformers and not from pytorch-transformers. First, make sure transformers is installed:
> pip install transformers

Then, in your code:
from transformers import XLNetForSequenceClassification
model = XLNetForSequenceClassification.from_pretrained("xlnet-large-cased", num_labels = 2)

This should work.


Instruction: Get the input channels for the conv2d from previous layer?

Input:
I was wondering if there are many convolutional layers (conv1 --> conv2 ). How can we get the input channels parameter for the conv2 from the conv1 output channel?

class MyModel(nn.Module):
  def __init__(self, in_ch, num_features, out_ch2):
    super(MyModel, self).__init__()
    self.conv1 = nn.Conv2D(in_channels,num_features)
    self.conv2 = nn.Conv2D(in_channnels_from_out_channels_of_conv1,out_ch2)

Can I get the out_channels from the conv1 layer and use it as in_ch for conv2?

Output:
Second parameter of nn.Conv2D constructor is number of output channels:

self.conv1 = nn.Conv2D(in_channels,conv1_out_channels)
self.conv2 = nn.Conv2D(conv1_out_channels,out_ch2)

as described in the docs

Also it available as a property:

self.conv1.out_channels


Instruction: PyTorch Dataset and Conv1d using a ton of memory

Input:
I am trying to write a convolutional neural network in pytorch. I'm very new to machine learning and PyTorch, so I'm not very familiar with the package.

I have written a custom dataset and it has loaded my data from a csv file properly. However, when I load it into a data loader, my system monitor shows python using a huge amount of memory. I'm currently using only a fraction of my data set, and one instance uses about 5 gigs as a data loader.

My dataset is 1 dimensional tensors. Each one is very long - about 33 million values. I used
sys.getsizeof(train_set.sample_list[0][0].storage())
to check the size of the underlying tensor, and it was only 271 megabytes.

Additionally, if I continue on and create an instance of my CNN, the initializer eats up memory until my kernel crashes. The reason for this is unclear to me.

Here is the code for my custom Dataset:

    def __init__(self, csv_file, train):

        self.train = train
        self.df_tmp = pd.read_csv(csv_file, header=None, sep='\t')
        self.df_tmp.drop(self.df_tmp.shape[1]-1, axis=1, inplace=True)
        self.df = self.df_tmp.transpose()
        self.sample_list = []

        for i in range(self.df.shape[0]): #num rows,  33 million ish
            sample = torch.tensor([self.df.iloc[i][1:].values])
            label = torch.tensor(self.df.iloc[i][0])
            self.sample_list.append((sample, label))

    def __len__(self):
        return len(self.sample_list)

    def __getitem__(self, idx):
        return self.sample_list[idx]

And the code for the NN:

    #input batch shape is (9 x 33889258 x 1)
    def __init__(self):
        super(CNN, self).__init__()

        #input channels 1, output 3
        self.conv1 = torch.nn.Conv1d(1, out_channels=3, kernel_size=(100), stride=10, padding=1)

        #size in is 3,1,33889258
        self.pool = torch.nn.MaxPool1d(kernel_size=2, stride=2, padding=0)

        self.fc1 = torch.nn.Linear(45750366, 1000) #3 * 1 * 3388917

        self.fc2 = torch.nn.Linear(1000, 2)

    def forward(self, x):
        #size: (1x1x33889258) to (3x1x33889258)
        tmp = self.conv1(x.float())
        x = F.relu(tmp) #

        x = self.pool(x)
        #whatever shape comes out of here needs to go into x.view

        x = x.view(45750366) #-1, 1*1*3388927
        x = self.fc1(x)
        x = F.relu(x)

        x = self.fc2(x)
        return(x)

Some of my input sizes might be off, I'm still working that out but the memory issue is preventing me from making progress

Output:
You are storing all datapoints in list(i.e. in memory) so it kinda deafeats the purpose of custom dataset/dataloader. you should just keep the reference of dataframe in your dataset class and for each index return the correct data something like

def __init__(self, csv_file, train):

    self.train = train
    self.df_tmp = pd.read_csv(csv_file, header=None, sep='\t')
    self.df_tmp.drop(self.df_tmp.shape[1]-1, axis=1, inplace=True)
    self.df = self.df_tmp.transpose()

def __len__(self):
    return self.df.shape[0]

def __getitem__(self, idx):
    sample = torch.tensor([self.df.iloc[idx][1:].values])
    label = torch.tensor(self.df.iloc[idx][0])
    return sample, label

one small note: you are returning tensors from dataset's getitem method returning pure numpy array is prefered and easier because the dataloader will convert it into
a pytorch tensor.


Instruction: Unable to find a valid cuDNN algorithm to run convolution

Input:
I just got this message when trying to run a feed forward torch.nn.Conv2d, getting the following stacktrace:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-26-04bd4a00565d> in <module>
      3
      4 # call training function
----> 5 losses = train(D, G, n_epochs=n_epochs)

<ipython-input-24-b539315e0aa0> in train(D, G, n_epochs, print_every)
     46                 real_images = real_images.cuda()
     47
---> 48             D_real = D(real_images)
     49             d_real_loss = real_loss(D_real, True) # smoothing label 1 => 0.9
     50

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    548             result = self._slow_forward(*input, **kwargs)
    549         else:
--> 550             result = self.forward(*input, **kwargs)
    551         for hook in self._forward_hooks.values():
    552             hook_result = hook(self, input, result)

<ipython-input-14-bf68e57c25ff> in forward(self, x)
     48         """
     49
---> 50         x = self.leaky_relu(self.conv1(x))
     51         x = self.leaky_relu(self.conv2(x))
     52         x = self.leaky_relu(self.conv3(x))

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    548             result = self._slow_forward(*input, **kwargs)
    549         else:
--> 550             result = self.forward(*input, **kwargs)
    551         for hook in self._forward_hooks.values():
    552             hook_result = hook(self, input, result)

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/container.py in forward(self, input)
     98     def forward(self, input):
     99         for module in self:
--> 100             input = module(input)
    101         return input
    102

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    548             result = self._slow_forward(*input, **kwargs)
    549         else:
--> 550             result = self.forward(*input, **kwargs)
    551         for hook in self._forward_hooks.values():
    552             hook_result = hook(self, input, result)

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/conv.py in forward(self, input)
    347
    348     def forward(self, input):
--> 349         return self._conv_forward(input, self.weight)
    350
    351 class Conv3d(_ConvNd):

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/conv.py in _conv_forward(self, input, weight)
    344                             _pair(0), self.dilation, self.groups)
    345         return F.conv2d(input, weight, self.bias, self.stride,
--> 346                         self.padding, self.dilation, self.groups)
    347
    348     def forward(self, input):

RuntimeError: Unable to find a valid cuDNN algorithm to run convolution

Running nvidia-smi shows:

+-----------------------------------------------------------------------------+
| NVIDIA-SMI 440.33.01    Driver Version: 440.33.01    CUDA Version: 10.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  GeForce GTX 770     On   | 00000000:01:00.0 N/A |                  N/A |
| 38%   50C    P8    N/A /  N/A |    624MiB /  4034MiB |     N/A      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|    0                    Not Supported                                       |
+-----------------------------------------------------------------------------+

I'm using Python 3.7, Pytorch 1.5, and GPU is Nvidia GeForce GTX 770, running on Ubuntu 18.04.2. I haven't found that error message anywhere. Does it ring any bell?.

Thanks a lot in advance.

Output:
According to this answer for similar issue with tensorflow, it could occur because the VRAM memory limit was hit (which is rather non-intuitive from the error message).
For my case with PyTorch model training, decreasing batch size helped. You could try this or maybe decrease your model size to consume less VRAM.


Instruction: Stacking binary mask frames in pytorch?

Input:
I am using Pytorch to attempt to create a 4-dimensional tensor (binary mask) using a "stack" of three-dimensional tensors that each hold binary masks.

The three-dimensional tensors have n instances of some segmented object in a binary mask that is 704 wide and 1080 high.

Lets lets say I have three of these 3-dimensional tensors. They are all size [n,704,1080], where n is the numbers of individual obejcts (and thus, individual binary masks) for the frame, 704 is height and 1080 is width:

t1.size = torch.Size([9, 704, 1280])
t2.size = torch.Size([12, 704, 1280])
t3.size = torch.Size([10, 704, 1280])

I want to create a stack of them by adding a fourth dimension, a. ie. [a, n, 704, 1280], where a is the original 3-dimensional tensor.

The goal is to have a 4-dimensional tensor that can hold the data of numerous 3-dimensional tensors.

I have tried to torch.stack([t1, t2, t3]) but that does not work since the second dimension, n, is not consistent between all the tensors.

How can I go about this since stack does not work?

Output:
You can't.
All tensor dimensions except first must be the same.
Only way to do this, append dummy rows to first and third tensor to make them the same size (12,704,1280)

Or you can stack it in one 3 -dim tensor.


Instruction: pytorch+tensorboard error " AttributeError: 'Tensor' object has no attribute 'items' "

Input:
Good afternoon.
I want to log the loss of the train using the tensorboard in pytorch.
I got an error there.

AttributeError: 'Tensor' object has no attribute 'items'

I want to solve this error and check the log using tensorboard.
Here I show my code.

l_mse = mseloss(img,decoder_out)
writer.add_scalars("MSE",l_mse,n_iter)

img is real image in GAN and decoder_out is Generator output.
then I have error blow.

Traceback (most recent call last):
  File "main.py", line 39, in <module>
    main()
  File "main.py", line 22, in main
    solover.train(dataloader)
  File "path to my file", line 239, in train
    writer.add_scalars("MSE",l_mse,n_iter)
  File "/~~/anaconda3/lib/python3.7/site-packages/torch/utils/tensorboard/writer.py", line 378, in add_scalars
    for tag, scalar_value in tag_scalar_dict.items():
AttributeError: 'Tensor' object has no attribute 'items'

I tried

writer.add_scalars("MSE",l_mse,n_iter).eval()
writer.add_scalars("MSE",l_mse.item(),n_iter)
writer.add_scalars("MSE",l_mse.detach().cpu().numpy(),n_iter)

but still not work well.

Output:
You are calling for writer.add_scalars with a s. From Pytorch Tensorboardx documentation you can see that this function expects a dictionary as input.

 add_scalars(main_tag, tag_scalar_dict, global_step=None, walltime=None)

tag_scalar_dict (dict) – Key-value pair storing the tag and corresponding values

writer = SummaryWriter()
r = 5
for i in range(100):
    writer.add_scalars('run_14h', {'xsinx':i*np.sin(i/r),
                                    'xcosx':i*np.cos(i/r),
                                    'tanx': np.tan(i/r)}, i)
writer.close()

Use writer.add_scalar instead

  To log a scalar value, use writer.add_scalar('myscalar', value, iteration). Note that the program complains if you feed a PyTorch tensor. Remember to extract the scalar value by x.item() if x is a torch scalar tensor.

writer.add_scalar("MSE", l_mse.item(), n_iter)


Instruction: CUDA error: CUBLAS_STATUS_ALLOC_FAILED when calling `cublasCreate(handle)`

Input:
I got the following error when I ran my pytorch deep learning model in colab

/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in linear(input, weight, bias)
   1370         ret = torch.addmm(bias, input, weight.t())
   1371     else:
-> 1372         output = input.matmul(weight.t())
   1373         if bias is not None:
   1374             output += bias

RuntimeError: CUDA error: CUBLAS_STATUS_ALLOC_FAILED when calling `cublasCreate(handle)`

I even reduced batch size from 128 to 64 i.e., reduced to half,  but still, I got this error. Earlier, I ran the same code with a batch size of 128 but didn't get any error like this.

Output:
No, batch size does not matter in this case
The most likely reason is that there is an inconsistency between number of labels and number of output units.

Try printing the size of the final output in the forward pass and check the size of the output

print(model.fc1(x).size())
Here fc1 would be replaced by the name of your model's last linear layer before returning

Make sure that label.size()  is equal to prediction.size() before calculating the loss

And even after fixing that problem, you'll have to restart the GPU runtime (I needed to do this in my case when using a colab GPU)
This answer might also be helpful


Instruction: How calculate the dice coefficient for multi-class segmentation task using Python?

Input:
I am wondering how can I calculate the dice coefficient for multi-class segmentation.

Here is the script that would calculate the dice coefficient for the binary segmentation task. How can I loop over each class and calculate the dice for each class?

Thank you in advance

import numpy

def dice_coeff(im1, im2, empty_score=1.0):

    im1 = numpy.asarray(im1).astype(numpy.bool)
    im2 = numpy.asarray(im2).astype(numpy.bool)

    if im1.shape != im2.shape:
        raise ValueError("Shape mismatch: im1 and im2 must have the same shape.")

    im_sum = im1.sum() + im2.sum()
    if im_sum == 0:
        return empty_score

    # Compute Dice coefficient
    intersection = numpy.logical_and(im1, im2)

    return (2. * intersection.sum() / im_sum)

Output:
You can use dice_score for binary classes and then use binary maps for all the classes repeatedly to get a multiclass dice score.

I'm assuming your images/segmentation maps are in the format (batch/index of image, height, width, class_map).

import numpy as np
import matplotlib.pyplot as plt

def dice_coef(y_true, y_pred):
    y_true_f = y_true.flatten()
    y_pred_f = y_pred.flatten()
    intersection = np.sum(y_true_f * y_pred_f)
    smooth = 0.0001
    return (2. * intersection + smooth) / (np.sum(y_true_f) + np.sum(y_pred_f) + smooth)

def dice_coef_multilabel(y_true, y_pred, numLabels):
    dice=0
    for index in range(numLabels):
        dice += dice_coef(y_true[:,:,:,index], y_pred[:,:,:,index])
    return dice/numLabels # taking average

num_class = 5

imgA = np.random.randint(low=0, high= 2, size=(5, 64, 64, num_class) ) # 5 images in batch, 64 by 64, num_classes map
imgB = np.random.randint(low=0, high= 2, size=(5, 64, 64, num_class) )

plt.imshow(imgA[0,:,:,0]) # for 0th image, class 0 map
plt.show()

plt.imshow(imgB[0,:,:,0]) # for 0th image, class 0 map
plt.show()

dice_score = dice_coef_multilabel(imgA, imgB, num_class)
print(f'For A and B {dice_score}')

dice_score = dice_coef_multilabel(imgA, imgA, num_class)
print(f'For A and A {dice_score}')


Instruction: Cannot import Pytorch [WinError 126] The specified module could not be found

Input:
I'm trying to do a basic install and import of Pytorch/Torchvision on Windows 10. I installed a Anaconda and created a new virtual environment named photo. I opened Anaconda prompt, activated the environment, and I ran:

(photo) C:\Users\<user>\anaconda3\envs>conda install pytorch torchvision cudatoolkit=10.2 -c pytorch**

This installed pytorch successfully. Running conda list I see:

  pytorch            pytorch/win-64::pytorch-1.5.0-py3.7_cuda102_cudnn7_0

  torchvision        pytorch/win-64::torchvision-0.6.0-py37_cu102

Then I open a python command prompt while in the virtual environment, and type:

import torch

The following error is printed:

  Traceback (most recent call last):
    File "", line 1, in
    File "C:\Users\njord\anaconda3\envs\photo\lib\site-packages\torch__init__.py", line 81, in
      ctypes.CDLL(dll)
    File "C:\Users\njord\anaconda3\envs\photo\lib\ctypes__init__.py", line 364, in init
      self._handle = _dlopen(self._name, mode)
  OSError: [WinError 126] The specified module could not be found

I have uninstalled/reinstalled python and anaconda but still run into the same issue. Advice appreciated.

Output:
Refer to below link:
https://discuss.pytorch.org/t/cannot-import-torch-on-jupyter-notebook/79334

This is most probably because you are using a CUDA variant of PyTorch on a system that doesn’t have GPU driver installed. That is to say, if you don’t have a Nvidia GPU card, please install the cpu-only package according to the commands on https://pytorch.org.

Conda

conda install pytorch torchvision cpuonly -c pytorch

Pip

pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html


Instruction: Resuming pytorch model training raises error “CUDA out of memory”

Input:
My goal is to save the model at every epoch as I have to stop the training during the night and I don't want to lose progress.
After I trained my model for 1 epoch I interrupted the process via terminal with CTRL+Z.
When I tried to resume the training I got this error

THCudaCheck FAIL file=/opt/conda/conda-bld/pytorch_1525909934016/work/aten/src/THC/generic/THCStorage.cu line=58 error=2 : out of memory
Traceback (most recent call last):
  File "train.py", line 174, in <module>
    train(train_loader, model, optimizer, epoch)
  File "train.py", line 97, in train
    loss1 = CE(atts, gts)
  File "/home/albytree/miniconda3/envs/cpd-wandb/lib/python2.7/site-packages/torch/nn/modules/module.py", line 491, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/albytree/miniconda3/envs/cpd-wandb/lib/python2.7/site-packages/torch/nn/modules/loss.py", line 500, in forward
    reduce=self.reduce)
  File "/home/albytree/miniconda3/envs/cpd-wandb/lib/python2.7/site-packages/torch/nn/functional.py", line 1516, in binary_cross_entropy_with_logits
    max_val = (-input).clamp(min=0)
RuntimeError: cuda runtime error (2) : out of memory at /opt/conda/conda-bld/pytorch_1525909934016/work/aten/src/THC/generic/THCStorage.cu:58

The code that manages everything is this one

import wandb
import torch
import torch.nn.functional as F
from torch.autograd import Variable

import numpy as np
import pdb, os, argparse
from datetime import datetime

from model.CPD_models import CPD_VGG
from model.CPD_ResNet_models import CPD_ResNet
from data import get_loader
from utils import clip_gradient, adjust_lr

parser = argparse.ArgumentParser()
parser.add_argument('--epoch', type=int, default=10, help='epoch number')
parser.add_argument('--lr', type=float, default=1e-4, help='learning rate')
parser.add_argument('--batchsize', type=int, default=1, help='training batch size')
parser.add_argument('--trainsize', type=int, default=352, help='training dataset size')
parser.add_argument('--clip', type=float, default=0.5, help='gradient clipping margin')
parser.add_argument('--is_ResNet', type=bool, default=False, help='VGG or ResNet backbone')
parser.add_argument('--decay_rate', type=float, default=0.1, help='decay rate of learning rate')
parser.add_argument('--decay_epoch', type=int, default=50, help='every n epochs decay learning rate')
parser.add_argument('--model_id', type=str, required=True, help='required unique id for trained model name')
parser.add_argument('--resume', type=str, default='', help='path to resume model training from checkpoint')
parser.add_argument('--wandb', type=bool, default=False, help='enable wandb tracking model training')
opt = parser.parse_args()

model_id = opt.model_id
WANDB_EN = opt.wandb
if WANDB_EN:
    wandb.init(entity="albytree", project="cpd-train")

# Add all parsed config in one line
if WANDB_EN:
    wandb.config.update(opt)
tot_epochs = opt.epoch
print("Training Info")
print("EPOCHS: {}".format(opt.epoch))
print("LEARNING RATE: {}".format(opt.lr))
print("BATCH SIZE: {}".format(opt.batchsize))
print("TRAIN SIZE: {}".format(opt.trainsize))
print("CLIP: {}".format(opt.clip))
print("USING ResNet backbone: {}".format(opt.is_ResNet))
print("DECAY RATE: {}".format(opt.decay_rate))
print("DECAY EPOCH: {}".format(opt.decay_epoch))
print("MODEL ID: {}".format(opt.model_id))

# build models
if opt.is_ResNet:
    model = CPD_ResNet()
else:
    model = CPD_VGG()

model.cuda()
params = model.parameters()
optimizer = torch.optim.Adam(params, opt.lr)
# If no previous training, 0 epochs passed
last_epoch = 0
resume_model_path = opt.resume;
if resume_model_path:
    print("Loading previous trained model:"+resume_model_path)
    checkpoint = torch.load(resume_model_path)
    model.load_state_dict(checkpoint['model_state_dict'])
    optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
    last_epoch = checkpoint['epoch']
    last_loss = checkpoint['loss']

dataset_name = 'ECSSD'
image_root = '../../DATASETS/TEST/'+dataset_name+'/im/'
gt_root = '../../DATASETS/TEST/'+dataset_name+'/gt/'
train_loader = get_loader(image_root, gt_root, batchsize=opt.batchsize, trainsize=opt.trainsize)
total_step = len(train_loader)
print("Total step per epoch: {}".format(total_step))

CE = torch.nn.BCEWithLogitsLoss()

####################################################################################################

def train(train_loader, model, optimizer, epoch):
    model.train()
    for i, pack in enumerate(train_loader, start=1):
        optimizer.zero_grad()
        images, gts = pack
        images = Variable(images)
        gts = Variable(gts)
        images = images.cuda()
        gts = gts.cuda()

        atts, dets = model(images)
        loss1 = CE(atts, gts)
        loss2 = CE(dets, gts)
        loss = loss1 + loss2
        loss.backward()

        clip_gradient(optimizer, opt.clip)
        optimizer.step()
        if WANDB_EN:
            wandb.log({'Loss': loss})
        if i % 100 == 0 or i == total_step:
            print('{} Epoch [{:03d}/{:03d}], Step [{:04d}/{:04d}], Loss1: {:.4f} Loss2: {:0.4f}'.
                  format(datetime.now(), epoch, opt.epoch, i, total_step, loss1.data, loss2.data))

    # Save model and optimizer training data
    trained_model_data = {
        'model_state_dict': model.state_dict(),
        'optimizer_state_dict': optimizer.state_dict(),
        'epoch': epoch,
        'loss': loss
    }

    if opt.is_ResNet:
        save_path = 'models/CPD_Resnet/'
    else:
        save_path = 'models/CPD_VGG/'

    if not os.path.exists(save_path):
        print("Making trained model folder [{}]".format(save_path))
        os.makedirs(save_path)

    torch_model_ext = '.pth'
    wandb_model_ext = '.h5'
    model_unique_id = model_id+'_'+'ep'+'_'+'%d' % epoch
    trained_model_name = 'CPD_train'
    save_full_path_torch = save_path + trained_model_name + '_' + model_unique_id + torch_model_ext
    save_full_path_wandb = save_path + trained_model_name + '_' + model_unique_id + wandb_model_ext
    if os.path.exists(save_full_path_torch):
        print("Torch model with name ["+save_full_path_torch+"] already exists!")
        answ = raw_input("Do you want to replace it? [y/n] ")
        if("y" in answ):
            torch.save(trained_model_data, save_full_path_torch)
            print("Saved torch model in "+save_full_path_torch)
    else:
            torch.save(trained_model_data, save_full_path_torch)
            print("Saved torch model in "+save_full_path_torch)

    if WANDB_EN:
        if os.path.exists(save_full_path_wandb):
            print("Wandb model with name ["+save_full_path_wandb+"] already exists!")
            answ = raw_input("Do you want to replace it? [y/n] ")
            if("y" in answ):
                wandb.save(save_full_path_wandb)
                print("Saved wandb model in "+save_full_path_wandb)
        else:
                wandb.save(save_full_path_wandb)
                print("Saved wandb model in "+save_full_path_wandb)

####################################################################################################

print("Training on dataset: "+dataset_name)
print("Train images path: "+image_root)
print("Train gt path: "+gt_root)
print("Let's go!")

if WANDB_EN:
    wandb.watch(model, log="all")
for epoch in range(last_epoch+1, tot_epochs+1):
    adjust_lr(optimizer, opt.lr, epoch, opt.decay_rate, opt.decay_epoch)
    train(train_loader, model, optimizer, epoch)
print("TRAINING DONE!")

It seems that there's something wrong with the loss but I cannot understand what's the problem.

EDIT 1:

I trained the model for 2 epochs without errors and then I interrupted the process.
I also killed the process that was leaved in the gpu memory.
After I tried to resume the model saved at epoch 1 and epoch 2 I got the same cuda error but in a different part of the code

THCudaCheck FAIL file=/opt/conda/conda-bld/pytorch_1525909934016/work/aten/src/THC/generic/THCStorage.cu line=58 error=2 : out of memory
Traceback (most recent call last):
  File "train.py", line 191, in <module>
    train(train_loader, model, optimizer, epoch)
  File "train.py", line 112, in train
    atts, dets = model(images)
  File "/home/albytree/miniconda3/envs/cpd-wandb/lib/python2.7/site-packages/torch/nn/modules/module.py", line 491, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/albytree/TESI/CODICE/Workspace/ALGS/CPD/model/CPD_models.py", line 131, in forward
    detection = self.agg2(x5_2, x4_2, x3_2)
  File "/home/albytree/miniconda3/envs/cpd-wandb/lib/python2.7/site-packages/torch/nn/modules/module.py", line 491, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/albytree/TESI/CODICE/Workspace/ALGS/CPD/model/CPD_models.py", line 86, in forward
    x3_2 = torch.cat((x3_1, self.conv_upsample5(self.upsample(x2_2))), 1)
RuntimeError: cuda runtime error (2) : out of memory at /opt/conda/conda-bld/pytorch_1525909934016/work/aten/src/THC/generic/THCStorage.cu:58

Moreover I tried to test the saved model at epoch 1 and epoch 2 and got this error

Traceback (most recent call last):
  File "test.py", line 45, in <module>
    model.load_state_dict(torch.load(opt.model_path))
  File "/home/albytree/miniconda3/envs/cpd-wandb/lib/python2.7/site-packages/torch/nn/modules/module.py", line 721, in load_state_dict
    self.__class__.__name__, "\n\t".join(error_msgs)))
RuntimeError: Error(s) in loading state_dict for CPD_VGG:
    Missing key(s) in state_dict: "vgg.conv1.conv1_1.bias", "vgg.conv1.conv1_1.weight", "vgg.conv1.conv1_2.bias", "vgg.conv1.conv1_2.weight", "vgg.conv2.conv2_1.bias", "vgg.conv2.conv2_1.weight", "vgg.conv2.conv2_2.bias", "vgg.conv2.conv2_2.weight", "vgg.conv3.conv3_1.bias", "vgg.conv3.conv3_1.weight", "vgg.conv3.conv3_2.bias", "vgg.conv3.conv3_2.weight", "vgg.conv3.conv3_3.bias", "vgg.conv3.conv3_3.weight", "vgg.conv4_1.conv4_1_1.bias", "vgg.conv4_1.conv4_1_1.weight", "vgg.conv4_1.conv4_2_1.bias", "vgg.conv4_1.conv4_2_1.weight", "vgg.conv4_1.conv4_3_1.bias", "vgg.conv4_1.conv4_3_1.weight", "vgg.conv5_1.conv5_1_1.bias", "vgg.conv5_1.conv5_1_1.weight", "vgg.conv5_1.conv5_2_1.bias", "vgg.conv5_1.conv5_2_1.weight", "vgg.conv5_1.conv5_3_1.bias", "vgg.conv5_1.conv5_3_1.weight", "vgg.conv4_2.conv4_1_2.bias", "vgg.conv4_2.conv4_1_2.weight", "vgg.conv4_2.conv4_2_2.bias", "vgg.conv4_2.conv4_2_2.weight", "vgg.conv4_2.conv4_3_2.bias", "vgg.conv4_2.conv4_3_2.weight", "vgg.conv5_2.conv5_1_2.bias", "vgg.conv5_2.conv5_1_2.weight", "vgg.conv5_2.conv5_2_2.bias", "vgg.conv5_2.conv5_2_2.weight", "vgg.conv5_2.conv5_3_2.bias", "vgg.conv5_2.conv5_3_2.weight", "rfb3_1.branch0.0.bias", "rfb3_1.branch0.0.weight", "rfb3_1.branch1.0.bias", "rfb3_1.branch1.0.weight", "rfb3_1.branch1.1.bias", "rfb3_1.branch1.1.weight", "rfb3_1.branch1.2.bias", "rfb3_1.branch1.2.weight", "rfb3_1.branch1.3.bias", "rfb3_1.branch1.3.weight", "rfb3_1.branch2.0.bias", "rfb3_1.branch2.0.weight", "rfb3_1.branch2.1.bias", "rfb3_1.branch2.1.weight", "rfb3_1.branch2.2.bias", "rfb3_1.branch2.2.weight", "rfb3_1.branch2.3.bias", "rfb3_1.branch2.3.weight", "rfb3_1.branch3.0.bias", "rfb3_1.branch3.0.weight", "rfb3_1.branch3.1.bias", "rfb3_1.branch3.1.weight", "rfb3_1.branch3.2.bias", "rfb3_1.branch3.2.weight", "rfb3_1.branch3.3.bias", "rfb3_1.branch3.3.weight", "rfb3_1.conv_cat.bias", "rfb3_1.conv_cat.weight", "rfb3_1.conv_res.bias", "rfb3_1.conv_res.weight", "rfb4_1.branch0.0.bias", "rfb4_1.branch0.0.weight", "rfb4_1.branch1.0.bias", "rfb4_1.branch1.0.weight", "rfb4_1.branch1.1.bias", "rfb4_1.branch1.1.weight", "rfb4_1.branch1.2.bias", "rfb4_1.branch1.2.weight", "rfb4_1.branch1.3.bias", "rfb4_1.branch1.3.weight", "rfb4_1.branch2.0.bias", "rfb4_1.branch2.0.weight", "rfb4_1.branch2.1.bias", "rfb4_1.branch2.1.weight", "rfb4_1.branch2.2.bias", "rfb4_1.branch2.2.weight", "rfb4_1.branch2.3.bias", "rfb4_1.branch2.3.weight", "rfb4_1.branch3.0.bias", "rfb4_1.branch3.0.weight", "rfb4_1.branch3.1.bias", "rfb4_1.branch3.1.weight", "rfb4_1.branch3.2.bias", "rfb4_1.branch3.2.weight", "rfb4_1.branch3.3.bias", "rfb4_1.branch3.3.weight", "rfb4_1.conv_cat.bias", "rfb4_1.conv_cat.weight", "rfb4_1.conv_res.bias", "rfb4_1.conv_res.weight", "rfb5_1.branch0.0.bias", "rfb5_1.branch0.0.weight", "rfb5_1.branch1.0.bias", "rfb5_1.branch1.0.weight", "rfb5_1.branch1.1.bias", "rfb5_1.branch1.1.weight", "rfb5_1.branch1.2.bias", "rfb5_1.branch1.2.weight", "rfb5_1.branch1.3.bias", "rfb5_1.branch1.3.weight", "rfb5_1.branch2.0.bias", "rfb5_1.branch2.0.weight", "rfb5_1.branch2.1.bias", "rfb5_1.branch2.1.weight", "rfb5_1.branch2.2.bias", "rfb5_1.branch2.2.weight", "rfb5_1.branch2.3.bias", "rfb5_1.branch2.3.weight", "rfb5_1.branch3.0.bias", "rfb5_1.branch3.0.weight", "rfb5_1.branch3.1.bias", "rfb5_1.branch3.1.weight", "rfb5_1.branch3.2.bias", "rfb5_1.branch3.2.weight", "rfb5_1.branch3.3.bias", "rfb5_1.branch3.3.weight", "rfb5_1.conv_cat.bias", "rfb5_1.conv_cat.weight", "rfb5_1.conv_res.bias", "rfb5_1.conv_res.weight", "agg1.conv_upsample1.bias", "agg1.conv_upsample1.weight", "agg1.conv_upsample2.bias", "agg1.conv_upsample2.weight", "agg1.conv_upsample3.bias", "agg1.conv_upsample3.weight", "agg1.conv_upsample4.bias", "agg1.conv_upsample4.weight", "agg1.conv_upsample5.bias", "agg1.conv_upsample5.weight", "agg1.conv_concat2.bias", "agg1.conv_concat2.weight", "agg1.conv_concat3.bias", "agg1.conv_concat3.weight", "agg1.conv4.bias", "agg1.conv4.weight", "agg1.conv5.bias", "agg1.conv5.weight", "rfb3_2.branch0.0.bias", "rfb3_2.branch0.0.weight", "rfb3_2.branch1.0.bias", "rfb3_2.branch1.0.weight", "rfb3_2.branch1.1.bias", "rfb3_2.branch1.1.weight", "rfb3_2.branch1.2.bias", "rfb3_2.branch1.2.weight", "rfb3_2.branch1.3.bias", "rfb3_2.branch1.3.weight", "rfb3_2.branch2.0.bias", "rfb3_2.branch2.0.weight", "rfb3_2.branch2.1.bias", "rfb3_2.branch2.1.weight", "rfb3_2.branch2.2.bias", "rfb3_2.branch2.2.weight", "rfb3_2.branch2.3.bias", "rfb3_2.branch2.3.weight", "rfb3_2.branch3.0.bias", "rfb3_2.branch3.0.weight", "rfb3_2.branch3.1.bias", "rfb3_2.branch3.1.weight", "rfb3_2.branch3.2.bias", "rfb3_2.branch3.2.weight", "rfb3_2.branch3.3.bias", "rfb3_2.branch3.3.weight", "rfb3_2.conv_cat.bias", "rfb3_2.conv_cat.weight", "rfb3_2.conv_res.bias", "rfb3_2.conv_res.weight", "rfb4_2.branch0.0.bias", "rfb4_2.branch0.0.weight", "rfb4_2.branch1.0.bias", "rfb4_2.branch1.0.weight", "rfb4_2.branch1.1.bias", "rfb4_2.branch1.1.weight", "rfb4_2.branch1.2.bias", "rfb4_2.branch1.2.weight", "rfb4_2.branch1.3.bias", "rfb4_2.branch1.3.weight", "rfb4_2.branch2.0.bias", "rfb4_2.branch2.0.weight", "rfb4_2.branch2.1.bias", "rfb4_2.branch2.1.weight", "rfb4_2.branch2.2.bias", "rfb4_2.branch2.2.weight", "rfb4_2.branch2.3.bias", "rfb4_2.branch2.3.weight", "rfb4_2.branch3.0.bias", "rfb4_2.branch3.0.weight", "rfb4_2.branch3.1.bias", "rfb4_2.branch3.1.weight", "rfb4_2.branch3.2.bias", "rfb4_2.branch3.2.weight", "rfb4_2.branch3.3.bias", "rfb4_2.branch3.3.weight", "rfb4_2.conv_cat.bias", "rfb4_2.conv_cat.weight", "rfb4_2.conv_res.bias", "rfb4_2.conv_res.weight", "rfb5_2.branch0.0.bias", "rfb5_2.branch0.0.weight", "rfb5_2.branch1.0.bias", "rfb5_2.branch1.0.weight", "rfb5_2.branch1.1.bias", "rfb5_2.branch1.1.weight", "rfb5_2.branch1.2.bias", "rfb5_2.branch1.2.weight", "rfb5_2.branch1.3.bias", "rfb5_2.branch1.3.weight", "rfb5_2.branch2.0.bias", "rfb5_2.branch2.0.weight", "rfb5_2.branch2.1.bias", "rfb5_2.branch2.1.weight", "rfb5_2.branch2.2.bias", "rfb5_2.branch2.2.weight", "rfb5_2.branch2.3.bias", "rfb5_2.branch2.3.weight", "rfb5_2.branch3.0.bias", "rfb5_2.branch3.0.weight", "rfb5_2.branch3.1.bias", "rfb5_2.branch3.1.weight", "rfb5_2.branch3.2.bias", "rfb5_2.branch3.2.weight", "rfb5_2.branch3.3.bias", "rfb5_2.branch3.3.weight", "rfb5_2.conv_cat.bias", "rfb5_2.conv_cat.weight", "rfb5_2.conv_res.bias", "rfb5_2.conv_res.weight", "agg2.conv_upsample1.bias", "agg2.conv_upsample1.weight", "agg2.conv_upsample2.bias", "agg2.conv_upsample2.weight", "agg2.conv_upsample3.bias", "agg2.conv_upsample3.weight", "agg2.conv_upsample4.bias", "agg2.conv_upsample4.weight", "agg2.conv_upsample5.bias", "agg2.conv_upsample5.weight", "agg2.conv_concat2.bias", "agg2.conv_concat2.weight", "agg2.conv_concat3.bias", "agg2.conv_concat3.weight", "agg2.conv4.bias", "agg2.conv4.weight", "agg2.conv5.bias", "agg2.conv5.weight", "HA.gaussian_kernel".
    Unexpected key(s) in state_dict: "loss", "optimizer_state_dict", "model_state_dict", "epoch".

Maybe I'm not saving the states as intended ?
The weird thing is that before adding the resume training code I was just saving the model at every epoch only with torch.save(model.state_dict(), save_full_path_torch) : I managed to train the model in 10 epochs and it still works during testing.

Output:
Although this question has been posted 5 months ago, in case if anyone else comes across a similar issue, here is a simple solution.
As explained in Pytorch FAQ, tensors defining the loss is accumulating history across the training loop because loss is a differentiable variable here.
One simple solution is to typecast the loss with float.
Secondly, make sure that you do not use the loss anywhere else but loss.item() while you are printing the loss or logging it to the wandb.


Instruction: Cannot import torch - Image not found

Input:
I'm trying to import torch but failed because of Image not Found error. Here is the error when I entered import torch:

---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
 in
----> 1 import torch

/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/torch/__init__.py in
    134     # See Note [Global dependencies]
    135     _load_global_deps()
--> 136     from torch._C import *
    137
    138 __all__ += [name for name in dir(_C)

ImportError: dlopen(/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/torch/_C.cpython-38-darwin.so, 2): Library not loaded: @rpath/libc++.1.dylib
  Referenced from: /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/torch/_C.cpython-38-darwin.so
  Reason: image not found

Thank you very much!

EDIT: This works for me:

>>> install_name_tool -add_rpath /usr/lib /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/torch/_C.cpython-38-darwin.so

Output:
I solved this by doing this way, suppose you are using the virtual environment.
Replace YOUR_PATH_TO_PYTHON_ENV with your python environment path.

install_name_tool -add_rpath /usr/lib YOUR_PATH_TO_PYTHON_ENV/venv/lib/python3.8/site-packages/torch/_C.cpython-38-darwin.so

If you are using your local python, maybe it will look like.

install_name_tool -add_rpath /usr/lib /Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/torch/_C.cpython-38-darwin.so

Basically, you need to add rpath of your library to torch in your python environment.


Instruction: How to save custom embedding matrix to .txt file format?

Input:
I have made a dictionary which contains word and its corresponding word vector in the following format:

{'word1': array([ 4.530e-02, -1.170e-02, -1.201e-01,  2.439e-01,  4.670e-02d], type=float32),
'word2': array([ 4.530e-02, -1.170e-02, -1.201e-01,  2.439e-01,  4.670e-02d], type=float32)}

I would like to save this dictionary to custom_embeddings.txt file in the following format:

The format of your custom_embeddings.txt file needs to be the token followed by the values of each of the dimensions for the embedding, all separated by a single space, e.g. here's two tokens with 5 dimensional embeddings:

word1 4.530e-02 -1.170e-02 -1.201e-01  2.439e-01  4.670e-02d
word2 4.530e-02 -1.170e-02 -1.201e-01  2.439e-01  4.670e-02d

It will be really helpful if you could tell me how to achieve this result?

Thanks in advance

Output:
Python's .items() call is an elegant way to loop over all the words in your dictionary. This will save the output as lines of a text file:

txt_filename = 'output.txt'

with open(txt_filename, 'w') as f:
    for word, vec in my_wordvec_dict.items():
        f.write('{} {}\n'.format(word, ' '.join(['{:e}'.format(item) for item in vec])))


Instruction: Why do we need state_dict = state_dict.copy()

Input:
I want to load the weights of a pre-trained model on my local model. I don’t understand why state_dict = state_dict.copy() is necessary if the two networks have the same name state_dict.

# copy state_dict so _load_from_state_dict can modify it
metadata = getattr(state_dict, '_metadata', None)
state_dict = state_dict.copy()
if metadata is not None:
    state_dict._metadata = metadata

def load(module, prefix=''):

    local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {})
    module._load_from_state_dict(
        state_dict, prefix, local_metadata, True, missing_keys, unexpected_keys, error_msgs)
    for name, child in module._modules.items():
        if child is not None:
            load(child, prefix + name + '.')
start_prefix = ''
# print("hasattr(model, 'bert')",hasattr(model, 'bert')  ) :false
if not hasattr(model, 'bert') and any(s.startswith('bert.') for s in state_dict.keys()):
    start_prefix = 'bert.'
load(model, prefix=start_prefix)

Note: the above code is from Hugging Face.

Output:
state_dict = state_dict.copy()

does exactly what you tell him to do: it copies in place the state_dict. State dict are all the parameters of your model, and copying it allows to make them independant. One should be careful whether you need a copy or a deepcopy though !


Instruction: Can conditionality be added inside Pytorch nn.Sequential()

Input:
Is there a way to add conditional statements inside the nn.Sequential(). Something similar to the code below.

import torch

class Building_Blocks(torch.nn.Module):
  def conv_block (self, in_features, out_features, kernal_size, upsample=False):
    block = torch.nn.Sequential(
        torch.nn.Conv2d(in_features, out_features, kernal_size),
        torch.nn.ReLU(inplace = True),
        torch.nn.Conv2d(out_features, out_features, kernal_size),
        torch.nn.ReLU(inplace = True),
        if(upsample):
          torch.nn.ConvTranspose2d(out_features, out_features, kernal_size)
        )
    return block

  def __init__(self):
    super(Building_Blocks, self).__init__()
    self.contracting_layer1 = self.conv_block(3, 64, 3, upsample=True)

  def forward(self, x):
    x=self.contracting_layer1(x)
    return x

Output:
No, but in your case it's easy to take if out of nn.Sequential:

class Building_Blocks(torch.nn.Module):
    def conv_block(self, in_features, out_features, kernal_size, upsample=False):
        layers = [
            torch.nn.Conv2d(in_features, out_features, kernal_size),
            torch.nn.ReLU(inplace=True),
            torch.nn.Conv2d(out_features, out_features, kernal_size),
            torch.nn.ReLU(inplace=True),
        ]
        if upsample:
            layers.append(
                torch.nn.ConvTranspose2d(out_features, out_features, kernal_size)
            )
        block = torch.nn.Sequential(*layers)
        return block

    def __init__(self):
        super(Building_Blocks, self).__init__()
        self.contracting_layer1 = self.conv_block(3, 64, 3, upsample=True)

    def forward(self, x):
        x = self.contracting_layer1(x)
        return x

You can always construct a list containing layers however you want and unpack it into torch.nn.Sequential afterwards.


Instruction: CNN vs SVM for smile intensity detection training?

Input:
I have a dataset made up of images of faces, with the corresponding landmarks that make up the mouth.
These landmarks are sets of 2D points (x,y pixel position).
Each image-landmark set data pair is tagged as either a smile, or neutral.

What i would like to do is train a deep learning model to return a smile intensity for a new image-landmark data pair.

What should I be searching for to help me with the next step?
Is it a CNN that i need? In my limited understanding, the usual training input is just an image, where I would be passing the landmark sets to train with. Or would an SVM approach be more accurate?

I am looking for maximum accuracy, as much as is possible.

What is the approach that I need called?

I am happy to use PyTorch, Dlib or any framework, I am just a little stuck on the search terms to help me move forward.

Thank you.

Output:
It's hard to tell without looking into the dataset and experimenting. But hopefully, the following research materials will guide you in the right direction.

Machine learning-based approach:
https://www.researchgate.net/publication/266672947_Estimating_smile_intensity_A_better_way
Deep learning (CNN): https://arxiv.org/pdf/1602.00172.pdf
A list of awesome papers for smile and smile intensity detection:
https://github.com/EvelynFan/AWESOME-FER/blob/master/README.md
SmileNet project: https://sites.google.com/view/sensingfeeling/

Now, I'm assuming you don't have any label for actual smile intensity.

In such a scenario, the existing smile detection methods can be used directly, you'll use the last activation output (sigmoid) as a confidence score for smiling. If the confidence is higher, the intensity should be higher.

Now, you can use the facial landmark points as separate features (pass them through an LSTM block) and concatenate to the CNN at an early stage/ or later to improve the performance of your model.

If you have the label for smiling intensity, you can just solve it as a regression problem, the CNN will have one output, will try to regress the smile intensity (the normalized smile intensity with sigmoid in this case).


Instruction: pytorch: How to do layer wise multiplication?

Input:
I have a tensor containing five 2x2 matrices - shape (1,5,2,2), and a tensor containing 5 elements -  shape ([5]). I want to multiply each 2x2 matrix(in the former tensor) with the corresponding value (in the latter tensor). The resultant tensor should be of shape (1,5,2,2). How to do that?

Getting the following error when I run this code

a = torch.rand(1,5,2,2)
print(a.shape)
b = torch.rand(5)
print(b.shape)
mul = a*b

RuntimeError: The size of tensor a (2) must match the size of tensor b (5) at non-singleton dimension 3

Output:
You can use either a * b or torch.mul(a, b) but you must use permute() before and after you multiply, in order to have the compatible shape:

import torch
a = torch.ones(1,5,2,2)
b = torch.rand(5)
a.shape # torch.Size([1, 5, 2, 2])
b.shape # torch.Size([5])

c = (a.permute(0,2,3,1) * b).permute(0,3,1,2)
c.shape # torch.Size([1, 5, 2, 2])

# OR #
c = torch.mul(a.permute(0,2,3,1), b).permute(0,3,1,2)
c.shape # torch.Size([1, 5, 2, 2])

The permute() function transposes the dimention in the order of it's arguments. I.e, a.permute(0,2,3,1) will be of shape torch.Size([1, 2, 2, 5]) which fits the shape of b (torch.Size([5])) for matrix multiplication, since the last dimention of a equals the first dimention of b. After we finish the multiplication we transpose it again, using permute(), to the. desired shape of torch.Size([1, 5, 2, 2]) by permute(0,3,1,2).

You can read about permute() in the docs. But it works with it's arguments numbering the current shape of [1, 5, 2, 2] by 0 to 3, and permutes as you inserted the arguments, meaning for a.permute(0,2,3,1) it will keep the first dimention in its place, since the first argument is 0, the second dimention it will move to the forth dimention, since the index 1 is the forth argument. And the third and forth dimention will move to the second and third dimention, because the 2 and 3 indices are located in the second and third place. Remember when talking about the 4th dimention for instance, its representation as an argument is 3 (not 4).

EDIT
If you want to element-wise multiply tensors of shape [32,5,2,2] and [32,5] for example, such that each 2x2 matrix will be multiplied by the corresponding value, you could rearrange the dimentions as [2,2,32,5] by permute(2,3,0,1), then perform the multiplication by a * b and then return to the original shape by permute(2,3,0,1) again. The key here, is that the last n dimentions of the first matrix, need to align with the first n dimentions of the second matrix. In our case n=2.

Hope that helps.


Instruction: RuntimeError: stack expects each tensor to be equal size, but got [32, 1] at entry 0 and [32, 0] at entry 1

Input:
I have a very large tensor of shape (512,3,224,224). I input it to model in batches of 32 and I then save the scores corresponding to the target label which is 2. in each iteration, after every slice, the shape of scores changes. Which leads to the following error. What am I doing wrong and how to fix it.
label = torch.ones(1)*2

def sub_forward(self, x):
    x = self.vgg16(x)
    x = self.bn1(x)
    x = self.linear1(x)
    x = self.linear2(x)
    return x

def get_scores(self, imgs, targets):
    b, _, _, _ = imgs.shape
    batch_size = 32
    total_scores = []
    for i in range(0, b, batch_size):
      scores = self.sub_forward(imgs[i:i+batch_size,:,:,:])
      scores = F.softmax(scores)
      labels = targets[i:i+batch_size]
      labels = labels.long()
      scores = scores[:,labels]
      print(i," scores: ", scores)
      total_scores.append(scores)
      print(i," total_socres: ", total_scores)
    total_scores = torch.stack(total_scores)
    return scores

0  scores:  tensor([[0.0811],
        [0.0918],
        [0.0716],
        [0.1680],
        [0.1689],
        [0.1319],
        [0.1556],
        [0.2966],
        [0.0913],
        [0.1238],
        [0.1480],
        [0.1215],
        [0.2524],
        [0.1283],
        [0.1603],
        [0.1282],
        [0.2668],
        [0.1146],
        [0.2043],
        [0.2475],
        [0.0865],
        [0.1869],
        [0.0860],
        [0.1979],
        [0.1677],
        [0.1983],
        [0.2623],
        [0.1975],
        [0.1894],
        [0.3299],
        [0.1970],
        [0.1094]], device='cuda:0')
0  total_socres:  [tensor([[0.0811],
        [0.0918],
        [0.0716],
        [0.1680],
        [0.1689],
        [0.1319],
        [0.1556],
        [0.2966],
        [0.0913],
        [0.1238],
        [0.1480],
        [0.1215],
        [0.2524],
        [0.1283],
        [0.1603],
        [0.1282],
        [0.2668],
        [0.1146],
        [0.2043],
        [0.2475],
        [0.0865],
        [0.1869],
        [0.0860],
        [0.1979],
        [0.1677],
        [0.1983],
        [0.2623],
        [0.1975],
        [0.1894],
        [0.3299],
        [0.1970],
        [0.1094]], device='cuda:0')]
32  scores:  tensor([], device='cuda:0', size=(32, 0))
32  total_socres:  [tensor([[0.0811],
        [0.0918],
        [0.0716],
        [0.1680],
        [0.1689],
        [0.1319],
        [0.1556],
        [0.2966],
        [0.0913],
        [0.1238],
        [0.1480],
        [0.1215],
        [0.2524],
        [0.1283],
        [0.1603],
        [0.1282],
        [0.2668],
        [0.1146],
        [0.2043],
        [0.2475],
        [0.0865],
        [0.1869],
        [0.0860],
        [0.1979],
        [0.1677],
        [0.1983],
        [0.2623],
        [0.1975],
        [0.1894],
        [0.3299],
        [0.1970],
        [0.1094]], device='cuda:0'), tensor([], device='cuda:0', size=(32, 0))]

> RuntimeError: stack expects each tensor to be equal size, but got [32, 1] at entry 0 and [32, 0] at entry 1

Output:
I don't know what happen to your code but you shouldn't do the batching like that honestly. Please use Dataset:

import torch

class MyDataloader(torch.utils.data.Dataset):
    def __init__(self):
        self.images = torch.Tensor(512, 3, 224, 224)

    def __len__(self):
        return 512

    def __getitem__(self, idx):
        return self.images[idx, :, :, :], torch.ones(1) * 2

train_data = MyDataloader()
train_loader = torch.utils.data.DataLoader(train_data,
                                           shuffle=True,
                                           num_workers=2,
                                           batch_size=32)
for batch_images, targets in train_loader:
    print(batch_images.shape)  # should be 32*3*224*224

    ... # let train your model
    logits = model(batch_images, targets)


Instruction: Problem with Dataloader object not subscriptable

Input:
I am now running a Python program using Pytorch. I use my own dataset, not torch.data.dataset. I download data from a pickle file extracted from feature extraction. But the following errors appear:

Traceback (most recent call last):
  File "C:\Users\hp\Downloads\efficient_densenet_pytorch-master\demo-emotion.py", line 326, in <module>
    fire.Fire(demo)
  File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\fire\core.py", line 138, in Fire
    component_trace = _Fire(component, args, parsed_flag_args, context, name)
  File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\fire\core.py", line 468, in _Fire
    target=component.__name__)
  File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\fire\core.py", line 672, in _CallAndUpdateTrace
    component = fn(*varargs, **kwargs)
  File "C:\Users\hp\Downloads\efficient_densenet_pytorch-master\demo-emotion.py", line 304, in demo
    train(model,train_set1, valid_set=valid_set, test_set=test1, save=save, n_epochs=n_epochs,batch_size=batch_size,seed=seed)
  File "C:\Users\hp\Downloads\efficient_densenet_pytorch-master\demo-emotion.py", line 172, in train
    n_epochs=n_epochs,
  File "C:\Users\hp\Downloads\efficient_densenet_pytorch-master\demo-emotion.py", line 37, in train_epoch
    loader=np.asarray(list(loader))
  File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\utils\data\dataloader.py", line 345, in __next__
    data = self._next_data()
  File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\utils\data\dataloader.py", line 385, in _next_data
    data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
  File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\utils\data\_utils\fetch.py", line 44, in fetch
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\utils\data\_utils\fetch.py", line 44, in <listcomp>
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "C:\Users\hp\Anaconda3\envs\tf-gpu\lib\site-packages\torch\utils\data\dataset.py", line 257, in __getitem__
    return self.dataset[self.indices[idx]]
TypeError: 'DataLoader' object is not subscriptable

The code is:

train_set1 = Owndata()

train1, test1 = train_set1 .get_splits()
# prepare data loaders
train_dl = torch.utils.data.DataLoader(train1, batch_size=32, shuffle=True)
test_dl =torch.utils.data.DataLoader(test1, batch_size=1024, shuffle=False)
test_set1 = Owndata()
'''print('test_set# ',test_set)'''
if valid_size:
    valid_set = Owndata()
    indices = torch.randperm(len(train_set1))
    train_indices = indices[:len(indices) - valid_size]
    valid_indices = indices[len(indices) - valid_size:]
    train_set1 = torch.utils.data.Subset(train_dl, train_indices)
    valid_set = torch.utils.data.Subset(valid_set, valid_indices)
else:
    valid_set = None
model = DenseNet(
    growth_rate=growth_rate,
    block_config=block_config,
    num_classes=10,
    small_inputs=True,
    efficient=efficient,
)
train(model,train_set1, valid_set=valid_set, test_set=test1, save=save, n_epochs=n_epochs, batch_size=batch_size, seed=seed)

Any help is appreciated! Thanks a lot in advance!!

Output:
It is not the line giving you an error as it's the very last train function you are not showing.

You are confusing two things:

torch.utils.data.Dataset object is indexable (dataset[5] works fine for example). It is a simple object which defines how to get a single (usually single) sample of data.
torch.utils.data.DataLoader - non-indexable, only iterable, usually returns batches of data from above Dataset. Can work in parallel using num_workers. It's what you are trying to index while you should use dataset for that.

Please see PyTorch documentation about data to get a better grasp on how those work.


Instruction: Why pandas.core.series.Series sometimes cannot convert to torch tensor in Python?

Input:
I have a dataframe in which I pick two columns:

X_train, X_test, y_train, y_test = train_test_split(df["EnergyFront"], df["particle"], test_size=0.2)

the type of both X_train and X_test is pandas.core.series.Series, the results are quite similar:

IMAGE

I can transform X_train to torch tensor:

X_train = torch.Tensor(X_train)

but, when I try to do the same with X_test:

X_test = torch.Tensor(X_test)

I get the following error:

ValueError                                Traceback (most recent call last)
<ipython-input-174-14117eb3ce4e> in <module>()
----> 1 X_test = torch.Tensor(X_test)

ValueError: could not determine the shape of object type 'Series'

How can I solve it?
By the way, I am running on Google Colaboratory.

Output:
This issue is described here: https://github.com/pytorch/pytorch/pull/7583
In order to determine the shape of the series, they try to access the element with index 0. If that element is not found, this error occurs. In your case, presumably this happens because your X_test doesn't contain the first element of the whole Series.

I believe a valid fix for your case would be to convert your X_test to an array like so:

X_test = torch.Tensor(X_test.to_numpy())


Instruction: Can not import fastai [WinError 126] The specified module could not be foun

Input:
Firstly I run the following command: conda install -c pytorch -c fastai fastai. After finishing install. I import this:

from fastai.imports import *

I got an error like this:

Traceback (most recent call last):

  File "C:\Users\acer\Desktop\Reddit Bot\demo.py", line 70, in <module>
    from fastai.imports import *

  File "C:\Users\acer\anaconda3\lib\site-packages\fastai\imports\__init__.py", line 2, in <module>
    from .torch import *

  File "C:\Users\acer\anaconda3\lib\site-packages\fastai\imports\torch.py", line 1, in <module>
    import torch, torch.nn.functional as F

  File "C:\Users\acer\anaconda3\lib\site-packages\torch\__init__.py", line 81, in <module>
    ctypes.CDLL(dll)

  File "C:\Users\acer\anaconda3\lib\ctypes\__init__.py", line 364, in __init__
    self._handle = _dlopen(self._name, mode)

OSError: [WinError 126] The specified module could not be found

I watch some post on stackoverflow but can't solve this. How can I solve this? Please help me.

Output:
This probably happened if you, like me - are using a machine without a NVIDIA GPU card. conda install -c pytorch -c fastai fastai will install a version that uses the GPU.

To resolve this, I uninstalled pytorch and then conda installed the CPU version using conda install pytorch torchvision cpuonly -c pytorch. It worked for me! You can check the pytorch installation guide on the official website :)


Instruction: An out of bounds index error when using Pytorch gather

Input:
I have Two Tensors

I am trying to gather one from each row with the column being specified by these indices. So I am trying to get:

[0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1]

This is my code for this:

self.manDistMat.gather(1, state.unsqueeze(-1)))
self.manDistMat

being the 16x16 matrix and state.unsqueeze(-1) being the other matrix.

When I try this I get this error.

  RuntimeError: index 578437695752307201 is out of bounds for dimension 1 with size 16

What am I doing wrong?

Output: I encountered the similar problem. It appears to be a bug in pytorch.


Instruction: Pytorch: Memory Efficient weighted sum with weights shared along channels

Input:
Inputs:

1) I = Tensor of dim (N, C, X)      (Input)

2) W = Tensor of dim (N, X, Y)    (Weight)

Output:

1) O = Tensor of dim (N, C, Y)     (Output)

I want to compute:

I = I.view(N, C, X, 1)
W = W.view(N, 1, X, Y)
PROD = I*W
O = PROD.sum(dim=2)
return O

without incurring N * C * X * Y memory overhead.

Basically I want to calculate the weighted sum of a feature map wherein the weights are the same along the channel dimension, without incurring memory overhead per channel.

Maybe I could use

from itertools import product

O = torch.zeros(N, C, Y)
for n, x, y in product(range(N), range(X), range(Y)):
    O[n, :, y] += I[n, :, x]*W[n, x, y]
return O

but that would be slower (no broadcasting) and I'm not sure how much memory overhead would be incurred by saving variables for the backward pass.

Output:
You can use torch.bmm (https://pytorch.org/docs/stable/torch.html#torch.bmm). Just do torch.bmm(I,W)

To verify the results :

import torch
N, C, X, Y= 100, 10, 9, 8

i = torch.rand(N,C,X)
w = torch.rand(N,X,Y)
o = torch.bmm(i,w)

# desired result code
I = i.view(N, C, X, 1)
W = w.view(N, 1, X, Y)
PROD = I*W
O = PROD.sum(dim=2)

print(torch.allclose(O,o)) # should output True if outputs are same.

EDIT: Ideally, I would assume using pytorch's internal matrix multiplication is efficient. However, you can also measure the memory usage with tracemalloc (at least on CPU). See https://discuss.pytorch.org/t/measuring-peak-memory-usage-tracemalloc-for-pytorch/34067 for GPU.

import torch
import tracemalloc
tracemalloc.start()
N, C, X, Y= 100, 10, 9, 8

i = torch.rand(N,C,X)
w = torch.rand(N,X,Y)
o = torch.bmm(i,w)
# output is a tuple indicating current memory and peak memory
print(tracemalloc.get_traced_memory())

You can do the same with other code and see the bmm implementation is indeed efficient.

import torch
import tracemalloc
tracemalloc.start()
N, C, X, Y= 100, 10, 9, 8

i = torch.rand(N,C,X)
w = torch.rand(N,X,Y)

I = i.view(N, C, X, 1)
W = w.view(N, 1, X, Y)
PROD = I*W
O = PROD.sum(dim=2)
# output is a tuple indicating current memory and peak memory
print(tracemalloc.get_traced_memory())


Instruction: Pytorch | I don't know why it is throwing an error? (Beginner)

Input:
import torch.nn as nn
import torch.nn.functional as F

## TODO: Define the NN architecture
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # linear layer (784 -> 1 hidden node)
        self.fc1 = nn.Linear(28 * 28, 512)
        self.fc2 = nn.Linear(512 * 512)
        self.fc3 = nn.Linear(512 * 10)

    def forward(self, x):
        # flatten image input
        x = x.view(-1, 28 * 28)
        # add hidden layer, with relu activation function
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = F.relu(self.fc3(x))
        return x

# initialize the NN
model = Net()
print(model)

When I run this, it throws this error. Why?

TypeError: __ init __() missing 1 required positional argument: 'out_features'

Output:
This error is because you have not provided the output size of the fully connected layer in your fc2 and fc3.
Below is the modified code. I added the output size, I am not sure if this is the output size architecture you want. But for the demonstration, I put the output size. Please edit the code and add the output size as per your requirement.

Remember that the output size of the previous fully connected layer should be the input size of the next FC layer. Else it will throw size mismatch error.

import torch.nn as nn
import torch.nn.functional as F

## TODO: Define the NN architecture
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        # linear layer (784 -> 1 hidden node)
        self.fc1 = nn.Linear(28 * 28, 512)
        self.fc2 = nn.Linear(512 ,512*10)
        self.fc3 = nn.Linear(512 * 10,10)

    def forward(self, x):
        # flatten image input
        x = x.view(-1, 28 * 28)
        # add hidden layer, with relu activation function
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = F.relu(self.fc3(x))
        return x

# initialize the NN
model = Net()
print(model)