Instruction: Unexpected and missing keys in state_dict when converting pytorch to onnx

Input:
When I convert a '.pth' model from PyTorch to ONNX, an error like Unexpected keys and missing keys occur.

This is my model:

    1 import torch
    2 import torch.onnx
    3 from mmcv import runner
    4 import torch.`enter code here`nn as nn
    5 from mobilenet import MobileNet
    6 # A model class instance (class not shown)
    7 md=MobileNet(1,2)
    8 model = md
    9 device_ids = [0,2,6,7,8]
    10 model = nn.DataParallel(model,device_ids)
    11 #torch.backends.cudnn.benchmark = True
    12 # Load the weights from a file (.pth usually)
    13 runner.load_checkpoint(model,'../mmdetection-        master/work_dmobile/faster_rcnn_r50_fpn_1x/epoch_60.pth')
    14 #model = MMDataParallel(model, device_ids=[0])
    15 #state_dict=torch.load('../mmdetection-master/r.pkl.json')
    16 # Load the weights now into a model net architecture defined by our class
    17 #model.load_state_dict(state_dict)
    18 #model = runner.load_state_dict(state_dict)
    19 model=runner.load_state_dict({k.replace('module.',' '):v for k,v in state_dict['state_dict'].items()})
    20 # Create the right input shape (e.g. for an image)
    21 dummy_input = torch.randn(1, 64, 512, 256)
    22
    23 torch.onnx.export(model, dummy_input, "onnx_model_name.onnx")

And this is the error:

unexpected key in source state_dict: backbone.stem.0.conv.weight, backbone.stem.0.bn.weight, backbone.stem.0.bn.bias, backbone.stem.0.bn.running_mean, backbone.stem.0.bn.running_var, backbone.stem.0.bn.num_batches_tracked, backbone.stem.1.depthwise.0.weight, backbone.stem.1.depthwise.1.weight, backbone.stem.1.depthwise.1.bias, backbone.stem.1.depthwise.1.running_mean, backbone.stem.1.depthwise.1.running_var, backbone.stem.1.depthwise.1.num_batches_tracked, backbone.stem.1.pointwise.0.weight, backbone.stem.1.pointwise.0.bias, backbone.stem.1.pointwise.1.weight, backbone.stem.1.pointwise.1.bias, backbone.stem.1.pointwise.1.running_mean, backbone.stem.1.pointwise.1.running_var, backbone.stem.1.pointwise.1.num_batches_tracked, backbone.conv1.0.depthwise.0.weight, backbone.conv1.0.depthwise.1.weight, backbone.conv1.0.depthwise.1.bias, backbone.conv1.0.depthwise.1.running_mean, backbone.conv1.0.depthwise.1.running_var, backbone.conv1.0.depthwise.1.num_batches_tracked, backbone.conv1.0.pointwise.0.weight, backbone.conv1.0.pointwise.0.bias, backbone.conv1.0.pointwise.1.weight, backbone.conv1.0.pointwise.1.bias, backbone.conv1.0.pointwise.1.running_mean, backbone.conv1.0.pointwise.1.running_var, backbone.conv1.0.pointwise.1.num_batches_tracked, backbone.conv1.1.depthwise.0.weight, backbone.conv1.1.depthwise.1.weight, backbone.conv1.1.depthwise.1.bias, backbone.conv1.1.depthwise.1.running_mean, backbone.conv1.1.depthwise.1.running_var, backbone.conv1.1.depthwise.1.num_batches_tracked, backbone.conv1.1.pointwise.0.weight, backbone.conv1.1.pointwise.0.bias, backbone.conv1.1.pointwise.1.weight, backbone.conv1.1.pointwise.1.bias, backbone.conv1.1.pointwise.1.running_mean, backbone.conv1.1.pointwise.1.running_var, backbone.conv1.1.pointwise.1.num_batches_tracked, backbone.conv2.0.depthwise.0.weight, backbone.conv2.0.depthwise.1.weight, backbone.conv2.0.depthwise.1.bias, backbone.conv2.0.depthwise.1.running_mean, backbone.conv2.0.depthwise.1.running_var, backbone.conv2.0.depthwise.1.num_batches_tracked, backbone.conv2.0.pointwise.0.weight, backbone.conv2.0.pointwise.0.bias, backbone.conv2.0.pointwise.1.weight, backbone.conv2.0.pointwise.1.bias, backbone.conv2.0.pointwise.1.running_mean, backbone.conv2.0.pointwise.1.running_var, backbone.conv2.0.pointwise.1.num_batches_tracked, backbone.conv2.1.depthwise.0.weight, backbone.conv2.1.depthwise.1.weight, backbone.conv2.1.depthwise.1.bias, backbone.conv2.1.depthwise.1.running_mean, backbone.conv2.1.depthwise.1.running_var, backbone.conv2.1.depthwise.1.num_batches_tracked, backbone.conv2.1.pointwise.0.weight, backbone.conv2.1.pointwise.0.bias, backbone.conv2.1.pointwise.1.weight, backbone.conv2.1.pointwise.1.bias, backbone.conv2.1.pointwise.1.running_mean, backbone.conv2.1.pointwise.1.running_var, backbone.conv2.1.pointwise.1.num_batches_tracked, backbone.conv3.0.depthwise.0.weight, backbone.conv3.0.depthwise.1.weight, backbone.conv3.0.depthwise.1.bias, backbone.conv3.0.depthwise.1.running_mean, backbone.conv3.0.depthwise.1.running_var, backbone.conv3.0.depthwise.1.num_batches_tracked, backbone.conv3.0.pointwise.0.weight, backbone.conv3.0.pointwise.0.bias, backbone.conv3.0.pointwise.1.weight, backbone.conv3.0.pointwise.1.bias, backbone.conv3.0.pointwise.1.running_mean, backbone.conv3.0.pointwise.1.running_var, backbone.conv3.0.pointwise.1.num_batches_tracked, backbone.conv3.1.depthwise.0.weight, backbone.conv3.1.depthwise.1.weight, backbone.conv3.1.depthwise.1.bias, backbone.conv3.1.depthwise.1.running_mean, backbone.conv3.1.depthwise.1.running_var, backbone.conv3.1.depthwise.1.num_batches_tracked, backbone.conv3.1.pointwise.0.weight, backbone.conv3.1.pointwise.0.bias, backbone.conv3.1.pointwise.1.weight, backbone.conv3.1.pointwise.1.bias, backbone.conv3.1.pointwise.1.running_mean, backbone.conv3.1.pointwise.1.running_var, backbone.conv3.1.pointwise.1.num_batches_tracked, backbone.conv3.2.depthwise.0.weight, backbone.conv3.2.depthwise.1.weight, backbone.conv3.2.depthwise.1.bias, backbone.conv3.2.depthwise.1.running_mean, backbone.conv3.2.depthwise.1.running_var, backbone.conv3.2.depthwise.1.num_batches_tracked, backbone.conv3.2.pointwise.0.weight, backbone.conv3.2.pointwise.0.bias, backbone.conv3.2.pointwise.1.weight, backbone.conv3.2.pointwise.1.bias, backbone.conv3.2.pointwise.1.running_mean, backbone.conv3.2.pointwise.1.running_var, backbone.conv3.2.pointwise.1.num_batches_tracked, backbone.conv3.3.depthwise.0.weight, backbone.conv3.3.depthwise.1.weight, backbone.conv3.3.depthwise.1.bias, backbone.conv3.3.depthwise.1.running_mean, backbone.conv3.3.depthwise.1.running_var, backbone.conv3.3.depthwise.1.num_batches_tracked, backbone.conv3.3.pointwise.0.weight, backbone.conv3.3.pointwise.0.bias, backbone.conv3.3.pointwise.1.weight, backbone.conv3.3.pointwise.1.bias, backbone.conv3.3.pointwise.1.running_mean, backbone.conv3.3.pointwise.1.running_var, backbone.conv3.3.pointwise.1.num_batches_tracked, backbone.conv3.4.depthwise.0.weight, backbone.conv3.4.depthwise.1.weight, backbone.conv3.4.depthwise.1.bias, backbone.conv3.4.depthwise.1.running_mean, backbone.conv3.4.depthwise.1.running_var, backbone.conv3.4.depthwise.1.num_batches_tracked, backbone.conv3.4.pointwise.0.weight, backbone.conv3.4.pointwise.0.bias, backbone.conv3.4.pointwise.1.weight, backbone.conv3.4.pointwise.1.bias, backbone.conv3.4.pointwise.1.running_mean, backbone.conv3.4.pointwise.1.running_var, backbone.conv3.4.pointwise.1.num_batches_tracked, backbone.conv3.5.depthwise.0.weight, backbone.conv3.5.depthwise.1.weight, backbone.conv3.5.depthwise.1.bias, backbone.conv3.5.depthwise.1.running_mean, backbone.conv3.5.depthwise.1.running_var, backbone.conv3.5.depthwise.1.num_batches_tracked, backbone.conv3.5.pointwise.0.weight, backbone.conv3.5.pointwise.0.bias, backbone.conv3.5.pointwise.1.weight, backbone.conv3.5.pointwise.1.bias, backbone.conv3.5.pointwise.1.running_mean, backbone.conv3.5.pointwise.1.running_var, backbone.conv3.5.pointwise.1.num_batches_tracked, backbone.conv4.0.depthwise.0.weight, backbone.conv4.0.depthwise.1.weight, backbone.conv4.0.depthwise.1.bias, backbone.conv4.0.depthwise.1.running_mean, backbone.conv4.0.depthwise.1.running_var, backbone.conv4.0.depthwise.1.num_batches_tracked, backbone.conv4.0.pointwise.0.weight, backbone.conv4.0.pointwise.0.bias, backbone.conv4.0.pointwise.1.weight, backbone.conv4.0.pointwise.1.bias, backbone.conv4.0.pointwise.1.running_mean, backbone.conv4.0.pointwise.1.running_var, backbone.conv4.0.pointwise.1.num_batches_tracked, backbone.conv4.1.depthwise.0.weight, backbone.conv4.1.depthwise.1.weight, backbone.conv4.1.depthwise.1.bias, backbone.conv4.1.depthwise.1.running_mean, backbone.conv4.1.depthwise.1.running_var, backbone.conv4.1.depthwise.1.num_batches_tracked, backbone.conv4.1.pointwise.0.weight, backbone.conv4.1.pointwise.0.bias, backbone.conv4.1.pointwise.1.weight, backbone.conv4.1.pointwise.1.bias, backbone.conv4.1.pointwise.1.running_mean, backbone.conv4.1.pointwise.1.running_var, backbone.conv4.1.pointwise.1.num_batches_tracked, neck.lateral_convs.0.conv.weight, neck.lateral_convs.0.conv.bias, neck.lateral_convs.1.conv.weight, neck.lateral_convs.1.conv.bias, neck.lateral_convs.2.conv.weight, neck.lateral_convs.2.conv.bias, neck.fpn_convs.0.conv.weight, neck.fpn_convs.0.conv.bias, neck.fpn_convs.1.conv.weight, neck.fpn_convs.1.conv.bias, neck.fpn_convs.2.conv.weight, neck.fpn_convs.2.conv.bias, rpn_head.rpn_conv.weight, rpn_head.rpn_conv.bias, rpn_head.rpn_cls.weight, rpn_head.rpn_cls.bias, rpn_head.rpn_reg.weight, rpn_head.rpn_reg.bias, bbox_head.fc_cls.weight, bbox_head.fc_cls.bias, bbox_head.fc_reg.weight, bbox_head.fc_reg.bias, bbox_head.shared_fcs.0.weight, bbox_head.shared_fcs.0.bias, bbox_head.shared_fcs.1.weight, bbox_head.shared_fcs.1.bias

missing keys in source state_dict: conv2.1.depthwise.1.weight, conv4.0.depthwise.0.weight, conv4.1.pointwise.1.weight, conv3.2.depthwise.0.weight, conv3.1.pointwise.0.weight, conv3.4.pointwise.1.bias, conv3.5.depthwise.1.bias, conv2.1.pointwise.1.weight, stem.1.pointwise.1.running_mean, conv3.3.pointwise.1.weight, conv3.3.depthwise.1.running_mean, conv3.1.depthwise.1.num_batches_tracked, conv3.0.depthwise.1.num_batches_tracked, conv2.1.depthwise.1.running_var, conv1.0.depthwise.1.weight, conv3.5.depthwise.1.running_var, stem.0.bn.bias, conv3.2.depthwise.1.num_batches_tracked, conv2.0.depthwise.0.weight, conv2.1.pointwise.0.bias, conv3.1.pointwise.1.bias, conv3.2.pointwise.1.bias, conv2.0.pointwise.1.num_batches_tracked, stem.1.pointwise.0.weight, conv2.0.depthwise.1.weight, stem.1.depthwise.0.weight, conv1.1.pointwise.1.weight, conv3.5.pointwise.0.weight, conv3.4.depthwise.1.running_var, conv1.0.pointwise.0.bias, conv3.3.depthwise.1.running_var, conv3.0.pointwise.1.weight, conv4.0.pointwise.1.num_batches_tracked, conv4.1.depthwise.1.running_var, stem.1.depthwise.1.running_var, conv3.0.pointwise.1.running_var, conv3.4.depthwise.0.weight, conv3.4.pointwise.1.num_batches_tracked, conv4.0.depthwise.1.num_batches_tracked, conv3.0.depthwise.1.weight, conv3.3.pointwise.0.bias, conv3.0.depthwise.1.running_mean, conv3.2.pointwise.1.running_mean, conv3.1.pointwise.0.bias, conv3.5.depthwise.1.num_batches_tracked, conv3.5.pointwise.1.running_mean, conv3.1.pointwise.1.running_var, conv1.0.depthwise.1.running_mean, stem.1.pointwise.1.bias, conv1.0.depthwise.0.weight, conv3.2.pointwise.0.weight, conv4.0.pointwise.1.running_mean, conv2.1.pointwise.1.running_mean, stem.1.pointwise.1.weight, conv4.1.depthwise.1.weight, conv4.0.pointwise.0.weight, conv1.1.depthwise.1.bias, conv3.2.pointwise.1.num_batches_tracked, conv4.1.depthwise.0.weight, conv3.4.depthwise.1.running_mean, conv1.0.depthwise.1.bias, conv2.0.pointwise.0.bias, conv3.4.depthwise.1.num_batches_tracked, conv4.1.pointwise.1.running_mean, conv2.1.depthwise.1.bias, conv3.2.depthwise.1.weight, conv2.0.pointwise.1.weight, conv1.0.pointwise.0.weight, conv3.1.depthwise.1.running_var, conv2.0.pointwise.1.bias, conv4.0.depthwise.1.bias, conv3.3.pointwise.1.running_var, conv3.4.pointwise.1.weight, conv4.0.pointwise.0.bias, conv3.4.depthwise.1.bias, conv4.1.depthwise.1.num_batches_tracked, conv2.0.pointwise.1.running_mean, conv1.1.depthwise.1.weight, conv2.0.pointwise.1.running_var, stem.1.depthwise.1.running_mean, conv3.4.pointwise.1.running_var, stem.1.depthwise.1.num_batches_tracked, conv3.3.depthwise.1.weight, stem.1.pointwise.1.running_var, conv4.1.depthwise.1.bias, conv3.0.pointwise.1.bias, conv2.0.depthwise.1.running_mean, conv1.1.pointwise.1.bias, conv4.1.pointwise.0.bias, conv3.2.pointwise.0.bias, conv1.1.pointwise.0.weight, conv1.0.pointwise.1.weight, conv1.0.pointwise.1.running_mean, stem.0.conv.weight, stem.1.depthwise.1.bias, conv3.3.depthwise.0.weight, conv1.1.depthwise.1.num_batches_tracked, conv3.3.pointwise.1.num_batches_tracked, conv3.2.pointwise.1.running_var, conv3.2.depthwise.1.running_mean, conv3.3.depthwise.1.bias, conv4.1.pointwise.1.num_batches_tracked, conv2.0.depthwise.1.num_batches_tracked, conv3.0.pointwise.0.bias, conv3.1.depthwise.1.running_mean, conv3.1.depthwise.1.weight, conv3.0.pointwise.1.num_batches_tracked, conv3.1.pointwise.1.weight, conv4.0.pointwise.1.bias, conv3.3.depthwise.1.num_batches_tracked, conv3.4.pointwise.0.weight, stem.1.pointwise.0.bias, conv3.0.depthwise.1.bias, conv1.1.pointwise.0.bias, conv4.0.pointwise.1.running_var, stem.0.bn.weight, conv1.0.pointwise.1.num_batches_tracked, conv2.1.depthwise.1.running_mean, conv4.1.depthwise.1.running_mean, conv1.1.pointwise.1.running_var, conv2.1.pointwise.1.num_batches_tracked, conv2.0.depthwise.1.running_var, conv3.5.depthwise.1.weight, conv3.0.depthwise.0.weight, conv4.0.depthwise.1.running_mean, stem.0.bn.num_batches_tracked, conv3.3.pointwise.1.running_mean, conv2.1.pointwise.1.running_var, conv3.0.pointwise.1.running_mean, conv1.1.depthwise.1.running_var, conv3.0.depthwise.1.running_var, conv1.0.depthwise.1.running_var, stem.1.pointwise.1.num_batches_tracked, conv4.0.pointwise.1.weight, conv1.1.pointwise.1.running_mean, conv2.1.depthwise.0.weight, conv1.0.depthwise.1.num_batches_tracked, conv1.0.pointwise.1.running_var, conv3.5.pointwise.1.weight, conv3.5.depthwise.1.running_mean, conv3.1.depthwise.1.bias, conv3.1.depthwise.0.weight, conv1.1.depthwise.1.running_mean, conv2.0.pointwise.0.weight, conv4.1.pointwise.1.bias, conv3.2.depthwise.1.running_var, conv3.5.pointwise.0.bias, conv3.4.depthwise.1.weight, conv3.2.depthwise.1.bias, stem.0.bn.running_mean, conv4.0.depthwise.1.running_var, conv1.1.depthwise.0.weight, stem.0.bn.running_var, conv4.1.pointwise.0.weight, conv2.1.pointwise.1.bias, conv3.4.pointwise.0.bias, conv1.0.pointwise.1.bias, conv3.5.pointwise.1.running_var, conv1.1.pointwise.1.num_batches_tracked, conv3.1.pointwise.1.running_mean, conv2.1.depthwise.1.num_batches_tracked, conv2.1.pointwise.0.weight, stem.1.depthwise.1.weight, conv3.5.pointwise.1.bias, conv3.5.pointwise.1.num_batches_tracked, conv3.1.pointwise.1.num_batches_tracked, conv3.2.pointwise.1.weight, conv3.5.depthwise.0.weight, conv3.3.pointwise.0.weight, conv2.0.depthwise.1.bias, conv3.0.pointwise.0.weight, conv3.3.pointwise.1.bias, conv3.4.pointwise.1.running_mean, conv4.0.depthwise.1.weight, conv4.1.pointwise.1.running_var

Output:
In line 19, try using model=runner.load_state_dict(..., strict=False).

Using the parameter strict=False tells the load_state_dict function that there might be missing keys in the checkpoint, which usually come from the BatchNorm layer as I see in this case.


Instruction: How to convert tensor to image array?

Input:
I would like to convert a tensor to image array and use tensor.data() method.

But it doesn't work.

#include <torch/script.h> // One-stop header.

#include <iostream>
#include <memory>
#include <sstream>
#include <string>
#include <vector>

#include "itkImage.h"
#include "itkImageFileReader.h"
#include "itkImageFileWriter.h"
#include "itkImageRegionIterator.h"

//////////////////////////////////////////////////////
//Goal: load jit script model and segment myocardium
//Step: 1. load jit script model
//      2. load input image
//      3. predict by model
//      4. save the result to file
//////////////////////////////////////////////////////
typedef short                                               PixelType;
const unsigned int Dimension = 3;
typedef itk::Image<PixelType, Dimension>                    ImageType;
typedef itk::ImageFileReader<ImageType>                     ReaderType;
typedef itk::ImageRegionIterator<ImageType>                 IteratorType;

bool itk2tensor(ImageType::Pointer itk_img, torch::Tensor &tensor_img) {

    typename ImageType::RegionType region = itk_img->GetLargestPossibleRegion();
    const typename ImageType::SizeType size = region.GetSize();
    std::cout << "Input size: " << size[0] << ", " << size[1]<< ", " << size[2] << std::endl;

    int len = size[0] * size[1] * size[2];
    short rowdata[len];
    int count = 0;
    IteratorType iter(itk_img, itk_img->GetRequestedRegion());

    // convert itk to array
    for (iter.GoToBegin(); !iter.IsAtEnd(); ++iter) {
        rowdata[count] = iter.Get();
        count++;
    }
    std::cout << "Convert itk to array DONE!" << std::endl;

    // convert array to tensor
    tensor_img = torch::from_blob(rowdata, {1, 1, (int)size[0], (int)size[1], (int)size[2]}, torch::kShort).clone();
    tensor_img = tensor_img.toType(torch::kFloat);
    tensor_img = tensor_img.to(torch::kCUDA);
    tensor_img.set_requires_grad(0);

    return true;
}

bool tensor2itk(torch::Tensor &t, ImageType::Pointer itk_img) {

    std::cout << "tensor dtype = " << t.dtype() << std::endl;
    std::cout << "tensor size = " << t.sizes() << std::endl;
    t = t.toType(torch::kShort);
    short * array = t.data<short>();

    ImageType::IndexType start;
    start[0] = 0;  // first index on X
    start[1] = 0;  // first index on Y
    start[2] = 0;  // first index on Z

    ImageType::SizeType  size;
    size[0] = t.size(2);
    size[1] = t.size(3);
    size[2] = t.size(4);

    ImageType::RegionType region;
    region.SetSize( size );
    region.SetIndex( start );

    itk_img->SetRegions( region );
    itk_img->Allocate();

    int len = size[0] * size[1] * size[2];

    IteratorType iter(itk_img, itk_img->GetRequestedRegion());
    int count = 0;
    // convert array to itk
    std::cout << "start!" << std::endl;
    for (iter.GoToBegin(); !iter.IsAtEnd(); ++iter) {
        short temp = *array++;    //  ERROR!
        std::cout << temp << " ";
        iter.Set(temp);
        count++;
    }
    std::cout << "end!" << std::endl;

    return true;
}

int main(int argc, const char* argv[]) {
    int a, b, c;
    if (argc != 4) {
        std::cerr << "usage: automyo input jitmodel output\n";
        return -1;
    }

    std::cout << "=========  jit start  =========\n";
    // 1. load jit script model
    std::cout << "Load script module: " << argv[2] << std::endl;
    std::shared_ptr<torch::jit::script::Module> module = torch::jit::load(argv[2]);
    module->to(at::kCUDA);

    // assert(module != nullptr);
    std::cout << "Load script module DONE" << std::endl;

    // 2. load input image
    const char* img_path = argv[1];
    std::cout << "Load image: " << img_path << std::endl;

    ReaderType::Pointer reader = ReaderType::New();

    if (!img_path) {
        std::cout << "Load input file error!" << std::endl;
        return false;
    }

    reader->SetFileName(img_path);
    reader->Update();

    std::cout << "Load image DONE!" << std::endl;

    ImageType::Pointer itk_img = reader->GetOutput();

    torch::Tensor tensor_img;
    if (!itk2tensor(itk_img, tensor_img)) {
        std::cerr << "itk2tensor ERROR!" << std::endl;
    }
    else {
        std::cout << "Convert array to tensor DONE!" << std::endl;
    }

    std::vector<torch::jit::IValue> inputs;
    inputs.push_back(tensor_img);

    // 3. predict by model
    torch::Tensor y = module->forward(inputs).toTensor();
    std::cout << "Inference DONE!" << std::endl;

    // 4. save the result to file
    torch::Tensor seg = y.gt(0.5);
    // std::cout << seg << std::endl;

    ImageType::Pointer out_itk_img = ImageType::New();
    if (!tensor2itk(seg, out_itk_img)) {
        std::cerr << "tensor2itk ERROR!" << std::endl;
    }
    else {
        std::cout << "Convert tensor to itk DONE!" << std::endl;
    }

    std::cout << out_itk_img << std::endl;

    return true;
}

The runtime log is showed below:

  Load script module:model_myo_jit.pt

  Load script module DONE Load

  image: patch_6.nii.gz

  Load image DONE!

  Input size: 128, 128, 128

  Convert itk to array DONE!

  Convert array to tensor DONE!

  Inference DONE!

  tensor dtype = unsigned char

  tensor size = [1, 1, 96, 96, 96]

  start!

  Segmentation fault (core dumped)

Why and how to convert?

Output: I have found the solution. When I convert the y to kCPU, it works. Because it in CUDA before.


Instruction: Why am I getting different results after saving and loading model weights in pytorch?

Input:
I have written a model, the architecture is follows:

CNNLSTM(
  (cnn): CNNText(
    (embed): Embedding(19410, 300, padding_idx=0)
    (convs1): ModuleList(
      (0): Conv2d(1, 32, kernel_size=(3, 300), stride=(1, 1))
      (1): Conv2d(1, 32, kernel_size=(5, 300), stride=(1, 1))
      (2): Conv2d(1, 32, kernel_size=(7, 300), stride=(1, 1))
    )
    (dropout): Dropout(p=0.6)
    (fc1): Linear(in_features=96, out_features=1, bias=True)
  )
  (lstm): RNN(
    (embedding): Embedding(19410, 300, padding_idx=0)
    (rnn): LSTM(300, 150, batch_first=True, bidirectional=True)
    (attention): Attention(
      (dense): Linear(in_features=300, out_features=1, bias=True)
      (tanh): Tanh()
      (softmax): Softmax()
    )
    (fc1): Linear(in_features=300, out_features=50, bias=True)
    (dropout): Dropout(p=0.5)
    (fc2): Linear(in_features=50, out_features=1, bias=True)
  )
  (fc1): Linear(in_features=146, out_features=1, bias=True)
)

I have used the RNN and the CNN differently on the same dataset and I have the weights saved. In the mixed model, I load the weights using the following function:

def load_pretrained_weights(self, model='cnn', path=None):
    if model not in ['cnn', 'rnn']:
        raise AttributeError("Model must be either rnn or cnn")
    if model == 'cnn':
        self.cnn.load_state_dict(torch.load(path))
    if model == 'rnn':
        self.lstm.load_state_dict(torch.load(path))

And freeze the sub modules using the function:

def freeze(self):
    for p in self.cnn.parameters():
        p.requires_grad = False
    for p in self.lstm.parameters():
        p.requires_grad = False

Then I train the model, and got better result compared to the each submodule trained and
evaluated alone.
I used an early-stopping technique in my epoch loop to save the best parameters.
After training I made a new instance of the same class and when I load the saved “best” parameters I am not getting similar result.
I tried the same thing with each submodule (RNN and CNNText here) alone, it worked. But in this case it is not giving the same performance.

Please help me understand it what is happening here. I am new to Deep Learning concepts.
Thank you.

Few Experiments I tried:

I loaded the saved weights of each submodule and loaded the best parameters, got somehow close to the best result.
Took the hidden layer from each submodule before applying the dropout, got better than the previous, but not the best!

EDIT

The init function of my class is as follows.
And the RNN and CNN are just usual implementations.

class CNNLSTM(nn.Module):

    def __init__(self, vocab_size, embedding_dim, embedding_weight, rnn_arch, isCuda=True, class_num=1, kernel_num=32, kernel_sizes=[3,4,5],train_wv=False, rnn_num_layers=1, rnn_bidirectional=True, rnn_use_attention=True):

        super(CNNLSTM, self).__init__()
        self.cnn = CNNText(vocab_size, embedding_dim, embedding_weight, class_num, kernel_num = kernel_num, kernel_sizes=kernel_sizes, static=train_wv,dropout=0.6)
        self.lstm = RNN(rnn_arch, vocab_size, embedding_dim, embedding_weight, num_layers=rnn_num_layers, rnn_unit='lstm', embedding_train=train_wv, isCuda=isCuda, bidirectional=rnn_bidirectional, use_padding=True, use_attention=rnn_use_attention, num_class=class_num)
        self.fc1 = nn.Linear(rnn_arch[-1] + len(kernel_sizes) * kernel_num , class_num)

After declaring the object

I Loaded individual pre-trained submodule as,

model.load_pretrained_weights('rnn', 'models/bilstm_2_atten.pth')
model.load_pretrained_weights('cnn', 'models/cnn2.pth')

model.freeze()

Then I train the last linear layer.
I saved the model parameter values as

torch.save(model.state_dict(),path)

So at 3rd/4th from last epoch I am getting the 'best' result.
And after training I loaded the parameters for best result with,

state_dict = torch.load(MODEL_PATH)
model.load_state_dict(state_dict)

Output:
After loading the model, you need to write model.eval().
state_dict = torch.load(MODEL_PATH)
model.load_state_dict(state_dict)
model.eval()

Reference : Pytorch Documentation
This is what it says:

When saving a model for inference, it is only necessary to save the trained model’s learned parameters. Saving the model’s state_dict with the torch.save() function will give you the most flexibility for restoring the model later, which is why it is the recommended method for saving models.
A common PyTorch convention is to save models using either a .pt or .pth file extension.
Remember that you must call model.eval() to set dropout and batch normalization layers to evaluation mode before running inference. Failing to do this will yield inconsistent inference results.


Instruction: what is the difference between unsqueez_ in pytorch and epxand_dim in keras and what will be the shape of output after using it?

Input:
I am a beginner in keras and I have a pytorch code that I need to change it to keras, but I could not understand some part of it. specially I have problems in the size of the output shape.  the shape of image is (:, 3,32,32) and the first dimension of image is the size of the batch. now, my question is: what this line do and what is the output shape:

    image_yuv_ch = image[:, channel, :, :].unsqueeze_(1)

it adds a dimension in position 1? what is the output shape?:(
the size of filters was (64,8,8) and then we have filters.unsqueez_(1), is this means the new shape of filters is (64,1,8,8)?
what does this line do?  image_conv = F.conv2d(image_yuv_ch, filters, stride=8) is it the same as conv2d in keras what is the shape of output tensor from it? I also could not understand what view do? I know it tries to show tensor in new shape but in the below code I could not understand the output shape after each unsqueez_, permute or view. could you please tell me what is the output shape of each line? Thank you in advance.

import torch.nn.functional as F
def apply_conv(self, image, filter_type: str):

        if filter_type == 'dct':
            filters = self.dct_conv_weights
        elif filter_type == 'idct':
            filters = self.idct_conv_weights
        else:
            raise('Unknown filter_type value.')

        image_conv_channels = []
        for channel in range(image.shape[1]):
            image_yuv_ch = image[:, channel, :, :].unsqueeze_(1)
            image_conv = F.conv2d(image_yuv_ch, filters, stride=8)
            image_conv = image_conv.permute(0, 2, 3, 1)
            image_conv = image_conv.view(image_conv.shape[0], image_conv.shape[1], image_conv.shape[2], 8, 8)
            image_conv = image_conv.permute(0, 1, 3, 2, 4)
            image_conv = image_conv.contiguous().view(image_conv.shape[0],
                                                  image_conv.shape[1]*image_conv.shape[2],
                                                  image_conv.shape[3]*image_conv.shape[4])

            image_conv.unsqueeze_(1)

            # image_conv = F.conv2d()
            image_conv_channels.append(image_conv)

        image_conv_stacked = torch.cat(image_conv_channels, dim=1)

        return image_conv_stacked

Output:
It seems like you are Keras-user or Tensorflow-user and trying to learn Pytorch.
You should go to the website of Pytorch document to understand more about each operation.

unsqueeze is to expand the dim by 1 of the tensor. The underscore in unsqueeze_() means this is in-place function.
view() can be understood as .reshape() in keras.
permute() is to switch multiple dimensions of tensor. For example:

x = torch.randn(1,2,3) # shape [1,2,3]
x = torch.permute(2,0,1) # shape [3,1,2]

In order to know the shape of the tensor after each operation, just simply add print(x.size()). For example:

image_conv = image_conv.permute(0, 2, 3, 1)
print(image_conv.size())

image_conv = image_conv.view(image_conv.shape[0], image_conv.shape[1],
print(image_conv.size())

image_conv.shape[2], 8, 8)
print(image_conv.size())

image_conv = image_conv.permute(0, 1, 3, 2, 4)
print(image_conv.size())

The big difference between Pytorch and Tensorflow (back-end of Keras) is that Pytorch will generate a dynamic graph, rather than a static graph as Tensorflow. Your way of defining a model would not work properly in Pytorch since the weights of conv will not be save in model.parameters() which can't be optimized during the backpropagation.

One more comment, please check this link to learn how to define a proper model using Pytorch:

import torch.nn as nn
import torch.nn.functional as F
class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.conv1 = nn.Conv2d(1, 20, 5)
        self.conv2 = nn.Conv2d(20, 20, 5)

    def forward(self, x):
       x = F.relu(self.conv1(x))
       return F.relu(self.conv2(x))

The code for the comment:

import torch

x = torch.randn(8, 3, 32, 32)
print(x.shape)
torch.Size([8, 3, 32, 32])
channel = 1
y = x[:, channel, :, :]
print(y.shape)
torch.Size([8, 32, 32])

y = y.unsqueeze_(1)
print(y.shape)
torch.Size([8, 1, 32, 32])

Hope this helps and enjoy your learning!


Instruction: PyTorch Design: Why does torch.distributions.multivariate_normal have methods outside of its class?

Input:
I'm trying to understand the design of pytorch a little bit better. I was trying to draw samples from a multivariate normal, and found torch.distributions.multivariate_normal, which to my surprise is a module with many protected functions defined outside of its MultivariateNormal() class.

I was confused as to why this was the case. Why not just define all of these functions as class methods inside the MultivariateNormal() class? That way, we could instantiate an object of this class by

torch.distributions.multivariate_normal(mu,sigma)

rather than

torch.distributions.multivariate_normal.MultivariateNormal(mu,sigma).

Any thoughts?

Thanks.

Output:
You can call MultivariateNormal directly:

import torch
gaussian = torch.distributions.MultivariateNormal(torch.ones(2),torch.eye(2))

But the class MultivariateNormal is implemented in the file "torch/distributions/multivariate_normal.py", so both calls are correct


Instruction: Why does loss continue decreasing but performance keep unchanged?

Input:
I am using bert-lstm-crf model, with bert model from https://github.com/huggingface/pytorch-pretrained-BERT/ and lstm crf models are written by myself.

After training bert-lstm-crf model for 25 epochs, the performance on training set, dev set and test set keep unchanged but the loss continue decreasing. Where should I make a change?

Here is performance:

25th epoch:

tensor(10267.6279, device='cuda:0')
(0.42706720346856614, 0.4595134955014995, 0.4426966292134832)
(0.43147208121827413, 0.4271356783919598, 0.42929292929292934)
(0.4460093896713615, 0.4668304668304668, 0.4561824729891957)

26th epoch:

tensor(10219.3398, device='cuda:0')
(0.44544364508393286, 0.4951682772409197, 0.46899163642101943)
(0.4469135802469136, 0.4547738693467337, 0.45080946450809467)
(0.45871559633027525, 0.4914004914004914, 0.4744958481613286)

27th epoch:

tensor(10169.0742, device='cuda:0')
(0.44544364508393286, 0.4951682772409197, 0.46899163642101943)
(0.4469135802469136, 0.4547738693467337, 0.45080946450809467)
(0.45871559633027525, 0.4914004914004914, 0.4744958481613286)

more epochs:
lower loss with same performance:

(0.44544364508393286, 0.4951682772409197, 0.46899163642101943)
(0.4469135802469136, 0.4547738693467337, 0.45080946450809467)
(0.45871559633027525, 0.4914004914004914, 0.4744958481613286)

It is really a weird problem, I have no idea how to handle this. Any suggestion would be of great help.

Here is related code:

  for epoch in tqdm(range(200)):

  {loss = train_one_epoch(dataloader=source_train_dataloader,
  model=model, optimizer=optimizer)

  train_perf = test_one_epoch(dataloader=source_train_dataloader_for_test,
  model=model)

  dev_perf = test_one_epoch(dataloader=source_dev_dataloader, model=model)

  test_perf = test_one_epoch(dataloader=source_test_dataloader,
  model=model)

  base_result_loc = "bert_char_ps/bert_char_result"

  # store performance result

  add_model_result(
    base_result_loc,
    epoch,
    loss,
    train_perf,
    dev_perf,
    test_perf)
  }

The performance should change with loss, but now it does not

Output:
I have modified this PyTorch implementation of BERT-NER model and added CRF. I have the following class which works fine in my case.

class BertWithCRF(BertPreTrainedModel):

    def __init__(self, config, labels, dropout=0.1):
        super(BertWithCRF, self).__init__(config)
        self.tagset_size = len(labels)
        self.tag_to_ix = {k: v for v, k in enumerate(labels)}
        self.bert = BertModel(config)
        self.dropout = nn.Dropout(dropout)
        self.classifier = nn.Linear(config.hidden_size, self.tagset_size)
        self.apply(self.init_bert_weights)
        self.transitions = nn.Parameter(
            torch.zeros(self.tagset_size, self.tagset_size))
        self.transitions.data[self.tag_to_ix[START_TAG], :] = -10000
        self.transitions.data[:, self.tag_to_ix[STOP_TAG]] = -10000

    def _batch_forward_alg(self, feats, mask):
        assert mask is not None
        # calculate in log domain
        # feats is batch_size * len(sentence) * tagset_size
        # initialize alpha with a Tensor with values all equal to -10000.
        score = torch.Tensor(feats.size(0), self.tagset_size).fill_(-10000.)
        score[:, self.tag_to_ix[START_TAG]] = 0.
        if feats.is_cuda:
            score = score.cuda()
        mask = mask.float()
        trans = self.transitions.unsqueeze(0)  # [1, C, C]
        for t in range(feats.size(1)):  # recursion through the sequence
            mask_t = mask[:, t].unsqueeze(1)
            emit_t = feats[:, t].unsqueeze(2)  # [B, C, 1]
            score_t = score.unsqueeze(1) + emit_t + trans  # [B, 1, C] -> [B, C, C]
            score_t = batch_log_sum_exp(score_t)  # [B, 1, C] -> [B, C, C]
            score = score_t * mask_t + score * (1 - mask_t)
        score = batch_log_sum_exp(score + self.transitions[self.tag_to_ix[STOP_TAG]])
        return score  # partition function

    def _batch_score_sentence(self, feats, tags, mask):
        assert mask is not None
        score = torch.Tensor(feats.size(0)).fill_(0.)
        if feats.is_cuda:
            score = score.cuda()
        feats = feats.unsqueeze(3)
        mask = mask.float()
        trans = self.transitions.unsqueeze(2)
        add_start_tags = torch.empty(tags.size(0), 1).fill_(self.tag_to_ix[START_TAG]).type_as(tags)
        tags = torch.cat([add_start_tags, tags], dim=-1)
        for t in range(feats.size(1)):  # recursion through the sequence
            mask_t = mask[:, t]
            emit_t = torch.cat([h[t, y[t + 1]] for h, y in zip(feats, tags)])
            trans_t = torch.cat([trans[y[t + 1], y[t]] for y in tags])
            score += (emit_t + trans_t) * mask_t
        last_tag = tags.gather(1, mask.sum(1).long().unsqueeze(1)).squeeze(1)
        score += self.transitions[self.tag_to_ix[STOP_TAG], last_tag]
        return score

    def _batch_viterbi_decode(self, feats, mask):
        # initialize backpointers and viterbi variables in log space
        bptr = torch.LongTensor()
        score = torch.Tensor(feats.size(0), self.tagset_size).fill_(-10000.)
        score[:, self.tag_to_ix[START_TAG]] = 0.
        if feats.is_cuda:
            score = score.cuda()
            bptr = bptr.cuda()
        mask = mask.float()
        for t in range(feats.size(1)):  # recursion through the sequence
            mask_t = mask[:, t].unsqueeze(1)
            score_t = score.unsqueeze(1) + self.transitions  # [B, 1, C] -> [B, C, C]
            score_t, bptr_t = score_t.max(2)  # best previous scores and tags
            score_t += feats[:, t]  # plus emission scores
            bptr = torch.cat((bptr, bptr_t.unsqueeze(1)), 1)
            score = score_t * mask_t + score * (1 - mask_t)
        score += self.transitions[self.tag_to_ix[STOP_TAG]]
        best_score, best_tag = torch.max(score, 1)

        # back-tracking
        bptr = bptr.tolist()
        best_path = [[i] for i in best_tag.tolist()]
        for b in range(feats.size(0)):
            x = best_tag[b]  # best tag
            y = int(mask[b].sum().item())  # no. of non-pad tokens
            for bptr_t in reversed(bptr[b]):
                x = bptr_t[x]
                best_path[b].append(x)
            best_path[b].pop()
            best_path[b].reverse()

        best_path = torch.LongTensor(best_path)
        if feats.is_cuda:
            best_path = best_path.cuda()
        return best_path

    def _get_bert_features(self, input_ids, token_type_ids, attention_mask):
        sequence_output, _ = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False)
        sequence_output = self.dropout(sequence_output)
        bert_feats = self.classifier(sequence_output)
        return bert_feats

    def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None):
        bert_feats = self._get_bert_features(input_ids, token_type_ids, attention_mask)
        if labels is not None:
            forward_score = self._batch_forward_alg(bert_feats, attention_mask)
            gold_score = self._batch_score_sentence(bert_feats, labels, attention_mask)
            return (forward_score - gold_score).mean()
        else:
            tag_seq = self._batch_viterbi_decode(bert_feats, attention_mask)
            return tag_seq


Instruction: Why doesn't the Discriminator's and Generators' loss change?

Input:
I'm trying to implement a Generative Adversarial Network (GAN) for the MNIST-Dataset.
I use Pytorch for this. My problem is, that after one epoch the Discriminator's and the Generator's loss doesn't change.

I already tried two other methods to build the network, but they cause all the same problem :/

import os
import torch
import matplotlib.pyplot as plt
import matplotlib.gridspec as grd
import numpy as np
import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F
import torchvision #Datasets
from torchvision.utils import save_image
import torchvision.transforms as transforms
from torch.autograd import Variable
import pylab

#Parameter
batch_size = 64
epochs = 50000
image_size = 784
hidden_size = 392
sample_dir = 'samples'
save_dir = 'save'
noise_size = 100
lr = 0.001

# Image processing
transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.5,),(0.5,))])

# Discriminator
D = nn.Sequential(
    nn.Linear(image_size, hidden_size),
    nn.ReLU(),
    nn.Linear(hidden_size, 1),
    nn.Sigmoid()
)

# Generator
G = nn.Sequential(
    nn.Linear(noise_size, hidden_size),
    nn.ReLU(),
    nn.Linear(hidden_size, image_size),
    nn.Sigmoid()
)

# Lossfunction and optimizer (sigmoid cross entropy with logits and Adam)
criterion = nn.BCEWithLogitsLoss()
d_optimizer = torch.optim.Adam(D.parameters(), lr = lr)
g_optimizer = torch.optim.Adam(G.parameters(), lr = lr)

def reset_grad():
    d_optimizer.zero_grad()
    g_optimizer.zero_grad()

# Statistics to be saved
d_losses = np.zeros(epochs)
g_losses = np.zeros(epochs)
real_scores = np.zeros(epochs)
fake_scores = np.zeros(epochs)

# Start training
total_step = len(data_loader)
for epoch in range(epochs):
    for i, (images, _) in enumerate(data_loader):
        if images.shape[0] != 64:
            continue
        images = images.view(batch_size, -1).cuda()
        images = Variable(images)
        # Create the labels which are later used as input for the BCE loss
        real_labels = torch.ones(batch_size, 1).cuda()
        real_labels = Variable(real_labels)
        fake_labels = torch.zeros(batch_size, 1).cuda()
        fake_labels = Variable(fake_labels)

        # Train discriminator

        # Compute BCE_WithLogitsLoss using real images
        outputs = D(images)
        d_loss_real = criterion(outputs, real_labels)
        real_score = outputs

        # Compute BCE_WithLogitsLoss using fake images
        # First term of the loss is always zero since fake_labels == 0
        z = torch.randn(batch_size, noise_size).cuda()
        z = Variable(z)
        fake_images = G(z)
        outputs = D(fake_images)
        d_loss_fake = criterion(outputs, fake_labels)
        fake_score = outputs

        # Backprop and optimize
        # If D is trained so well, then don't update
        d_loss = d_loss_real + d_loss_fake
        reset_grad()
        d_loss.backward()
        d_optimizer.step()

        # Train generator

        # Compute loss with fake images
        z = torch.randn(batch_size, noise_size).cuda()
        z = Variable(z)
        fake_images = G(z)
        outputs = D(fake_images)

        # We train G to maximize log(D(G(z)) instead of minimizing log(1 -D(G(z)))
        # For the reason, see the last paragraph of section 3. https://arxiv.org/pdf/1406.2661.pdf
        g_loss = criterion(outputs, real_labels)

        # Backprop and optimize
        # if G is trained so well, then don't update
        reset_grad()
        g_loss.backward()
        g_optimizer.step()

        # Update statistics

        d_losses[epoch] = d_losses[epoch]*(i/(i+1.)) + d_loss.item()*(1./(i+1.))
        g_losses[epoch] = g_losses[epoch]*(i/(i+1.)) + g_loss.item()*(1./(i+1.))
        real_scores[epoch] = real_scores[epoch]*(i/(i+1.)) + real_score.mean().item()*(1./(i+1.))
        fake_scores[epoch] = fake_scores[epoch]*(i/(i+1.)) + fake_score.mean().item()*(1./(i+1.))

    # print results
    print('Epoch [{}/{}], d_loss: {:.4f}, g_loss: {:.4f}, D(x): {:.2f}, D(G(z)): {:.2f}'
            .format(epoch, epochs, d_loss.item(), g_loss.item(),
                    real_score.mean().item(), fake_score.mean().item()))

The Generator's and Discriminator's loss should change from epoch to epoch, but they don't.

Epoch [0/50000], d_loss: 1.0069, g_loss: 0.6927, D(x): 1.00, D(G(z)): 0.00
Epoch [1/50000], d_loss: 1.0065, g_loss: 0.6931, D(x): 1.00, D(G(z)): 0.00
Epoch [2/50000], d_loss: 1.0064, g_loss: 0.6931, D(x): 1.00, D(G(z)): 0.00
Epoch [3/50000], d_loss: 1.0064, g_loss: 0.6931, D(x): 1.00, D(G(z)): 0.00
Epoch [4/50000], d_loss: 1.0064, g_loss: 0.6931, D(x): 1.00, D(G(z)): 0.00
Epoch [5/50000], d_loss: 1.0064, g_loss: 0.6931, D(x): 1.00, D(G(z)): 0.00

Thank's for your help.

Output:
I found out the solution of the problem.
BCEWithLogitsLoss() and Sigmoid() doesn't work together, because BCEWithLogitsLoss() includes the Sigmoid activation.
So you can use BCEWithLogitsLoss() without Sigmoid() or you can use Sigmoid() and BCELoss()


Instruction: pytorch parallelize for loop of Cross Validation

Input:
I have a cuda9-docker with tensorflow and pytorch installed, I am doing cross validation on an image dataset. Currently I am using a for loop to do the cross validation. Something like

for data_train, data_test in sklearn.kfold(5, all_data):
  train(data_train)
  test(data_test)

But the for loop takes too long, will the following code work to parallelize the for loop? Maybe there is already a solution. But this is not Data Parallelization.

from multiprocessing import Pool

def f(trainset, testset):
    train_result = train(trainset)
    test_result = test(testset)
    save_train_result()
    save_test_result()

if __name__ == '__main__':
    with Pool(5) as p:
        print(p.map(f, sklearn.cvfold(5, all_data)))

I am not sure if the multiprocessing will only paralize the cpu or both cpu and gpu? This might be easiler than doing parallel in side  a model i guess like https://discuss.pytorch.org/t/parallelize-simple-for-loop-for-single-gpu/33701
since in my case, there is no need to communicate across each process?

Output:
You use try horovod with PyTorch.
ResNet50 example is here:
https://github.com/horovod/horovod/blob/master/examples/pytorch/pytorch_imagenet_resnet50.py
horovod-related changes should be small and isolated.


Instruction: FastAI v1 PyTorch Custom Model

Input:
i have been trying to use fastai with a custom torch model. My code is as follow:

X_train = np.load(dirpath + 'X_train.npy')
X_valid = np.load(dirpath + 'X_valid.npy')
Y_train = np.load(dirpath + 'Y_train.npy')
Y_valid = np.load(dirpath + 'Y_valid.npy')

X_train's shape is : (240, 122, 96),
and Y_train's shape is : (240,1)

Then i convert these to torch tensors ,

# Converting data to torch tensors
def to_torch_data(x,np_type,tch_type):
    return torch.from_numpy(x.astype(np_type)).to(tch_type)

X_train = to_torch_data(X_train,float,torch.float32)
X_valid = to_torch_data(X_valid,float,torch.float32)
Y_train = to_torch_data(Y_train,float,torch.float32)
Y_valid = to_torch_data(Y_valid,float,torch.float32)

Creating TensorDataSets for fastai DataBunch wrapper,

# Creating torch tensor datasets so that data can be used
# on ImageDataBunch function for fastai
train_ds = tdatautils.TensorDataset(X_train,Y_train)
valid_ds = tdatautils.TensorDataset(X_valid,Y_valid)

# Creating DataBunch object to be used as data in fastai methods.
batch_size = 24
my_data_bunch = DataBunch.create(train_ds,valid_ds,bs=batch_size)

And this is my custom torch model :

# Creating corresponding torch model
import torch.nn.functional as F
class Net(nn.Module):
    def __init__(self,droprate=0,activationF=None):
        super(Net, self).__init__()
        self.lstm_0 = nn.LSTM(96, 720)
        self.activation_0 = nn.ELU()
        self.dropout_0 = nn.Dropout(p=droprate)
        self.lstm_1 = nn.LSTM(720,480)
        self.activation_1 = nn.ELU()
        self.batch_norm_1 = nn.BatchNorm1d(122)
        self.fc_2 = nn.Linear(480,128)
        self.dropout_2 = nn.Dropout(p=droprate)
        self.last = nn.Linear(128,1)
        self.last_act = nn.ReLU()

    def forward(self, x):
        out,hid1 = self.lstm_0(x)
        out = self.dropout_0(self.activation_0(out))
        out,hid2 = self.lstm_1(out)
        out = out[:,-1,:]
        out = self.batch_norm_1(self.activation_1(out))
        out = self.dropout_2(self.fc_2(out))
        out = self.last_act(self.last(out))
        return out

#create instance of model
net = Net(droprate=train_droprate,activationF=train_activation) #.cuda()
print(net)

After all these, i run the learn for lr_find method. And i get this error :

Empty                                     Traceback (most recent call last)
C:\Anaconda3\envs\fastai\lib\site-packages\torch\utils\data\dataloader.py in _try_get_batch(self, timeout)
    510         try:
--> 511             data = self.data_queue.get(timeout=timeout)
    512             return (True, data)

C:\Anaconda3\envs\fastai\lib\queue.py in get(self, block, timeout)
    171                     if remaining <= 0.0:
--> 172                         raise Empty
    173                     self.not_empty.wait(remaining)

Empty:

During handling of the above exception, another exception occurred:

RuntimeError                              Traceback (most recent call last)
<ipython-input-35-e4b7603c0a82> in <module>
----> 1 my_learner.lr_find()

~\Desktop\fastai\fastai\fastai\train.py in lr_find(learn, start_lr, end_lr, num_it, stop_div, wd)
     30     cb = LRFinder(learn, start_lr, end_lr, num_it, stop_div)
     31     epochs = int(np.ceil(num_it/len(learn.data.train_dl)))
---> 32     learn.fit(epochs, start_lr, callbacks=[cb], wd=wd)
     33
     34 def to_fp16(learn:Learner, loss_scale:float=None, max_noskip:int=1000, dynamic:bool=True, clip:float=None,

~\Desktop\fastai\fastai\fastai\basic_train.py in fit(self, epochs, lr, wd, callbacks)
    197         callbacks = [cb(self) for cb in self.callback_fns + listify(defaults.extra_callback_fns)] + listify(callbacks)
    198         if defaults.extra_callbacks is not None: callbacks += defaults.extra_callbacks
--> 199         fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
    200
    201     def create_opt(self, lr:Floats, wd:Floats=0.)->None:

~\Desktop\fastai\fastai\fastai\basic_train.py in fit(epochs, learn, callbacks, metrics)
     97             cb_handler.set_dl(learn.data.train_dl)
     98             cb_handler.on_epoch_begin()
---> 99             for xb,yb in progress_bar(learn.data.train_dl, parent=pbar):
    100                 xb, yb = cb_handler.on_batch_begin(xb, yb)
    101                 loss = loss_batch(learn.model, xb, yb, learn.loss_func, learn.opt, cb_handler)

C:\Anaconda3\envs\fastai\lib\site-packages\fastprogress\fastprogress.py in __iter__(self)
     70         self.update(0)
     71         try:
---> 72             for i,o in enumerate(self._gen):
     73                 if i >= self.total: break
     74                 yield o

~\Desktop\fastai\fastai\fastai\basic_data.py in __iter__(self)
     73     def __iter__(self):
     74         "Process and returns items from `DataLoader`."
---> 75         for b in self.dl: yield self.proc_batch(b)
     76
     77     @classmethod

C:\Anaconda3\envs\fastai\lib\site-packages\torch\utils\data\dataloader.py in __next__(self)
    574         while True:
    575             assert (not self.shutdown and self.batches_outstanding > 0)
--> 576             idx, batch = self._get_batch()
    577             self.batches_outstanding -= 1
    578             if idx != self.rcvd_idx:

C:\Anaconda3\envs\fastai\lib\site-packages\torch\utils\data\dataloader.py in _get_batch(self)
    541         elif self.pin_memory:
    542             while self.pin_memory_thread.is_alive():
--> 543                 success, data = self._try_get_batch()
    544                 if success:
    545                     return data

C:\Anaconda3\envs\fastai\lib\site-packages\torch\utils\data\dataloader.py in _try_get_batch(self, timeout)
    517             if not all(w.is_alive() for w in self.workers):
    518                 pids_str = ', '.join(str(w.pid) for w in self.workers if not w.is_alive())
--> 519                 raise RuntimeError('DataLoader worker (pid(s) {}) exited unexpectedly'.format(pids_str))
    520             if isinstance(e, queue.Empty):
    521                 return (False, None)

RuntimeError: DataLoader worker (pid(s) 9584, 7236, 5108, 932, 13228, 13992, 4576, 13204) exited unexpectedly

I have researched about DataLoader but couldn't find anything useful.

Output:
Although I didn't understand the error message you posted, I see one problem in your code.

out = out[:,-1,:] # batch_size x 480
out = self.batch_norm_1(self.activation_1(out))

But you declared batch_norm_1 as:

self.batch_norm_1 = nn.BatchNorm1d(122)

Which should be:

self.batch_norm_1 = nn.BatchNorm1d(480)


Instruction: Training and testing CNN with pytorch. With and without model.eval()

Input:
I have two questions:-

I am trying to train a convolution neural network initialized with some pre trained weights (Netwrok contains batch normalization layers as well) (taking reference from here). Before training I want to calculate a validation error using loss_fn = torch.nn.MSELoss().cuda().
And in the reference, the author is using model.eval() before calculating  the validation error. But with that result, the CNN model is off from what it should be however when I comment out model.eval(), the output is good (what it should be with pre-trained weights). What could be reason behind it as I have read on many posts that model.eval should be used before testing the model and model.train() before training it.
While calculating the validation error with pre-trained weights and above mentioned loss function what should be the batch size. Shouldn't it be 1 as i want output on each of my input, calculate error with ground truth and in the end take average of all results. If i use higher batch size error is increased. So question is can i use higher batch size if yes what should be the right way. In given code i have given err = float(loss_local) / num_samples but i observed without averaging i.e err = float(loss_local). Error is different for different batch size. I am doing this without model.eval right now.

    batch_size = 1
    data_path = 'path_to_data'
    dtype = torch.FloatTensor
    weight_file = 'path_to_weight_file'
    val_loader = torch.utils.data.DataLoader(NyuDepthLoader(data_path, val_lists),batch_size=batch_size, shuffle=True, drop_last=True)
    model = Model(batch_size)
    model.load_state_dict(load_weights(model, weight_file, dtype))
    loss_fn = torch.nn.MSELoss().cuda()
    # model.eval()

    with torch.no_grad():
        for input, depth in val_loader:
            input_var = Variable(input.type(dtype))
            depth_var = Variable(depth.type(dtype))

            output = model(input_var)

            input_rgb_image = input_var[0].data.permute(1, 2, 0).cpu().numpy().astype(np.uint8)
            input_gt_depth_image = depth_var[0][0].data.cpu().numpy().astype(np.float32)
            pred_depth_image = output[0].data.squeeze().cpu().numpy().astype(np.float32)
            print (format(type(depth_var)))
            pred_depth_image_resize = cv2.resize(pred_depth_image, dsize=(608, 456), interpolation=cv2.INTER_LINEAR)
            target_depth_transform = transforms.Compose([flow_transforms.ArrayToTensor()])
            pred_depth_image_tensor = target_depth_transform(pred_depth_image_resize)
            #both inputs to loss_fn are 'torch.Tensor'
            loss_local += loss_fn(pred_depth_image_tensor, depth_var)

            num_samples += 1
            print ('num_samples {}'.format(num_samples))

    err = float(loss_local) / num_samples
    print('val_error before train:', err)

Output:
  What could be reason behind it as I have read on many posts that model.eval should be used before testing the model and model.train() before training it.

Note: testing the model is called inference.

As explained in the official documentation:

  Remember that you must call model.eval() to set dropout and batch normalization layers to evaluation mode before running inference. Failing to do this will yield inconsistent inference results.

So this code must be present once you load the model from a file and do inference.

# Model class must be defined somewhere
model = torch.load(PATH)
model.eval()

This is because dropout works as a regularization for preventing overfitting during training, it is not needed for inference. Same for the batch norms.
When you use eval() this just sets module train label to False and affects only certain types of modules in particular Dropout and BatchNorm.


Instruction: Transfer learning with OpenNMT

Input:
I'm training a transformer model with OpenNMT-py on MIDI music files, but results are poor because I only have access to a small dataset pertaining to the style I want to study. To help the model learn something useful, I would like to use a much larger dataset of other styles of music for a pre-training and then fine-tune the results using the small dataset.

I was thinking of freezing the encoder side of the transformer after the pre-training and letting the decoder part free to do the fine-tuning. How would one do this with OpenNMT-py?

Output:
Please be more specific about your questions and show some code which will help you to get a productive response from the SO community.

If I were in your place and wanted to freeze a neural network component, I would simply do:

for name, param in self.encoder.named_parameters():
    param.requires_grad = False

Here I assume you have a NN module like as follows.

class Net(nn.Module):
    def __init__(self, params):
        super(Net, self).__init__()

        self.encoder = TransformerEncoder(num_layers,
                                        d_model,
                                        heads,
                                        d_ff,
                                        dropout,
                                        embeddings,
                                        max_relative_positions)

    def foward(self):
        # write your code


Instruction: GAN, discriminator output only 0 or 1

Input:
I'm trying to train SRGAN. (Super Resolution GAN)
However, the discriminator's output converge to 0 or 1 whatever the input is.
Discriminator's loss function is only

D_loss = 0.5*(D_net(fake) + 1 - D_net(real))

D_net(fake) and D_net(real) both becomes 0 or 1. (sigmoid)

How can I fix it?

    for epoch_idx in range(epoch_num):
      for batch_idx, data in enumerate(data_loader):
        D_net.zero_grad()
        #### make real, low, fake
        real = data[0]
        for img_idx in range(batch_size):
            low[img_idx] = trans_low_res(real[img_idx])
        fake = G_net(Variable(low).cuda())

        #### get Discriminator loss and train Discriminator
        real_D_out = D_net(Variable(real).cuda()).mean()
        fake_D_out = D_net(Variable(fake).cuda()).mean()

        D_loss = 0.5*(fake_D_out + 1 - real_D_out)
        D_loss.backward()
        D_optim.step()

        #### train Generator

        G_net.zero_grad()
        #### get new fake D out with updated Discriminator
        fake_D_out = D_net(Variable(fake).cuda()).mean()
        G_loss = generator_criterion(fake_D_out.cuda(), fake.cuda(), real.cuda())
        G_loss.backward()
        G_optim.step()

  Batch : [10/6700] Discriminator_Loss: 0.0860 Generator_Loss : 0.1393

  Batch : [20/6700] Discriminator_Loss: 0.0037 Generator_Loss : 0.1282

  Batch : [30/6700] Discriminator_Loss: 0.0009 Generator_Loss : 0.0838

  Batch : [40/6700] Discriminator_Loss: 0.0002 Generator_Loss : 0.0735

  Batch : [50/6700] Discriminator_Loss: 0.0001 Generator_Loss : 0.0648

  Batch : [60/6700] Discriminator_Loss: 0.5000 Generator_Loss : 0.0634

  Batch : [70/6700] Discriminator_Loss: 0.5000 Generator_Loss : 0.0706

  Batch : [80/6700] Discriminator_Loss: 0.5000 Generator_Loss : 0.0691

  Batch : [90/6700] Discriminator_Loss: 0.5000 Generator_Loss : 0.0538
  ...

Output:
I am not sure if I understand your problem correctly. You meant that the sigmoid output from the discriminator is either 0 or 1?

In your loss function: D_loss = 0.5 * (fake_D_out + 1 - real_D_out), you are directly optimizing on the sigmoid output and looks like the discriminator overfits to your data that it can accurately predict 0 and 1 for fake and real examples respectively.

There are some GAN hacks suggested by experts in this subject matters. You can find a list of tips and tricks here. I would like to suggest you use soft-labels rather than hard-labels (see ref).

You can use BCEWithLogitsLoss() and compute loss based on soft labels instead of hard labels.

Difference between hard and soft labels:

# hard labels
real = 1
fake = 0

# soft labels
real = np.random.uniform(0.7, 1.0)  # 1
fake = np.random.uniform(0.0, 0.3)  # 0


Instruction: Is there a difference between "torch.nn.CTCLoss" supported by PYTORCH and "CTCLoss" supported by torch_baidu_ctc?

Input:
Is there a difference between "torch.nn.CTCLoss" supported by PYTORCH and "CTCLoss" supported by torch_baidu_ctc?

i think, I didn't notice any difference when I compared the tutorial code.

Does anyone know the true?

Tutorial code is located below.

import torch
from torch_baidu_ctc import ctc_loss, CTCLoss

# Activations. Shape T x N x D.
# T -> max number of frames/timesteps
# N -> minibatch size
# D -> number of output labels (including the CTC blank)
x = torch.rand(10, 3, 6)
# Target labels
y = torch.tensor([
# 1st sample
1, 1, 2, 5, 2,
# 2nd
1, 5, 2,
# 3rd
4, 4, 2, 3,
],
dtype=torch.int,
)
# Activations lengths
xs = torch.tensor([10, 6, 9], dtype=torch.int)
# Target lengths
ys = torch.tensor([5, 3, 4], dtype=torch.int)

# By default, the costs (negative log-likelihood) of all samples are
summed.
# This is equivalent to:
#   ctc_loss(x, y, xs, ys, average_frames=False, reduction="sum")
loss1 = ctc_loss(x, y, xs, ys)

# You can also average the cost of each sample among the number of
frames.
# The averaged costs are then summed.
loss2 = ctc_loss(x, y, xs, ys, average_frames=True)

# Instead of summing the costs of each sample, you can perform
# other `reductions`: "none", "sum", or "mean"
#
# Return an array with the loss of each individual sample
losses = ctc_loss(x, y, xs, ys, reduction="none")
#
# Compute the mean of the individual losses
loss3 = ctc_loss(x, y, xs, ys, reduction="mean")
#
# First, normalize loss by number of frames, later average losses
loss4 = ctc_loss(x, y, xs, ys, average_frames=True, reduction="mean")

# Finally, there's also a nn.Module to use this loss.
ctc = CTCLoss(average_frames=True, reduction="mean", blank=0)
loss4_2 = ctc(x, y, xs, ys)

# Note: the `blank` option is also available for `ctc_loss`.
# By default it is 0.

torch.nn.CTCLoss

T = 50      # Input sequence length
C = 20      # Number of classes (excluding blank)
N = 16      # Batch size
S = 30      # Target sequence length of longest target in batch
S_min = 10  # Minimum target length, for demonstration purposes

# Initialize random batch of input vectors, for *size = (T,N,C)
input = torch.randn(T, N, C).log_softmax(2).detach().requires_grad_()

# Initialize random batch of targets (0 = blank, 1:C+1 = classes)
target = torch.randint(low=1, high=C+1, size=(N, S), dtype=torch.long)

input_lengths = torch.full(size=(N,), fill_value=T, dtype=torch.long)
target_lengths = torch.randint(low=S_min, high=S, size=(N,),
dtype=torch.long)
ctc_loss = nn.CTCLoss()
loss = ctc_loss(input, target, input_lengths, target_lengths)
loss.backward()

I am Korean. English is not my first language. So I'm not good at English. If there's anything that hasn't been delivered well, please leave a comment. I'll change the sentence as soon as possible.

Output:
CTC loss only part of PyTorch since the 1.0 version and it is a better way to go because it is natively part of PyTorch. If you are using PyTorch 1.0 or newer, use torch.nn.CTCLoss.

warp-ctc does not seem to be maintained, the last commits changing the core code are from 2017. Later, they only fixed bindings for (an already obsolete version of) TensorFlow.


Instruction: Best Way to Overcome Early Convergence for Machine Learning Model

Input:
I have a machine learning model built that tries to predict weather data, and in this case I am doing a prediction on whether or not it will rain tomorrow (a binary prediction of Yes/No).

In the dataset there is about 50 input variables, and I have 65,000 entries in the dataset.

I am currently running a RNN with a single hidden layer, with 35 nodes in the hidden layer. I am using PyTorch's NLLLoss as my loss function, and Adaboost for the optimization function. I've tried many different learning rates, and 0.01 seems to be working fairly well.

After running for 150 epochs, I notice that I start to converge around .80 accuracy for my test data. However, I would wish for this to be even higher. However, it seems like the model is stuck oscillating around some sort of saddle or local minimum. (A graph of this is below)

What are the most effective ways to get out of this "valley" that the model seems to be stuck in?

Output:
Not sure why exactly you are using only one hidden layer and what is the shape of your history data but here are the things you can try:

Try more than one hidden layer
Experiment with LSTM and GRU layer and combination of these layers together with RNN.
Shape of your data i.e. the history you look at to predict the weather.
Make sure your features are scaled properly since you have about 50 input variables.


Instruction: How to fine tune BERT on its own tasks?

Input: I wanted to pre-train BERT with the data from my own language since multilingual (which includes my language) model of BERT is not successful. Since whole pre-training costs a lot, I decided to fine tune it on its own 2 tasks: masked language model and next sentence prediction. There are previous implementation on different tasks (NER, sentiment analysis etc.), but I couldn't find any fine tuning on its own tasks. Is there an implementation that I couldn't see? If not, where should I start? I need some initial help.

Output:
A wonderful resource for BERT is: https://github.com/huggingface/pytorch-pretrained-BERT. This repository contains op-for-op PyTorch reimplementations, pre-trained models and fine-tuning examples for Google's BERT model.

You can find the language model fine-tuning examples in the following link. The three example scripts in this folder can be used to fine-tune a pre-trained BERT model using the pretraining objective (the combination of masked language modeling and next sentence prediction loss).

https://github.com/huggingface/pytorch-pretrained-BERT/tree/master/examples/lm_finetuning

By the way, BERT multilingual is available for 104 languages (ref), and it is found to be surprisingly effective in many cross-lingual NLP tasks (ref). So, make sure you use BERT appropriately in your task.


Instruction: How to normalize set of images between (-1,1)

Input:
i have a dataset on images and i would like to normalize them betwwen (-1,1) before feeding them to NN how can i do that ?

x=sample
#Normalized Data
normalized = (x-min(x))/(max(x)-min(x))

# Histogram of example data and normalized data
par(mfrow=c(1,2))
hist(x,          breaks=10, xlab="Data",            col="lightblue", main="")
hist(normalized, breaks=10, xlab="Normalized Data", col="lightblue", main="")

i found this code online but it did not solve my problem since  i have image dataset

Output:
Assuming your image img_array is an np.array :

normalized_input = (img_array - np.amin(img_array)) / (np.amax(img_array) - np.amin(img_array))

Will normalize your data between 0 and 1.

Then, 2*normalized_input-1 will shift it between -1 and 1

If you want to normalize multiple images, you can make it a function :

def normalize_negative_one(img):
    normalized_input = (img - np.amin(img)) / (np.amax(img) - np.amin(img))
    return 2*normalized_input - 1

Then iterate over a e.g. list, tuple of images called imgs :

for i,img in enumerate(imgs):
    imgs[i] = normalize_negative_one(img)


Instruction: Why TensorBoard summary is not updating?

Input:
I use tensorboard with pytorch1.1 to log loss values.

I use writer.add_scalar("loss", loss.item(), global_step) in every for- loop body.

However, the plotting graph does not update while the training is processing.

Every time I want to see the latest loss, I have to restart the tensorboard server.

The code is here

import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
from torch.utils.tensorboard import SummaryWriter
from torchvision import datasets, transforms

# Writer will output to ./runs/ directory by default
writer = SummaryWriter()

transform = transforms.Compose(
    [transforms.ToTensor(), transforms.Normalize((0.5,), (0.5,))]
)
trainset = datasets.MNIST("mnist_train", train=True, download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True)
model = torchvision.models.resnet50(False)
# Have ResNet model take in grayscale rather than RGB
model.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)
model.fc = nn.Linear(2048, 10, True)

criterion = nn.CrossEntropyLoss()

epochs = 100

opt = torch.optim.Adam(model.parameters())

niter = 0

for epoch in range(epochs):
    for step, (x, y) in enumerate(trainloader):
        yp = model(x)
        loss = criterion(yp, y)
        opt.zero_grad()
        loss.backward()
        opt.step()
        writer.add_scalar("loss", loss.item(), niter)
        niter += 1
        print(loss.item())

grid = torchvision.utils.make_grid(images)
writer.add_image("images", grid, 0)
writer.add_graph(model, images)
writer.close()

The training is still going on, and the global steps has already been 3594. However, the tensorboard still shows around 1900.

Output: Also for those who have multiple event log files for a single run, you need to start your tensorboard with --reload_multifile True


Instruction: why can't I reimplement my tensorflow model with pytorch?

Input: I am developing a model in tensorflow and find that it is good on my specific evaluation method. But when I transfer to pytorch, I can't achieve the same results. I have checked the model architecture, the weight init method, the lr schedule, the weight decay, momentum and epsilon used in BN layer, the optimizer, and the data preprocessing. All things are the same. But I can't get the same results as in tensorflow. Anybody have met the same problem?

Output:
I did a similar conversion recently.

First you need to make sure that the forward path produces the same results: disable all randomness, initialize with the same values, give it a very small input and compare. If there is a discrepancy, disable parts of the network and compare enabling layers one by one.

When the forward path is confirmed, check the loss, gradients, and updates after one forward-backward cycle.


Instruction: RuntimeError: Assertion `cur_target >= 0 && cur_target < n_classes' failed

Input:
I get:

  RuntimeError: Assertion `cur_target >= 0 && cur_target < n_classes'
  failed.  at
  /opt/conda/conda-bld/pytorch_1550796191843/work/aten/src/THNN/generic/ClassNLLCriterion.c:93

When running this code:

    criterion = nn.CrossEntropyLoss()
    #Define the optimizer
    optimizer=optim.SGD(net.parameters(),lr=0.01,momentum=0.9)
    epochs=20
    for epoch in range(epochs):
        print ("epoch #", epoch)
        running_loss=0.0
        for i, data in enumerate(train_loader,0):
            inputs,labels=data
            inputs,labels= inputs.to(device),labels.to(device)
            optimizer.zero_grad()
            #train
            output=net(inputs)
            loss=criterion(output,labels)

    print ("loss: ", loss.item())
    running_loss+=loss.item()
    loss.backward()
    optimizer.step()
    print ('Finished Training')

Output:
The exception says that one of your labels is out of bounds.
Maybe they start from 1 instead of 0? Try printing them out.


Instruction: How to correctly access elements in a 3D-Pytorch-Tensor?

Input:
I am trying to access multiple elements in a 3D-Pytorch-Tensor, but the number of elements that are returned is wrong.

This is my code:

import torch

a = torch.FloatTensor(4,3,2)
print("a = {}".format(a))
print("a[:][:][0] = {}".format(a[:][:][0]))

This is the output:

a = tensor([[[-4.8569e+36,  3.0760e-41],
         [ 2.7953e+20,  1.6928e+22],
         [ 3.1692e-40,  7.2945e-15]],

        [[ 2.5011e+24,  1.3173e-39],
         [ 1.7229e-07,  4.1262e-08],
         [ 4.1490e-08,  6.4103e-10]],

        [[ 3.1728e-40,  5.8258e-40],
         [ 2.8776e+32,  6.7805e-10],
         [ 3.1764e-40,  5.4229e+08]],

        [[ 7.2424e-37,  1.3697e+07],
         [-2.0362e-33,  1.8146e+11],
         [ 3.1836e-40,  1.9670e+34]]])
a[:][:][0] = tensor([[-4.8569e+36,  3.0760e-41],
        [ 2.7953e+20,  1.6928e+22],
        [ 3.1692e-40,  7.2945e-15]])

I would expect something like this:

a[:][:][0] = tensor([[-4.8569e+36,  2.7953e+20, 3.1692e-40,
          2.5011e+24, 1.7229e-07, 4.1490e-08,
          3.1728e-40, 2.8776e+32, 3.1764e-40,
          7.2424e-37, -2.0362e-33, 3.1836e-40]])

Can anyone explain to me how I can come to this result?
Thank you very much in advance!

I get exactly the expected result on performing:

for i in range(4):
   for j in range(3):
      print("a[{}][{}][0] = {}".format(i,j, a[i][j][0]))

Output:
Short answer, you need to use a[:, :, 0]

More explanation:
When you do a[:] it returns a itself. So a[:][:][0] is same as doing a[0] which will give you the elements at the 0th position of the first axis (hence the size is (3,2)). What you want are elements from the 0th position of the last axis for which you need to do a[:, :, 0].


Instruction: conv2d function in pytorch

Input:
I'm trying to use the function torch.conv2d from Pytorch but can't get a result I understand...

Here is a simple example where the kernel (filt) is the same size as the input (im) to explain what I'm looking for.

import pytorch

filt = torch.rand(3, 3)
im = torch.rand(3, 3)

I want to compute a simple convolution with no padding, so the result should be a scalar (i.e. a 1x1 tensor).

I tried this with conv2d:

# I have to convert image and kernel to 4 dimensions tensors to use conv2d
im_torch = im.reshape((im_height, filt_height, 1, 1))
filt_torch = filt.reshape((filt_height, im_height, 1, 1))
out = torch.nn.functional.conv2d(im_torch, filt_torch, stride=1, padding=0)
print(out)

But the result is not what I expected:

tensor([[[[0.6067]], [[0.3564]], [[0.5397]]],
    [[[0.2557]], [[0.0493]], [[0.2562]]],
    [[[0.6067]], [[0.3564]], [[0.5397]]]])

To give an idea of what I'd like, I want to reproduce scipy convolve2d behavior:

import scipy.signal
out_scipy = scipy.signal.convolve2d(im.detach().numpy(), filt.detach().numpy(), 'valid')
print(out_scipy)

which prints:

array([[1.195723]], dtype=float32)

Output:
The tensor shape of your input and the filter should be:

(batch, dim_ch, width, height)

and NOT:

(width, height, 1, 1)

e.g.

import torch
import torch.nn.functional as F
x = torch.randn(1,1,4,4);
y = torch.randn(1,1,4,4);
z = F.conv2d(x,y);

Output shape of z:

torch.Size([1,1,1,1])


Instruction: Inplace arithmetic operation versus normal arithmetic operation in PyTorch Tensors

Input:
I am trying to build Linear regression using Pytorch framework and while implementing Gradient Descent, I observed two different outputs based on how I use arithmetic operation in Python code. Below is the code:

 #X and Y are input and target labels respectively
 X = torch.randn(100,1)*10
 Y = X + 3*torch.randn(100,1) +2
 plt.scatter(X.numpy(),Y.numpy())

 #Initialiation of weight and bias
 w = torch.tensor(1.0,requires_grad=True)
 b = torch.tensor(1.0,requires_grad=True)

 #forward pass
 def forward_feed(x):
   y = w*x +b
   return y

 #Parameters Learning
 epochs = 100
 lr = 0.00008
 loss_list = []
 for epoch in range(epochs):
   print('epoch',epoch)
   Y_pred = forward_feed(X)
   loss = torch.sum((Y - Y_pred)**2)
   loss_list.append(loss)
   loss.backward()
   with torch.no_grad():
     w -= lr*w.grad
     b -= lr*b.grad
     w.grad.zero_()
     b.grad.zero_()

If I use this code, I get the expected results i.e my code is able to estimate weight and bias. However, If I change the gradient descent code line like below:

  w =w- lr*w.grad
  b =b- lr*b.grad

I get the below error:

 AttributeError                            Traceback (most recent call
 last)
 <ipython-input-199-84b86804d4d5> in <module>()
 ---> 41         w.grad.zero_()
      42         b.grad.zero_()

 AttributeError: 'NoneType' object has no attribute 'zero_'

Can anyone please help me with this?

I did try checking answers on google and found a related link: https://github.com/pytorch/pytorch/issues/7731. But this is exactly opposite to what I am facing. As per this link, they say that inplace assignment is causing a problem because tensors share the same storage. However, For my code, Inplace operation works not the normal operation.

Output:
I think the reason is simple. When you do:

w = w - lr * w.grad
b = b - lr * b.grad

The w and b in the left-hand side are two new tensors and their .grad is None.

However, when you do inplace operation, you do not create any new tensor, you just update the value of the concerned tensor. So, in this scenario, inplace operation is required.


Instruction: Vectorize to Apply Function to 3d Array

Input:
I am trying to apply a function to 3d torch tensor while the function is applied to 2d tensor which is read through the axis 1 of the 3d torch tensor.

For example, I have a torch tensor of the shape (51, 128, 20100) (a variable with name autoencode_logprob) and the function(rawid2sentence) runs on the input of the shape (51, 20100).

Right now I wrote the code to run with naive for loop, looping one by one with range(128).

However, it’s too slow. Following is the code part that matters.

autoencode_logprobs is the 3d tensor and I need to apply rawids2sentence function along its second axis. Any help to vectorize it?

for i in range(128):
    output_sent = self.dictionary.rawids2sentence(
        autoencode_logprobs[:, i].max(1)[
            1].data.cpu().numpy(),
        oov_dicts[i],
    )
    output_sent_encoding = ifst_model.encode([output_sent])

Output:
Since I do not know what rawids2sentence or encode function does, I can help you with to do the max operation.

In the following statement,

autoencode_logprobs[:, i].max(1)[1]

You identify the index of the maximum values along dim=1 for each 51 x 20100 tensor. So, the output is a vector of size 51.

You can perform the same operation in your full tensor of shape 51 x 128 x 20100 and get the output as 128 x 51 tensor.

autoencode_logprobs.transpose(0, 1).max(2)[1] # 128 x 51

So, if your rawids2sentence or encode methods can tackle batch inputs, the above change should work for you without any loop.


Instruction: 'An attempt has been made to start' in lr_find with fast.ai

Input:
I am running this small piece of code to identify learning rate:

import cv2
from fastai.vision import *
from fastai.callbacks.hooks import *
path = untar_data(URLs.CAMVID)
path_lbl = path/'labels'
path_img = path/'images'
fnames = get_image_files(path_img)
lbl_names = get_image_files(path_lbl)
img_f = fnames[0]
img = open_image(img_f)
get_y_fn = lambda x: path_lbl/f'{x.stem}_P{x.suffix}'
mask = open_mask(get_y_fn(img_f))
src_size = np.array(mask.shape[1:])
src_size,mask.data
codes = np.loadtxt(path/'codes.txt', dtype=str); codes
size = src_size//2
bs=4
src = (SegmentationItemList.from_folder(path_img)
       .split_by_fname_file('../valid.txt')
       .label_from_func(get_y_fn, classes=codes))
data = (src.transform(get_transforms(), size=size, tfm_y=True)
        .databunch(bs=bs)
        .normalize(imagenet_stats))
name2id = {v:k for k,v in enumerate(codes)}
void_code = name2id['Void']
def acc_camvid(input, target):
    target = target.squeeze(1)
    mask = target != void_code
    return (input.argmax(dim=1)[mask]==target[mask]).float().mean()
wd=1e-2
learn = unet_learner(data, models.resnet34, metrics=acc_camvid, wd=wd)
lr_find(learn)
print("end")

And I get this error:

RuntimeError:
        An attempt has been made to start a new process before the
        current process has finished its bootstrapping phase.

        This probably means that you are not using fork to start your
        child processes and you have forgotten to use the proper idiom
        in the main module:

            if __name__ == '__main__':
                freeze_support()
                ...

        The "freeze_support()" line can be omitted if the program
        is not going to be frozen to produce an executable.

and also this one:

Traceback (most recent call last):
  File "C:\Program Files\JetBrains\PyCharm Community Edition 2018.1.4\helpers\pydev\pydevd.py", line 1664, in <module>
    main()
  File "C:\Program Files\JetBrains\PyCharm Community Edition 2018.1.4\helpers\pydev\pydevd.py", line 1658, in main
    globals = debugger.run(setup['file'], None, None, is_module)
  File "C:\Program Files\JetBrains\PyCharm Community Edition 2018.1.4\helpers\pydev\pydevd.py", line 1068, in run
    pydev_imports.execfile(file, globals, locals)  # execute the script
  File "C:\Program Files\JetBrains\PyCharm Community Edition 2018.1.4\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "C:/Users/steve/Project/fastai_unet/main.py", line 32, in <module>
    lr_find(learn)
  File "C:\Users\steve\Miniconda3\lib\site-packages\fastai\train.py", line 32, in lr_find
    learn.fit(epochs, start_lr, callbacks=[cb], wd=wd)
  File "C:\Users\steve\Miniconda3\lib\site-packages\fastai\basic_train.py", line 199, in fit
    fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
  File "C:\Users\steve\Miniconda3\lib\site-packages\fastai\basic_train.py", line 99, in fit
    for xb,yb in progress_bar(learn.data.train_dl, parent=pbar):
  File "C:\Users\steve\Miniconda3\lib\site-packages\fastprogress\fastprogress.py", line 72, in __iter__
    for i,o in enumerate(self._gen):
  File "C:\Users\steve\Miniconda3\lib\site-packages\fastai\basic_data.py", line 75, in __iter__
    for b in self.dl: yield self.proc_batch(b)
  File "C:\Users\steve\Miniconda3\lib\site-packages\torch\utils\data\dataloader.py", line 193, in __iter__
    return _DataLoaderIter(self)
  File "C:\Users\steve\Miniconda3\lib\site-packages\torch\utils\data\dataloader.py", line 469, in __init__
    w.start()
  File "C:\Users\steve\Miniconda3\lib\multiprocessing\process.py", line 105, in start
    self._popen = self._Popen(self)
  File "C:\Users\steve\Miniconda3\lib\multiprocessing\context.py", line 223, in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
  File "C:\Users\steve\Miniconda3\lib\multiprocessing\context.py", line 322, in _Popen
    return Popen(process_obj)
  File "C:\Users\steve\Miniconda3\lib\multiprocessing\popen_spawn_win32.py", line 65, in __init__
    reduction.dump(process_obj, to_child)
  File "C:\Users\steve\Miniconda3\lib\multiprocessing\reduction.py", line 60, in dump
    ForkingPickler(file, protocol).dump(obj)
BrokenPipeError: [Errno 32] Broken pipe

how can I fix this?

Output:
Oh, the solution is to wrap code in a method and invoke it:

import cv2
from fastai.vision import *
from fastai.callbacks.hooks import *

def main():
    path = untar_data(URLs.CAMVID)
    path_lbl = path/'labels'
    path_img = path/'images'
    fnames = get_image_files(path_img)
    lbl_names = get_image_files(path_lbl)
    get_y_fn = lambda x: path_lbl/f'{x.stem}_P{x.suffix}'
    mask = open_mask(get_y_fn(img_f))
    src_size = np.array(mask.shape[1:])
    src_size,mask.data
    codes = np.loadtxt(path/'codes.txt', dtype=str); codes
    size = src_size//2
    bs=4
    src = (SegmentationItemList.from_folder(path_img)
           .split_by_fname_file('../valid.txt')
           .label_from_func(get_y_fn, classes=codes))
    data = (src.transform(get_transforms(), size=size, tfm_y=True)
            .databunch(bs=bs)
            .normalize(imagenet_stats))
    name2id = {v:k for k,v in enumerate(codes)}
    void_code = name2id['Void']
    def acc_camvid(input, target):
        target = target.squeeze(1)
        mask = target != void_code
        return (input.argmax(dim=1)[mask]==target[mask]).float().mean()
    wd=1e-2
    learn = unet_learner(data, models.resnet34, metrics=acc_camvid, wd=wd)
    lr_find(learn)
    print("end")
if __name__ == '__main__':
    main()


Instruction: Validation loss not moving with MLP in Regression

Input:
Given input features as such, just raw numbers:

tensor([0.2153, 0.2190, 0.0685, 0.2127, 0.2145, 0.1260, 0.1480, 0.1483, 0.1489,
        0.1400, 0.1906, 0.1876, 0.1900, 0.1925, 0.0149, 0.1857, 0.1871, 0.2715,
        0.1887, 0.1804, 0.1656, 0.1665, 0.1137, 0.1668, 0.1168, 0.0278, 0.1170,
        0.1189, 0.1163, 0.2337, 0.2319, 0.2315, 0.2325, 0.0519, 0.0594, 0.0603,
        0.0586, 0.0067, 0.0624, 0.2691, 0.0617, 0.2790, 0.2805, 0.2848, 0.2454,
        0.1268, 0.2483, 0.2454, 0.2475], device='cuda:0')

And the expected output is a single real number output, e.g.

tensor(-34.8500, device='cuda:0')

Full code on https://www.kaggle.com/alvations/pytorch-mlp-regression

I've tried creating a simple 2 layer network with:

class MLP(nn.Module):
    def __init__(self, input_size, output_size, hidden_size):
        super(MLP, self).__init__()
        self.linear = nn.Linear(input_size, hidden_size)
        self.classifier = nn.Linear(hidden_size, output_size)

    def forward(self, inputs, hidden=None, dropout=0.5):
        inputs = F.dropout(inputs, dropout) # Drop-in.
        # First Layer.
        output = F.relu(self.linear(inputs))

        # Matrix manipulation magic.
        batch_size, sequence_len, hidden_size = output.shape
        # Technically, linear layer takes a 2-D matrix as input, so more manipulation...
        output = output.contiguous().view(batch_size * sequence_len, hidden_size)
        # Apply dropout.
        output = F.dropout(output, dropout)

        # Put it through the classifier
        # And reshape it to [batch_size x sequence_len x vocab_size]
        output = self.classifier(output).view(batch_size, sequence_len, -1)

        return output

And training as such:

# Training routine.
def train(num_epochs, dataloader, valid_dataset, model, criterion, optimizer):
    losses = []
    valid_losses = []
    learning_rates = []
    plt.ion()
    x_valid, y_valid = valid_dataset
    for _e in range(num_epochs):
        for batch in tqdm(dataloader):
            # Zero gradient.
            optimizer.zero_grad()
            #print(batch)
            this_x = torch.tensor(batch['x'].view(len(batch['x']), 1, -1)).to(device)
            this_y = torch.tensor(batch['y'].view(len(batch['y']), 1, 1)).to(device)

            # Feed forward.
            output = model(this_x)

            prediction, _ = torch.max(output, dim=1)
            loss = criterion(prediction, this_y.view(len(batch['y']), -1))
            loss.backward()
            optimizer.step()
            losses.append(torch.sqrt(loss.float()).data)

            with torch.no_grad():
                # Zero gradient.
                optimizer.zero_grad()
                output = model(x_valid.view(len(x_valid), 1, -1))
                prediction, _ = torch.max(output, dim=1)
                loss = criterion(prediction, y_valid.view(len(y_valid), -1))
                valid_losses.append(torch.sqrt(loss.float()).data)

            clear_output(wait=True)
            plt.plot(losses, label='Train')
            plt.plot(valid_losses, label='Valid')
            plt.legend()
            plt.pause(0.05)

Tuning several hyperparameters, it looks like the model doesn't train well, the validation loss doesn't move at all e.g.

hyperparams = Hyperparams(input_size=train_dataset.x.shape[1],
                          output_size=1,
                          hidden_size=150,
                          loss_func=nn.MSELoss,
                          learning_rate=1e-8,
                          optimizer=optim.Adam,
                          batch_size=500)

And it's loss curve:

Any idea what's wrong with the network?

Am I training the regression model with the wrong loss? Or I've just not yet found the right hyperparameters?

Or am I validating the model wrongly?

Output:
From the code you provided, it is tough to say why the validation loss is constant but I see several problems in your code.

Why do you validate for each training mini-batch? Instead, you should validate your model after you do the training for one complete epoch (iterating over your full dataset once). So, the skeleton should be like:

for _e in range(num_epochs):
    for batch in tqdm(train_dataloader):
        # training code

    with torch.no_grad():
        for batch in tqdm(valid_dataloader):
            # validation code

    # plot your loss values

Also, you can plot after each epoch, not after each mini-batch training.

Did you check whether the model parameters are getting updated after optimizer.step() during training? How many validation examples do you have? Why don't you use mini-batch computation during validation?
Why do you do: optimizer.zero_grad() during validation? It doesn't make sense because, during validation, you are not going to do anything related to optimization.
You should use model.eval() during validation to turn off the dropouts. See PyTorch documentation to learn about .train() and .eval() methods.
The learning rate is set to 1e-8, isn't it too small? Why don't you use the default learning rate for Adam (1e-3)?

The following requires some reasoning.

Why are you using such a large batch size? What is your training dataset size?
You can directly plot the MSELoss, instead of taking the square root.

My suggestion would be: use some existing resources for MLP in PyTorch. Don't do it from scratch if you do not have sufficient knowledge at this point. It would make you suffer a lot.


Instruction: How to convert keras LSTM to pytorch LSTM?

Input:
I have a very simple LSTM example written in Keras that I am trying to port to pytorch. But it does not seem to be able to learn at all. I am an absolute beginning so any advice is appreciated.

KERAS:

X_train_lmse has shape (1691, 1, 1), I am essentially running X(t) with X(t-1) as single feature

lstm_model = Sequential()
lstm_model.add(LSTM(7, input_shape=(1, X_train_lmse.shape[1]), activation='relu', kernel_initializer='lecun_uniform', return_sequences=False))
lstm_model.add(Dense(1))
lstm_model.compile(loss='mean_squared_error', optimizer='adam')
early_stop = EarlyStopping(monitor='loss', patience=2, verbose=1)
history_lstm_model = lstm_model.fit(X_train_lmse, y_train, epochs=100, batch_size=1, verbose=1, shuffle=False, callbacks=[early_stop])

Output:

Epoch 1/100
1691/1691 [==============================] - 10s 6ms/step - loss: 0.0236
Epoch 2/100
1691/1691 [==============================] - 9s 5ms/step - loss: 0.0076
Epoch 3/100
...

PYTORCH:

X_train_tensor has same shape as in keras (1691, 1, 1). I am specifying batch_first to be true below so I think it should be ok.

class LSTM_model(nn.Module):

    def __init__(self):
        super(LSTM_model, self).__init__()
        self.lstm = nn.LSTM(input_size=1, hidden_size=7, num_layers=1, batch_first=True)
        self.dense = nn.Linear(7, 1)

    def forward(self, x):
        out, states = self.lstm(x)
        out = self.dense(out)
        return out

lstm_model = LSTM_model()
loss_function = nn.MSELoss()
optimizer = optim.Adam(lstm_model.parameters())

for t in range(100):
    y_pred = lstm_model(X_train_tensor)
    loss = loss_function(y_pred, Y_train_tensor)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    print('Train Epoch ', t, ' Loss = ', loss)

Output:

Train Epoch  0  Loss =  tensor(0.2834, grad_fn=<MseLossBackward>)
Train Epoch  1  Loss =  tensor(0.2812, grad_fn=<MseLossBackward>)
Train Epoch  2  Loss =  tensor(0.2790, grad_fn=<MseLossBackward>)
Train Epoch  3  Loss =  tensor(0.2768, grad_fn=<MseLossBackward>)
Train Epoch  4  Loss =  tensor(0.2746, grad_fn=<MseLossBackward>)
Train Epoch  5  Loss =  tensor(0.2725, grad_fn=<MseLossBackward>)
Train Epoch  6  Loss =  tensor(0.2704, grad_fn=<MseLossBackward>)
Train Epoch  7  Loss =  tensor(0.2683, grad_fn=<MseLossBackward>)
...

As you can see, the error barely moves in Pytorch. Also each epoch runs much much faster than keras.

I must be doing something stupid. I checked the input data and they look identical in both implementations. Thanks!

Output:
You miss the relu activation function in your PyTorch model (See Relu layer in PyTorch). Also, you seem to be using a customized kernel_initalizer for the weights. You can pass your initialization weights in the model call:

...
y_pred = lstm_model(X_train_tensor, (hn, cn))
...


Instruction: Windows installing pytorch 0.3

Input: I need to install pytorch==0.3 (I'm using conda), but when I run the command line, conda says that the needed packages are not available. Is there a way I can install it (possibly without using ubuntu)?

Output: peterjc123 released the version for windows here: https://anaconda.org/peterjc123/pytorch


Instruction: Input Tensors not being moved to GPU in pytorch

Input:
When running my code, I get the error:

Input and parameter tensors are not at the same device, found input tensor at cpu and parameter tensor at cuda:0

even though I'm using .cuda() on my inputs.

Google Colab link

Code:

use_cuda = True
if use_cuda and torch.cuda.is_available():
   model.cuda()

def test():
model.eval()
avgLoss = 0
for dataPoint in range(len(testData)):
    lstmInput = testData[dataPoint][0]
    lstmInput = torch.Tensor(lstmInput)
    lstmInput = lstmInput.view(len(testData[dataPoint][0]), 1, 5)
    label = testData[dataPoint][1]
    label = torch.Tensor(label)
    lstmInput = Variable(lstmInput)
    label = Variable(label)

    if use_cuda and torch.cuda.is_available():
          lstmInput.cuda()
          label.cuda()

    pred_label = model(lstmInput)
    loss = loss_fn(label, pred_label)
    avgLoss += loss.item()
return avgLoss / len(testData)

def train(num_epochs):
model.train()
for epoch in range(num_epochs):
    avgLoss = 0.0
    for datapoint in range(len(trainData)):
        model.hidden = model.init_hidden()
        optimizer.zero_grad()

        lstmInput = trainData[datapoint][0]
        lstmInput = torch.Tensor(lstmInput)
        lstmInput = lstmInput.view(len(trainData[datapoint][0]), 1, 5)
        label = torch.Tensor(trainData[datapoint][1])
        label = label.view(1, 5)
        lstmInput = Variable(lstmInput)
        label = Variable(label)

        if use_cuda and torch.cuda.is_available():
          print("happens")
          lstmInput.cuda()
          label.cuda()

        pred_label = model(lstmInput)
        loss = loss_fn(pred_label, label)
        # print(label, pred_label)
        avgLoss += loss.item()
        loss.backward()
        optimizer.step()
    print("Epoch: ", epoch, "MSELoss: ", avgLoss / len(trainData), "Test Acc: ", test())

Output:
The cuda()method returns the tensor on the right gpu so you need to assign it back to your input variable:

lstmInput, label = lstimInput.cuda(), label.cuda()


Instruction: How to calculate gradients on a tensor in PyTorch?

Input:
I want to calculate the gradient of a tensor and however, it gives error as

RunTimeerror: grad can be implicitly created only for scalar outputs

and here is what I am trying to code:

x = torch.full((2,3), 4,requires_grad=True)
y = (2*x**2+3)
y.backward()

And at this point, it throws an error.

Output:
Since there is no summing up/reducing the loss-value , like .sum()
Hence the issue could be fixed by:

y.backward(torch.ones_like(x))

which performs a Jacobian-vector product with a tensor of all ones and get the gradient.