Instruction: RuntimeError: Expected number of channels in input to be divisible by num_groups, but got input of shape [64, 16, 32, 32] and num_groups=32

Input:
I have EfficientNet working fine on my dataset. Now, I changed all the batch norm layers into group norm layers. I have already done this process with other networks like vgg16 and resnet18 and all was ok. On EfficientNet I have this error RuntimeError: Expected number of channels in input to be divisible by num_groups, but got input of shape [64, 16, 32, 32] and num_groups=32
Basically I have done this:
efficientnet_b0 = torchvision.models.efficientnet_b0(pretrained=False)

efficientnet_b0.classifier = nn.Linear(in_features=1280, out_features=10, bias=True)

efficientnet_b0.features[0][1] = nn.GroupNorm(32, 32)
efficientnet_b0.features[1][0].block[0][1] = nn.GroupNorm(32, 32)
efficientnet_b0.features[1][0].block[2][1] = nn.GroupNorm(32, 16)
efficientnet_b0.features[2][0].block[0][1] = nn.GroupNorm(32, 96)
efficientnet_b0.features[2][0].block[1][1] = nn.GroupNorm(32, 96)
efficientnet_b0.features[2][0].block[3][1] = nn.GroupNorm(32, 24)
efficientnet_b0.features[2][1].block[0][1] = nn.GroupNorm(32, 144)
efficientnet_b0.features[2][1].block[1][1] = nn.GroupNorm(32, 144)
efficientnet_b0.features[2][1].block[3][1] = nn.GroupNorm(32, 24)
efficientnet_b0.features[3][0].block[0][1] = nn.GroupNorm(32, 144)
efficientnet_b0.features[3][0].block[1][1] = nn.GroupNorm(32, 144)
efficientnet_b0.features[3][0].block[3][1] = nn.GroupNorm(32, 40)
efficientnet_b0.features[3][1].block[0][1] = nn.GroupNorm(32, 240)
efficientnet_b0.features[3][1].block[1][1] = nn.GroupNorm(32, 240)
efficientnet_b0.features[3][1].block[3][1] = nn.GroupNorm(32, 40)
efficientnet_b0.features[4][0].block[0][1] = nn.GroupNorm(32, 240)
efficientnet_b0.features[4][0].block[1][1] = nn.GroupNorm(32, 240)
efficientnet_b0.features[4][0].block[3][1] = nn.GroupNorm(32, 80)
efficientnet_b0.features[4][1].block[0][1] = nn.GroupNorm(32, 480)
efficientnet_b0.features[4][1].block[1][1] = nn.GroupNorm(32, 480)
efficientnet_b0.features[4][1].block[3][1] = nn.GroupNorm(32, 80)
efficientnet_b0.features[4][2].block[0][1] = nn.GroupNorm(32, 480)
efficientnet_b0.features[4][2].block[1][1] = nn.GroupNorm(32, 480)
efficientnet_b0.features[4][2].block[3][1] = nn.GroupNorm(32, 80)
efficientnet_b0.features[5][0].block[0][1] = nn.GroupNorm(32, 480)
efficientnet_b0.features[5][0].block[1][1] = nn.GroupNorm(32, 480)
efficientnet_b0.features[5][0].block[3][1] = nn.GroupNorm(32, 112)
efficientnet_b0.features[5][1].block[0][1] = nn.GroupNorm(32, 672)
efficientnet_b0.features[5][1].block[1][1] = nn.GroupNorm(32, 672)
efficientnet_b0.features[5][1].block[3][1] = nn.GroupNorm(32, 112)
efficientnet_b0.features[5][2].block[0][1] = nn.GroupNorm(32, 672)
efficientnet_b0.features[5][2].block[1][1] = nn.GroupNorm(32, 672)
efficientnet_b0.features[5][2].block[3][1] = nn.GroupNorm(32, 112)
efficientnet_b0.features[6][0].block[0][1] = nn.GroupNorm(32, 672)
efficientnet_b0.features[6][0].block[1][1] = nn.GroupNorm(32, 672)
efficientnet_b0.features[6][0].block[3][1] = nn.GroupNorm(32, 192)
efficientnet_b0.features[6][1].block[0][1] = nn.GroupNorm(32, 1152)
efficientnet_b0.features[6][1].block[1][1] = nn.GroupNorm(32, 1152)
efficientnet_b0.features[6][1].block[3][1] = nn.GroupNorm(32, 192)
efficientnet_b0.features[6][2].block[0][1] = nn.GroupNorm(32, 1152)
efficientnet_b0.features[6][2].block[1][1] = nn.GroupNorm(32, 1152)
efficientnet_b0.features[6][2].block[3][1] = nn.GroupNorm(32, 192)
efficientnet_b0.features[6][3].block[0][1] = nn.GroupNorm(32, 1152)
efficientnet_b0.features[6][3].block[1][1] = nn.GroupNorm(32, 1152)
efficientnet_b0.features[6][3].block[3][1] = nn.GroupNorm(32, 192)
efficientnet_b0.features[7][0].block[0][1] = nn.GroupNorm(32, 1152)
efficientnet_b0.features[7][0].block[1][1] = nn.GroupNorm(32, 1152)
efficientnet_b0.features[7][0].block[3][1] = nn.GroupNorm(32, 320)
efficientnet_b0.features[8][1] = nn.GroupNorm(32, 1280)

The original efficientnet is this:
EfficientNet(
  (features): Sequential(
    (0): ConvNormActivation(
      (0): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (2): SiLU(inplace=True)
    )
    (1): Sequential(
      (0): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)
            (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(32, 8, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(8, 32, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (2): ConvNormActivation(
            (0): Conv2d(32, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.0, mode=row)
      )
    )
    (2): Sequential(
      (0): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(16, 96, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96, bias=False)
            (1): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(96, 4, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(4, 96, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.0125, mode=row)
      )
      (1): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(24, 144, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(144, 144, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=144, bias=False)
            (1): BatchNorm2d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(144, 6, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(6, 144, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(144, 24, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(24, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.025, mode=row)
      )
    )
    (3): Sequential(
      (0): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(24, 144, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(144, 144, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=144, bias=False)
            (1): BatchNorm2d(144, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(144, 6, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(6, 144, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(144, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.037500000000000006, mode=row)
      )
      (1): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(40, 240, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(240, 240, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=240, bias=False)
            (1): BatchNorm2d(240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(240, 10, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(10, 240, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(240, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.05, mode=row)
      )
    )
    (4): Sequential(
      (0): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(40, 240, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(240, 240, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=240, bias=False)
            (1): BatchNorm2d(240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(240, 10, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(10, 240, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(240, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(80, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.0625, mode=row)
      )
      (1): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(480, 480, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=480, bias=False)
            (1): BatchNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(480, 20, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(20, 480, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(480, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(80, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.07500000000000001, mode=row)
      )
      (2): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(480, 480, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=480, bias=False)
            (1): BatchNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(480, 20, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(20, 480, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(480, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(80, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.08750000000000001, mode=row)
      )
    )
    (5): Sequential(
      (0): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(480, 480, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=480, bias=False)
            (1): BatchNorm2d(480, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(480, 20, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(20, 480, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(480, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(112, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.1, mode=row)
      )
      (1): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(672, 672, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=672, bias=False)
            (1): BatchNorm2d(672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(112, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.1125, mode=row)
      )
      (2): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(672, 672, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=672, bias=False)
            (1): BatchNorm2d(672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(112, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.125, mode=row)
      )
    )
    (6): Sequential(
      (0): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(672, 672, kernel_size=(5, 5), stride=(2, 2), padding=(2, 2), groups=672, bias=False)
            (1): BatchNorm2d(672, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(672, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.1375, mode=row)
      )
      (1): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(1152, 1152, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1152, bias=False)
            (1): BatchNorm2d(1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(1152, 48, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(48, 1152, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.15000000000000002, mode=row)
      )
      (2): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(1152, 1152, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1152, bias=False)
            (1): BatchNorm2d(1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(1152, 48, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(48, 1152, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.1625, mode=row)
      )
      (3): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(1152, 1152, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1152, bias=False)
            (1): BatchNorm2d(1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(1152, 48, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(48, 1152, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.17500000000000002, mode=row)
      )
    )
    (7): Sequential(
      (0): MBConv(
        (block): Sequential(
          (0): ConvNormActivation(
            (0): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (1): ConvNormActivation(
            (0): Conv2d(1152, 1152, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1152, bias=False)
            (1): BatchNorm2d(1152, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (2): SiLU(inplace=True)
          )
          (2): SqueezeExcitation(
            (avgpool): AdaptiveAvgPool2d(output_size=1)
            (fc1): Conv2d(1152, 48, kernel_size=(1, 1), stride=(1, 1))
            (fc2): Conv2d(48, 1152, kernel_size=(1, 1), stride=(1, 1))
            (activation): SiLU(inplace=True)
            (scale_activation): Sigmoid()
          )
          (3): ConvNormActivation(
            (0): Conv2d(1152, 320, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (1): BatchNorm2d(320, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (stochastic_depth): StochasticDepth(p=0.1875, mode=row)
      )
    )
    (8): ConvNormActivation(
      (0): Conv2d(320, 1280, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (1): BatchNorm2d(1280, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (2): SiLU(inplace=True)
    )
  )
  (avgpool): AdaptiveAvgPool2d(output_size=1)
  (classifier): Sequential(
    (0): Dropout(p=0.2, inplace=True)
    (1): Linear(in_features=1280, out_features=1000, bias=True)
  )
)

So, basically I changed all the batch norm layers into group norm layers. Each GN layer has 32 as num_groups and the number of channels is exactly the same of batch norm.

Output: I solved: basically, num_channels must be divisible by num_groups, so I used 8 in each layer rather than 32 as num_groups.


Instruction: Pytorch torchvision.transforms execute randomly?

Input:
I am doing this transformation:
self.transform = transforms.Compose( {
            transforms.Resize((224, 224)),
            transforms.ToTensor(),
            transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
        } )

and then
image = Image.open(img_name)

if self.transform:
    image = self.transform(image)

this works for the first epoch then how the hell it crashes for the second epoch?
why the f normalize getting PIL-image and not torch.tensor? is the execution of each transforms Compose items random?

Traceback (most recent call last):   File
"/home/ubuntu/projects/ssl/src/train_supervised.py", line 63, in

main()   File "/home/ubuntu/projects/ssl/src/train_supervised.py", line 60, in main
train()   File "/home/ubuntu/projects/ssl/src/train_supervised.py", line 45, in train
for i, data in enumerate(tqdm_):   File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/tqdm/std.py",
line 1195, in iter
for obj in iterable:   File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/utils/data/dataloader.py",
line 530, in next
data = self._next_data()   File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/utils/data/dataloader.py",
line 1224, in _next_data
return self._process_data(data)   File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/utils/data/dataloader.py",
line 1250, in _process_data
data.reraise()   File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/_utils.py",
line 457, in reraise
raise exception TypeError: Caught TypeError in DataLoader worker process 0. Original Traceback (most recent call last):   File
"/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/utils/data/_utils/worker.py", line 287, in _worker_loop
data = fetcher.fetch(index)   File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py", line 49, in fetch
data = [self.dataset[idx] for idx in possibly_batched_index]   File
"/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py", line 49, in
data = [self.dataset[idx] for idx in possibly_batched_index]   File "/home/ubuntu/projects/ssl/src/data_loader.py", line 44, in
getitem
image = self.transform(image)   File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torchvision/transforms/transforms.py", line 95, in call
img = t(img)   File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torch/nn/modules/module.py",
line 1110, in _call_impl
return forward_call(*input, **kwargs)   File "/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torchvision/transforms/transforms.py", line 270, in forward
return F.normalize(tensor, self.mean, self.std, self.inplace)   File
"/home/ubuntu/anaconda3/envs/pytorch-1.11.0/lib/python3.9/site-packages/torchvision/transforms/functional.py", line 341, in normalize
raise TypeError(f"Input tensor should be a torch tensor. Got {type(tensor)}.") TypeError: Input tensor should be a torch tensor.
Got <class 'PIL.Image.Image'>.

Output: Python set iteration order is not deterministic. Kindly use list instead ([] rather than {}).


Instruction: converting tf.data.Dataset.from_tensor_slices to pytorch

Input:
I am trying to convert this model from tensorflow to pytorch. Unfortunately, I don't know tensorflow very well. I have a problem transferring ti data_loader from here and in general converting this function to pytorch
def _components_train_step(self, importance_weights, old_means, old_chol_precisions):
    for i in range(self._model.num_components):
        dt = (self._train_contexts, importance_weights[:, i], old_means, old_chol_precisions)
        data = tf.data.Dataset.from_tensor_slices(dt)
        data = data.shuffle(self._train_contexts.shape[0]).batch(self.c.components_batch_size)

        for context_batch, iw_batch, old_means_batch, old_chol_precisions_batch in data:
            iw_batch = iw_batch / tf.reduce_sum(iw_batch)
            with tf.GradientTape() as tape:
                 samples = self._model.components[i].sample(context_batch)
                 losses = - tf.squeeze(self._dre(tf.concat([context_batch, samples], axis=-1)))
                 kls = self._model.components[i].kls_other_chol_inv(context_batch, old_means_batch[:, i],
                                                                  old_chol_precisions_batch[:, i])
                loss = tf.reduce_mean(iw_batch * (losses + kls))
            gradients = tape.gradient(loss, self._model.components[i].trainable_variables)
            self._c_opts[i].apply_gradients(zip(gradients, self._model.components[i].trainable_variables))

I implemented this function as follows
def _components_train_step(self, importance_weights, old_means, old_chol_precisions):
        self._c_opts = [ torch.optim.Adam(self._model.components[i].trainable_variables, lr=self.c.components_learning_rate, betas=(0.5, 0.999)) for i in self._model.components]

        for i in range(self._model.num_components):
            dataset = torch.utils.data.TensorDataset(self._train_contexts, importance_weights[:, i], old_means, old_chol_precisions)
            loader = torch.utils.data.DataLoader( dataset, shuffle = True, batch_size=self.c.components_batch_siz)

            for batch_idx, (context_batch, iw_batch, old_means_batch, old_chol_precisions_batch) in enumerate(loader):
                iw_batch = iw_batch / torch.sum(iw_batch)
                samples = self._model.components[i].sample(context_batch)
                losses = - torch.squeeze(self._dre(torch.cat([context_batch, samples], dim=-1)))
                kls = self._model.components[i].kls_other_chol_inv(context_batch, old_means_batch[:, i],
                                                                  old_chol_precisions_batch[:, i])
                loss = torch.mean(iw_batch * (losses + kls))
                loss.backward()
                self._c_opts[i].zero_grad()
                self._c_opts[i].step()

Any suggestion or help?

Output:
I believe you can achieve a comparable result to tf.data.from_tensor_slices using PyTorch's data.TensorDataset which expects a tuple of tensors as input. This has the effect of zipping the different elements into a single dataset yielding tuple of the same length as there are elements.
Here is a minimal example:
feats = torch.tensor([[[1, 3], [2, 3]], [[2, 1], [1, 2]], [[3, 3], [3, 2]]])
tf_feats = tf.convert_to_tensor(feats.numpy())

labels = torch.tensor([[10, 10], [20, 20], [10, 20]])
tf_labels = tf.convert_to_tensor(labels.numpy())

Using TensorFlow:
>>> dataset = Dataset.from_tensor_slices((tf_feats, tf_labels))
>>> for x in dataset.as_numpy_iterator():
...     print(x)
(array([[1, 3],
        [2, 3]]), array([10, 10]))
(array([[2, 1],
        [1, 2]]), array([20, 20]))
(array([[3, 3],
        [3, 2]]), array([10, 20]))

Using PyTorch:
 >>> dataset = data.TensorDataset(feats, labels)
 >>> for x in dataset:
 ...     print(x)
(tensor([[1, 3],
         [2, 3]]), tensor([10, 10]))
(tensor([[2, 1],
         [1, 2]]), tensor([20, 20]))
(tensor([[3, 3],
         [3, 2]]), tensor([10, 20]))


Instruction: How can I apply cuda to custom model in pytorch?

Input:
The type of inputs is dictionary of tensors. So while training I convert device to cuda to use gpu. And my custom model is like above. Also I assigned cuda to the model.
class EmbeddingLayer(nn.Module):
  def __init__(self):
    super(EmbeddingLayer, self).__init__()

    # other features
    self.other_features_embedding = []
    for feature_name in OTHER_FEATURES:
      vocabulary = CATEGORICAL_FEATURES_WITH_VOCABULARY[feature_name]
      embedding_dims = int(math.sqrt(len(vocabulary)))
      embedding = nn.Embedding(len(vocabulary)+1, embedding_dims)
      self.other_features_embedding.append(embedding)

    # transformer features
    item_vocabulary = CATEGORICAL_FEATURES_WITH_VOCABULARY['item']
    self.item_embedding_dims = int(math.sqrt(len(item_vocabulary)))
    self.item_embedding = nn.Embedding(len(item_vocabulary)+1, self.item_embedding_dims)

  def forward(self, inputs):

    # other features
    encoded_other_features = []
    for i, feature_name in enumerate(OTHER_FEATURES):
      embedding = self.other_features_embedding[i](inputs[feature_name])
      encoded_other_features.append(embedding)

    encoded_other_features = torch.cat(encoded_other_features, -1)

    # transformer features
    encoded_sequence_item = self.item_embedding(inputs['sequence_item'])
    encoded_target_item = self.item_embedding(inputs['target_item'])

    positions = inputs['target_timestamp'].repeat(sequence_length-1, 1).transpose(0, 1) - inputs['sequence_timestamp']
    encoded_positions = positions.repeat(1, self.item_embedding_dims).reshape(-1, self.item_embedding_dims, sequence_length-1).transpose(1,2)

    encoded_sequence_item_with_position = encoded_sequence_item + encoded_positions
    encoded_transformer_features = torch.cat((encoded_sequence_item_with_position, encoded_target_item.reshape(-1, 1, self.item_embedding_dims)), 1)

    return encoded_other_features, encoded_transformer_features

class BST(nn.Module):
  def __init__(self, hidden_units, dropout, num_heads):
    super(BST, self).__init__()

    ...

    self.embedding_layer = EmbeddingLayer()

    ...

  def forward(self, inputs):
    other_features, transformer_features = self.embedding_layer(inputs)

    ...

    return self.output(features)

model = BST([256, 128], 0.3, 1)
model.to(device)

def train(model, optimizer, dataloader):
  model.train()
  for inputs in tqdm(dataloader, total=len(dataloader)):

    for k, v in inputs.items():
      inputs[k] = v.to(device)

    model.zero_grad()
    pred = model(inputs)

    ...

But following error occurs:
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cpu and cuda:0! (when checking argument for argument index in method wrapper__index_select)

I think the error occurs at embedding in EmbeddingLayer. How can I fix this error to use gpu while training?

Output:
Your list of nn.Module is not registering the embedding layers as sub modules of your layer. In order to properly register a list of modules you should use nn.ModuleList. Therefore, you should add the following right after the loop in your __init__ function:
embeddings = []
for feature_name in OTHER_FEATURES:
    vocabulary = CATEGORICAL_FEATURES_WITH_VOCABULARY[feature_name]
    embedding_dims = int(math.sqrt(len(vocabulary)))
    embedding = nn.Embedding(len(vocabulary)+1, embedding_dims)
    embeddings.append(embedding)

self.other_features_embedding = nn.ModuleList(embeddings)


Instruction: Unable to build vocab for a torchtext text classification

Input:
I'm trying to prepare a custom dataset loaded from a csv file in order to use in a torchtext text binary classification problem. It's a basic dataset with news headlines and a market sentiment label assigned "positive" or "negative". I've been following some online tutorials on PyTorch to get this far but they've made some significant changes in the latest torchtext package so most of the stuff is out of date.
Below I've successfully parsed my csv file into a pandas dataframe with two columns - text headline and a label which is either 0 or 1 for positive/negative, split into a training and test dataset then wrapped them as a PyTorch dataset class:
train, test = train_test_split(eurusd_df, test_size=0.2)
class CustomTextDataset(Dataset):
def __init__(self, text, labels):
    self.text = text
    self.labels = labels

def __getitem__(self, idx):
    label = self.labels.iloc[idx]
    text = self.text.iloc[idx]
    sample = {"Label": label, "Text": text}
    return sample

def __len__(self):
    return len(self.labels)
train_dataset = CustomTextDataset(train['Text'], train['Labels'])
test_dataset = CustomTextDataset(test['Text'], test['Labels'])

I'm now trying to build a vocabulary of tokens following this tutorial https://coderzcolumn.com/tutorials/artificial-intelligence/pytorch-simple-guide-to-text-classification and the official pytorch tutorial https://pytorch.org/tutorials/beginner/text_sentiment_ngrams_tutorial.html .
However using the below code
from torchtext.data.utils import get_tokenizer
from torchtext.vocab import build_vocab_from_iterator

tokenizer = get_tokenizer('basic_english')
train_iter = train_dataset

def yield_tokens(data_iter):
    for _, text in data_iter:
        yield tokenizer(text)

vocab = build_vocab_from_iterator(yield_tokens(train_iter), specials=["<unk>"])
vocab.set_default_index(vocab["<unk>"])

yields a very small length of vocabulary, and applying the example vocab(['here', 'is', 'an', 'example']) on a text field taken from the original dataframe yields a list of 0s, implying the vocab is being built from the label field, containing only 0s and 1s, not the text field. Could anyone review and show me how to build the vocab targeting the text field?

Output:
The very small length of vocabulary is because under the hood, build_vocab_from_iterator uses a Counter from the Collections standard library, and more specifically its update function. This function is used in a way that assumes that what you are passing to build_vocab_from_iterator is an iterable wrapping an iterable containing words/tokens.
This means that in its current state, because strings can be iterated upon, your code will create a vocab able to encode all letters, not words, comprising your dataset, hence the very small vocab size.
I do not know if that is intended by Python/Pytorch devs, but because of this you need to wrap your simple iterator in a list, for example like this :
vocab = build_vocab_from_iterator([yield_tokens(train_iter)], specials=["<unk>"])

Note : If your vocab gives only zeros, it is not because it is taking from the label field, it is just returning the integer corresponding to an unknown token, since all words that are not just a character will be unknown to it.
Hope this helps!


Instruction: Mnist model performing very badly on custom data

Input:
I have used the resnet50 prebuilt and pretrained model from pytorch, on the MNIST dataset,
from torch import nn
from torchvision.models import ResNet50_Weights, resnet50

class Model(nn.Module):
  def __init__(self):
    super(Model, self).__init__()

    self.model = resnet50(weights=ResNet50_Weights.DEFAULT)

    self.model.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)

    num_ftrs = self.model.fc.in_features
    self.model.fc = nn.Linear(num_ftrs, 10)

  def forward(self, x):
    return self.model(x)

it performs very well and after training for 10 epochs it has achieved an incredible 99.895% accuracy on the 50,000 test images.
model.eval()

with torch.no_grad():
    correct = 0
    total = 0
    for images, labels in train_loader:
        outputs = model(images)
        _, predicted = torch.max(outputs.data, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()

    print('Accuracy of the network on the {} train images: {} %'.format(50000, 100 * correct / total))

[out]: Accuracy of the network on the 50000 train images: 99.895 %

I have used pygame to easily create my own numbers to input to the model. I start off with a very basic program just placing circles while the left mouse button is held, then I save the generated image into a png format.
    if event.type == pg.MOUSEMOTION:
        if (drawing):
            mouse_position = pg.mouse.get_pos()
            pg.draw.circle(screen, color, mouse_position, w)
    elif event.type == pg.MOUSEBUTTONUP:
        mouse_position = (0, 0)
        drawing = False
        last_pos = None
    elif event.type == pg.MOUSEBUTTONDOWN:
        drawing = True

I convert the image to grayscale and scale it down to 28x28 and into a tensor using PIL and torch.PILToTensor().
image = Image.open("image.png").convert("L").resize((28,28),Image.Resampling.LANCZOS)

transform = Compose([
    PILToTensor(),
    Lambda(lambda image: image.view(-1, 1, 28, 28))
])

img_tensor = transform(image).to(torch.float)

Then I feed this image to the network. I get no errors or anything the model just predicts really badly. For example when I gave it this
image of a 2
this code outputed:
with torch.no_grad():
    outputs = model(img_tensor)
    print(outputs)
    _, predicted = torch.max(outputs.data, 1)
    print(predicted)

[out]: tensor([[ 20.6237,   0.4952, -15.5033,   8.5165,   1.0938,   2.8278,   2.0153,
           3.2825,  -6.2655,  -0.6992]])
tensor([0])

The sureness is outputted as list with the sureness for each class 0, 1, 2, 3... so as you can see the sureness for "2" is actually negative, does anyone know why this could be and how I could solve it?
Thank you very much

Output: I have solved this, the problem was that when I converted the image to a tensor it had values from 0-255 instead of 0-1, that's why the model was behaving so unpredictably.


Instruction: How can I use DataGenerator with my model training?

Input:
I am having a bit of trouble trying to use DataGenerator class with my model training during the model.fit() function. The reason I am using the DataGenerator is to help with dealing with my large amount of images and labels for my object detection task.
All of my images are split into train, test and validation sets. I have converted the images and my labels to a numpy array and then preprocessed accordingly for my RESNET50 model, which works fine.
val_images = np.array(val_images)
train_images = np.array(train_images)

However, when I try to use the DataGenerator function for my val and training images, it does not seem to work.
training_generator = DataGenerator(train_images, train_targets)
validation_generator = DataGenerator(val_images, val_targets)

I tried to use the model.fit() function but it does not work and ends up showing an error.
resnet_model = model.fit_generator(
    training_generator,
    epochs=4,
    validation_data=validation_generator)

TypeError: expected str, bytes or os.PathLike object, not ndarray
Full traceback:
https://www.toptal.com/developers/hastebin/gusicucali.yaml
I am not sure if this is the right way to use the DataGenerator class as I am quite new with it. I have 6000+ images with labels (xmin,ymin,ymax,xmax) accordingly.
The reason why I am trying to use it is to help make my model training a bit more efficient since I do not have a GPU.

Output:
I think you should aim at having a code similar to this below. I used ImageDataGenerator. If you have a look at the documentation it lets you pick lots of arguments to get a data augmentation of your images. I copied only a few.
val_images = np.array(val_images)
train_images = np.array(train_images)

dataGenerator = ImageDataGenerator(rotation_range=0,
                                   zoom_range=0,
                                   width_shift_range=0,
                                   height_shift_range=0,
                                   horizontal_flip=False,
                                   fill_mode="nearest")
# train
model = model.fit_generator(dataGenerator.flow(train_images, train_targets, batch_size=BATCH_SIZE),
                            validation_data=(val_images, val_targets),
                            steps_per_epoch=len(train_images),
                            epochs=EPOCHS)

This should work. Also I think that recently fit_generator has been deprecated, but you could substitute it with fit without changing anything.


Instruction: How to make pytorch lightning module have injected, nested models?

Input:
I have some nets, such as the following (augmented) resnet18:
num_classes = 10
resnet = models.resnet18(pretrained=True)
for param in resnet.parameters():
    param.requires_grad = True
num_ftrs = resnet.fc.in_features
resnet.fc = nn.Linear(num_ftrs, num_classes)

And I want to use them inside a lightning module, and have it handle all optimizations, to_device, stages and so on. In other words, I want to register those modules for my lightning module.
I also want to be able to access their public members.
class MyLightning(LightningModule):
    def __init__(self, resnet):
        super().__init__()
        self._resnet = resnet
        self._criterion = lambda x: 1.0

    def forward(self, x):
        resnet_out = self._resnet(x)

        loss =  self._criterion(resnet_out)
        return loss

my_lightning = MyLightning(resnet)

The above doesn't optimize any parameters.
Trying
def __init__(self, resnet)
    ...
    _layers = list(resnet.children())[:-1]
    self._resnet = nn.Sequential(*_layers)

Doesn't take resnet.fc into account. This also doesn't make sense to be the intended way of nesting models inside pytorch lightning.

How to nest models in pytorch lightning, and have them fully accessible and handled by the framework?

Output:
The training loop and optimization process is handles by the Trainer class. You can do so by initializing a new instance:
>>> trainer = Trainer()

And wrapping your PyTorch Lightning module with it. This way you can perform fitting, tuning, validating, and testing on that instance provided a DataLoader or LightningDataModule:
>>> trainer.fit(my_lightning, train_dataloader, val_dataloader)

You will have to implement the following functions on your Lightning module (i.e. in your case MyLightning):

Name
Description

init
Define computations here

forward
Use for inference only (separate from training_step)

training_step
the complete training loop

validation_step
the complete validation loop

test_step
the complete test loop

predict_step
the complete prediction loop

configure_optimizers
define optimizers and LR schedulers

source LightningModule documentation page.
Keep in mind a LightningModule is a nn.Module, so whenever you define a nn.Module as attribute to a LightningModule in the __init__ function, this module will end being registered as a sub-module to the parent pytorch lightning module.


Instruction: Loss key needs to be present Pytorch

Input:
I am following this repo:
https://github.com/NVIDIA/NeMo/tree/main/examples/nlp/entity_linking
Here is a small tutorial:
https://colab.research.google.com/github/NVIDIA/NeMo/blob/v1.0.2/tutorials/nlp/Entity_Linking_Medical.ipynb
Before starting this tutorial change branch to r1.10.0
When I train this model on entire UMLS dataset given the commands it gives the following error:
In automatic_optimization, when `training_step` returns a dict, the 'loss' key needs to be present

I checked the training steps method and it is fine:
def training_step(self, batch, batch_idx):
        """
        Lightning calls this inside the training loop with the data from the training dataloader
        passed in as `batch`.
        """
        input_ids, token_type_ids, attention_mask, concept_ids = batch
        logits = self.forward(input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask)
        train_loss = self.loss(logits=logits, labels=concept_ids)

        # No hard examples found in batch,
        # shouldn't use this batch to update model weights
        if train_loss == 0:
            train_loss = None
            lr = None

        else:
            lr = self._optimizer.param_groups[0]["lr"]
            self.log("train_loss", train_loss)
            self.log("lr", lr, prog_bar=True)

        return {"loss": train_loss, "lr": lr}

Here is a full stacktrace:
[NeMo I 2022-07-29 18:29:27 multi_similarity_loss:91] Encountered zero loss in multisimloss, loss = 0.0. No hard examples found in the batch
Error executing job with overrides: ['project_dir=.']
Traceback (most recent call last):
  File "self_alignment_pretraining.py", line 38, in main
    trainer.fit(model)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 769, in fit
    self._fit_impl, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 719, in _call_and_handle_interrupt
    return self.strategy.launcher.launch(trainer_fn, *args, trainer=self, **kwargs)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/strategies/launchers/subprocess_script.py", line 93, in launch
    return function(*args, **kwargs)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 809, in _fit_impl
    results = self._run(model, ckpt_path=self.ckpt_path)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1234, in _run
    results = self._run_stage()
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1321, in _run_stage
    return self._run_train()
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1351, in _run_train
    self.fit_loop.run()
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/loops/base.py", line 204, in run
    self.advance(*args, **kwargs)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/loops/fit_loop.py", line 268, in advance
    self._outputs = self.epoch_loop.run(self._data_fetcher)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/loops/base.py", line 204, in run
    self.advance(*args, **kwargs)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py", line 208, in advance
    batch_output = self.batch_loop.run(batch, batch_idx)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/loops/base.py", line 204, in run
    self.advance(*args, **kwargs)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/loops/batch/training_batch_loop.py", line 88, in advance
    outputs = self.optimizer_loop.run(split_batch, optimizers, batch_idx)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/loops/base.py", line 204, in run
    self.advance(*args, **kwargs)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 207, in advance
    self.optimizer_idx,
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 256, in _run_optimization
    self._optimizer_step(optimizer, opt_idx, batch_idx, closure)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 378, in _optimizer_step
    using_lbfgs=is_lbfgs,
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 1593, in _call_lightning_module_hook
    output = fn(*args, **kwargs)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/core/lightning.py", line 1644, in optimizer_step
    optimizer.step(closure=optimizer_closure)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/core/optimizer.py", line 168, in step
    step_output = self._strategy.optimizer_step(self._optimizer, self._optimizer_idx, closure, **kwargs)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/strategies/ddp.py", line 278, in optimizer_step
    optimizer_output = super().optimizer_step(optimizer, opt_idx, closure, model, **kwargs)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/strategies/strategy.py", line 193, in optimizer_step
    return self.precision_plugin.optimizer_step(model, optimizer, opt_idx, closure, **kwargs)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/plugins/precision/native_amp.py", line 85, in optimizer_step
    closure_result = closure()
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 148, in __call__
    self._result = self.closure(*args, **kwargs)
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 134, in closure
    step_output = self._step_fn()
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 437, in _training_step
    training_step_output, self.trainer.accumulate_grad_batches
  File "/home/umair/miniconda3/envs/aemap/lib/python3.7/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py", line 75, in from_training_step_output
    "In automatic_optimization, when `training_step` returns a dict, the 'loss' key needs to be present"
pytorch_lightning.utilities.exceptions.MisconfigurationException: In automatic_optimization, when `training_step` returns a dict, the 'loss' key needs to be present

Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.

Output:
You get this error message about "loss key needs to be present" because in some training steps you return the dict {"loss": None}. This happens in your code here
if train_loss == 0:
    train_loss = None
    lr = None

where you set train_loss = None. Lightning does not like that, because it wants loss to be a tensor with a graph attached.
If you wish to skip the optimization step completely, just return None from the training_step method, like this:
if train_loss == 0:
    return None


Instruction: Rearrange a 5D tensor in PyTorch

Input: I have a 5D tensor in the shape of (N,C,T,H,W). I want to rearrange it using PyTorch to the form of (N,T,HW,C). How can I do that?

Output:
Naturally you can reshape the last two dimensions of your tensor by flattening your tensor from dim=-2, this will produce a shape of (N,C,T,HW):
>>> x.flatten(-2)

Then you can permute the dimensions around:
>>> x.flatten(-2).permute(0,2,3,1)


Instruction: Shuffling two 2D tensors in PyTorch and maintaining same order correlation

Input:
Is it possible to shuffle two 2D tensors in PyTorch by their rows, but maintain the same order for both? I know you can shuffle a 2D tensor by rows with the following code:
a=a[torch.randperm(a.size()[0])]

To elaborate:
If I had 2 tensors
a = torch.tensor([[1, 1, 1, 1, 1],
            [2, 2, 2, 2, 2],
            [3, 3, 3, 3, 3]])

b = torch.tensor([[4, 4, 4, 4, 4],
            [5, 5, 5, 5, 5],
            [6, 6, 6, 6, 6]])

And ran them through some function/block of code to shuffle randomly but maintain correlation and produce something like the following
a = torch.tensor([[2, 2, 2, 2, 2],
            [1, 1, 1, 1, 1],
            [3, 3, 3, 3, 3]])

b = torch.tensor([[5, 5, 5, 5, 5],
            [4, 4, 4, 4, 4],
            [6, 6, 6, 6, 6]])

My current solution is converting to a list, using the random.shuffle() function like below.
a_list = a.tolist()
b_list = b.tolist()
temp_list = list(zip(a_list , b_list ))
random.shuffle(temp_list) # Shuffle
a_temp, b_temp = zip(*temp_list)
a_list, b_list = list(a_temp), list(b_temp)

# Convert back to tensors
a = torch.tensor(a_list)
b = torch.tensor(b_list)

This takes quite a while and was wondering if there is a better way.

Output:
You mean
indices = torch.randperm(a.size()[0])
a=a[indices]
b=b[indices]

?


Instruction: Applying torch.combinations on multidimensional tensor or tuple of tensors in PyTorch?

Input:
Using PyTorch, torch.combinations will only take a 1D tensor as input but I would like to apply it to each 1D tensor in a multidimensional tensor.
inp = torch.tensor([[1, 2, 3],
                    [2, 3, 4]])

torch.combinations((inp), r=2)

The result is an error saying I can't apply it to that shape but I want to apply it to [1, 2, 3] and [2, 3, 4] individually. I can't do it one by one because the idea is to apply this to large sets of data.
inp = torch.tensor([[1,2,3],[2,3,4]])
inp_tuple = torch.unbind(inp)
print(inp_tuple)
(tensor([1, 2, 3]), tensor([2, 3, 4]))

torch.combinations((inp_tuple), r=2)

I also tried unbinding the tensor and applying it to the tuple of tensors but it gives an error saying it can't be applied to a tuple.
Is there any way that I can get torch.combinations to automatically apply to each individual 1D tensor in a multidimensional tensor or each tensor in a tuple of tensors? If not are there any alternatives to achieve all combinations of each individual part of a multidimensional tensor?

Output:
Function torch.combinations returns all possible combinations of size r of the elements contained in the 1D input vector. The reason why multi-dimensional inputs are not supported is probably that you have no guarantee that the different vectors in your input have the exact same number of unique elements. Obviously if one of the vectors has a duplicate element then you would end up with one set of combinations bigger than another which is simply not possible to represent with a homogenous PyTorch tensor.
So from there on, I will assume that the input tensor inp is a 2D tensor shaped (N, C) where each of its N vectors contains C unique elements. The example you gave would fit to this requirement since both vectors have three unique elements each: {1, 2, 3} and {2, 3, 4}.
>>> inp = torch.tensor([[1,2,3],[2,3,4]])

The idea is to apply torch.combinations on an arrangement tensor of length equal to that of our vectors. We can then use those as indices to gather values in our different vectors in our input tensor.
We can retrieve all combinations of an arrangement with the following:
>>> c = torch.combinations(torch.arange(inp.size(1)), r=2)
tensor([[0, 1],
        [0, 2],
        [1, 2]])

Then we need to reshape and expand both inp and c such that they match in number of dimensions:
>>> x = inp[:,None].expand(-1,len(c),-1)
tensor([[[1, 2, 3],
         [1, 2, 3],
         [1, 2, 3]],

        [[2, 3, 4],
         [2, 3, 4],
         [2, 3, 4]]])

>>> idx = c[None].expand(len(x), -1, -1)
tensor([[[0, 1],
         [0, 2],
         [1, 2]],

        [[0, 1],
         [0, 2],
         [1, 2]]])

Finally we can apply torch.gather on x and idx on dim=2. This will return a 3D tensor out such that:
out[i][j][k] = x[i][j][index[i][j][k]]

Let's make our call on torch.gather:
>>> x.gather(dim=2, index=idx)
tensor([[[1, 2],
         [1, 3],
         [2, 3]],

        [[2, 3],
         [2, 4],
         [3, 4]]])

Which is the desired result.


Instruction: Inner workings of pytorch autograd.grad for inner derivatives

Input:
Consider the following code:
x = torch.tensor(2.0, requires_grad=True)

y = torch.square(x)
grad = autograd.grad(y, x)

x = x + grad[0]

y = torch.square(x)
grad2 = autograd.grad(y, x)

First, we have that ∇(x^2)=2x. In my understanding, grad2=∇((x + ∇(x^2))^2)=∇((x+2x)^2)=∇((3x)^2)=9∇x^2=18x .  As expected, grad=4.0=2x, but grad2=12.0=6x, which I don't understand where it comes from. It feels as though the 3 comes from the expression I had, but it is not squared, and the 2 comes from the traditional derivative. Could somebody help me understand why this is happening? Furthermore, how far back does the computational graph that stores the gradients go?
Specifically, I am coming from a meta learning perspective, where one is interested in computing a quantity of the following form ∇ L(theta - alpha * ∇ L(theta))=(1 + ∇^2 L(theta)) ∇L(theta - alpha * ∇ L(theta) (here the derivative is with respect to theta). Therefore, the computation, let's call it A, includes a second derivative. Computation is quite different than the following ∇_{theta - alpha ∇ L(theta)}L(\theta - alpha * ∇ L(theta))=∇_beta L(beta), which I will call B.
Hopefully, it is clear how the snippet I had is related to what I described in the second paragraph. My overall question is: under what circumstances does pytorch realize computation A vs computation B when using autograd.grad? I'd appreciate any explanation that goes into technical details about how this particular case is handled by autograd.
PD. The original code I was following made me wonder this is here; in particular, lines 69 through 106, and subsequently line 193, which is when they use autograd.grad. For the code is even more unclear because they do a lot of model.clone() and so on.
If the question is unclear in any way, please let me know.

Output:
I made a few changes:

I am not sure what torch.rand(2.0) is supposed to do. According to the text I simply set it to 2.
An intermediate variable z is added so that we can compute gradient w.r.t. to the original variable. Yours is overwritten.
Set create_graph=True to compute higher order gradients. See https://pytorch.org/docs/stable/generated/torch.autograd.grad.html

import torch
from torch import autograd

x = torch.ones(1, requires_grad=True)*2

y = torch.square(x)
grad = autograd.grad(y, x, create_graph=True)
z = x + grad[0]

y = torch.square(z)
grad2 = autograd.grad(y, x)
# yours is more like autograd.grad(y, z)

print(x)
print(grad)
print(grad2)


Instruction: Visual Transformer - understand if a model is pre-trained or not

Input:
I am working on a binary image classifier, and I am now testing using a Visual Transformer for the task. As a reference, I am using the model reimplementation by lucidrains. In the code snippet used to declare the model (see here and below), is the model pretrained or do we do the training from scratch? Thanks!
import torch
from vit_pytorch import SimpleViT

v = SimpleViT(
    image_size = 256,
    patch_size = 32,
    num_classes = 1000,
    dim = 1024,
    depth = 6,
    heads = 16,
    mlp_dim = 2048
)

Output: If you look into the implementation code, there is no weight loading involved.


Instruction: When to use prepare_data vs setup in pytorch lightning?

Input:
Pytorch's docs on Dataloaders only say, in the code
def prepare_data(self):
    # download
    ...

and
def setup(self, stage: Optional[str] = None):
    # Assign train/val datasets for use in dataloaders

Please explain the intended separation between prepare_data and setup, what callbacks may occur between them, and why put something in one over the other.

Output:
If you look at the pseudo for the Trainer.fit function provided in the documentation page of LightningModule at § Hooks, you can read:
def fit(self):
    if global_rank == 0:
        # prepare data is called on GLOBAL_ZERO only
        prepare_data()                                 ## <-- prepare_data

    configure_callbacks()

    with parallel(devices):
        # devices can be GPUs, TPUs, ...
        train_on_device(model)

def train_on_device(model):
    # called PER DEVICE
    on_fit_start()
    setup("fit")                                       ## <-- setup
    configure_optimizers()

    # the sanity check runs here

    on_train_start()
    for epoch in epochs:
        fit_loop()
    on_train_end()

    on_fit_end()
    teardown("fit")

You can see prepare_data being called only for global_rank == 0, i.e. it is only called by a single processor. It turns out you can read from the documentation description of prepare_data:

LightningModule.prepare_data()
Use this to download and prepare data. Downloading and saving data with multiple processes (distributed settings) will result in corrupted data. Lightning ensures this method is called only within a single process, so you can safely add your downloading logic within.

Whereas setup is called on all processes as you can read from the pseudo-code above as well as its documentation description:

LightningModule.setup(stage=None)Called at the beginning of fit (train + validate), validate, test, or predict. This is a good hook when you need to build models dynamically or adjust something about them. This hook is called on every process when using DDP.


Instruction: How to get total test accuracy for pytorch lightning?

Input:
How can the trainer.test method be used to get total accuracy over all batches?
I know I can implement model.test_step but that is for a single batch only. I need the accuracy over the whole data set. I can use torchmetrics.Accuracy to accumulate accuracy. But what is the proper way to combine that and get the total accuracy out? What is model.test_step supposed to return anyway since batchwise test scores are not very useful? I could hack it somehow, but I'm surprised that I couldn't find any example on the internet that demonstrates how to get accuracy with the pytorch-lightning native way.

Output:
You can see here (https://pytorch-lightning.readthedocs.io/en/stable/extensions/logging.html#automatic-logging) that the on_epoch argument in log automatically accumulates and logs at the end of the epoch. The right way of doing this would be:
from torchmetrics import Accuracy

def validation_step(self, batch, batch_idx):
    x, y = batch
    preds = self.forward(x)
    loss = self.criterion(preds, y)
    accuracy = Accuracy()
    acc = accuracy(preds, y)
    self.log('accuracy', acc, on_epoch=True)
    return loss

If you want a custom reduction function you can set it using the reduce_fx argument, the default is torch.mean(). log() can be called from any method in you LightningModule


Instruction: AttributeError: 'list' object has no attribute 'size'

Input:
I'm trying to prepare some audio data for a Dataloader but I'm struggling with a few points. At the moment my data is organised into two lists inputs and target, which are both length 32, but have different dimensions for their elements; inputs[0].shape = (8, 3690288) (8 mono audio tracks) and target[0].shape = (2, 3690288) (a single stereo mix).
I've converted each array to a tensor by:
tensor_inputs = torch.Tensor(inputs)
tensor_target = torch.Tensor(target)

which seems to work: tensor_inputs.shape = torch.Size([32, 8, 3690288]). I've then tried to convert each of these to a melspectrogram:
melspectrogram = torchaudio.transforms.melspectrogram(
        sr=44100,
        n_fft=1024,
        hop_length=512,
        n_mels=64)

tensor_input_specs = []

for i in range(len(tensor_inputs)):
    spec = mel_spectrogram(tensor_inputs[i])
    tensor_input_specs.append(spec)

tensor_target_specs = []

for i in range(len(tensor_target)):
    spec = mel_spectrogram(tensor_target[i])
    tensor_target_specs.append(spec)

and then move these into a Dataloader by doing:
dataset = TensorDataset(tensor_input_specs,tensor_target_specs)
iter = DataLoader(dataset)

However I get the following error: AttributeError: 'list' object has no attribute 'size', which I imagine is due to the fact that I'm appending the spectrograms to a list, but I'm not sure how else to achieve this.
EDIT:
AttributeError                            Traceback (most recent call last)
C:\Users\BRUDAL~1\AppData\Local\Temp/ipykernel_24968/2240294361.py in <module>
     14     tensor_target_specs.append(spec)
     15
---> 16 dataset = TensorDataset(tensor_input_specs,tensor_target_specs)
     17 iter = DataLoader(dataset) # create your dataloader

~\anaconda3\lib\site-packages\torch\utils\data\dataset.py in __init__(self, *tensors)
    165
    166     def __init__(self, *tensors: Tensor) -> None:
--> 167         assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors), "Size mismatch between tensors"
    168         self.tensors = tensors
    169

~\anaconda3\lib\site-packages\torch\utils\data\dataset.py in <genexpr>(.0)
    165
    166     def __init__(self, *tensors: Tensor) -> None:
--> 167         assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors), "Size mismatch between tensors"
    168         self.tensors = tensors
    169

AttributeError: 'list' object has no attribute 'size'

Output:
The most straightforward method I've found is by stacking the list after the for loops, by using torch.stack:
tensor_input_specs = []

for i in range(len(tensor_inputs)):
    spec = mel_spectrogram(tensor_inputs[i])
    tensor_input_specs.append(spec)

tensor_input_specs = torch.stack(train_tensor_input_specs)

tensor_input_specs.shape

>>> torch.size([32, 8, 64, 7208])


Instruction: 'No module named 'engine' error in Pytorch tutorial

Input:
I am following a tutorial on the PyTorch website and I can't figure out what package this import uses:
from engine import train_one_epoch, evaluate

I get an error saying:
Traceback (most recent call last):
  File "C:\Users\...\tv-training-code.py", line 13, in <module>
    from engine import train_one_epoch, evaluate
ModuleNotFoundError: No module named 'engine'

For reference, I am using Conda to run the program and I have the latest PyTorch version installed. Any ideas for what package I need to install to make this work?

Output:
There is an obvious omission to the subject tutorial, which has caused justified confusion to others, too; this question has been raised in the Pytorch forum as well - here is the accepted answer:

In references/detection/, we have a number of helper functions to simplify training and evaluating detection models. Here, we will use references/detection/engine.py, references/detection/utils.py and references/detection/transforms.py. Just copy them to your folder and use them here.

Essentially, the necessary steps are shown in the colab notebook accompanying the tutorial:
%%shell

# Download TorchVision repo to use some files from
# references/detection
git clone https://github.com/pytorch/vision.git
cd vision
git checkout v0.8.2

cp references/detection/utils.py ../
cp references/detection/transforms.py ../
cp references/detection/coco_eval.py ../
cp references/detection/engine.py ../
cp references/detection/coco_utils.py ../

which must be executed before you attempt to import anything from the engine module.


Instruction: How to find features and targets for subset of data?

Input:
I'm trying to find the specific amount of features in a subset of a dataset. Below is an example code:
# import the required modules
import torch
import torchvision
from torchvision.datasets import CIFAR10
from collections import Counter

trainset = CIFAR10(root='./data', train=True, download=True, transform=torchvision.transforms.ToTensor())

indices = torch.arange(3000)
new_trainset_split = torch.utils.data.Subset(trainset, indices)

This outputs a 3000 datapoints, which is exactly what I want. However when I try the next line of code to see, how many types of each features there are (in this case, how many of the datapoints are 1's, 2's, 3's etc..), it gives me the error:
print(dict(Counter(new_trainset_split.targets)))

AttributeError: 'Subset' object has no attribute 'targets'

How do I find features and targets for subset of data?

Output:
The reason why you are not getting access to targets is because data.Subset simply doesn't implement the attributes as the wrapped data.Dataset (in your case datasets.CIFAR10) implements.
However, an easy workaround is to simply filter your initial dataset's targets with indices directly:
>>> Counter(trainset.targets[i] for i in indices)
Counter({0: 299,
         1: 287,
         2: 322,
         3: 285,
         4: 311,
         5: 279,
         6: 312,
         7: 297,
         8: 308,
         9: 300})


Instruction: How can I add some error to linear layer in pytorch?

Input:
I want to add some errors to parameters like the following code.
import torch
import torch.nn as nn

layer = nn.Linear(10, 100)
plus_weight = torch.randn(100, 10)
layer.weight += plus_weight

But I got this error.
RuntimeError: a leaf Variable that requires grad is being used in an in-place operation.

How can I avoid this error? And why can't I change the layer parameter value directly?

Output:
you can use .data to modify the value of leaf node
layer.weight.data += plus_weight

if you want initialize or update parameters, assigning to .data is the way to go.
But it is very dangerous to take a in-place operation to modify the value of a leaf node in the middle of a forward pass since it may broken the backward result.
For example
import torch
a = torch.randn(1,1,requires_grad=True)
print(a)
b = a**2
c = torch.sum(b**2)
a.data += 1
c.backward()
print(a.grad)

the output result will be
tensor([[0.9123]], requires_grad=True)
tensor([[6.3666]])

with the code above, we have c = b**2 = a**2。 Derivation of c with respect to a should should be 4*a**3. but since you update the value of a during middle of forward process, the b will be 0.9123**2 not (0.9123+1)**2. and the final result with a.grad will be 2*(0.9123**2)*2*1.9123


Instruction: ValueError: out_channels must be divisible by groups

Input:
I want to implement these three layers in a single class, but I face this problem

ValueError: out_channels must be divisible by groups
I don't understand where is the mistake?!

THis is the code:
class Block(nn.Module):
def __init__(self, in_channels, out_channels, exp=1, stride=1, type=''):
    super().__init__()
    self.t = type
    self.stride = stride
    self.inc, self.outc = in_channels, out_channels
    self.exp = exp
    self.blockc = nn.Sequential(
        nn.Conv2d(self.inc, self.inc* self.exp, kernel_size=1),
        nn.ReLU(),
        nn.Conv2d(self.inc * self.exp, self.outc, kernel_size=3, groups= self.inc * self.exp, stride= self.stride),
        nn.ReLU(),
        nn.Conv2d(self.outc, self.outc * self.exp, kernel_size=1),
        nn.Linear(self.outc * self.exp, self.outc * self.exp))

def forward(self, x):
    out = self.blockc(x)
    if self.t == 'A':
        out = torch.sum(out,x)
    return out

and here is the layers consists of depthwise convolution and conv 1x1 also there is a param exp which means out_channels = exp* in_channles

Output:
Here is a possible implementation, you will have to adjust the channels and padding for your needs:
class BType(Enum):
    A = 0
    B = 1
    C = 2

class Block(nn.Module):
    def __init__(self, c_in: int, c_out: int, btype: BType) -> nn.Module:
        super().__init__()
        self.btype = btype

        if btype == BType.A:
            assert c_in == c_out

        self.c1 = nn.Sequential(
            nn.Conv2d(c_in, c_in, kernel_size=1),
            nn.ReLU())

        self.c2 = nn.Sequential(
            nn.Conv2d(c_in, c_in, kernel_size=3, groups=c_in,
                      stride=2 if btype == BType.C else 1,
                      padding=2 if btype == BType.C else 1),
            nn.ReLU())

        self.c3 = nn.Conv2d(c_in, c_out, kernel_size=1)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        out = self.c1(x)
        out = self.c2(out)
        out = self.c3(out)

        if self.btype == BType.A:
            out += x
        return out

Here is a test with all three block types:

block A:
>>> block = Block(3, 3, BType.A)
>>> block(torch.rand(2,3,10,10)).shape
torch.Size([2, 3, 10, 10])

block B:
>>> block = Block(3, 10, BType.B)
>>> block(torch.rand(2,3,10,10)).shape
torch.Size([2, 10, 10, 10])

block C:
>>> block = Block(3, 10, BType.C)
>>> block(torch.rand(2,3,10,10)).shape
torch.Size([2, 10, 6, 6])


Instruction: RuntimeError: Found dtype Char but expected Float

Input:
I am using PyTorch in my program(Binary Classification).
The output from my model and actual labels are
model outputs are: (tensor([[0.4512],
        [0.2273],
        [0.4710],
        [0.2965]], grad_fn=<SigmoidBackward0>), torch.float32),
actuall labels are (tensor([[0],
        [1],
        [0],
        [1]], dtype=torch.int8), torch.int8)

When I calculate the Binary Cross Entropy, it gives me the error
RuntimeError: Found dtype Char but expected Float

I have no idea how it is finding the Char dtype.
Even If calculate it manually, it gives me this error.
import torch
cri = torch.nn.BCELoss()
cri(torch.tensor([[0.4470],[0.5032],[0.3494],[0.5057]], dtype=torch.float), torch.tensor([[0],[1],[0],[0]], dtype=torch.int8))

My DataLoader is
# CREATING DATA LOADER

class MyDataset(torch.utils.data.Dataset):
    def __init__(self, dataframe, subset='train'):
        self.subset = subset

        self.dataframe = dataframe
        self.transforms = transforms.Compose([
                                        transforms.RandomResizedCrop(224),
                                        transforms.RandomHorizontalFlip(),
                                        transforms.Grayscale(),
                                        transforms.ToTensor(),
                                        transforms.Normalize((0.5), (0.5))
                                    ])

    def __len__(self):
        return len(self.dataframe)

    def __getitem__(self, index):
        row = self.dataframe.iloc[index]
        img = Image.open(os.path.join('/kaggle/input/mura-v11',row['path']))

        if self.subset=='train':
#             print(torch.tensor(0, dtype=torch.int8) if row['labels'] == 'negative' else torch.tensor(1, dtype=torch.int8))
            return (self.transforms(img), torch.tensor(0, dtype=torch.int8) if row['labels'] == 'negative' else torch.tensor(1, dtype=torch.int8))
        else:
            tensor_img = torchvision.transforms.functional.to_tensor(img)
            return (tensor_img, torch.tensor(0, dtype=torch.int8) if row['labels'] == 'negative' else torch.tensor(1, dtype=torch.int8))

my training loop is
def train_model(model, criterion, optimizer, scheduler, num_epochs=25):
    since = time.time()

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0

    for epoch in range(num_epochs):
        print(f'Epoch {epoch}/{num_epochs - 1}')
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'val']:
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device)
                labels = labels.to(device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)
                    print(f"model outputs are: {outputs, outputs.dtype}, \nmodel labels are {labels.view(labels.shape[0],1), labels.dtype}")
                    loss = criterion(outputs, labels.view(labels.shape[0], 1))

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)
            if phase == 'train':
                scheduler.step()

            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]

            print(f'{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')

            # deep copy the model
            if phase == 'val' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())

        print()

    time_elapsed = time.time() - since
    print(f'Training complete in {time_elapsed // 60:.0f}m {time_elapsed % 60:.0f}s')
    print(f'Best val Acc: {best_acc:4f}')

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model

And my Model is
class MuraModel(torch.nn.Module):
    def __init__(self):
        """
        In the constructor we instantiate four parameters and assign them as
        member parameters.
        """
        super().__init__()
        self.inp = torch.nn.Conv2d(1, 3, 3) # Change the num of channels to 3
        self.backbone = models.resnet18(pretrained=True)
        num_ftrs = self.backbone.fc.in_features
        self.backbone.fc = nn.Linear(num_ftrs, 1)
        self.act = nn.Sigmoid()

    def forward(self, x):
        """
        In the forward function we accept a Tensor of input data and we must return
        a Tensor of output data. We can use Modules defined in the constructor as
        well as arbitrary operators on Tensors.
        """
        three_channel = self.inp(x)
        back_out = self.backbone(three_channel)
        return self.act(back_out)

# inp = nn.Conv2d(1, 3, 3)
# model_ft = models.resnet18(pretrained=True)(inp)
# num_ftrs = model_ft.fc.in_features
# model_ft.fc = nn.Linear(num_ftrs, 2)

# model_ft = model_ft.to(device)

criterion = nn.BCELoss()
model = MuraModel()

optimizer_ft = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)

# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)

model = train_model(model, criterion, optimizer_ft, exp_lr_scheduler,
                       num_epochs=25)

How to overcome it.
EDIT
Trace back on train_model function:
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
/tmp/ipykernel_17/2718774237.py in <module>
      1 model = train_model(model, criterion, optimizer_ft, exp_lr_scheduler,
----> 2                        num_epochs=25)

/tmp/ipykernel_17/2670448577.py in train_model(model, criterion, optimizer, scheduler, num_epochs)
     33                     _, preds = torch.max(outputs, 1)
     34                     print(f"model outputs are: {outputs, outputs.dtype}, \nmodel labels are {labels.view(labels.shape[0],1), labels.dtype}")
---> 35                     loss = criterion(outputs, labels.view(labels.shape[0], 1))
     36
     37                     # backward + optimize only if in training phase

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1108         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110             return forward_call(*input, **kwargs)
   1111         # Do not call functions when jit is used
   1112         full_backward_hooks, non_full_backward_hooks = [], []

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
    610
    611     def forward(self, input: Tensor, target: Tensor) -> Tensor:
--> 612         return F.binary_cross_entropy(input, target, weight=self.weight, reduction=self.reduction)
    613
    614

/opt/conda/lib/python3.7/site-packages/torch/nn/functional.py in binary_cross_entropy(input, target, weight, size_average, reduce, reduction)
   3063         weight = weight.expand(new_size)
   3064
-> 3065     return torch._C._nn.binary_cross_entropy(input, target, weight, reduction_enum)
   3066
   3067
RuntimeError: Found dtype Char but expected Float

Trace back on calculating loss individually
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
/tmp/ipykernel_17/4156819471.py in <module>
      1 import torch
      2 cri = torch.nn.BCELoss()
----> 3 cri(torch.tensor([[0.4470],[0.5032],[0.3494],[0.5057]], dtype=torch.float), torch.tensor([[0],[1],[0],[0]], dtype=torch.int8))

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
   1108         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
   1109                 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1110             return forward_call(*input, **kwargs)
   1111         # Do not call functions when jit is used
   1112         full_backward_hooks, non_full_backward_hooks = [], []

/opt/conda/lib/python3.7/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
    610
    611     def forward(self, input: Tensor, target: Tensor) -> Tensor:
--> 612         return F.binary_cross_entropy(input, target, weight=self.weight, reduction=self.reduction)
    613
    614

/opt/conda/lib/python3.7/site-packages/torch/nn/functional.py in binary_cross_entropy(input, target, weight, size_average, reduce, reduction)
   3063         weight = weight.expand(new_size)
   3064
-> 3065     return torch._C._nn.binary_cross_entropy(input, target, weight, reduction_enum)
   3066
   3067

RuntimeError: Found dtype Char but expected Float

Output: BCELoss() expects float labels. Yours are int8 (aka char). Converting them to float in the last line of__getitem__()should fix the issue.


Instruction: Reshape original X from LSTM with predictions

Input:
I have a tensor with my LSTM inputs X (in PyTorch) as well as the matching predictions Y_hat
I want to add the Y_hat as a column to the original X.
The problem is that LSTM a sliding window with seq_length. In case seq. length is 3 and I have 6 variables in X, and 2 variables in Y_hat, I have something like this:
First entry of Tensor:
X1:
[1 2 3 4 5 6]
[5 4 6 7 8 9]
[3 6 8 7 4 8]

Y_hat1
[0 1]

X2:
[5 4 6 7 8 9] (repeated from X1)
[3 6 8 7 4 8] (repeated from X1)
[4 8 7 9 8 4]
Y_hat1
[1 1]

and so on.
Is there an easy pythonesk command to reshape the X with Y so I can quickly get:
[3 6 8 7 4 8 0 1]
[4 8 7 9 8 4 1 1]

Output:
Having defined X1, X2, Y_hat1, and Y_hat2:
X1 = torch.tensor([[1, 2, 3, 4, 5, 6],
                   [5, 4, 6, 7, 8, 9],
                   [3, 6, 8, 7, 4, 8]])
Y_hat1 = torch.tensor([0,1])

X2 = torch.tensor([[5, 4, 6, 7, 8, 9],
                   [3, 6, 8, 7, 4, 8],
                   [4, 8, 7, 9, 8, 4]])
Y_hat2 = torch.tensor([1,1])

As well as stacks of inputs (X) and targets (Y):
X = torch.stack([X1, X2])
Y = torch.stack([Y_hat1, Y_hat2])

You can the desired operation using concatenation and stacking operators:
>>> X = torch.stack([X1, X2])
>>> Y = torch.stack([Y_hat1, Y_hat2])
>>> torch.stack(tuple(torch.cat((x[-1], y)) for x, y in zip(X, Y)))

Which expanded corresponds to:
>>> torch.stack((
        torch.cat((X1[-1], Y_hat1)),
        torch.cat((X2[-1], Y_hat2))))

In a more vectorized fashion you can do:
>>> torch.hstack((X[:,-1], Y))


Instruction: How to seamlesly load different ML models in python?

Input:
Let's say I have my original .pt weights and I export them to ONNX, OpenVINO and TFLite. Is there a way of loading these models without needing to write a custom class that checks its type of instance and loads it accordingly?
OpenVINO model loading example:
from openvino.runtime import Core

ie = Core()
classification_model_xml = "model/classification.xml"

model = ie.read_model(model=classification_model_xml)
compiled_model = ie.compile_model(model=model, device_name="CPU")

TFlite model loading example:
class TestModel(tf.Module):
  def __init__(self):
    super(TestModel, self).__init__()

  @tf.function(input_signature=[tf.TensorSpec(shape=[1, 10], dtype=tf.float32)])
  def add(self, x):
    '''
    Simple method that accepts single input 'x' and returns 'x' + 4.
    '''
    # Name the output 'result' for convenience.
    return {'result' : x + 4}

SAVED_MODEL_PATH = 'content/saved_models/test_variable'
TFLITE_FILE_PATH = 'content/test_variable.tflite'

# Save the model
module = TestModel()
# You can omit the signatures argument and a default signature name will be
# created with name 'serving_default'.
tf.saved_model.save(
    module, SAVED_MODEL_PATH,
    signatures={'my_signature':module.add.get_concrete_function()})

# Convert the model using TFLiteConverter
converter = tf.lite.TFLiteConverter.from_saved_model(SAVED_MODEL_PATH)
tflite_model = converter.convert()
with open(TFLITE_FILE_PATH, 'wb') as f:
  f.write(tflite_model)

# Load the TFLite model in TFLite Interpreter
interpreter = tf.lite.Interpreter(TFLITE_FILE_PATH)
# There is only 1 signature defined in the model,
# so it will return it by default.
# If there are multiple signatures then we can pass the name.
my_signature = interpreter.get_signature_runner()

# my_signature is callable with input as arguments.
output = my_signature(x=tf.constant([1.0], shape=(1,10), dtype=tf.float32))
# 'output' is dictionary with all outputs from the inference.
# In this case we have single output 'result'.
print(output['result'])

pt model loading example:
model = TheModelClass(*args, **kwargs)
model.load_state_dict(torch.load(PATH))

Output:
The closest solution I could find for this, atm, is Ivy . However, you have to write your model using their framework agnostic operations and the set of available frameworks are quite limited: jnp, tf, np, mx, torch
I found a quite good adapter that I will base my own on, here


Instruction: Identifying the loudest part of an audio track and cropping (Librosa or torchaudio)

Input:
I've built a U-Net model to perform audio mixing of multitrack audio, for which I've used 20s clips of the audio tracks (converted into spectrograms) as input in training the model. However the training process is incredibly long, so I think it would be better to take 2s clips from each track to train the model.
The data is organised as 8 stems (individual instrument tracks) as the inputs and a single mixture of the stems as the target (all have sr=44100). I want to find the most energetic 2s section of the mixture track and crop all tracks (input and mixture) this specific 2s part. I'm mainly using librosa in my data preparation but I'm unsure what functions to use to find the start point of the loudest (I understand this is ambiguous) 88200 sample segment (2s).

Output:
If I am following the question well enough, the below code might be useful as a starting point.  It takes in one sound file and locates where it is "loudest" (as you allude to in the question, defining what bit is loudest is not entirely straight-forward) using librosa.feature.rms and then cuts a two second slice out of the original sound file centered on that point:
import librosa

FILENAME = 'soundfile.wav'  # change to path of your sound file
FRAME_LENGTH = 2048
HOP_LENGTH = 512
NUM_SECONDS_OF_SLICE = 2

sound, sr = librosa.load(FILENAME, sr=None)

clip_rms = librosa.feature.rms(y=sound,
                               frame_length=FRAME_LENGTH,
                               hop_length=HOP_LENGTH)

clip_rms = clip_rms.squeeze()
peak_rms_index = clip_rms.argmax()
peak_index = peak_rms_index * HOP_LENGTH + int(FRAME_LENGTH/2)

half_slice_width = int(NUM_SECONDS_OF_SLICE * sr / 2)
left_index = max(0, peak_index - half_slice_width)
right_index = peak_index + half_slice_width
sound_slice = sound[left_index:right_index]


Instruction: Using Transposed convolution for upsampling in PyTorch

Input:
I have a 4D tensor of (2,1024,4,6). I want to use transposed convolution for upsampling spatial dimensions of such tensor by factor of two and reducing the channel numbers from 1024 to the 512. I want to have a 4D tensor like this (2,512,8,12). How can I do that? Also, is the transposed convolution a good idea for reducing the channel numbers? For example I used the following script but it is not working:
nn.ConvTranspose3d(in_channels=1024, out_channels=512, kernel_size=(1,2,2), stride=(1,3,2), padding=(0,1,1))

Output:
It seems you should be using ConvTranspose2d instead of ConvTranspose3d since your input tensor is 4D, shaped NCHW.
There are different ways of getting to these results but one straightforward approach is to use a kernel size of 2 with a matching stride:
>>> conv = nn.ConvTranspose2d(1024, 512, kernel_size=2, stride=2)

Here is an inference example:
>>> conv(torch.rand(2, 1024, 4, 6)).shape
torch.Size([2, 512, 8, 12])


Instruction: How does torch differentiate between batch and single values?

Input:
Here’s my neural network.
from torch import nn
from torch.utils.data import DataLoader
class NeuralNetwork(nn.Module):
    def __init__(self, state_size, action_size):
        super(NeuralNetwork, self).__init__()
        self.state_size = state_size
        self.action_size = action_size
#         self.flatten = nn.Flatten()
        self.linear_relu_stack = nn.Sequential(
            nn.Linear(1, 30),
            nn.ReLU(),
            nn.Linear(30, 30),
            nn.ReLU(),
            nn.Linear(30, action_size)
        )
    def forward(self, x):
        x = self.linear_relu_stack(x)
        return x

As you can imagine, it can compute tensor inputs of the shape torch.Size([1]). However, when I try to feed it batch data, for instance, shape torch.Size([10]) it throws the following error -
RuntimeError: mat1 and mat2 shapes cannot be multiplied (1x10 and 1x30)

For instance, this code works -
net = NeuralNetwork(10, 5)
x1 = torch.rand(1)
print(x1.shape)
out = net(x1)

But this fails -
x2 = torch.rand(10)
print(x2.shape)
out = net(x2)

Output:
You just need to change your inputs a little bit. Your code is expecting the net() to have a second dimension of 1, so it can multiply by nn.linear(1,30).  The inner dimensions must match for matrix multiplication to occur.  I.e. 10x1 * 1x30:
x2 = torch.rand(10,1)
print(x2.shape)
out = net(x2)
torch.Size([10, 1])

Try this with any dimension x2 = torch.rand(100,1) etc. it still works.


Instruction: Load estimator from model artifact in s3 bucket aws

Input:
I have used estimator for a pytorch model and have saved the artifacts in s3 bucket. using below code
estimator = PyTorch(
    entry_point="train_deploy.py",
    source_dir="code",
    role=role,
    framework_version="1.3.1",
    py_version="py3",
    instance_count=1,  # this script only support distributed training for GPU instances.
    instance_type="ml.m5.12xlarge",
    output_path=output_path,
    hyperparameters={
        "epochs": 1,
        "num_labels": 7,
        "backend": "gloo",
    },
    disable_profiler=False, # disable debugger
)
estimator.fit({"training": inputs_train, "testing": inputs_test})

The model works well and there are no issues with it. However i would like to re use this model later for inference, how do i do that. i am looking for something like below
estimator = PyTorch.load(input_path = "<xyz>")

Output:
I was able to solve this by the following steps
model_data=output_path
from sagemaker.pytorch.model import PyTorchModel

pytorch_model = PyTorchModel(model_data=model_data,
                             role=role,
                             framework_version="1.3.1",
                             source_dir="code",
                             py_version="py3",
                             entry_point="train_deploy.py")

predictor = pytorch_model.deploy(initial_instance_count=1, instance_type="ml.m4.2xlarge")
predictor.serializer = sagemaker.serializers.JSONSerializer()
predictor.deserializer = sagemaker.deserializers.JSONDeserializer()
result = predictor.predict("<text that needs to be predicted>")
print("predicted class: ", np.argmax(result, axis=1))