Instruction: ValueError('need at least one array to stack')

Input:
When I Processed video and its audio，I encountered an error:

Original Traceback (most recent call last):
  File "/home/yzx/anaconda3/envs/pytorch/lib/python3.8/site-packages/torch/utils/data/_utils/worker.py", line 202, in _worker_loop
    data = fetcher.fetch(index)
  File "/home/yzx/anaconda3/envs/pytorch/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 44, in fetch
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "/home/yzx/anaconda3/envs/pytorch/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 44, in
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "/home/yzx/lunwen/PseudoBinaural_CVPR2021-master/data/Augment_dataset.py", line 66, in __getitem__
    data_ret, data_ret_sep = self._get_pseudo_item(index)
  File "/home/yzx/lunwen/PseudoBinaural_CVPR2021-master/data/Pseudo_dataset.py", line 263, in _get_pseudo_item
    stereo = self.construct_stereo_ambi(pst_sources)
  File "/home/yzx/lunwen/PseudoBinaural_CVPR2021-master/data/Pseudo_dataset.py", line 120, in construct_stereo_ambi
    signals = np.stack([src.signal for src in pst_sources], axis=1) # signals shape: [Len, n_signals]
  File "", line 5, in stack
  File "/home/yzx/anaconda3/envs/pytorch/lib/python3.8/site-packages/numpy/core/shape_base.py", line 423, in stack
    raise ValueError('need at least one array to stack')
ValueError: need at least one array to stack

The list_samlple_file below is the content of train.txt. Every line has the same form as the first line "/home/yzx/lunwen/datasets/FAIR-PLAY/binaural_audios/000383.wav,/home/yzx/lunwen/datasets/FAIR-PLAY/frames/000383". The former path is the path of each audio,and the latter is the folder of the frames of each video.I extracted 10 frames per second in each 10 seconds video.They are splited to 'audio_file' and 'img_folder' in Psedo_dataset.py.I checked this path is true.But I don't know why the error generate.I need your help.
The content of clss Augment_dataset.py is:
class AugmentDataset(StereoDataset, PseudoDataset):
    def __init__(self, opt, list_sample_file):
        self.opt = opt
        PseudoDataset.__init__(self, opt, list_sample_file)
        StereoDataset.__init__(self, opt)

        if "MUSIC" in self.opt.datalist:
            dup_times = 1
            print("dup_times:", dup_times)
        else:
            dup_times = 2
            print("dup_times:", dup_times)
        self.total_samples *= dup_times # in order to align with the length of FAIR-Play dataset

        random.shuffle(self.audios)
        random.shuffle(self.total_samples)

    def __getitem__(self, index):
        if random.uniform(0,1) < self.opt.pseudo_ratio:
            data_ret, data_ret_sep = self._get_pseudo_item(index)
        else:
            data_ret = self._get_stereo_item(index)
            _, data_ret_sep = self._get_pseudo_item(index)
        data_ret.update(data_ret_sep)

        return data_ret

    def __len__(self):
        return min(len(self.audios), len(self.total_samples))

    def name(self):
        return 'AugmentDataset'

The content of Psedo_dataset.py is:
def audio_normalize(samples, desired_rms = 0.1, eps = 1e-4):
    rms = np.maximum(eps, np.sqrt(np.mean(samples**2)))
    samples = samples * (desired_rms / rms)
    return rms / desired_rms, samples

def generate_spectrogram(audio):
    spectro = librosa.core.stft(audio, n_fft=512, hop_length=160, win_length=400, center=True)
    real = np.expand_dims(np.real(spectro), axis=0)
    imag = np.expand_dims(np.imag(spectro), axis=0)
    spectro_two_channel = np.concatenate((real, imag), axis=0)
    return spectro_two_channel

def process_image(image, augment, square=False):
    if square:
        iH, iW = 240, 240
        H, W = 224, 224
    else:
        iH, iW = 240, 480
        H, W = 224, 448
    image = mmcv.imresize(image, (iW,iH))
    h,w,_ = image.shape
    w_offset = w - W
    h_offset = h - H
    left = random.randrange(0, w_offset + 1)
    upper = random.randrange(0, h_offset + 1)
    image = mmcv.imcrop(image, np.array([left, upper, left+W-1, upper+H-1]))

    if augment:
        enhancer = ImageEnhance.Brightness(Image.fromarray(image))
        image = enhancer.enhance(random.random()*0.6 + 0.7)
        enhancer = ImageEnhance.Color(image)
        image = enhancer.enhance(random.random()*0.6 + 0.7)
    return image

class PseudoDataset(data.Dataset):
    def __init__(self, opt, list_sample_file):
        super().__init__()

        self.opt = opt
        self.total_samples = mmcv.list_from_file(list_sample_file)

        normalize = transforms.Normalize(
        mean=[0.485, 0.456, 0.406],
        std=[0.229, 0.224, 0.225]
        )
        vision_transform_list = [transforms.ToTensor(), normalize]
        self.vision_transform = transforms.Compose(vision_transform_list)

        random.seed(self.opt.seed)

        # load background, just one large-sizeimage
        self.bkg_img = mmcv.imread('./data/bkg.png')

        # build binauralizer
        hrtf_dir = "./data/subject03"
        # binauralizer = SourceBinauralizer(use_hrtfs=True, cipic_dir=hrtf_dir)
        self.hrir_db = CIPIC_HRIR(hrtf_dir)
        # encode to ambisonics, and then stereo
        speakers_phi = (2. * np.arange(2*4) / float(2*4) - 1.) * np.pi
        self.speakers_pos = [Position(phi, 0, 1, 'polar') for phi in speakers_phi]
        self.sph_mat = spherical_harmonics_matrix(self.speakers_pos, max_order=1) # shape: [N_array_speakers, 4]

        # parameter of loading audio
        self.exp_audio_len = int(self.opt.audio_length * self.opt.audio_sampling_rate)
        #print("Now load {} box info".format(self.opt.mode))
        #self.box_info = mmcv.load('./dataset/ASMR/scripts/results/{}_box_info.pkl'.format(self.opt.mode))

        if opt.fov == '1/3':
            self.fov = 1/3.
        elif opt.fov == '1/2':
            self.fov = 1/2.
        elif opt.fov == '5/6':
            self.fov = 5/6.
        elif opt.fov == '1':
            self.fov = 1.
        else:
            self.fov = 2/3.

        #self.categories = ['acoustic_guitar', 'banjo', 'bass', 'cello', 'drum', 'harp', 'piano', 'trumpet', 'ukelele']

    def construct_stereo_direct(self, pst_sources):
        stereo = np.zeros((2, self.exp_audio_len))
        for src in pst_sources:
            left_hrir, right_hrir = self.hrir_db.get_closest(src.position)[1:]
            left_signal = np.convolve(src.signal, np.flip(left_hrir, axis=0), 'valid')
            right_signal = np.convolve(src.signal, np.flip(right_hrir, axis=0), 'valid')

            n_valid, i_start = left_signal.shape[0], left_hrir.shape[0] - 1
            stereo[0, i_start:(i_start + n_valid)] += left_signal
            stereo[1, i_start:(i_start + n_valid)] += right_signal

        return stereo

    def construct_stereo_ambi(self, pst_sources):
        # encode to ambisonics
        Y = spherical_harmonics_matrix([src.position for src in pst_sources], max_order=1) # Y shape: [n_signals, 4]
        signals = np.stack([src.signal for src in pst_sources], axis=1) # signals shape: [Len, n_signals]
        ambisonic = np.dot(signals, Y) # shape: [Len, 4]

        array_speakers_sound = np.dot(ambisonic, self.sph_mat.T)
        #array_speakers_sound = np.dot(ambisonic, np.linalg.pinv(self.sph_mat))
        array_sources = [PositionalSource(array_speakers_sound[:, i], speaker_pos, \
            self.opt.audio_sampling_rate) for i, speaker_pos in enumerate(self.speakers_pos)]

        return self.construct_stereo_direct(array_sources)

    def construct_stereo_ambi_direct(self, pst_sources):
        # encode to ambisonics
        Y = spherical_harmonics_matrix([src.position for src in pst_sources], max_order=1)
        signals = np.stack([src.signal for src in pst_sources], axis=1)
        ambisonic = np.dot(signals, Y) # shape: [Len, 4]

        stereo = np.stack((
            ambisonic[:, 0] / 2 + ambisonic[:, 1] / 2,
            ambisonic[:, 0] / 2 - ambisonic[:, 1] / 2
        ))

        return stereo

    def _get_pseudo_item(self, index):
        # ensure the number of audios in a scene
        N = np.random.choice([1,2,3], p=[0.4, 0.5, 0.1])
        chosen_samples = [self.total_samples[index]]
        # avoid repeat sample
        for _ in range(1, N):
            while True:
                new_sample = random.choice(self.total_samples)
                if new_sample not in chosen_samples:
                    chosen_samples.append(new_sample)
                    break

        audio_margin = 0
        init_H = 360
        init_W = 640
        pst_sources = []

        if self.opt.not_use_background:
            cur_bkg_img = np.zeros((init_H, init_W, 3)).astype(np.uint8)
        else:
            # crop background img, exp_shape: [init_H, init_W, 3]
            bkg_start_x = np.random.randint(low=0, high=self.bkg_img.shape[1] - init_W)
            bkg_start_y = np.random.randint(low=0, high=self.bkg_img.shape[0] - init_H)
            cur_bkg_img = mmcv.imcrop(self.bkg_img.copy(),
                np.array([bkg_start_x, bkg_start_y, bkg_start_x+init_W-1, bkg_start_y+init_H-1]))
            #H_bkg, W_bkg, _ = cur_bkg_img.shape

        corner_record = []
        patch_size_record = []
        center_x_record = []
        audio_list = []
        patch_list = []
        actual_N = 0
        #load audio
        for idx, chosen_sample in enumerate(chosen_samples):
            audio_file, img_folder = chosen_sample.split(',')
            # audio part
            audio, audio_rate = librosa.load(audio_file, sr=self.opt.audio_sampling_rate, mono=True)
            #randomly get a start time for the audio segment from the original clip
            audio_len = len(audio) / audio_rate
            assert audio_len - self.opt.audio_length - audio_margin > audio_margin
            audio_start_time = random.uniform(audio_margin, audio_len - self.opt.audio_length - audio_margin)
            audio_end_time = audio_start_time + self.opt.audio_length
            audio_start = int(audio_start_time * self.opt.audio_sampling_rate)
            audio_end = audio_start + self.exp_audio_len
            audio = audio[audio_start:audio_end]
            if self.opt.audio_normal:
                normalizer0, audio = audio_normalize(audio)

            # video part
            # load img **patches**, copy bkg_img and construct a new image
            # load accurate frame
            cur_img_list = natsort.natsorted(glob(osp.join(img_folder, '*.jpg')))
            # get the closest frame to the audio segment
            frame_idx = (audio_start_time + audio_end_time) / 2 * 10
            frame_idx = int(np.clip(frame_idx, 0, len(cur_img_list) - 1))
            img_file = cur_img_list[frame_idx]
            img_patch = mmcv.imread(img_file)

            if self.opt.patch_resize:
                h_patch, w_patch, _= img_patch.shape
                resize_ratio = min(1/normalizer0, init_H / h_patch, init_W / 2 / w_patch)
                img_patch = mmcv.imrescale(img_patch, resize_ratio * random.uniform(0.8, 1))
            H_new, W_new, _ = img_patch.shape

            # just consider the overlap in the horizontal axis
            occupy_matrix = np.ones((init_W))
            # avoid cross border in x dim
            occupy_matrix[:(-W_new + 1)] = 0
            # avoid overlap
            for last_corner_x, W_last in zip(corner_record, patch_size_record):
                occupy_x = max(0, last_corner_x - W_new)
                occupy_matrix[occupy_x : last_corner_x + W_last] = 1

            # random sample position for this mono audio
            free_x_positions = np.where(occupy_matrix == 0)[0]
            if len(free_x_positions) < 2:
                break
            actual_N += 1
            corner_x = random.choice(free_x_positions)
            corner_record.append(corner_x)
            patch_size_record.append(W_new)
            corner_y = random.randint(0, init_H - H_new)

            center_y = corner_y + H_new // 2
            center_x = corner_x + W_new // 2
            center_x_record.append(center_x)
            azimuth = (init_W // 2 - center_x) / init_W * pi * self.fov
            elevation = (init_H // 2 - center_y) / init_H * pi / 2

            if self.opt.visualize_data:
                output_dir = 'others/dataset_visual/{:d}_{:d}'.format(N, index)
                if not osp.exists(output_dir):
                    os.mkdir(output_dir)
                if librosa.__version__ >= '0.8.0':
                    import soundfile as sf
                    sf.write(osp.join(output_dir, '{:d}.wav'.format(idx)), audio.transpose(), audio_rate)
                else:
                    librosa.output.write_wav(osp.join(output_dir, '{:d}.wav'.format(idx)), audio, sr=audio_rate)
            audio_list.append(audio)
            patch_list.append(img_file)
            pst_sources.append(PositionalSource(audio, Position(azimuth, elevation, 3, 'polar'), audio_rate))
            pdb.set_trace()

            if self.opt.blending:
                center = (center_x, center_y)
                mask = 255 * np.ones(img_patch.shape, img_patch.dtype)
                cur_bkg_img = cv2.seamlessClone(img_patch, cur_bkg_img, mask, center, cv2.NORMAL_CLONE)
            else:
                patch_in_start_x = corner_x
                patch_in_start_y = corner_y
                assert patch_in_start_x >= 0
                assert patch_in_start_y >= 0
                cur_bkg_img[patch_in_start_y : patch_in_start_y + H_new, patch_in_start_x : patch_in_start_x + W_new] = img_patch

        if self.opt.stereo_mode == 'direct':
            #print("use direct")
            stereo = self.construct_stereo_direct(pst_sources)
        elif self.opt.stereo_mode == 'ambisonic':
            #print("use ambisonic")
            stereo = self.construct_stereo_ambi(pst_sources)
        elif self.opt.stereo_mode == 'ambidirect':
            #print("use ambidirect")
            stereo = self.construct_stereo_ambi_direct(pst_sources)
        else:
            raise ValueError("please choose right stereo mode")

        normalizer, _ = audio_normalize(stereo[0] + stereo[1])
        stereo = stereo / normalizer
        audio_channel1, audio_channel2 = stereo
        frame = cur_bkg_img
        if self.opt.visualize_data:
            output_dir = 'others/dataset_visual/{:d}_{:d}'.format(N, index)
            if librosa.__version__ >= '0.8.0':
                import soundfile as sf
                sf.write(osp.join(output_dir, 'input_binaural.wav'), stereo.transpose(), audio_rate)
            else:
                librosa.output.write_wav(osp.join(output_dir, 'input_binaural.wav'), stereo, sr=audio_rate)
            mmcv.imwrite(frame, osp.join(output_dir, 'reference.jpg'))
        frame = process_image(frame, self.opt.enable_data_augmentation)
        frame = self.vision_transform(frame)

        #passing the spectrogram of the difference
        audio_diff_spec = torch.FloatTensor(generate_spectrogram(audio_channel1 - audio_channel2))
        audio_mix_spec = torch.FloatTensor(generate_spectrogram(audio_channel1 + audio_channel2))

        data_ret = {'frame': frame, 'audio_diff_spec':audio_diff_spec, 'audio_mix_spec':audio_mix_spec}

        # incorporate separation part
        assert len(patch_list) == len(audio_list)
        left_channel = np.zeros(self.exp_audio_len).astype(np.float32)
        right_channel = np.zeros(self.exp_audio_len).astype(np.float32)
        if len(audio_list) >= 2:
            for cur_audio, center_x in zip(audio_list, center_x_record):
                if center_x < init_W // 3:
                    left_channel += cur_audio
                elif center_x > init_W // 3:
                    right_channel += cur_audio
                else:
                    left_channel += cur_audio
                    right_channel += cur_audio

            _, left_channel = audio_normalize(left_channel)
            _, right_channel = audio_normalize(right_channel)
        else:
            left_channel = audio_channel1
            right_channel = audio_channel2

        if self.opt.visualize_data:
            output_dir = 'others/dataset_visual/{:d}_{:d}'.format(N, index)
            if librosa.__version__ >= '0.8.0':
                import soundfile as sf
                sf.write(osp.join(output_dir, 'gt_left.wav'), left_channel, audio_rate)
                sf.write(osp.join(output_dir, 'gt_right.wav'), right_channel, audio_rate)
            else:
                librosa.output.write_wav(osp.join(output_dir, 'gt_left.wav'), left_channel, audio_rate)
                librosa.output.write_wav(osp.join(output_dir, 'gt_right.wav'), right_channel, audio_rate)

        sep_mix_spec = audio_mix_spec
        sep_diff_spec = torch.FloatTensor(generate_spectrogram(left_channel - right_channel))
        frame_sep_list = frame

        if self.opt.mode == 'train':
            data_ret_sep = {'frame_sep': frame_sep_list, 'sep_diff_spec': sep_diff_spec, 'sep_mix_spec': sep_mix_spec}
        else:
            data_ret_sep = {'frame_sep': frame_sep_list, 'sep_diff_spec': sep_diff_spec, 'sep_mix_spec': sep_mix_spec, 'left_audio': left_channel, 'right_audio': right_channel}

        return data_ret, data_ret_sep

    def __getitem__(self, index):
        data_ret, data_ret_sep = self._get_pseudo_item(index)
        if self.opt.dataset_mode == 'Pseudo_stereo':
            return data_ret
        elif self.opt.dataset_mode == 'Pseudo_sep':
            return data_ret_sep
        else:
            data_ret.update(data_ret_sep)
            return data_ret

    def __len__(self):
        return len(self.total_samples)

    def name(self):
        return 'PseudoDataset'

    def initialize(self, opt):
        pass

Output:
Your issue is here:
signals = np.stack([src.signal for src in pst_sources], axis=1) # signals shape: [Len, n_signals]

It looks like pst_sources is empty, and so you are trying to stack an empty list.


Instruction: Train on top of a Torchscript model

Input: I currently have a Torchscript model I load via torch.jit.load. I would like to take some data I have and train on top of these weights, however I cannot find out how to train a serialised torchscript model.

Output: Turns out that the returned ScriptModule does actually support training: https://pytorch.org/docs/stable/generated/torch.jit.ScriptModule.html#torch.jit.ScriptModule.train


Instruction: Delete and Reinitialize pertained BERT weights / parameters

Input:
I tried to fine-tune BERT for a classification downstream task.
Now I loaded the model again and I run into the following warning:
Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight']

This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).

[Screen Shot][1]
[1]: https://i.stack.imgur.com/YJZVc.png
I already deleted and reinstalled transformers==4.6.0 but nothing helped.
I thought maybe through the parameter "force_download=True" it might get the original weights back but nothing helped.
Shall I continue and ignore the warning? Is there a way to delete the model checkpoints such when the model is downloaded the weights are fixed again?
Thanks in advance!
Best,
Alex

Output:
As long as you're fine-tuning a model for a downstream task this warning can be ignored. The idea is that the [CLS] token weights from the pretrained model aren't going to be useful for downstream tasks and need to be fine-tuned.
Huggingface randomly initializes them because you're using bert-base-cased which is a BertForPretraing model and you're created a BertModel from it. The warning is to ensure that you understand the difference of directly using the pretrained model directly or if you're planning on finetuning them for a different task.
On that note if you plan working on a classification task I'd recommend using their BertForSequenceClassification class instead.
TL;DR you can ignore it as long as you're finetuning.


Instruction: Computing the loss of a function of predictions with pytorch

Input:
I have a convolutional neural network that predicts 3 quantities: Ux, Uy, and P. These are the x velocity, y-velocity, and pressure field. They are all 2D arrays of size [100,60], and my batch size is 10.
I want to compute the loss and update the network by calculating the CURL of the predicted velocity with the CURL of the target velocity. I have a function that does this: v = curl(Ux_pred, Uy_pred). Given the predicted Ux and Uy, I want to compute the loss by comparing it to ground truth targets that I have: true_curl = curl(Ux_true, Uy_true) - I've already computed the true curl and added it to my Y data, as the fourth channel.
However, I want my network to only predict Ux, Uy, and P. I want my NN parameters to update based on the LOSS of the curls to improve the accuracy of Ux and Uy. The loss of the curl has to be in terms of Ux and Uy. I have been trying to do this using Pytorch autograd, and have already read many similar questions, but I just can't get it to work. This is my code so far:
        print("pred_Curl shape:", np.shape(pred_curl))
        print("pred_Ux shape:", np.shape(pred[:,0,:,:]))
        print("pred_Uy shape:", np.shape(pred[:,1,:,:]))
        true_curl = torch.from_numpy(y[:,3,:,:]) # not sure where to use the true curl?

        pred_curl = Variable(pred_curl, requires_grad=True)

        pred_ux = pred[:,0,:,:]
        pred_uy = pred[:,1,:,:]

        pred_ux = Variable(pred_ux, requires_grad=True)
        pred_uy = Variable(pred_uy, requires_grad=True)

        grad_tensor = torch.autograd.grad(outputs=pred_curl, inputs=(pred_ux, pred_uy),
                       grad_outputs=torch.ones_like(pred_curl),
                       retain_graph=True,create_graph=True)

        loss = torch.sum(grad_tensor)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

This has the following output:
pred_Curl shape: torch.Size([10, 100, 60])
pred_Ux shape: torch.Size([10, 100, 60])
pred_Uy shape: torch.Size([10, 100, 60])

RuntimeError: One of the differentiated Tensors appears to not have been used in the graph.
Set allow_unused=True if this is the desired behavior.

Any help would be appreciated!
Edit: Here is my curl function:
    def discrete_curl(self,x,y,new_arr):
            for m in range(100):
                for n in range(60):
                    if n <= 58:
                        if m <= 98:
                            if x[m,n] != 0 and y[m,n] != 0:
                                new_arr[m,n] = ((y[m+1,n] - y[m-1,n]) / 2*1) - ((x[m,n+1] - x[m,n-1]) / 2*1)
            return new_arr

Where x and y are Ux ad Uy, and new_arr is the curl output.

Output:
You could try something like this:
def discrete_curl(self, pred):
        new_arr = torch.zeros((pred.shape[0],100,60))
        for pred_idx in range(pred.shape[0]):
            for m in range(100):
                for n in range(60):
                    if n <= 58:
                        if m <= 98:
                            if pred[pred_idx,0,m,n] != 0 and pred[pred_idx,1,m,n] != 0:
                                new_arr[pred_idx,m,n] = ((pred[pred_idx,1,m+1,n] - pred[pred_idx,1,m-1,n]) / 2*1) - ((pred[pred_idx,0,m,n+1] - pred[pred_idx,0,m,n-1]) / 2*1)
        return new_arr

pred_curl = discrete_curl(pred)
true_curl = torch.from_numpy(y[:,3,:,:])
loss = torch.nn.functional.mse_loss(pred_curl, true_curl)
optimizer.zero_grad()
loss.backward()
optimizer.step()

I think the curl computation can be optimized, but I tried to stick to your structure for the most part.


Instruction: RuntimeError: No CUDA GPUs are available

Input:
I want to train a gpt2 model in my laptop and I have a GPU in it and my os is windows , but I always got this error in python:
torch._C._cuda_init()
RuntimeError: No CUDA GPUs are available

when I tried to check the availability of GPU in the python console, I got true:
import torch
torch.cuda.is_available()
Out[4]: True

but I can't get the version by
nvcc version
#or  nvcc --version
NameError: name 'nvcc' is not defined

I use this command to install CUDA
conda install pytorch torchvision torchaudio cudatoolkit=10.2 -c pytorch

What can I do to make the GPU available for python?

Output:
In my case the problem was that the CUDA drivers that I was trying to install, didn't support my GPU model. In your case, please check which CUDA driver supports your GPU model. You are now installing 10.2. In my case CUDA 11.0 and 11.2 supported my GPU model but not 11.3 which I was trying to install.
If you got the same error after a while, which can happen if you run a cloud vm which hardware can be updated automatically, here is how to solve it:
Remove CUDA drivers
sudo apt-get remove --purge nvidia*

Then reinstall the drivers as follows. Note! in this case I have debian distro on x64 system.
wget https://developer.download.nvidia.com/compute/cuda/repos/debian11/x86_64/cuda-keyring_1.0-1_all.deb
sudo dpkg -i cuda-keyring_1.0-1_all.deb
sudo add-apt-repository contrib
sudo apt-get update
sudo apt-get -y install cuda

Get the correct commands that works for your distro and system from the link: https://developer.nvidia.com/cuda-downloads?target_os=Linux
Good luck!


Instruction: Binary classification - BCELoss and model output size not corresponding

Input:
I'm doing a binary classification, hence I used a binary cross entropy loss:
criterion = torch.nn.BCELoss()

However, I'm getting an error:
Using a target size (torch.Size([64, 1])) that is different to the input size (torch.Size([64, 2])) is deprecated. Please ensure they have the same size.

My model ends with:
    x = self.wave_block6(x)
    x = self.sigmoid(self.fc(x))
    return x.squeeze()

I tried removing the squeeze, but to no avail. My batch size is 64. It seems like I'm doing something simple wrong here. Is my model giving 1 output and BCE loss expecting 2 inputs? Which loss should I use then?

Output:
Binary Cross-Entropy Loss  (BCELoss) is used for binary classification tasks. Therefore if N is your batch size, your model output should be of shape [64, 1] and your labels must be of shape [64].Therefore just squeeze your output at the 2nd dimension and pass it to the loss function -
Here is a minimal working example
import torch
a = torch.randn((64, 1))
b = torch.randn((64))
loss = torch.nn.BCELoss()

b = torch.round(torch.sigmoid(b)) # just to create some labels
a = torch.sigmoid(a).squeeze(1)
l = loss(a, b)

Update - Basing on the conversation in the comments, focal loss can be defined as follows -
class focalLoss(nn.Module):
    def __init__(self, alpha=0.25, gamma=3):
        super(focalLoss, self).__init__()
        self.alpha = alpha
        self.gamma = gamma

    def forward(self, pred_logits: torch.Tensor, target: torch.Tensor):
        batch_size = pred_logits.shape[0]
        pred = pred.view(batch_size, -1)
        target = target.view(batch_size, -1)
        pred = pred_logits.sigmoid()
        ce = F.binary_cross_entropy(pred_logits, target, reduction='none')
        alpha = target * self.alpha + (1. - target) * (1. - self.alpha)
        pt = torch.where(target == 1, pred, 1 - pred)
        return alpha * (1. - pt) ** self.gamma * ce


Instruction: FastAI fastbook - what does it do and why do I need to setup a book?

Input:
I tried running on my google colab notebook:
!pip install -Uqq fastbook
import fastbook

as it is written in the FastAI book, chapter 2.
but nor the book or anywhere on google there is an explanation on what is this liberty at all.
amazingly, the page for it does not include any explanation on what fastbook does-  only about some course for deep learning.
so,

what does it do?

also, when I run:
fastbook.setup_book()

what does that do? in which way does it setup a book, and what kind of book is it?

ty.

Output:
fastbook.setup_book()

It is used setup when you are using google colab specifically and working with FastAI library. It helps to connect the colab notebook to google drive using an authentication token.


Instruction: AWS Sagemaker custom PyTorch model inference on raw image input

Input:
I am new to AWS Sagemaker. I have custom CV PyTorch model locally and deployed it to Sagemaker endpoint. I used custom inference.py code to define model_fn, input_fn, output_fn and predict_fn methods. So, I'm able to generate predictions on json input, which contains url to the image, the code is quite straigtforward:
def input_fn(request_body, content_type='application/json'):

    logging.info('Deserializing the input data...')

    image_transform = transforms.Compose([
            transforms.Resize(size=(224, 224)),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ])

    if content_type:
        if content_type == 'application/json':
            input_data = json.loads(request_body)
            url = input_data['url']
            logging.info(f'Image url: {url}')
            image_data = Image.open(requests.get(url, stream=True).raw)

        return image_transform(image_data)
    raise Exception(f'Requested unsupported ContentType in content_type {content_type}')

Then I am able to invoke endpoint with code:
client = boto3.client('runtime.sagemaker')
inp = {"url":url}
inp = json.loads(json.dumps(inp))

response = client.invoke_endpoint(EndpointName='ENDPOINT_NAME',
                                  Body=json.dumps(inp),
                                  ContentType='application/json')

The problem is, I see, that locally url request return slightly different image array comparing to the one on Sagemaker. Which is why on the same URL I obtain slightly different predictions. To check that at least model weights are the same I want to generate predictions on image itself, downloaded locally and to Sagemaker. But I fail trying to put image as input to endpoint. E.g.:
def input_fn(request_body, content_type='application/json'):

    logging.info('Deserializing the input data...')

    image_transform = transforms.Compose([
            transforms.Resize(size=(224, 224)),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ])

    if content_type == 'application/x-image':
        image_data = request_body

        return image_transform(image_data)
    raise Exception(f'Requested unsupported ContentType in content_type {content_type}')

Invoking endpoint I experience the error:

ParamValidationError: Parameter validation failed:
Invalid type for parameter Body, value: {'img': <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=630x326 at 0x7F78A61461D0>}, type: <class 'dict'>, valid types: <class 'bytes'>, <class 'bytearray'>, file-like object

Does anybody know how to generate Sagemaker predictions by Pytorch model on images?

Output:
As always, after asking I found a solution. Actually, as the error suggested, I had to convert input to bytes or bytearray. For those who may need the solution:
from io import BytesIO

img = Image.open(open(PATH, 'rb'))
img_byte_arr = BytesIO()
img.save(img_byte_arr, format=img.format)
img_byte_arr = img_byte_arr.getvalue()

client = boto3.client('runtime.sagemaker')

response = client.invoke_endpoint(EndpointName='ENDPOINT_NAME
                                  Body=img_byte_arr,
                                  ContentType='application/x-image')
response_body = response['Body']
print(response_body.read())


Instruction: Can I reduce number of GPUs without terminating the training?

Input:
Let's say I am using multiple GPUs (0,1,2,3) on one machine and later someone else also needs to use GPUs on this machine. Is there a way for me to reduce the number of gpu usage (i.e. only use 0 and 1) from my training without terminating the training and start over again? I don't want to waste the training I already did.
This sounds like a common need in a team. Is that possible?

Output:
I do not think that this is possible. You should save checkpoints so that you can later continue training where you left. This is possible with Hugging Face API.
training_args = Seq2SeqTrainingArguments(
        output_dir=model_directory,
        num_train_epochs=args.epochs,
        do_eval=True,
        evaluation_strategy='epoch',
        load_best_model_at_end=True, # the last checkpoint is the best model wrt metric_for_best_model
        metric_for_best_model='eval_loss',
        greater_is_better=False,
        save_total_limit=args.epochs
    )

The save_total_limit is the number of checkpoints it will save. In the above case it would save a single checkpoint after each epoch. You can adjust the number based on your memory.


Instruction: How does one run PyTorch on a A40 GPU without errors (with DDP too)?

Input:
I tried running my pytorch code but got this error:
A40 with CUDA capability sm_86 is not compatible with the current PyTorch installation.
The current PyTorch install supports CUDA capabilities sm_37 sm_50 sm_60 sm_61 sm_70 sm_75 compute_37.
If you want to use the A40 GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/

  warnings.warn(incompatible_device_warn.format(device_name, capability, " ".join(arch_list), device_name))
Using backend: pytorch
/home/miranda9/miniconda3/envs/metalearningpy1.7.1c10.2/lib/python3.8/site-packages/torch/cuda/__init__.py:104: UserWarning:
A40 with CUDA capability sm_86 is not compatible with the current PyTorch installation.
The current PyTorch install supports CUDA capabilities sm_37 sm_50 sm_60 sm_61 sm_70 sm_75 compute_37.
If you want to use the A40 GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/

  warnings.warn(incompatible_device_warn.format(device_name, capability, " ".join(arch_list), device_name))
Traceback (most recent call last):
  File "/home/miranda9/ML4Coq/ml4coq-proj-src/embeddings_zoo/tree_nns/main_brando.py", line 305, in <module>
    main_distributed()
  File "/home/miranda9/ML4Coq/ml4coq-proj-src/embeddings_zoo/tree_nns/main_brando.py", line 201, in main_distributed
    mp.spawn(fn=train, args=(opts,), nprocs=opts.world_size)
  File "/home/miranda9/miniconda3/envs/metalearningpy1.7.1c10.2/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 199, in spawn
    return start_processes(fn, args, nprocs, join, daemon, start_method='spawn')
  File "/home/miranda9/miniconda3/envs/metalearningpy1.7.1c10.2/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 157, in start_processes
    while not context.join():
  File "/home/miranda9/miniconda3/envs/metalearningpy1.7.1c10.2/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 118, in join
    raise Exception(msg)
Exception:

-- Process 0 terminated with the following error:
Traceback (most recent call last):
  File "/home/miranda9/miniconda3/envs/metalearningpy1.7.1c10.2/lib/python3.8/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap
    fn(i, *args)
  File "/home/miranda9/ML4Coq/ml4coq-proj-src/embeddings_zoo/tree_nns/main_brando.py", line 210, in train
    setup_process(opts, rank, master_port=opts.master_port, world_size=opts.world_size)
  File "/home/miranda9/ultimate-utils/ultimate-utils-proj-src/uutils/torch/distributed.py", line 165, in setup_process
    dist.init_process_group(backend, rank=rank, world_size=world_size)
  File "/home/miranda9/miniconda3/envs/metalearningpy1.7.1c10.2/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 455, in init_process_group
    barrier()
  File "/home/miranda9/miniconda3/envs/metalearningpy1.7.1c10.2/lib/python3.8/site-packages/torch/distributed/distributed_c10d.py", line 1960, in barrier
    work = _default_pg.barrier()
RuntimeError: NCCL error in: /opt/conda/conda-bld/pytorch_1607369981906/work/torch/lib/c10d/ProcessGroupNCCL.cpp:31, unhandled cuda error, NCCL version 2.7.8

but then it sends me to download it for my mac...? which is weird. What version of
pytorch, cuda, cudnn, nccl and other things do I need for a GPU A40?
to see the code I ran and conda env info see this: https://github.com/pytorch/pytorch/issues/58794

related links

https://github.com/pytorch/pytorch/issues/45021
https://github.com/pytorch/pytorch/issues/45028
https://github.com/pytorch/pytorch/issues/58794
How does one install pytorch 1.9 in an HPC that seems to refuse to cooperate?

Output:
My guess is the following:
A40 gpus have CUDA capability of sm_86 and they are only compatible with CUDA >= 11.0. But CUDA >= 11.0 is only compatible with PyTorch >= 1.7.0 I believe.
So do:
conda install pytorch==1.7.1 torchvision==0.8.2 torchaudio==0.7.2 cudatoolkit=11.0 -c pytorch

or
conda install pytorch torchvision torchaudio cudatoolkit=11.0 -c pytorch

or
conda install pytorch torchvision torchaudio cudatoolkit=11.1 -c pytorch

if you are in an HPC you might want to do:
module load gcc/9.2.0

#module load cuda-toolkit/10.2
module load cuda-toolkit/11.1

this seemed to work:
(metalearning) miranda9~/automl-meta-learning $ python -c "import uutils; uutils.torch_uu.gpu_test()"

device name: A40

Success, no Cuda errors means it worked see:
out=tensor([[2.3272],
        [5.6796]], device='cuda:0')
(metalearning) miranda9~/automl-meta-learning $
(metalearning) miranda9~/automl-meta-learning $
(metalearning) miranda9~/automl-meta-learning $
(metalearning) miranda9~/automl-meta-learning $ conda list | grep torch
_pytorch_select           0.1                       cpu_0
pytorch                   1.7.1           py3.9_cuda11.0.221_cudnn8.0.5_0    pytorch
torchaudio                0.7.2                      py39    pytorch
torchmeta                 1.7.0                    pypi_0    pypi
torchvision               0.8.2           cpu_py39ha229d99_0


Instruction: AttributeError: module 'torch' has no attribute 'rfft' with PyTorch

Input:
I am getting an error using a code that should work according to the documentation.
The goal is to calculate the Feature Similarity Index Measure (FSIM) using the piq Python library.
Terminal Output:
TiffPage 1: ByteCounts tag is missing
Traceback (most recent call last):
  File "...\.venv\lib\site-packages\IPython\core\interactiveshell.py", line 3441, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-2-3044cfc208ce>", line 1, in <module>
    runfile('.../stackoverflow.py', wdir='...')
  File "...\plugins\python\helpers\pydev\_pydev_bundle\pydev_umd.py", line 197, in runfile
    pydev_imports.execfile(filename, global_vars, local_vars)  # execute the script
  File "...\plugins\python\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File ".../stackoverflow.py", line 15, in <module>
    main()
  File "...\.venv\lib\site-packages\torch\autograd\grad_mode.py", line 27, in decorate_context
    return func(*args, **kwargs)
  File ".../stackoverflow.py", line 10, in main
    fsim_index: torch.Tensor = piq.fsim(x, y, data_range=1., reduction='none')
  File "...\.venv\lib\site-packages\piq\fsim.py", line 84, in fsim
    pc_x = _phase_congruency(
  File "...\.venv\lib\site-packages\piq\fsim.py", line 241, in _phase_congruency
    imagefft = torch.rfft(x, 2, onesided=False)
AttributeError: module 'torch' has no attribute 'rfft'

Code:
from skimage import io
import torch
import piq

@torch.no_grad()
def main():
    x = torch.tensor(io.imread('scikit_image\cover\cover_1.tiff')).permute(2, 0, 1)[None, ...] / 255.
    y = torch.tensor(io.imread('scikit_image\stego\stego_1.tiff')).permute(2, 0, 1)[None, ...] / 255.

    fsim_index: torch.Tensor = piq.fsim(x, y, data_range=1., reduction='none')

    print(fsim_index)

if __name__ == "__main__":
    main()

Output:
The latest version of pytorch implements all fast fourier functions in the module torch.fft, apparently piq rely on an older version of pytorch, so if you want to run piq consider downgrading your pytorch version, for example:
pip3 install torch==1.7.1 torchvision==0.8.2


Instruction: How to fix an error with the quickstart tutorial for pytorch?

Input:
I am trying to follow the tutorial on pytorch HERE, but there seems to be a problem. I have created a custom dataloader named training_data that returns an object as required HERE which is a dictionary
{"image": image, "label": label}

where image is a tensor and label is a string. I then follow the tutorial and create a DataLoader as follows:
train_dataloader = DataLoader(training_data, batch_size=batch_size)

and use that DataLoader in the method train:
def train(dataloader, model, loss_fn, optimizer):
    size = len(dataloader)
    for batch, (X, y) in enumerate(dataloader):
        X, y = X.to(device), y.to(device)

        # Compute prediction error
        pred = model(X)
        loss = loss_fn(pred, y)

        # Backpropagation
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if batch % 100 == 0:
            loss, current = loss.item(), batch * len(X)
            print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")

However, when I call the training method for a batch
train(train_dataloader, model, loss_fn, optimizer)

I get an error
Traceback (most recent call last):
  File "train_network.py", line 110, in <module>
    train(train_dataloader, model, loss_fn, optimizer)
  File "train_network.py", line 76, in train
    X, y = X.to(device), y.to(device)
AttributeError: 'str' object has no attribute 'to'

as y is a string with the content label. What am I doing wrong?

Output:
Your labels y need to be torch tensors. Since you currently have strings, and assuming you are doing classification among n classes, you can simply map them using a list. For example, with three classes, inside the __init__ of your Dataset class:
self.label_names = ["class1", "class2", "class3"]

Then, in __getitem__, you could add:
label = torch.tensor(label_names.index(label))

where label previously stored a string.


Instruction: Pytorch List of all gradients in a model

Input:
I'm trying to clip my gradients in a simple deep network model (for RL). But for that I want to fetch statistics of gradients in each epochs, e.g. mean, max etc. Through this I will be able to determine the threshold value to clip my gradients to.
So the way I can approach this was if there was any way to fetch all the calculated gradients as an array after model.backwards() step.
How can I do this? Or is there any other way to determine this hyper-parameter?

Output:
You can iterate over the parameters to obtain their gradients. For example,
for param in model.parameters():
    print(param.grad)

The example above just prints the gradient, but you can apply it suitably to compute the information you need.


Instruction: Running BERT SQUAD model on GPU

Input:
I am using the BERT Squad model to ask the same question on a collection of documents (>20,000). The model currently runs on my CPU and it takes around a minute to process a single document - which means that I'll need several days to complete the program.
I was wondering if I could speed this up by running the model on a GPU. However, I am new to GPUs and I don't know how to send these inputs and the model to the device (Titan xp).
The code is borrowed from Chris McChormick.
import torch
import tensorflow as tf
from transformers import BertForQuestionAnswering
from transformers import BertTokenizer

model = BertForQuestionAnswering.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad')
tokenizer = BertTokenizer.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad')

'question' and 'answer_text' are the question and the context string respectively.
    input_ids = tokenizer.encode(question, answer_text)

    # ======== Set Segment IDs ========
    # Search the input_ids for the first instance of the `[SEP]` token.
    sep_index = input_ids.index(tokenizer.sep_token_id)

    if len(input_ids)>512:
        input_ids=input_ids[:512]

    num_seg_a = sep_index + 1
    num_seg_b = len(input_ids) - num_seg_a

    # Construct the list of 0s and 1s.
    segment_ids = [0]*num_seg_a + [1]*num_seg_b

    # There should be a segment_id for every input token.
    assert len(segment_ids) == len(input_ids)

    # ======== Evaluate ========
    # Run our example through the model.
    outputs = model(torch.tensor([input_ids]), # The tokens representing our input text.
                    token_type_ids=torch.tensor([segment_ids]), # The segment IDs to differentiate question from answer_text
                    return_dict=True)

    start_scores = outputs.start_logits
    end_scores = outputs.end_logits

I know that I can send the model to the GPU using model.tocuda(). But how do I send the inputs, train the model, and the retreive output from the GPU?

Output:
It's been a while, but I'll answer anyway in the hope that maybe it will help someone.
You can copy each tensor to the GPU using the to method.
For example your batch contains 4 pytorch tensors: input ids, attention masks, segment ids and labels
device = torch.device("cuda")
b_input_ids = batch[0].to(device)
b_input_mask = batch[1].to(device)
b_seg_ids = batch[2].to(device)
b_labels = batch[2].to(device)

Then,You can use the .cpu() to transfer the logits and labels from the gpu back to the cpu. In example;
start_logits = start_logits.detach().cpu()
end_logits = end_logits.detach().cpu()

or similarly to(device) you can use
start_logits = start_logits.to('cpu')
end_logits = end_logits.to('cpu')

Note that: Since you will be using them in the model, you will probably need to add .numpy() to the end and convert them to a numpy array.
Source:https://discuss.pytorch.org/t/time-to-transform-gpu-to-cpu-with-cpu/18856


Instruction: Loss is nan, stopping training when training Mask-RCNN multi-class segmentation

Input:
number of train data: 346
number of test data: 69
Epoch: [0] [0/346] eta: 0:35:20 lr: 0.000019 loss: -312.6024 (-312.6024) loss_classifier: 1.5789 (1.5789) loss_box_reg: 0.1299 (0.1299) loss_mask: -314.3485 (-314.3485) loss_objectness: 0.0266 (0.0266) loss_rpn_box_reg: 0.0106 (0.0106) time: 6.1275 data: 0.1599 max mem: 0
Loss is nan, stopping training
{‘loss_classifier’: tensor (nan, grad_fn = ), ‘loss_box_reg’: tensor (nan, grad_fn = ), ‘loss_mask’: tensor (nan, grad_fn = ), ’ tensor (nan, grad_fn = ), ‘loss_rpn_box_reg’: tensor (nan, grad_fn = )}
An exception has occurred, use% tb to see the full traceback.

SystemExit : 1

And this is the dataset code:

class maskrcnn_Dataset(torch.utils.data.Dataset):
   def __init__(self, root, transforms=None):
    self.root = root
    self.transforms = transforms
    # load all image files, sorting them to
    # ensure that they are aligned
    self.imgs = list(sorted(os.listdir(os.path.join(root, "images"))))
    self.masks = list(sorted(os.listdir(os.path.join(root, "masks"))))
    #self.class_masks = list(sorted(os.listdir(os.path.join(root, "SegmentationClass"))))

def __getitem__(self, idx):
    # load images ad masks
    img_path = os.path.join(self.root, "images", self.imgs[idx])
    x=self.imgs[idx].split('.')
    mask_path = os.path.join(self.root, "masks", self.masks[idx])
    #class_mask_path = os.path.join(self.root, "SegmentationClass", self.class_masks[idx])

    #read and convert image to RGB
    img = cv2.imread(img_path)
    mask_for_all=[]
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    # note that we haven't converted the mask to RGB,
    # because each color corresponds to a different instance
    # with 0 being background
    # mask = Image.open(mask_path)
    mask_folder=os.path.join(self.root,"masks")
    source_mask = os.path.join(mask_folder, x[0])
    #print(os.listdir(source_mask))
    boxes = []
    xx=trier(os.listdir(source_mask))
    #print(xx)
    for file_name in xx:
        mask = Image.open(os.path.join(source_mask,file_name))
        mask = np.array(mask)
        mask_for_all.append(mask)
        obj_ids = np.unique(mask)
        obj_ids = obj_ids[1:]
        masks = mask == obj_ids[:, None, None]
        num_objs = len(obj_ids)
        for i in range(num_objs):
            pos = np.where(masks[i])
            xmin = np.min(pos[1])
            xmax = np.max(pos[1])
            ymin = np.min(pos[0])
            ymax = np.max(pos[0])
            boxes.append([xmin, ymin, xmax, ymax])

    num_objs=len(boxes)
    boxes = torch.as_tensor(boxes, dtype=torch.float32)
    # there is only one class
    if(self.root.find("train")!=-1):
        #print("bisgltjf")

        labels =class_ids_train[class_ids_train_names.index(self.imgs[idx])]
        #print(labels)
    else:
        labels =class_ids_val[class_ids_val_names.index(self.imgs[idx])]
        #print('l3assba')

    #labels = np.array([])
    #for i in range(masks.shape[0]):
     #   labels = np.append(labels, (masks[i] * class_mask).max())

    labels = torch.as_tensor(labels, dtype=torch.int64)
    #print(boxes,":",labels)
    masks = torch.as_tensor(mask_for_all, dtype=torch.uint8)
    #print(labels)
    #print(masks)
    #print(masks.shape)

    image_id = torch.tensor([idx])
    area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])
    # suppose all instances are not crowd
    iscrowd = torch.zeros((num_objs,), dtype=torch.int64)
    #print(img.shape)
    #print(self.imgs[idx])

    target = {}
    target["boxes"] = boxes
    #print(boxes)
    target["labels"] = labels
    #print(labels.shape)
    target["masks"] = masks
    #print(masks.shape)
    target["image_id"] = image_id
    #print(image_id.shape)
    target["area"] = area
    #print(area)
    target["iscrowd"] = iscrowd
    #print(iscrowd.shape)

    if self.transforms is not None:
        img, target = self.transforms(img, target)

    return img, target

def __len__(self):
    return len(self.imgs)

Output:
There can be two issues:

Check the coordinate of boxes, make sure [xmin, ymin, xmax, ymax] is positive

Make sure the mask's length is the same as boxes.


Instruction: Data Loading in Pytorch for a dataset having all the classes in same folder

Input:
I am new to deep learning and Pytorch. I have data set of 6000 images that have all four classes in a single folder. I used the following snippet to upload my data.

torchvision.datasets.ImageFolder(root='/content/drive/My Drive/DFU/base_dir/train_dir', transform=None)

I read that for ImageFolder, the images should be organized into sub-folders based on class labels. However, my dataset has all four class images in a single folder. I have a .csv file that contains the one-hot-encoded class label for each image. How to load my dataset to Pytorch?

Output:
The simplest solution would be to reorganise the images into class-subfolders based on the csv file, and load as intended by ImageFolder:
import pandas as pd
from pathlib import Path

root = '/content/drive/My Drive/DFU/base_dir/train_dir'
my_csv_file = ...

# Loading csv as {image:class,...} format
df = pd.read_csv(my_csv_file).set_index('images')
class_dict = df.idxmax(axis="columns").to_dict()

# Moving files to class-named subfolders
for path in Path(root).iterdir():
    if path.is_file() and path.name in class_dict.keys():
        path.rename(Path(path.parent, class_dict[path.name], path.name)

# Loading dataset
dataset = torchvision.datasets.ImageFolder(root=root, transform=None)


Instruction: Is torch.empty_like() dependent on the input value as well as input size?

Input:
The description in the torch docs for torch.empty_like says:

torch.empty_like(input, *, dtype=None, layout=None, device=None, requires_grad=False, memory_format=torch.preserve_format) → Tensor
Returns an uninitialized tensor with the same size as input. torch.empty_like(input) is equivalent to torch.empty(input.size(), dtype=input.dtype, layout=input.layout, device=input.device).
Parameters
input (Tensor) – the size of input will determine size of the output tensor.

What I do is :
>>> torch.empty(3,4)
tensor([[-1.8597e+15,  4.5657e-41, -1.8597e+15,  4.5657e-41],
        [ 4.4842e-44,  0.0000e+00,  8.9683e-44,  0.0000e+00],
        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00]])

>>> c1
tensor([[ 0,  1,  2,  3],
        [ 4,  5,  6,  7],
        [ 8,  9, 10, 11]])

>>> torch.empty_like(c1)
tensor([[139942262173040,  93851872482144,               1,               0],
        [              0,               0,  93851872492496,               0],
        [              0,               0,               0,               0]])

>>> d
tensor([[0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.]])

>>> torch.empty_like(d)
tensor([[-8.6092e-25,  3.0620e-41,  0.0000e+00,  0.0000e+00],
        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00],
        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00]])

It seems that the tensor returned by torch.empty_like depends on input value, contrary to the description in the docs. Can someone explain this?

Output:
The docs description is correct. I am not sure if you are confused by torch.empty_like returning different outputs on different calls, but you can see this is also the behaviour of torch.empty by calling e.g. torch.empty((2,3), dtype=torch.int64) multiple times.
Note torch.empty_like does depend on the dtype of the input (but not its specific values).


Instruction: Load pytorch model from S3 bucket

Input:
I want to load a pytorch model (model.pt) from a S3 bucket. I wrote the following code:
from smart_open import open as smart_open
import io

load_path = "s3://serial-no-images/yolo-models/model4/model.pt"
with smart_open(load_path) as f:
    buffer = io.BytesIO(f.read())
    model.load_state_dict(torch.load(buffer))

This results in the following error:
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 64: invalid start byte

One solution would be to download the model locally, but I want to avoid this and load the model directly from S3. Unfortunately, I couldn't find a good solution for that online. Can someone help me out here?

Output:
According to the documentation, the following works:
from smart_open import open as smart_open
import io

load_path = "s3://serial-no-images/yolo-models/model4/model.pt"
with smart_open(load_path, 'rb') as f:
    buffer = io.BytesIO(f.read())
    model.load_state_dict(torch.load(buffer))

I have tried this before, but didn't see that I have to set 'rb' as argument.


Instruction: issue in loading Model using PyTorch in google-collaboratory

Input:
I am trying to Load the Model in google_collaboratory to get evaluate it and generate all the statistics results.
My trying
import torch
import torch.nn.functional as F
import torch.optim as optim
import torch.backends.cudnn as cudnn
import torch.backends.cudnn as cudnn
import numpy as np
import torch.nn as nn
import os

def load_checkpoint(filepath):
    checkpoint = torch.load(filepath)
    model = fc_model.Network(checkpoint['input_size'],
                             checkpoint['output_size'],
                             checkpoint['hidden_layers'])
    model.load_state_dict(checkpoint['state_dict'])

    return model

PATH = "/content/gdrive/MyDrive/best.pt"
state_dict = load_checkpoint(PATH)

The Error
---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-24-0515f2edfa1a> in <module>()
     18
     19 PATH = "/content/gdrive/MyDrive/best.pt"
---> 20 state_dict = load_checkpoint(PATH)

2 frames
/usr/local/lib/python3.7/dist-packages/torch/serialization.py in _load(zip_file, map_location, pickle_module, pickle_file, **pickle_load_args)
    849     unpickler = pickle_module.Unpickler(data_file, **pickle_load_args)
    850     unpickler.persistent_load = persistent_load
--> 851     result = unpickler.load()
    852
    853     torch._utils._validate_loaded_sparse_tensors()

ModuleNotFoundError: No module named 'models'

---------------------------------------------------------------------------
NOTE: If your import is failing due to a missing package, you can
manually install dependencies using either !pip or !apt.

To view examples of installing some common dependencies, click the
"Open Examples" button below.
---------------------------------------------------------------------------

I tried to install some library but it gives me the same is there anyways to load models inside google collaboratory.

Output:
This problem is that when you save the weight you actually uses torch.save(model instead of model.state_dict()
One way to solve this is import the models "the same way you did when train". This is important as when you save the whole model it save the name reference along with the weight.
Maybe you'll need to upload it if models is a file. If it's an object then just put it to a cell and it'll work.


Instruction: PyTorch does not make initial weights random

Input:
I created a Neural Network that takes two greyscale images 14x14 pixels portraying a digit (from MNIST database) and returns 1 if the first digit is less or equal to the second digit, returns 0 otherwise. The code runs, but every time the initial weights are the same. They should be random
Forcing the initial weights to be random, by using the following line of code in the Net class, does not help.
torch.nn.init.normal_(self.layer1.weight, mean=0.0, std=0.01)

Here is the code of the "main.py" file:
import os; os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
import torch
import torch.nn as nn
from dlc_practical_prologue import *

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.layer1 = nn.Linear(2*14*14, 32)
        #torch.nn.init.normal_(self.layer1.weight, mean=0.0, std=0.01)
        #self.layer2 = nn.Linear(100, 100)
        #self.layer3 = nn.Linear(100, 100)
        self.layer2 = nn.Linear(32, 1)

    def forward(self, x):
        x = torch.relu(self.layer1(x))
        #x = torch.relu(self.layer2(x))
        #x = torch.relu(self.layer3(x))
        x = torch.sigmoid(self.layer2(x))
        return x

if __name__ == '__main__':
    # Data initialization
    N = 1000
    train_input, train_target, train_classes, _, _, _, = generate_pair_sets(N)
    _, _, _, test_input, test_target, test_classes = generate_pair_sets(N)

    train_input = train_input.view(-1, 2*14*14)
    test_input = test_input.view(-1, 2*14*14)

    train_target = train_target.view(-1, 1)
    test_target = test_target.view(-1, 1)

    # I convert the type to torch.float32
    train_input, train_target, train_classes, test_input, test_target, test_classes = \
        train_input.type(torch.float32), train_target.type(torch.float32), train_classes.type(torch.long), \
        test_input.type(torch.float32), test_target.type(torch.float32), test_classes.type(torch.long)

    # Create the neural network
    net = Net()

    # Training
    learning_rate = 0.01
    # Use MSELoss
    loss = nn.MSELoss()
    # Use Adam optimizer
    optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)

    EPOCHS = 50

    for param in net.parameters():
        print(param)

    for epoch in range(EPOCHS):
        target_predicted = net(train_input)
        l = loss(train_target, target_predicted)  #loss = nn.MSELoss()
        #l = loss(target_predicted, train_target)
        l.backward()
        optimizer.step()
        optimizer.zero_grad()

        #print(l)

    # Testing
    total = 1000
    correct = 0
    with torch.no_grad():
        correct = ( test_target == net(test_input).round() ).sum()

    print("Accuracy %.2f%%" % (correct / total * 100))

Here is the code for "dlc_practical_monologue.py":
import os; os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"
import torch
from torchvision import datasets

import argparse
import os

import urllib

######################################################################

parser = argparse.ArgumentParser(description='DLC prologue file for practical sessions.')

parser.add_argument('--full',
                    action='store_true', default=False,
                    help = 'Use the full set, can take ages (default False)')

parser.add_argument('--tiny',
                    action='store_true', default=False,
                    help = 'Use a very small set for quick checks (default False)')

parser.add_argument('--seed',
                    type = int, default = 0,
                    help = 'Random seed (default 0, < 0 is no seeding)')

parser.add_argument('--cifar',
                    action='store_true', default=False,
                    help = 'Use the CIFAR data-set and not MNIST (default False)')

parser.add_argument('--data_dir',
                    type = str, default = None,
                    help = 'Where are the PyTorch data located (default $PYTORCH_DATA_DIR or \'./data\')')

# Timur's fix
parser.add_argument('-f', '--file',
                    help = 'quick hack for jupyter')

args = parser.parse_args()

if args.seed >= 0:
    torch.manual_seed(args.seed)

######################################################################
# The data

def convert_to_one_hot_labels(input, target):
    tmp = input.new_zeros(target.size(0), target.max() + 1)
    tmp.scatter_(1, target.view(-1, 1), 1.0)
    return tmp

def load_data(cifar = None, one_hot_labels = False, normalize = False, flatten = True):

    if args.data_dir is not None:
        data_dir = args.data_dir
    else:
        data_dir = os.environ.get('PYTORCH_DATA_DIR')
        if data_dir is None:
            data_dir = './data'

    if args.cifar or (cifar is not None and cifar):
        print('* Using CIFAR')
        cifar_train_set = datasets.CIFAR10(data_dir + '/cifar10/', train = True, download = True)
        cifar_test_set = datasets.CIFAR10(data_dir + '/cifar10/', train = False, download = True)

        train_input = torch.from_numpy(cifar_train_set.data)
        train_input = train_input.transpose(3, 1).transpose(2, 3).float()
        train_target = torch.tensor(cifar_train_set.targets, dtype = torch.int64)

        test_input = torch.from_numpy(cifar_test_set.data).float()
        test_input = test_input.transpose(3, 1).transpose(2, 3).float()
        test_target = torch.tensor(cifar_test_set.targets, dtype = torch.int64)

    else:
        print('* Using MNIST')

        ######################################################################
        # import torchvision

        # raw_folder = data_dir + '/mnist/raw/'
        # resources = [
            # ("https://fleuret.org/dlc/data/train-images-idx3-ubyte.gz", "f68b3c2dcbeaaa9fbdd348bbdeb94873"),
            # ("https://fleuret.org/dlc/data/train-labels-idx1-ubyte.gz", "d53e105ee54ea40749a09fcbcd1e9432"),
            # ("https://fleuret.org/dlc/data/t10k-images-idx3-ubyte.gz", "9fb629c4189551a2d022fa330f9573f3"),
            # ("https://fleuret.org/dlc/data/t10k-labels-idx1-ubyte.gz", "ec29112dd5afa0611ce80d1b7f02629c")
        # ]

        # os.makedirs(raw_folder, exist_ok=True)

        # # download files
        # for url, md5 in resources:
            # filename = url.rpartition('/')[2]
            # torchvision.datasets.utils.download_and_extract_archive(url, download_root=raw_folder, filename=filename, md5=md5)
        ######################################################################

        mnist_train_set = datasets.MNIST(data_dir + '/mnist/', train = True, download = True)
        mnist_test_set = datasets.MNIST(data_dir + '/mnist/', train = False, download = True)

        train_input = mnist_train_set.data.view(-1, 1, 28, 28).float()
        train_target = mnist_train_set.targets
        test_input = mnist_test_set.data.view(-1, 1, 28, 28).float()
        test_target = mnist_test_set.targets

    if flatten:
        train_input = train_input.clone().reshape(train_input.size(0), -1)
        test_input = test_input.clone().reshape(test_input.size(0), -1)

    if args.full:
        if args.tiny:
            raise ValueError('Cannot have both --full and --tiny')
    else:
        if args.tiny:
            print('** Reduce the data-set to the tiny setup')
            train_input = train_input.narrow(0, 0, 500)
            train_target = train_target.narrow(0, 0, 500)
            test_input = test_input.narrow(0, 0, 100)
            test_target = test_target.narrow(0, 0, 100)
        else:
            print('** Reduce the data-set (use --full for the full thing)')
            train_input = train_input.narrow(0, 0, 1000)
            train_target = train_target.narrow(0, 0, 1000)
            test_input = test_input.narrow(0, 0, 1000)
            test_target = test_target.narrow(0, 0, 1000)

    print('** Use {:d} train and {:d} test samples'.format(train_input.size(0), test_input.size(0)))

    if one_hot_labels:
        train_target = convert_to_one_hot_labels(train_input, train_target)
        test_target = convert_to_one_hot_labels(test_input, test_target)

    if normalize:
        mu, std = train_input.mean(), train_input.std()
        train_input.sub_(mu).div_(std)
        test_input.sub_(mu).div_(std)

    return train_input, train_target, test_input, test_target

######################################################################

def mnist_to_pairs(nb, input, target):
    input = torch.functional.F.avg_pool2d(input, kernel_size = 2)
    a = torch.randperm(input.size(0))
    a = a[:2 * nb].view(nb, 2)
    input = torch.cat((input[a[:, 0]], input[a[:, 1]]), 1)
    classes = target[a]
    target = (classes[:, 0] <= classes[:, 1]).long()
    return input, target, classes

######################################################################

def generate_pair_sets(nb):
    if args.data_dir is not None:
        data_dir = args.data_dir
    else:
        data_dir = os.environ.get('PYTORCH_DATA_DIR')
        if data_dir is None:
            data_dir = './data'

    train_set = datasets.MNIST(data_dir + '/mnist/', train = True, download = True)
    train_input = train_set.data.view(-1, 1, 28, 28).float()
    train_target = train_set.targets

    test_set = datasets.MNIST(data_dir + '/mnist/', train = False, download = True)
    test_input = test_set.data.view(-1, 1, 28, 28).float()
    test_target = test_set.targets

    return mnist_to_pairs(nb, train_input, train_target) + \
           mnist_to_pairs(nb, test_input, test_target)

######################################################################

Note that I have to add the following line of code to run the code on Windows 10, while it is not necessary to run it on Linux.
import os; os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"

Also on Linux I always get the same initial weights.
Please, can you help me?

Output:
Correct me if I'm wrong here but only the weights of the first layer should be the same each time you run this. The thing is when you import the dlc_practical_monologue.py there's this thing in it:
if args.seed >= 0:
    torch.manual_seed(args.seed)

which fires up if the seed is >=0 (default is 0).
This should only initialize the first layer with the same weights for each run. Check if this is the case.


Instruction: Using weights in CrossEntropyLoss and BCELoss (PyTorch)

Input:
I am training a PyTorch model to perform binary classification. My minority class makes up about 10% of the data, so I want to use a weighted loss function. The docs for BCELoss and CrossEntropyLoss say that I can use a 'weight' for each sample.
However, when I declare CE_loss = nn.BCELoss() or nn.CrossEntropyLoss() and then do CE_Loss(output, target, weight=batch_weights), where output, target, and batch_weights are Tensors of batch_size, I get the following error message:
forward() got an unexpected keyword argument 'weight'

Output:
Another way you could accomplish your goal is to use reduction=none when initializing the loss and then multiply the resulting tensor by your weights before computing the mean.
e.g.
loss = torch.nn.BCELoss(reduction='none')
model = torch.sigmoid

weights = torch.rand(10,1)
inputs = torch.rand(10,1)
targets = torch.rand(10,1)

intermediate_losses = loss(model(inputs), targets)
final_loss = torch.mean(weights*intermediate_losses)

Of course for your scenario you still would need to calculate the weights tensor. But hopefully this helps!


Instruction: apply a function over all combination of tensor rows in pytorch

Input:
I want to make a function f1(arg_tensor) which gets a pytorch tensor as an argument.
In this function I use another function:
f2(tensor_row_1, tensor_row_2) which gets two pytorch's tensor rows as an arguments and outputs a scalar.
f2(..) should be applied over all combinations of tensor's rows [1..n] (i.e. apply function f2(..) on tensor rows' indices: [0,1], [0,2], [0,3]...[0,n-1]...[n-1,0]..[n-1,n-1]).
The output of f1(..) should be a tensor such that at element [0,0] there will the output value of f2(tensor_rows[0], tensor_rows[0]) and so on...
Is there a way to perform it efficiently (and not with double for loop)?

Output:
Yes, one can do it with a simple broadcasting trick:
def f1(tensor):
    tensor = tensor.permute(1, 0)
    return torch.nn.functional.kl_div(
        tensor.unsqueeze(dim=2), tensor.unsqueeze(dim=1), reduction="none"
    ).mean(dim=0)

def manual_f1(tensor):
    result = []
    for row1 in tensor:
        for row2 in tensor:
            result.append(torch.nn.functional.kl_div(row1, row2))
    return torch.stack(result).reshape(tensor.shape[0], -1)

data = torch.randn(5, 7)

result = f1(data)
manual_result = manual_f1(data)

print(torch.all(result == manual_result).item())

Please notice, for more rows the result will differ due to numerical difference. You can:

print the values and inspect manually
use torch.isclose to verify similarity

In the second case, last print would become:
print(torch.all(torch.isclose(result, manual_result)).item())


Instruction: Constrain parameters to be -1, 0 or 1 in neural network in pytorch

Input:
I want to constrain the parameters of an intermediate layer in a neural network to prefer discrete values: -1, 0, or 1. The idea is to add a custom objective function that would increase the loss if the parameters take any other value. Note that, I want to constrain parameters of a particular layer, not all layers.
How can I implement this in pytorch? I want to add this custom loss to the total loss in the training loop, something like this:
custom_loss = constrain_parameters_to_be_discrete
loss = other_loss + custom_loss

May be using a Dirichlet prior might help, any pointer to this?

Output:
Extending upon @Shai answer and mixing it with this answer one could do it simpler via custom layer into which you could pass your specific layer.
First, the calculated derivative of torch.abs(x**2 - torch.abs(x)) taken from WolframAlpha (check here) would be placed inside regularize function.
Now the Constrainer layer:
class Constrainer(torch.nn.Module):
    def __init__(self, module, weight_decay=1.0):
        super().__init__()
        self.module = module
        self.weight_decay = weight_decay

        # Backward hook is registered on the specified module
        self.hook = self.module.register_full_backward_hook(self._weight_decay_hook)

    # Not working with grad accumulation, check original answer and pointers there
    # If that's needed
    def _weight_decay_hook(self, *_):
        for parameter in self.module.parameters():
            parameter.grad = self.regularize(parameter)

    def regularize(self, parameter):
        # Derivative of the regularization term created by @Shia
        sgn = torch.sign(parameter)
        return self.weight_decay * (
            (sgn - 2 * parameter) * torch.sign(1 - parameter * sgn)
        )

    def forward(self, *args, **kwargs):
        # Simply forward and args and kwargs to module
        return self.module(*args, **kwargs)

Usage is really simple (with your specified weight_decay hyperparameter if you need more/less force on the params):
constrained_layer = Constrainer(torch.nn.Linear(20, 10), weight_decay=0.1)

Now you don't have to worry about different loss functions and can use your model normally.


Instruction: What is the edifference between spacy.load('en_core_web_sm') vs spacy.load(en)

Input: I have seen both of these written down in Colab Notebooks, Can someone please explain the difference between them? Thanks

Output:
In spaCy v2, it was possible to use shorthand to refer to a model in some circumstances, so "en" could be the same as "en_core_web_sm".
The way this worked internally kind of relied on symlinks, which added file system state and caused issues on Windows. This caused troubleshooting problems and confusion, so it was decided the convenience of the short names wasn't worth it, and there are no short names in v3.
So if you see code using spacy.load("en") it's using v2. There's no meaningful difference in how it works though.


Instruction: How do I use the fastai saved model?

Input: I trained my model in google colab, and downloaded the .pkl file in my computer. Now, how do I use it?  How do I load the .pkl file and do I need to install fastai for it to work?

Output:
How do I load the .pkl file

Assuming you've saved your model using learner.save you can use complementary learner.load method.

do I need to install fastai for it to work

Yes, you need fastai if you saved it this way. You could also save PyTorch model itself contained inside learner via:
torch.save(learner.model, "/path/to/model.pt") # or save it's state_dict, better option
model = torch.load("/path/to/model.pt")

Either way you need those libraries as pickle stores the data but class definition and creation has to be provided code-wise.


Instruction: IndexError: too many indices for tensor of dimension 2

Input:
here is the dataset:
class price_dataset(Dataset):
    def __init__(self, transform=None):
        xy = pd.read_csv('data_balanced_full.csv')

        self.n_samples = xy.shape[0]

        xy = xy.to_numpy()
        self.x_data = torch.from_numpy(xy[:, 7:].astype(np.float32))
        self.y_data = torch.from_numpy(xy[:, 6].astype(np.float32))
        self.transform = transform

    def __getitem__(self, index):
        x = self.x_data[index]
        y = self.y_data[index]

        sample = {'data': x, 'label': y}
        if self.transform:
            sample = self.transform(sample)

        return sample

    # we can call len(dataset) to return the size
    def __len__(self):
        return self.n_samples

and I'm trying to split the dataset into testing and training:
dataset_normalized = price_dataset(transform=transforms.ToTensor())
train_dataset, test_dataset = train_test_split(dataset_normalized['data'], dataset_normalized['label'], test_size=0.10, random_state=0)

but I'm getting this error:
IndexError: too many indices for tensor of dimension 2

Output:
'data' and 'label' are not indices but keys of a dictionnary. This dictionary is accessible and calling __getitem__ as follows : dataset_normalized[idx] with idx an integer.
Moreover, you cannot invoke your transformation directly on a dictionary. You should call it on sample['data'] instead.
I advise you to carefully read this example of the PyTorch documentation which is very nice.


Instruction: Installation problem with PyTorch's Geometric. "torch-scatter" produces an error with exit status 1

Input:
Could anyone if used PyTorch geometric before, help me resolve this issue. I'm having trouble installing torch-scatter from PyTorch Geometric to deal with some tabular data for question answering task based on TAPAS model. I presume there is a compile error at source. I tried checking other forums and found no solution for this yet.
Procedure followed to produce the error:
pip3 install torch==1.8.1+cpu torchvision==0.9.1+cpu torchaudio===0.8.1 -f https://download.pytorch.org/whl/torch_stable.html

pip3 install torch-scatter

Console output:
ERROR: Command errored out with exit status 1:

I also tried using the python -f flag and specifically tried to pull from the source at:
pip3 install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.8.1+cpu.html

Following are my PyTorch and CUDA versions with the respective imports and console outputs:
python -c "import torch; print(torch.__version__)"

Output:
1.8.1+cpu

CUDA version:
python -c "import torch; print(torch.version.cuda)"

Output:
None

Python version:
Python 3.7.5

Thank you very much for your time and guidance.

Output: I had this problem too which is solved by install C++ build tools. You can install it from vs_buildtools.exe that is downloadable here


Instruction: Extremely poor accuracy upon training network from scratch

Input:
I am trying to retrain resnet50 from scratch using a dataset that is similar to ImageNet. I wrote the following training loop:
def train_network(epochs , train_loader , val_loader , optimizer , network):
    since = time.time ( )
    train_acc_history = []
    val_acc_history = []

    best_model_weights = copy.deepcopy (network.state_dict ( ))
    best_accuracy = 0.0

    for epoch in range (epochs):
        correct_train = 0
        correct_val = 0
        for x , t in train_loader:
            x = x.to (device)
            t = t.to (device)
            optimizer.zero_grad ( )
            z = network (x)
            J = loss (z , t)
            J.backward ( )
            optimizer.step ( )

            _ , y = torch.max (z , 1)
            correct_train += torch.sum (y == t.data)

        with torch.no_grad ( ):
            network.eval ( )
            for x_val , t_val in val_loader:
                x_val = x_val.to (device)
                t_val = t_val.to (device)
                z_val = network (x_val)
                _ , y_val = torch.max (z_val , 1)
                correct_val += torch.sum (y_val == t_val.data)

        network.train ( )
        train_accuracy = correct_train.float ( ) / len (train_loader.dataset)
        val_accuracy = correct_val.float ( ) / len (val_loader.dataset)
        print (
            F"Epoch: {epoch + 1} train_accuracy: {(train_accuracy.item ( ) * 100):.3f}% val_accuracy: {(val_accuracy.item ( ) * 100):.3f}%" ,
            flush = True)

        # time_elapsed_epoch = time.time() - since
        # print ('Time taken for Epoch {} is {:.0f}m {:.0f}s'.format (epoch + 1, time_elapsed_epoch // 60 , time_elapsed_epoch % 60))

        if val_accuracy > best_accuracy:
            best_accuracy = val_accuracy
            best_model_weights = copy.deepcopy (network.state_dict ( ))
        train_acc_history.append (train_accuracy)
        val_acc_history.append (val_accuracy)

    print ( )

    time_elapsed = time.time ( ) - since
    print ('Training complete in {:.0f}m {:.0f}s'.format (time_elapsed // 60 , time_elapsed % 60))
    print ('Best Validation Accuracy: {:3f}'.format (best_accuracy * 100))

    network.load_state_dict (best_model_weights)
    return network , train_acc_history , val_acc_history

But I am getting extremely poor training and validation accuracies as below:
> Epoch: 1 train_accuracy: 3.573% val_accuracy: 3.481%
> Epoch: 2 train_accuracy: 3.414% val_accuracy: 3.273%
> Epoch: 3 train_accuracy: 3.515% val_accuracy: 4.039%
> Epoch: 4 train_accuracy: 3.567% val_accuracy: 4.195%

Upon googling, I found that the accuracies of training from scratch are usually not so poor (in fact they start off from around 40% - 50%). I am finding it difficult to understand where the glitch might be. It would be great if someone could help me figure out where I might be going wrong.
Thanks

Output:
I tried your training loop without the weight checkpoint and got accuracy over 90% on fashionMNIST dataset using my own ResNet. So if you are using a good loss/optimizer I would suggest looking at the network architecture or creation of the data-loaders.
def train_network(epochs , train_loader , val_loader , optimizer , network):
    #since = time.time ( )
    train_acc_history = []
    val_acc_history = []
    loss = nn.CrossEntropyLoss()

    #best_model_weights = copy.deepcopy (network.state_dict ( ))
    #best_accuracy = 0.0

    for epoch in range (epochs):
        correct_train = 0
        correct_val = 0
        network.train ( )
        for x , t in train_loader:
            x = x.to (device)
            t = t.to (device)
            optimizer.zero_grad ( )
            z = network (x)
            J = loss (z , t)
            J.backward ( )
            optimizer.step ( )
            _ , y = torch.max (z , 1)
            correct_train += torch.sum (y == t.data)

        with torch.no_grad ( ):
            network.eval ( )
            for x_val , t_val in val_loader:
                x_val = x_val.to (device)
                t_val = t_val.to (device)
                z_val = network (x_val)
                _ , y_val = torch.max (z_val , 1)
                correct_val += torch.sum (y_val == t_val.data)

        network.train ( )
        train_accuracy = correct_train.float ( ) / len (train_loader.dataset)
        val_accuracy = correct_val.float ( ) / len (val_loader.dataset)
        print (
            F"Epoch: {epoch + 1} train_accuracy: {(train_accuracy.item ( ) * 100):.3f}% val_accuracy: {(val_accuracy.item ( ) * 100):.3f}%" ,
            flush = True)
    '''
        if val_accuracy > best_accuracy:
            best_accuracy = val_accuracy
            best_model_weights = copy.deepcopy (network.state_dict ( ))
        train_acc_history.append (train_accuracy)
        val_acc_history.append (val_accuracy)

    #time_elapsed = time.time ( ) - since
    #print ('Training complete in {:.0f}m {:.0f}s'.format (time_elapsed // 60 , time_elapsed % 60))
    print ('Best Validation Accuracy: {:3f}'.format (best_accuracy * 100))

    #network.load_state_dict (best_model_weights)
    '''
    return network , train_acc_history , val_acc_history

optimizer = optim.Adam(net.parameters(), lr = 0.01)
train_network(10,trainloader, testloader, optimizer, net)

Epoch: 1 train_accuracy: 83.703% val_accuracy: 86.820%
Epoch: 2 train_accuracy: 88.893% val_accuracy: 89.400%
Epoch: 3 train_accuracy: 90.297% val_accuracy: 89.700%
Epoch: 4 train_accuracy: 91.272% val_accuracy: 90.640%
Epoch: 5 train_accuracy: 91.948% val_accuracy: 91.250%
...

So, if you tested with the training loop I used (yours with small mods) and it still doesn't work I would check data loader and play around with network architecture.


Instruction: Failed to Build Torch-Scatter in Pytorch Geometry

Input:
I am very new to the concept of Graph Neural Networks. To learn more I tried installing torch geometric, but it is giving a huge error(which I can't even paste here).
My Versions:
>>> import torch
>>> torch.__version__
'1.8.1'
>>> torch.version.cuda
'10.1'

The command I used to install torch geometric:
pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.8.1+cu101.html

The Error Trace:
 C:/Program Files (x86)/Microsoft Visual Studio/2019/Community/VC/Tools/MSVC/14.29.30037/include\xutility(4424): error: function "torch::OrderedDict<Key, Value>::Item::operator=(const torch::OrderedDict<std::string, at::Tensor>::Item &) [with Key=std::string, Value=at::Tensor]" (declared implicitly) cannot be referenced -- it is a deleted function
              detected during:
                instantiation of "_OutIt std::_Move_unchecked(_InIt, _InIt, _OutIt) [with _InIt=torch::OrderedDict<std::string, at::Tensor>::Item *, _OutIt=torch::OrderedDict<std::string, at::Tensor>::Item *]"
    C:/Program Files (x86)/Microsoft Visual Studio/2019/Community/VC/Tools/MSVC/14.29.30037/include\vector(1419): here
                instantiation of "std::vector<_Ty, _Alloc>::iterator std::vector<_Ty, _Alloc>::erase(std::vector<_Ty, _Alloc>::const_iterator) [with _Ty=torch::OrderedDict<std::string, at::Tensor>::Item, _Alloc=std::allocator<torch::OrderedDict<std::string, at::Tensor>::Item>]"
    C:/Users/Gopu/anaconda3/envs/torch_geometry/lib/site-packages/torch/include\torch/csrc/api/include/torch/ordered_dict.h(419): here
                instantiation of "void torch::OrderedDict<Key, Value>::erase(const Key &) [with Key=std::string, Value=at::Tensor]"
    C:/Users/Gopu/anaconda3/envs/torch_geometry/lib/site-packages/torch/include/torch/csrc/api/include\torch/nn/modules/container/parameterdict.h(51): here

    C:/Program Files (x86)/Microsoft Visual Studio/2019/Community/VC/Tools/MSVC/14.29.30037/include\xutility(4424): error: function "torch::OrderedDict<Key, Value>::Item::operator=(const torch::OrderedDict<std::string, std::shared_ptr<torch::nn::Module>>::Item &) [with Key=std::string, Value=std::shared_ptr<torch::nn::Module>]" (declared implicitly) cannot be referenced -- it is a deleted function
              detected during:
                instantiation of "_OutIt std::_Move_unchecked(_InIt, _InIt, _OutIt) [with _InIt=torch::OrderedDict<std::string, std::shared_ptr<torch::nn::Module>>::Item *, _OutIt=torch::OrderedDict<std::string, std::shared_ptr<torch::nn::Module>>::Item *]"
    C:/Program Files (x86)/Microsoft Visual Studio/2019/Community/VC/Tools/MSVC/14.29.30037/include\vector(1419): here
                instantiation of "std::vector<_Ty, _Alloc>::iterator std::vector<_Ty, _Alloc>::erase(std::vector<_Ty, _Alloc>::const_iterator) [with _Ty=torch::OrderedDict<std::string, std::shared_ptr<torch::nn::Module>>::Item, _Alloc=std::allocator<torch::OrderedDict<std::string, std::shared_ptr<torch::nn::Module>>::Item>]"
    C:/Users/Gopu/anaconda3/envs/torch_geometry/lib/site-packages/torch/include\torch/csrc/api/include/torch/ordered_dict.h(419): here
                instantiation of "void torch::OrderedDict<Key, Value>::erase(const Key &) [with Key=std::string, Value=std::shared_ptr<torch::nn::Module>]"
    C:/Users/Gopu/anaconda3/envs/torch_geometry/lib/site-packages/torch/include/torch/csrc/api/include\torch/nn/modules/container/moduledict.h(196): here

    2 errors detected in the compilation of "C:/Users/Gopu/AppData/Local/Temp/tmpxft_00001354_00000000-7_scatter_cuda.cpp1.ii".
    scatter_cuda.cu
    error: command 'C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.1\\bin\\nvcc.exe' failed with exit status 1
    ----------------------------------------
ERROR: Command errored out with exit status 1: 'C:\Users\Gopu\anaconda3\envs\torch_geometry\python.exe' -u -c 'import io, os, sys, setuptools, tokenize; sys.argv[0] = '"'"'C:\\Users\\Gopu\\AppData\\Local\\Temp\\pip-install-amy5gxh5\\torch-scatter_f0827337be9443d09c6b48e753621f6e\\setup.py'"'"'; __file__='"'"'C:\\Users\\Gopu\\AppData\\Local\\Temp\\pip-install-amy5gxh5\\torch-scatter_f0827337be9443d09c6b48e753621f6e\\setup.py'"'"';f = getattr(tokenize, '"'"'open'"'"', open)(__file__) if os.path.exists(__file__) else io.StringIO('"'"'from setuptools import setup; setup()'"'"');code = f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record 'C:\Users\Gopu\AppData\Local\Temp\pip-record-h5amznql\install-record.txt' --single-version-externally-managed --compile --install-headers 'C:\Users\Gopu\anaconda3\envs\torch_geometry\Include\torch-scatter' Check the logs for full command output.

Output:
Try checking python version it should be less then 3.9 as wheel for torch-scatter for python 3.9 is not released yet.
Create new environment with python 3.8
install pytorch cuda version and then :-
pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.8.1+cu101.html
if still not working try
pip install --upgrade torch-scatter -f https://pytorch-geometric.com/whl/torch-1.8.1+cu101.html
hope its helpful


Instruction: Weighted random sampler - oversample or undersample?

Input:
Problem
I am training a deep learning model in PyTorch for binary classification, and I have a dataset containing unbalanced class proportions. My minority class makes up about 10% of the given observations. To avoid the model learning to just predict the majority class, I want to use the WeightedRandomSampler from torch.utils.data in my DataLoader.
Let's say I have 1000 observations (900 in class 0, 100 in class 1), and a batch size of 100 for my dataloader.
Without weighted random sampling, I would expect each training epoch to consist of 10 batches.
Questions

Will only 10 batches be sampled per epoch when using this sampler - and consequently, would the model 'miss' a large portion of the majority class during each epoch, since the minority class is now overrepresented in the training batches?
Will using the sampler result in more than 10 batches being sampled per epoch (meaning the same minority class observations may appear many times, and also that training would slow down)?

Output:
A small snippet of code to use WeightedRandomSampler
First, define the function:
def make_weights_for_balanced_classes(images, nclasses):
    n_images = len(images)
    count_per_class = [0] * nclasses
    for _, image_class in images:
        count_per_class[image_class] += 1
    weight_per_class = [0.] * nclasses
    for i in range(nclasses):
        weight_per_class[i] = float(n_images) / float(count_per_class[i])
    weights = [0] * n_images
    for idx, (image, image_class) in enumerate(images):
        weights[idx] = weight_per_class[image_class]
    return weights

And after this, use it in the following way:
import torch
dataset_train = datasets.ImageFolder(traindir)

# For unbalanced dataset we create a weighted sampler
weights = make_weights_for_balanced_classes(dataset_train.imgs, len(dataset_train.classes))
weights = torch.DoubleTensor(weights)
sampler = torch.utils.data.sampler.WeightedRandomSampler(weights, len(weights))

train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=args.batch_size, shuffle = True,
                                                             sampler = sampler, num_workers=args.workers, pin_memory=True)


Instruction: Pytorch - Use a UNet to perform Image Deblurring/Image Reconstruction

Input:
Currently, I'm working with a dataset where I have two kinds of images: "sharp version" of the image and "blurry version" of the same images, where a blur was added synthetically. My goal is to train a model that takes the blurry version of the images in and tries to deblur the image as much as it can so that the "deblurred image" is closer to the sharp version. In the literature, the UNet architecture seemed to be a model with good results. Additionally, I can use a pre-trained U-Net via Pytorch (https://pytorch.org/hub/mateuszbuda_brain-segmentation-pytorch_unet/).
My problem is now: When I train this pre-trained U-Net with my images and then try it on my test set, I get the following output:

The original image:

I know that this pre-trained model is usually used for biomedical image segmentation but I'm rather confused about how I have to modify the model to use it for an Image Deblurring/Reconstruction task. Does anyone have any advice on how to do this?
I would appreciate any feedback :)

Output:
The U-net you're using is for segmentation (classification of each pixels of the image) whereas you're trying to denoise the image (getting your image "sharper"/remove noise). It explains the results you got.
To get what you want you need and as DerekG said, you first need to modify the number of channels of the output. By modify it, you can't load all the pretrained model. You will have to copy parameters by parameters until the last one.
As the last layer is initialized randomly, you can retrained the model with your training set. You can freeze or not the pretrained parts.
Also, I'm not sure what your new dataset is but if it's really not related to biomedical images you should retrain your network from scratch (transfer learning shouldn't be done in these cases), maybe even change the encoder-decoder network.


Instruction: Installing geffnet with pip

Input:
I used a google colab notebook to run a certain model. It required me to install geffnet like this.
!pip -q install geffnet

How can I install geffnet locally?
I tried the line below but I get an error when trying to get efficientnet_b7. "RuntimeError: Unknown model (efficientnet_b7)
pip3 install geffnet

Output:
Were your other python installing commands work properly?
Try with a version likethis,
pip install geffnet==0.9.0
Still not working,try to use Pytorch instead of Colab, sometimes issue may be fixed


Instruction: Convert list of tensors into tensor pytorch

Input:
I have a list of embeddings. The list has N lists with M embedding (tensors) each.
list_embd = [[M embeddings], [M embeddings], ...]

(Each embedding is a tensor with size (1,512))
What I want to do is create a tensor size (N, M), where each "cell" is one embedding.
Tried this for numpy array.
array = np.zeros(n,m)

for i in range(n):
    for j in range(m):
        array[i, j] = list_embd[i][j]

But still got errors.
In pytorch tried to concat all M embeddings into one tensor size (1, M), and then concat all rows. But when I concat along dim 1 two of those M embeddings, I get a tensor shaped (1, 1028) instead (1, 2).
final = torch.tensor([])
    for i in range(n):
        interm = torch.tensor([])
        for j in range(m):
            interm = torch.cat((interm, list_embd[i][j]), 0)
        final = = torch.cat((final, interm), 1)

Any ideas or suggestions?
I need a matrix with the embeddings in each cell.

Output:
You can use torch.cat and torch.stack to create a final 3D tensor of shape (N, M, 512):
final = torch.stack([torch.cat(sub_list, dim=0) for sub_list in list_embd], dim=0)

First, you use torch.cat to create a list of N 2D tensors of shape (M, 512) from each list of M embeddings. Then torch.stack is used to stack these N 2D matrices into a single 3D tensor final.