multi-task-pretrain/dataloaders/brain_tumor_dataset.py
def tokenize(self, max_length=12):
        """Tokenizes the questions.

        This will add q_token in each entry of the dataset.
        -1 represent nil, and should be treated as padding_idx in embedding
        """
        for entry in self.entries:
            tokens = self.dictionary.tokenize(entry['question'], False)
            tokens = tokens[:max_length]
            if len(tokens) < max_length:
                # Note here we pad in front of the sentence
                padding = [self.dictionary.padding_idx] * (max_length - len(tokens))
                tokens = tokens + padding

            assert_eq(len(tokens), max_length)
            entry['q_token'] = tokens
Tokenizes the questions.

        This will add q_token in each entry of the dataset.
        -1 represent nil, and should be treated as padding_idx in embedding
        
Tokenizes the questions.
This will add q_token in each entry of the dataset.
1 represent nil, and should be treated as padding_idx in embedding
Tokenizes the questions . This will add q_token in each entry of the dataset . 1 represent nil and should be treated as padding_idx in embedding
def tokenize(self, max_length=12):
        for entry in self.entries:
            tokens = self.dictionary.tokenize(entry['question'], False)
            tokens = tokens[:max_length]
            if len(tokens) < max_length:
                padding = [self.dictionary.padding_idx] * (max_length - len(tokens))
                tokens = tokens + padding
            assert_eq(len(tokens), max_length)
            entry['q_token'] = tokens
def tokenize ( self , max_length = 12 ) : for entry in self . entries : tokens = self . dictionary . tokenize ( entry [ 'question' ] , False ) tokens = tokens [ : max_length ] if len ( tokens ) < max_length : padding = [ self . dictionary . padding_idx ] * ( max_length - len ( tokens ) ) tokens = tokens + padding assert_eq ( len ( tokens ) , max_length ) entry [ 'q_token' ] = tokens
Tokenizes the questions.
Tokenizes the questions .
"""Tokenizes the questions.

        This will add q_token in each entry of the dataset.
        -1 represent nil, and should be treated as padding_idx in embedding
        """ # Note here we pad in front of the sentence

self
max_length