mtp/machamp/dataset_readers/machamp_universal_reader.py
def read_unlabeled(self, dataset, path, is_train, max_sents):
        """
        Reads raw data to perform masked language modeling on. This is a separate function, because
        it also already masks the data.
        """
        # TODO make full use of batch size
        data = []
        sep_token = self.tokenizer.tokenize(self.tokenizer.tokenizer.sep_token)[0]
        cls_token = self.tokenizer.tokenize(self.tokenizer.tokenizer.cls_token)[0]
        mask_token = self.tokenizer.tokenize(self.tokenizer.tokenizer.mask_token)[0]

        for sent_idx, sent in enumerate(open(path, encoding='utf-8', mode='r')):
            if self.do_lowercase:
                sent = sent.lower()
            # skip empty lines
            if len(sent.strip()) == 0:
                continue
            if max_sents != 0 and sent_idx >= max_sents:
                break

            tokens = [cls_token] + self.tokenizer.tokenize(sent) + [sep_token]
            # R: 106 is taken from mbert, hope its sufficient in most cases?
            targets = mlm_mask(tokens, 106, self.tokenizer.tokenizer.vocab_size, mask_token)

            # set them to TOKENIZED, so that they are not tokenized again in the indexer/embedder
            new_tokens = []
            for token in tokens:
                new_tokens.append(Token(text=token.text, idx=token.idx, idx_end=token.idx_end,
                                        ent_type_='TOKENIZED', text_id=token.text_id, type_id=token.type_id))
            tokens = new_tokens
            if len(self.datasets[dataset]['tasks']) > 1:
                logger.error('currently MaChAmp does not support mlm mixed with other tasks on one dataset')
            task_name = list(self.datasets[dataset]['tasks'].items())[0][0]

            sent_tasks = {task_name: targets, 'tokens': tokens}
            # task2type = {task_name: 'unsupervised'}
            col_idxs = {task_name: -1}
            data.append(self.text_to_instance2(sent_tasks, col_idxs, is_train, dataset, sent))

        return data

        Reads raw data to perform masked language modeling on. This is a separate function, because
        it also already masks the data.
        
Reads raw data to perform masked language modeling on. This is a separate function, because
it also already masks the data.
Reads raw data to perform masked language modeling on . This is a separate function because it also already masks the data .
def read_unlabeled(self, dataset, path, is_train, max_sents):
        data = []
        sep_token = self.tokenizer.tokenize(self.tokenizer.tokenizer.sep_token)[0]
        cls_token = self.tokenizer.tokenize(self.tokenizer.tokenizer.cls_token)[0]
        mask_token = self.tokenizer.tokenize(self.tokenizer.tokenizer.mask_token)[0]
        for sent_idx, sent in enumerate(open(path, encoding='utf-8', mode='r')):
            if self.do_lowercase:
                sent = sent.lower()
            if len(sent.strip()) == 0:
                continue
            if max_sents != 0 and sent_idx >= max_sents:
                break
            tokens = [cls_token] + self.tokenizer.tokenize(sent) + [sep_token]
            targets = mlm_mask(tokens, 106, self.tokenizer.tokenizer.vocab_size, mask_token)
            new_tokens = []
            for token in tokens:
                new_tokens.append(Token(text=token.text, idx=token.idx, idx_end=token.idx_end,
                                        ent_type_='TOKENIZED', text_id=token.text_id, type_id=token.type_id))
            tokens = new_tokens
            if len(self.datasets[dataset]['tasks']) > 1:
                logger.error('currently MaChAmp does not support mlm mixed with other tasks on one dataset')
            task_name = list(self.datasets[dataset]['tasks'].items())[0][0]
            sent_tasks = {task_name: targets, 'tokens': tokens}
            col_idxs = {task_name: -1}
            data.append(self.text_to_instance2(sent_tasks, col_idxs, is_train, dataset, sent))
        return data
def read_unlabeled ( self , dataset , path , is_train , max_sents ) : data = [ ] sep_token = self . tokenizer . tokenize ( self . tokenizer . tokenizer . sep_token ) [ 0 ] cls_token = self . tokenizer . tokenize ( self . tokenizer . tokenizer . cls_token ) [ 0 ] mask_token = self . tokenizer . tokenize ( self . tokenizer . tokenizer . mask_token ) [ 0 ] for sent_idx , sent in enumerate ( open ( path , encoding = 'utf-8' , mode = 'r' ) ) : if self . do_lowercase : sent = sent . lower ( ) if len ( sent . strip ( ) ) == 0 : continue if max_sents != 0 and sent_idx >= max_sents : break tokens = [ cls_token ] + self . tokenizer . tokenize ( sent ) + [ sep_token ] targets = mlm_mask ( tokens , 106 , self . tokenizer . tokenizer . vocab_size , mask_token ) new_tokens = [ ] for token in tokens : new_tokens . append ( Token ( text = token . text , idx = token . idx , idx_end = token . idx_end , ent_type_ = 'TOKENIZED' , text_id = token . text_id , type_id = token . type_id ) ) tokens = new_tokens if len ( self . datasets [ dataset ] [ 'tasks' ] ) > 1 : logger . error ( 'currently MaChAmp does not support mlm mixed with other tasks on one dataset' ) task_name = list ( self . datasets [ dataset ] [ 'tasks' ] . items ( ) ) [ 0 ] [ 0 ] sent_tasks = { task_name : targets , 'tokens' : tokens } col_idxs = { task_name : - 1 } data . append ( self . text_to_instance2 ( sent_tasks , col_idxs , is_train , dataset , sent ) ) return data
Reads raw data to perform masked language modeling on.
Reads raw data to perform masked language modeling on .
"""
        Reads raw data to perform masked language modeling on. This is a separate function, because
        it also already masks the data.
        """ # TODO make full use of batch size # skip empty lines # R: 106 is taken from mbert, hope its sufficient in most cases? # set them to TOKENIZED, so that they are not tokenized again in the indexer/embedder # task2type = {task_name: 'unsupervised'}
self
dataset
path
is_train
max_sents