mtp/machamp/dataset_readers/machamp_universal_reader.py
def read_classification(self, dataset, path, is_train, max_sents):
        """
        Reads classification data, meaning that it reads input text from N columns, and
        a corresponding label from a specific column.
        """
        data = []
        sent_idxs = self.datasets[dataset]['sent_idxs']
        for sent_counter, instance in enumerate(lines2data(path, self.do_lowercase)):
            task2type = {}
            if max_sents != 0 and sent_counter > max_sents:
                break

            full_text = self.tokenizer.tokenize(instance[sent_idxs[0]].strip())
            for sent_idx in sent_idxs[1:]:
                new_text = self.tokenizer.tokenize(instance[sent_idx].strip())
                full_text = self.tokenizer.add_special_tokens(full_text, new_text)
            sent_tasks = {}
            if len(full_text) == 0:
                full_text = self.tokenizer.tokenize(self.tokenizer.tokenizer.unk_token)
            sent_tasks['tokens'] = full_text

            col_idxs = {}
            for task in self.datasets[dataset]['tasks']:
                task_idx = self.datasets[dataset]['tasks'][task]['column_idx']
                task_type = self.datasets[dataset]['tasks'][task]['task_type']
                task2type[task] = task_type
                col_idxs[task] = task_idx
                if task_type == 'classification':
                    sent_tasks[task] = instance[task_idx]
                else:
                    logger.error('Task type ' + task_type + ' for task ' + task + ' in dataset ' +
                                 dataset + ' is unknown')
            data.append(self.text_to_instance(sent_tasks, instance, col_idxs, is_train, task2type, dataset))
        return data

        Reads classification data, meaning that it reads input text from N columns, and
        a corresponding label from a specific column.
        
Reads classification data, meaning that it reads input text from N columns, and
a corresponding label from a specific column.
Reads classification data meaning that it reads input text from N columns and a corresponding label from a specific column .
def read_classification(self, dataset, path, is_train, max_sents):
        data = []
        sent_idxs = self.datasets[dataset]['sent_idxs']
        for sent_counter, instance in enumerate(lines2data(path, self.do_lowercase)):
            task2type = {}
            if max_sents != 0 and sent_counter > max_sents:
                break
            full_text = self.tokenizer.tokenize(instance[sent_idxs[0]].strip())
            for sent_idx in sent_idxs[1:]:
                new_text = self.tokenizer.tokenize(instance[sent_idx].strip())
                full_text = self.tokenizer.add_special_tokens(full_text, new_text)
            sent_tasks = {}
            if len(full_text) == 0:
                full_text = self.tokenizer.tokenize(self.tokenizer.tokenizer.unk_token)
            sent_tasks['tokens'] = full_text
            col_idxs = {}
            for task in self.datasets[dataset]['tasks']:
                task_idx = self.datasets[dataset]['tasks'][task]['column_idx']
                task_type = self.datasets[dataset]['tasks'][task]['task_type']
                task2type[task] = task_type
                col_idxs[task] = task_idx
                if task_type == 'classification':
                    sent_tasks[task] = instance[task_idx]
                else:
                    logger.error('Task type ' + task_type + ' for task ' + task + ' in dataset ' +
                                 dataset + ' is unknown')
            data.append(self.text_to_instance(sent_tasks, instance, col_idxs, is_train, task2type, dataset))
        return data
def read_classification ( self , dataset , path , is_train , max_sents ) : data = [ ] sent_idxs = self . datasets [ dataset ] [ 'sent_idxs' ] for sent_counter , instance in enumerate ( lines2data ( path , self . do_lowercase ) ) : task2type = { } if max_sents != 0 and sent_counter > max_sents : break full_text = self . tokenizer . tokenize ( instance [ sent_idxs [ 0 ] ] . strip ( ) ) for sent_idx in sent_idxs [ 1 : ] : new_text = self . tokenizer . tokenize ( instance [ sent_idx ] . strip ( ) ) full_text = self . tokenizer . add_special_tokens ( full_text , new_text ) sent_tasks = { } if len ( full_text ) == 0 : full_text = self . tokenizer . tokenize ( self . tokenizer . tokenizer . unk_token ) sent_tasks [ 'tokens' ] = full_text col_idxs = { } for task in self . datasets [ dataset ] [ 'tasks' ] : task_idx = self . datasets [ dataset ] [ 'tasks' ] [ task ] [ 'column_idx' ] task_type = self . datasets [ dataset ] [ 'tasks' ] [ task ] [ 'task_type' ] task2type [ task ] = task_type col_idxs [ task ] = task_idx if task_type == 'classification' : sent_tasks [ task ] = instance [ task_idx ] else : logger . error ( 'Task type ' + task_type + ' for task ' + task + ' in dataset ' + dataset + ' is unknown' ) data . append ( self . text_to_instance ( sent_tasks , instance , col_idxs , is_train , task2type , dataset ) ) return data
Reads classification data, meaning that it reads input text from N columns, and
a corresponding label from a specific column.
Reads classification data meaning that it reads input text from N columns and a corresponding label from a specific column .
"""
        Reads classification data, meaning that it reads input text from N columns, and
        a corresponding label from a specific column.
        """
self
dataset
path
is_train
max_sents