mtp/machamp/dataset_readers/machamp_universal_reader.py
def _read(self, file_path: str) -> Iterable[Instance]:
        """
        Main reading class, for each dataset it identifies the type of dataset to read,
        and calls the corresponding function. A trick is used to be able to use the same
        function universally, the actual path is read from the dataset_config based on
        which placeholder is saved in `file_path`.

        """
        is_train = file_path == 'TRAINPLACEHOLDER'
        is_dev = file_path == 'DEVPLACEHOLDER'
        is_test = file_path == 'TESTPLACEHOLDER'
        for dataset in self.datasets:
            if is_train:
                file_path = self.datasets[dataset]['train_data_path']
            if is_dev:
                file_path = self.datasets[dataset]['validation_data_path']
            if is_test:
                file_path = self.datasets[dataset]['test_data_path']

            num_classification = 0
            num_mlm = 0
            num_s2s = 0
            for task in self.datasets[dataset]['tasks']:
                is_clas = self.datasets[dataset]['tasks'][task]['task_type'] == 'classification'
                read_seq = self.datasets[dataset]['tasks'][task]['column_idx'] == -1 \
                    if 'column_idx' in self.datasets[dataset]['tasks'][task] else None
                if is_clas and not read_seq:
                    num_classification += 1
                if self.datasets[dataset]['tasks'][task]['task_type'] == 'unsupervised':
                    num_mlm += 1
                if self.datasets[dataset]['tasks'][task]['task_type'] == 'seq2seq':
                    num_s2s += 1

            num_tasks = len(self.datasets[dataset]['tasks'])
            if num_mlm not in [0, num_tasks]:
                logger.error('A dataset can only consists of 0 unsupervised tasks or all')
            if num_s2s not in [0, num_tasks]:
                logger.error('A dataset can only consists of 0 seq2seq tasks or all')
            if num_classification not in [0, num_tasks]:
                logger.error('A dataset can only consists of 0 classification tasks or all, if you combine both ' +
                             'word-level and text-level tasks, use column_idx: -1 for the text level tasks')

            max_sents = 0
            if 'max_sents' in self.datasets[dataset]:
                max_sents = self.datasets[dataset]['max_sents']

            # read raw input
            if self.is_raw:
                read_function = self.read_raw
            # read classification data
            elif num_tasks == num_classification:
                read_function = self.read_classification
            # read raw data for MLM
            elif num_mlm != 0:
                read_function = self.read_unlabeled
            # read seq2seq data
            elif num_s2s != 0:
                read_function = self.read_seq2seq
            # read word-level annotation (conll-like)
            else:
                read_function = self.read_sequence

            for item in read_function(dataset, file_path, is_train, max_sents):
                yield item

        Main reading class, for each dataset it identifies the type of dataset to read,
        and calls the corresponding function. A trick is used to be able to use the same
        function universally, the actual path is read from the dataset_config based on
        which placeholder is saved in `file_path`.

        
Main reading class, for each dataset it identifies the type of dataset to read,
and calls the corresponding function. A trick is used to be able to use the same
function universally, the actual path is read from the dataset_config based on
which placeholder is saved in `file_path`.
Main reading class for each dataset it identifies the type of dataset to read and calls the corresponding function . A trick is used to be able to use the same function universally the actual path is read from the dataset_config based on which placeholder is saved in ` file_path ` .
def _read(self, file_path: str) -> Iterable[Instance]:
        is_train = file_path == 'TRAINPLACEHOLDER'
        is_dev = file_path == 'DEVPLACEHOLDER'
        is_test = file_path == 'TESTPLACEHOLDER'
        for dataset in self.datasets:
            if is_train:
                file_path = self.datasets[dataset]['train_data_path']
            if is_dev:
                file_path = self.datasets[dataset]['validation_data_path']
            if is_test:
                file_path = self.datasets[dataset]['test_data_path']
            num_classification = 0
            num_mlm = 0
            num_s2s = 0
            for task in self.datasets[dataset]['tasks']:
                is_clas = self.datasets[dataset]['tasks'][task]['task_type'] == 'classification'
                read_seq = self.datasets[dataset]['tasks'][task]['column_idx'] == -1 \
                    if 'column_idx' in self.datasets[dataset]['tasks'][task] else None
                if is_clas and not read_seq:
                    num_classification += 1
                if self.datasets[dataset]['tasks'][task]['task_type'] == 'unsupervised':
                    num_mlm += 1
                if self.datasets[dataset]['tasks'][task]['task_type'] == 'seq2seq':
                    num_s2s += 1
            num_tasks = len(self.datasets[dataset]['tasks'])
            if num_mlm not in [0, num_tasks]:
                logger.error('A dataset can only consists of 0 unsupervised tasks or all')
            if num_s2s not in [0, num_tasks]:
                logger.error('A dataset can only consists of 0 seq2seq tasks or all')
            if num_classification not in [0, num_tasks]:
                logger.error('A dataset can only consists of 0 classification tasks or all, if you combine both ' +
                             'word-level and text-level tasks, use column_idx: -1 for the text level tasks')
            max_sents = 0
            if 'max_sents' in self.datasets[dataset]:
                max_sents = self.datasets[dataset]['max_sents']
            if self.is_raw:
                read_function = self.read_raw
            elif num_tasks == num_classification:
                read_function = self.read_classification
            elif num_mlm != 0:
                read_function = self.read_unlabeled
            elif num_s2s != 0:
                read_function = self.read_seq2seq
            else:
                read_function = self.read_sequence
            for item in read_function(dataset, file_path, is_train, max_sents):
                yield item
def _read ( self , file_path : str ) -> Iterable [ Instance ] : is_train = file_path == 'TRAINPLACEHOLDER' is_dev = file_path == 'DEVPLACEHOLDER' is_test = file_path == 'TESTPLACEHOLDER' for dataset in self . datasets : if is_train : file_path = self . datasets [ dataset ] [ 'train_data_path' ] if is_dev : file_path = self . datasets [ dataset ] [ 'validation_data_path' ] if is_test : file_path = self . datasets [ dataset ] [ 'test_data_path' ] num_classification = 0 num_mlm = 0 num_s2s = 0 for task in self . datasets [ dataset ] [ 'tasks' ] : is_clas = self . datasets [ dataset ] [ 'tasks' ] [ task ] [ 'task_type' ] == 'classification' read_seq = self . datasets [ dataset ] [ 'tasks' ] [ task ] [ 'column_idx' ] == - 1 if 'column_idx' in self . datasets [ dataset ] [ 'tasks' ] [ task ] else None if is_clas and not read_seq : num_classification += 1 if self . datasets [ dataset ] [ 'tasks' ] [ task ] [ 'task_type' ] == 'unsupervised' : num_mlm += 1 if self . datasets [ dataset ] [ 'tasks' ] [ task ] [ 'task_type' ] == 'seq2seq' : num_s2s += 1 num_tasks = len ( self . datasets [ dataset ] [ 'tasks' ] ) if num_mlm not in [ 0 , num_tasks ] : logger . error ( 'A dataset can only consists of 0 unsupervised tasks or all' ) if num_s2s not in [ 0 , num_tasks ] : logger . error ( 'A dataset can only consists of 0 seq2seq tasks or all' ) if num_classification not in [ 0 , num_tasks ] : logger . error ( 'A dataset can only consists of 0 classification tasks or all, if you combine both ' + 'word-level and text-level tasks, use column_idx: -1 for the text level tasks' ) max_sents = 0 if 'max_sents' in self . datasets [ dataset ] : max_sents = self . datasets [ dataset ] [ 'max_sents' ] if self . is_raw : read_function = self . read_raw elif num_tasks == num_classification : read_function = self . read_classification elif num_mlm != 0 : read_function = self . read_unlabeled elif num_s2s != 0 : read_function = self . read_seq2seq else : read_function = self . read_sequence for item in read_function ( dataset , file_path , is_train , max_sents ) : yield item
Main reading class, for each dataset it identifies the type of dataset to read,
and calls the corresponding function.
Main reading class for each dataset it identifies the type of dataset to read and calls the corresponding function .
"""
        Main reading class, for each dataset it identifies the type of dataset to read,
        and calls the corresponding function. A trick is used to be able to use the same
        function universally, the actual path is read from the dataset_config based on
        which placeholder is saved in `file_path`.

        """ # read raw input # read classification data # read raw data for MLM # read seq2seq data # read word-level annotation (conll-like)
self
file_path
str