mtp/machamp/dataset_readers/machamp_universal_reader.py
def read_sequence(self, dataset, path, is_train, max_sents):
        """
        Reads conllu-like files. It relies heavily on reader_utils.seqs2data.
        Can also read sentence classification tasks for which the labels should
        be specified in the comments.
        Note that this read corresponds to a variety of task_types, but the
        differences between them during data reading are kept minimal
        """
        data = []
        word_idx = self.datasets[dataset]['word_idx']
        sent_counter = 0
        tknzr = BasicTokenizer()

        for sent, full_data in seqs2data(path, self.do_lowercase):
            task2type = {}
            sent_counter += 1
            if max_sents != 0 and sent_counter > max_sents:
                break
            sent_tasks = {}
            tokens = [token[word_idx] for token in sent]
            for tokenIdx in range(len(tokens)):
                if len(tknzr._clean_text(tokens[tokenIdx])) == 0:
                    tokens[tokenIdx] = self.tokenizer.tokenizer.unk_token
            sent_tasks['tokens'] = [Token(token) for token in tokens]

            col_idxs = {'word_idx': word_idx}
            for task in self.datasets[dataset]['tasks']:
                sent_tasks[task] = []
                task_type = self.datasets[dataset]['tasks'][task]['task_type']
                task_idx = self.datasets[dataset]['tasks'][task]['column_idx']
                task2type[task] = task_type
                col_idxs[task] = task_idx
                if task_type == 'classification' and task_idx == -1:
                    start = '# ' + task + ': '
                    for line in full_data:
                        if line[0].startswith(start):
                            sent_tasks[task] = line[0][len(start):]
                elif task_type in ['seq', 'multiseq', 'seq_bio']:
                    for word_data in sent:
                        sent_tasks[task].append(word_data[task_idx])
                elif task_type == 'string2string':
                    for word_data in sent:
                        task_label = gen_lemma_rule(word_data[word_idx], word_data[task_idx])
                        sent_tasks[task].append(task_label)
                elif task_type == 'dependency':
                    heads = []
                    rels = []
                    for word_data in sent:
                        if not word_data[task_idx].isdigit():
                            logger.error("Your dependency file " + path + " seems to contain invalid structures sentence "  + str(sent_counter) + " contains a non-integer head: " +   word_data[task_idx] + "\nIf you directly used UD data, this could be due to special EUD constructions which we do not support, you can clean your conllu file by using scripts/misc/cleanconl.py")
                            exit(1)
                        heads.append(int(word_data[task_idx]))
                        rels.append(word_data[task_idx + 1])
                    sent_tasks[task] = list(zip(rels, heads))
                else:
                    logger.error('Task type ' + task_type + ' for task ' + task +
                                 ' in dataset ' + dataset + ' is unknown')
            data.append(self.text_to_instance(sent_tasks, full_data, col_idxs, is_train, task2type, dataset))
        return data

        Reads conllu-like files. It relies heavily on reader_utils.seqs2data.
        Can also read sentence classification tasks for which the labels should
        be specified in the comments.
        Note that this read corresponds to a variety of task_types, but the
        differences between them during data reading are kept minimal
        
Reads conllu-like files. It relies heavily on reader_utils.seqs2data.
Can also read sentence classification tasks for which the labels should
be specified in the comments.
Note that this read corresponds to a variety of task_types, but the
differences between them during data reading are kept minimal
Reads conllu - like files . It relies heavily on reader_utils . seqs2data . Can also read sentence classification tasks for which the labels should be specified in the comments . Note that this read corresponds to a variety of task_types but the differences between them during data reading are kept minimal
def read_sequence(self, dataset, path, is_train, max_sents):
        data = []
        word_idx = self.datasets[dataset]['word_idx']
        sent_counter = 0
        tknzr = BasicTokenizer()
        for sent, full_data in seqs2data(path, self.do_lowercase):
            task2type = {}
            sent_counter += 1
            if max_sents != 0 and sent_counter > max_sents:
                break
            sent_tasks = {}
            tokens = [token[word_idx] for token in sent]
            for tokenIdx in range(len(tokens)):
                if len(tknzr._clean_text(tokens[tokenIdx])) == 0:
                    tokens[tokenIdx] = self.tokenizer.tokenizer.unk_token
            sent_tasks['tokens'] = [Token(token) for token in tokens]
            col_idxs = {'word_idx': word_idx}
            for task in self.datasets[dataset]['tasks']:
                sent_tasks[task] = []
                task_type = self.datasets[dataset]['tasks'][task]['task_type']
                task_idx = self.datasets[dataset]['tasks'][task]['column_idx']
                task2type[task] = task_type
                col_idxs[task] = task_idx
                if task_type == 'classification' and task_idx == -1:
                    start = '# ' + task + ': '
                    for line in full_data:
                        if line[0].startswith(start):
                            sent_tasks[task] = line[0][len(start):]
                elif task_type in ['seq', 'multiseq', 'seq_bio']:
                    for word_data in sent:
                        sent_tasks[task].append(word_data[task_idx])
                elif task_type == 'string2string':
                    for word_data in sent:
                        task_label = gen_lemma_rule(word_data[word_idx], word_data[task_idx])
                        sent_tasks[task].append(task_label)
                elif task_type == 'dependency':
                    heads = []
                    rels = []
                    for word_data in sent:
                        if not word_data[task_idx].isdigit():
                            logger.error("Your dependency file " + path + " seems to contain invalid structures sentence "  + str(sent_counter) + " contains a non-integer head: " +   word_data[task_idx] + "\nIf you directly used UD data, this could be due to special EUD constructions which we do not support, you can clean your conllu file by using scripts/misc/cleanconl.py")
                            exit(1)
                        heads.append(int(word_data[task_idx]))
                        rels.append(word_data[task_idx + 1])
                    sent_tasks[task] = list(zip(rels, heads))
                else:
                    logger.error('Task type ' + task_type + ' for task ' + task +
                                 ' in dataset ' + dataset + ' is unknown')
            data.append(self.text_to_instance(sent_tasks, full_data, col_idxs, is_train, task2type, dataset))
        return data
def read_sequence ( self , dataset , path , is_train , max_sents ) : data = [ ] word_idx = self . datasets [ dataset ] [ 'word_idx' ] sent_counter = 0 tknzr = BasicTokenizer ( ) for sent , full_data in seqs2data ( path , self . do_lowercase ) : task2type = { } sent_counter += 1 if max_sents != 0 and sent_counter > max_sents : break sent_tasks = { } tokens = [ token [ word_idx ] for token in sent ] for tokenIdx in range ( len ( tokens ) ) : if len ( tknzr . _clean_text ( tokens [ tokenIdx ] ) ) == 0 : tokens [ tokenIdx ] = self . tokenizer . tokenizer . unk_token sent_tasks [ 'tokens' ] = [ Token ( token ) for token in tokens ] col_idxs = { 'word_idx' : word_idx } for task in self . datasets [ dataset ] [ 'tasks' ] : sent_tasks [ task ] = [ ] task_type = self . datasets [ dataset ] [ 'tasks' ] [ task ] [ 'task_type' ] task_idx = self . datasets [ dataset ] [ 'tasks' ] [ task ] [ 'column_idx' ] task2type [ task ] = task_type col_idxs [ task ] = task_idx if task_type == 'classification' and task_idx == - 1 : start = '# ' + task + ': ' for line in full_data : if line [ 0 ] . startswith ( start ) : sent_tasks [ task ] = line [ 0 ] [ len ( start ) : ] elif task_type in [ 'seq' , 'multiseq' , 'seq_bio' ] : for word_data in sent : sent_tasks [ task ] . append ( word_data [ task_idx ] ) elif task_type == 'string2string' : for word_data in sent : task_label = gen_lemma_rule ( word_data [ word_idx ] , word_data [ task_idx ] ) sent_tasks [ task ] . append ( task_label ) elif task_type == 'dependency' : heads = [ ] rels = [ ] for word_data in sent : if not word_data [ task_idx ] . isdigit ( ) : logger . error ( "Your dependency file " + path + " seems to contain invalid structures sentence " + str ( sent_counter ) + " contains a non-integer head: " + word_data [ task_idx ] + "\nIf you directly used UD data, this could be due to special EUD constructions which we do not support, you can clean your conllu file by using scripts/misc/cleanconl.py" ) exit ( 1 ) heads . append ( int ( word_data [ task_idx ] ) ) rels . append ( word_data [ task_idx + 1 ] ) sent_tasks [ task ] = list ( zip ( rels , heads ) ) else : logger . error ( 'Task type ' + task_type + ' for task ' + task + ' in dataset ' + dataset + ' is unknown' ) data . append ( self . text_to_instance ( sent_tasks , full_data , col_idxs , is_train , task2type , dataset ) ) return data
Reads conllu-like files.
Reads conllu - like files .
"""
        Reads conllu-like files. It relies heavily on reader_utils.seqs2data.
        Can also read sentence classification tasks for which the labels should
        be specified in the comments.
        Note that this read corresponds to a variety of task_types, but the
        differences between them during data reading are kept minimal
        """
self
dataset
path
is_train
max_sents