mtp/machamp/dataset_readers/machamp_universal_reader.py
def read_raw(self, dataset, path, is_train, max_sents):
        """
        Reads the data from a raw txt file. Assumes that each sentence is on a line, and
        the words are separated by a whitespace.
        """
        data = []
        word_idx = self.datasets[dataset]['word_idx']
        for sent_counter, sent in enumerate(open(path, encoding='utf-8', mode='r')):
            if max_sents != 0 and sent_counter > max_sents:
                break
            sent = sent.strip('\n')
            if self.do_lowercase:
                sent = sent.lower()
            # could also use basictokenizer?
            sent_tok = [word for word in sent.split(' ')]

            sent_tasks = {'tokens': []}
            for word in sent_tok:
                sent_tasks['tokens'].append(word)

            col_idxs = {'word_idx': word_idx}
            task2type = {}
            for task in self.datasets[dataset]['tasks']:
                task_idx = self.datasets[dataset]['tasks'][task]['column_idx']
                task_type = self.datasets[dataset]['tasks'][task]['task_type']
                task2type[task] = task_type
                col_idxs[task] = task_idx
            data.append(self.text_to_instance(sent_tasks, sent_tok, col_idxs, is_train, task2type, dataset))
        return data

        Reads the data from a raw txt file. Assumes that each sentence is on a line, and
        the words are separated by a whitespace.
        
Reads the data from a raw txt file. Assumes that each sentence is on a line, and
the words are separated by a whitespace.
Reads the data from a raw txt file . Assumes that each sentence is on a line and the words are separated by a whitespace .
def read_raw(self, dataset, path, is_train, max_sents):
        data = []
        word_idx = self.datasets[dataset]['word_idx']
        for sent_counter, sent in enumerate(open(path, encoding='utf-8', mode='r')):
            if max_sents != 0 and sent_counter > max_sents:
                break
            sent = sent.strip('\n')
            if self.do_lowercase:
                sent = sent.lower()
            sent_tok = [word for word in sent.split(' ')]
            sent_tasks = {'tokens': []}
            for word in sent_tok:
                sent_tasks['tokens'].append(word)
            col_idxs = {'word_idx': word_idx}
            task2type = {}
            for task in self.datasets[dataset]['tasks']:
                task_idx = self.datasets[dataset]['tasks'][task]['column_idx']
                task_type = self.datasets[dataset]['tasks'][task]['task_type']
                task2type[task] = task_type
                col_idxs[task] = task_idx
            data.append(self.text_to_instance(sent_tasks, sent_tok, col_idxs, is_train, task2type, dataset))
        return data
def read_raw ( self , dataset , path , is_train , max_sents ) : data = [ ] word_idx = self . datasets [ dataset ] [ 'word_idx' ] for sent_counter , sent in enumerate ( open ( path , encoding = 'utf-8' , mode = 'r' ) ) : if max_sents != 0 and sent_counter > max_sents : break sent = sent . strip ( '\n' ) if self . do_lowercase : sent = sent . lower ( ) sent_tok = [ word for word in sent . split ( ' ' ) ] sent_tasks = { 'tokens' : [ ] } for word in sent_tok : sent_tasks [ 'tokens' ] . append ( word ) col_idxs = { 'word_idx' : word_idx } task2type = { } for task in self . datasets [ dataset ] [ 'tasks' ] : task_idx = self . datasets [ dataset ] [ 'tasks' ] [ task ] [ 'column_idx' ] task_type = self . datasets [ dataset ] [ 'tasks' ] [ task ] [ 'task_type' ] task2type [ task ] = task_type col_idxs [ task ] = task_idx data . append ( self . text_to_instance ( sent_tasks , sent_tok , col_idxs , is_train , task2type , dataset ) ) return data
Reads the data from a raw txt file.
Reads the data from a raw txt file .
"""
        Reads the data from a raw txt file. Assumes that each sentence is on a line, and
        the words are separated by a whitespace.
        """ # could also use basictokenizer?
self
dataset
path
is_train
max_sents