mtp/machamp/dataset_readers/machamp_universal_reader.py
def read_seq2seq(self, dataset, path, is_train, max_sents):
        """
        Reads generation data. This means that both the input and the output can be a sequence
        of words. For now it only supports one input column, multiple tasks (outputs) on the
        same dataset are already supported though.
        """
        data = []
        self._source_max_exceeded = 0
        self._target_max_exceeded = 0
        logger.info("Reading instances from lines in file at: {}".format(path))
        for line_num, instance in enumerate(lines2data(path, self.do_lowercase)):
            if max_sents != 0 and line_num > max_sents:
                break

            source_string = instance[self.datasets[dataset]['sent_idxs'][0]]
            if len(self.datasets[dataset]['sent_idxs']) > 1:
                logger.error("unfortunately we do not support specifying multiple sent_idxs " +
                             "for seq2seq yet, try copying them to the same column")
            # TODO support more than 1 input? see read_classification on how
            if len(source_string) == 0:
                continue

            tokenized_source = self._source_tokenizer.tokenize(source_string.strip())
            tokenized_source = self.tokenizer.add_special_tokens(tokenized_source)
            if self._source_max_tokens and len(tokenized_source) > self._source_max_tokens:
                self._source_max_exceeded += 1
                tokenized_source = tokenized_source[: self._source_max_tokens]
            if self._source_add_start_token:
                tokenized_source.insert(0, copy.deepcopy(self._start_token))
            if self._source_add_end_token:
                tokenized_source.append(copy.deepcopy(self._end_token))

            sent_tasks = {'tokens': tokenized_source}
            task2type = {}
            col_idxs = {}
            for task in self.datasets[dataset]['tasks']:
                task_idx = self.datasets[dataset]['tasks'][task]['column_idx']
                task_type = self.datasets[dataset]['tasks'][task]['task_type']
                task2type[task] = task_type
                col_idxs[task] = task_idx
                target_string = instance[task_idx]
                if len(target_string) == 0:
                    continue
                tokenized_target = self._target_tokenizer.tokenize(target_string.strip())
                tokenized_target = self._target_tokenizer.add_special_tokens(tokenized_target)
                if self._target_max_tokens and len(tokenized_target) > self._target_max_tokens:
                    self._target_max_exceeded += 1
                    tokenized_target = tokenized_target[: self._target_max_tokens]
                if self._target_add_start_token:
                    tokenized_target.insert(0, copy.deepcopy(self._start_token))
                if self._target_add_end_token:
                    tokenized_target.append(copy.deepcopy(self._end_token))
                sent_tasks[task] = tokenized_target

            data.append(self.text_to_instance(sent_tasks, instance, col_idxs, is_train, task2type, dataset))

        if self._source_max_tokens and self._source_max_exceeded:
            logger.info(
                "In {0} instances, the source token length exceeded the max limit ({1}) and were truncated.".format(
                    self._source_max_exceeded,
                    self._source_max_tokens,
                ))
        if self._target_max_tokens and self._target_max_exceeded:
            logger.info(
                "In {0} instances, the target token length exceeded the max limit ({1}) and were truncated.".format(
                    self._target_max_exceeded,
                    self._target_max_tokens,
                ))
        return data

        Reads generation data. This means that both the input and the output can be a sequence
        of words. For now it only supports one input column, multiple tasks (outputs) on the
        same dataset are already supported though.
        
Reads generation data. This means that both the input and the output can be a sequence
of words. For now it only supports one input column, multiple tasks (outputs) on the
same dataset are already supported though.
Reads generation data . This means that both the input and the output can be a sequence of words . For now it only supports one input column multiple tasks ( outputs ) on the same dataset are already supported though .
def read_seq2seq(self, dataset, path, is_train, max_sents):
        data = []
        self._source_max_exceeded = 0
        self._target_max_exceeded = 0
        logger.info("Reading instances from lines in file at: {}".format(path))
        for line_num, instance in enumerate(lines2data(path, self.do_lowercase)):
            if max_sents != 0 and line_num > max_sents:
                break
            source_string = instance[self.datasets[dataset]['sent_idxs'][0]]
            if len(self.datasets[dataset]['sent_idxs']) > 1:
                logger.error("unfortunately we do not support specifying multiple sent_idxs " +
                             "for seq2seq yet, try copying them to the same column")
            if len(source_string) == 0:
                continue
            tokenized_source = self._source_tokenizer.tokenize(source_string.strip())
            tokenized_source = self.tokenizer.add_special_tokens(tokenized_source)
            if self._source_max_tokens and len(tokenized_source) > self._source_max_tokens:
                self._source_max_exceeded += 1
                tokenized_source = tokenized_source[: self._source_max_tokens]
            if self._source_add_start_token:
                tokenized_source.insert(0, copy.deepcopy(self._start_token))
            if self._source_add_end_token:
                tokenized_source.append(copy.deepcopy(self._end_token))
            sent_tasks = {'tokens': tokenized_source}
            task2type = {}
            col_idxs = {}
            for task in self.datasets[dataset]['tasks']:
                task_idx = self.datasets[dataset]['tasks'][task]['column_idx']
                task_type = self.datasets[dataset]['tasks'][task]['task_type']
                task2type[task] = task_type
                col_idxs[task] = task_idx
                target_string = instance[task_idx]
                if len(target_string) == 0:
                    continue
                tokenized_target = self._target_tokenizer.tokenize(target_string.strip())
                tokenized_target = self._target_tokenizer.add_special_tokens(tokenized_target)
                if self._target_max_tokens and len(tokenized_target) > self._target_max_tokens:
                    self._target_max_exceeded += 1
                    tokenized_target = tokenized_target[: self._target_max_tokens]
                if self._target_add_start_token:
                    tokenized_target.insert(0, copy.deepcopy(self._start_token))
                if self._target_add_end_token:
                    tokenized_target.append(copy.deepcopy(self._end_token))
                sent_tasks[task] = tokenized_target
            data.append(self.text_to_instance(sent_tasks, instance, col_idxs, is_train, task2type, dataset))
        if self._source_max_tokens and self._source_max_exceeded:
            logger.info(
                "In {0} instances, the source token length exceeded the max limit ({1}) and were truncated.".format(
                    self._source_max_exceeded,
                    self._source_max_tokens,
                ))
        if self._target_max_tokens and self._target_max_exceeded:
            logger.info(
                "In {0} instances, the target token length exceeded the max limit ({1}) and were truncated.".format(
                    self._target_max_exceeded,
                    self._target_max_tokens,
                ))
        return data
def read_seq2seq ( self , dataset , path , is_train , max_sents ) : data = [ ] self . _source_max_exceeded = 0 self . _target_max_exceeded = 0 logger . info ( "Reading instances from lines in file at: {}" . format ( path ) ) for line_num , instance in enumerate ( lines2data ( path , self . do_lowercase ) ) : if max_sents != 0 and line_num > max_sents : break source_string = instance [ self . datasets [ dataset ] [ 'sent_idxs' ] [ 0 ] ] if len ( self . datasets [ dataset ] [ 'sent_idxs' ] ) > 1 : logger . error ( "unfortunately we do not support specifying multiple sent_idxs " + "for seq2seq yet, try copying them to the same column" ) if len ( source_string ) == 0 : continue tokenized_source = self . _source_tokenizer . tokenize ( source_string . strip ( ) ) tokenized_source = self . tokenizer . add_special_tokens ( tokenized_source ) if self . _source_max_tokens and len ( tokenized_source ) > self . _source_max_tokens : self . _source_max_exceeded += 1 tokenized_source = tokenized_source [ : self . _source_max_tokens ] if self . _source_add_start_token : tokenized_source . insert ( 0 , copy . deepcopy ( self . _start_token ) ) if self . _source_add_end_token : tokenized_source . append ( copy . deepcopy ( self . _end_token ) ) sent_tasks = { 'tokens' : tokenized_source } task2type = { } col_idxs = { } for task in self . datasets [ dataset ] [ 'tasks' ] : task_idx = self . datasets [ dataset ] [ 'tasks' ] [ task ] [ 'column_idx' ] task_type = self . datasets [ dataset ] [ 'tasks' ] [ task ] [ 'task_type' ] task2type [ task ] = task_type col_idxs [ task ] = task_idx target_string = instance [ task_idx ] if len ( target_string ) == 0 : continue tokenized_target = self . _target_tokenizer . tokenize ( target_string . strip ( ) ) tokenized_target = self . _target_tokenizer . add_special_tokens ( tokenized_target ) if self . _target_max_tokens and len ( tokenized_target ) > self . _target_max_tokens : self . _target_max_exceeded += 1 tokenized_target = tokenized_target [ : self . _target_max_tokens ] if self . _target_add_start_token : tokenized_target . insert ( 0 , copy . deepcopy ( self . _start_token ) ) if self . _target_add_end_token : tokenized_target . append ( copy . deepcopy ( self . _end_token ) ) sent_tasks [ task ] = tokenized_target data . append ( self . text_to_instance ( sent_tasks , instance , col_idxs , is_train , task2type , dataset ) ) if self . _source_max_tokens and self . _source_max_exceeded : logger . info ( "In {0} instances, the source token length exceeded the max limit ({1}) and were truncated." . format ( self . _source_max_exceeded , self . _source_max_tokens , ) ) if self . _target_max_tokens and self . _target_max_exceeded : logger . info ( "In {0} instances, the target token length exceeded the max limit ({1}) and were truncated." . format ( self . _target_max_exceeded , self . _target_max_tokens , ) ) return data
Reads generation data.
Reads generation data .
"""
        Reads generation data. This means that both the input and the output can be a sequence
        of words. For now it only supports one input column, multiple tasks (outputs) on the
        same dataset are already supported though.
        """ # TODO support more than 1 input? see read_classification on how
self
dataset
path
is_train
max_sents