examples/text_summarization/prophetnet/train_prophetnet.py
def warpper(example):
        """convert an example into necessary features"""
        tokens = example['article']
        labels = example['highlights']
        src_ids, src_attention_mask_ids = tokens.split("$1$")
        src_ids = [int(i) for i in src_ids.split(" ")]
        src_attention_mask_ids = [
            int(i) for i in src_attention_mask_ids.split(" ")
        ]

        if not is_test:
            labels, decoder_input_attention_mask_ids = labels.split("$1$")
            labels = [int(i) for i in labels.split(" ")]
            decoder_input_attention_mask_ids = [
                int(i) for i in decoder_input_attention_mask_ids.split(" ")
            ]
            decoder_input_ids = [labels[-1]] + labels[:-1]
            return src_ids, src_attention_mask_ids, decoder_input_ids, decoder_input_attention_mask_ids, labels

        else:
            return src_ids, src_attention_mask_ids
convert an example into necessary features
convert an example into necessary features
convert an example into necessary features
def warpper(example):
        tokens = example['article']
        labels = example['highlights']
        src_ids, src_attention_mask_ids = tokens.split("$1$")
        src_ids = [int(i) for i in src_ids.split(" ")]
        src_attention_mask_ids = [
            int(i) for i in src_attention_mask_ids.split(" ")
        ]
        if not is_test:
            labels, decoder_input_attention_mask_ids = labels.split("$1$")
            labels = [int(i) for i in labels.split(" ")]
            decoder_input_attention_mask_ids = [
                int(i) for i in decoder_input_attention_mask_ids.split(" ")
            ]
            decoder_input_ids = [labels[-1]] + labels[:-1]
            return src_ids, src_attention_mask_ids, decoder_input_ids, decoder_input_attention_mask_ids, labels
        else:
            return src_ids, src_attention_mask_ids
def warpper ( example ) : tokens = example [ 'article' ] labels = example [ 'highlights' ] src_ids , src_attention_mask_ids = tokens . split ( "$1$" ) src_ids = [ int ( i ) for i in src_ids . split ( " " ) ] src_attention_mask_ids = [ int ( i ) for i in src_attention_mask_ids . split ( " " ) ] if not is_test : labels , decoder_input_attention_mask_ids = labels . split ( "$1$" ) labels = [ int ( i ) for i in labels . split ( " " ) ] decoder_input_attention_mask_ids = [ int ( i ) for i in decoder_input_attention_mask_ids . split ( " " ) ] decoder_input_ids = [ labels [ - 1 ] ] + labels [ : - 1 ] return src_ids , src_attention_mask_ids , decoder_input_ids , decoder_input_attention_mask_ids , labels else : return src_ids , src_attention_mask_ids
convert an example into necessary features
convert an example into necessary features
"""convert an example into necessary features"""
example