paddlenlp/taskflow/text_generation.py
def _preprocess(self, inputs, padding=True, add_special_tokens=True):
        """
        Transform the raw text to the model inputs, two steps involved:
           1) Transform the raw text to token ids.
           2) Generate the other model inputs from the raw text and token ids.
        """
        inputs = self._check_input_text(inputs)
        # Get the config from the kwargs
        batch_size = self.kwargs[
            'batch_size'] if 'batch_size' in self.kwargs else 1
        num_workers = self.kwargs[
            'num_workers'] if 'num_workers' in self.kwargs else 0
        generation_task = self.kwargs[
            'generation_task'] if 'generation_task' in self.kwargs else 'question_answering'
        max_seq_len = 32

        def select_few_shot_input(model_name, generation_task):
            pre_input = ""
            if generation_task not in [
                    'question_answering', 'poetry_generation'
            ]:
                raise ValueError(
                    "The generation task must be question or poetry")
            if model_name == "gpt-cpm-large-cn":
                if generation_task == "question_answering":
                    pre_input = '问题：中国的首都是哪里？答案：北京。\n问题：{} 答案：'
                else:
                    pre_input = '默写古诗: 大漠孤烟直，长河落日圆。\n{}'
            return pre_input

        pre_input = select_few_shot_input(self.model, generation_task)

        infer_data = []

        examples = []
        filter_inputs = []
        for input_text in inputs:
            if not (isinstance(input_text, str) and len(input_text) > 0):
                continue
            filter_inputs.append(input_text)
            few_shot_input = pre_input.format(input_text)
            ids = self._tokenizer(few_shot_input)["input_ids"]
            examples.append((ids, len(ids)))

        batchify_fn = lambda samples, fn=Tuple(
            Pad(axis=0, pad_val=0, dtype="int64"),
            Stack(dtype='int64'),  # seq_len
        ): fn(samples)

        batches = [
            examples[idx:idx + batch_size]
            for idx in range(0, len(examples), batch_size)
        ]
        outputs = {}
        outputs['text'] = filter_inputs
        outputs['data_loader'] = batches
        self._batchify_fn = batchify_fn
        return outputs

        Transform the raw text to the model inputs, two steps involved:
           1) Transform the raw text to token ids.
           2) Generate the other model inputs from the raw text and token ids.
        
Transform the raw text to the model inputs, two steps involved:
1) Transform the raw text to token ids.
2) Generate the other model inputs from the raw text and token ids.
Transform the raw text to the model inputs two steps involved : 1 ) Transform the raw text to token ids . 2 ) Generate the other model inputs from the raw text and token ids .
def _preprocess(self, inputs, padding=True, add_special_tokens=True):
        inputs = self._check_input_text(inputs)
        batch_size = self.kwargs[
            'batch_size'] if 'batch_size' in self.kwargs else 1
        num_workers = self.kwargs[
            'num_workers'] if 'num_workers' in self.kwargs else 0
        generation_task = self.kwargs[
            'generation_task'] if 'generation_task' in self.kwargs else 'question_answering'
        max_seq_len = 32
        def select_few_shot_input(model_name, generation_task):
            pre_input = ""
            if generation_task not in [
                    'question_answering', 'poetry_generation'
            ]:
                raise ValueError(
                    "The generation task must be question or poetry")
            if model_name == "gpt-cpm-large-cn":
                if generation_task == "question_answering":
                    pre_input = '问题：中国的首都是哪里？答案：北京。\n问题：{} 答案：'
                else:
                    pre_input = '默写古诗: 大漠孤烟直，长河落日圆。\n{}'
            return pre_input
        pre_input = select_few_shot_input(self.model, generation_task)
        infer_data = []
        examples = []
        filter_inputs = []
        for input_text in inputs:
            if not (isinstance(input_text, str) and len(input_text) > 0):
                continue
            filter_inputs.append(input_text)
            few_shot_input = pre_input.format(input_text)
            ids = self._tokenizer(few_shot_input)["input_ids"]
            examples.append((ids, len(ids)))
        batchify_fn = lambda samples, fn=Tuple(
            Pad(axis=0, pad_val=0, dtype="int64"),
            Stack(dtype='int64'),  
        ): fn(samples)
        batches = [
            examples[idx:idx + batch_size]
            for idx in range(0, len(examples), batch_size)
        ]
        outputs = {}
        outputs['text'] = filter_inputs
        outputs['data_loader'] = batches
        self._batchify_fn = batchify_fn
        return outputs
def _preprocess ( self , inputs , padding = True , add_special_tokens = True ) : inputs = self . _check_input_text ( inputs ) batch_size = self . kwargs [ 'batch_size' ] if 'batch_size' in self . kwargs else 1 num_workers = self . kwargs [ 'num_workers' ] if 'num_workers' in self . kwargs else 0 generation_task = self . kwargs [ 'generation_task' ] if 'generation_task' in self . kwargs else 'question_answering' max_seq_len = 32 def select_few_shot_input ( model_name , generation_task ) : pre_input = "" if generation_task not in [ 'question_answering' , 'poetry_generation' ] : raise ValueError ( "The generation task must be question or poetry" ) if model_name == "gpt-cpm-large-cn" : if generation_task == "question_answering" : pre_input = '问题：中国的首都是哪里？答案：北京。\n问题：{} 答案：' else : pre_input = '默写古诗: 大漠孤烟直，长河落日圆。\n{}' return pre_input pre_input = select_few_shot_input ( self . model , generation_task ) infer_data = [ ] examples = [ ] filter_inputs = [ ] for input_text in inputs : if not ( isinstance ( input_text , str ) and len ( input_text ) > 0 ) : continue filter_inputs . append ( input_text ) few_shot_input = pre_input . format ( input_text ) ids = self . _tokenizer ( few_shot_input ) [ "input_ids" ] examples . append ( ( ids , len ( ids ) ) ) batchify_fn = lambda samples , fn = Tuple ( Pad ( axis = 0 , pad_val = 0 , dtype = "int64" ) , Stack ( dtype = 'int64' ) , ) : fn ( samples ) batches = [ examples [ idx : idx + batch_size ] for idx in range ( 0 , len ( examples ) , batch_size ) ] outputs = { } outputs [ 'text' ] = filter_inputs outputs [ 'data_loader' ] = batches self . _batchify_fn = batchify_fn return outputs
Transform the raw text to the model inputs, two steps involved:
1) Transform the raw text to token ids.
Transform the raw text to the model inputs two steps involved : 1 ) Transform the raw text to token ids .
"""
        Transform the raw text to the model inputs, two steps involved:
           1) Transform the raw text to token ids.
           2) Generate the other model inputs from the raw text and token ids.
        """ # Get the config from the kwargs # seq_len
self
inputs
padding
add_special_tokens