Update README.md

README.md

@@ -29,7 +29,8 @@ from transformers import PegasusForConditionalGeneration,BertTokenizer
 class PegasusTokenizer(BertTokenizer):
     model_input_names = ["input_ids", "attention_mask"]
     def __init__(self, **kwargs):
-        super().__init__(pre_tokenizer=lambda x: jieba.cut(x, HMM=False), **kwargs)
+        # super().__init__(pre_tokenizer=lambda x: jieba.cut(x, HMM=False), **kwargs)
+        super().__init__(**kwargs)
         self.add_special_tokens({'additional_special_tokens':["<mask_1>"]})
 
     def build_inputs_with_special_tokens(