Update README.md

README.md

@@ -24,8 +24,7 @@ Task: Summarization
 ```python
 from transformers import PegasusForConditionalGeneration,BertTokenizer
 from typing import List, Optional
-import jieba
-jieba.initialize()
+
 # Need to download tokenizers_pegasus.py and other Python script from Fengshenbang-LM github repo in advance,
 # or you can mv download in tokenizers_pegasus.py and data_utils.py in https://huggingface.co/IDEA-CCNL/Randeng_Pegasus_523M_Summary/tree/main
 # Strongly recommend you git clone the Fengshenbang-LM repo:
@@ -35,7 +34,7 @@ jieba.initialize()
 # from tokenizers_pegasus import PegasusTokenizer
 class PegasusTokenizer(BertTokenizer):
     model_input_names = ["input_ids", "attention_mask"]
-    def __init__(self, pre_tokenizer=lambda x: jieba.cut(x, HMM=False), **kwargs):
+    def __init__(self, **kwargs):
         self.pre_tokenizer = pre_tokenizer
         super().__init__(pre_tokenizer=self.pre_tokenizer, **kwargs)
         self.add_special_tokens({'additional_special_tokens':["<mask_1>"]})