




"""simple docstring"""

import numpy as np

from transformers import BatchFeature
from transformers.testing_utils import require_tf, require_torch

from .test_feature_extraction_common import FeatureExtractionSavingTestMixin





class     lowercase_	(					__lowerCAmelCase							):



 '''simple docstring'''
 UpperCAmelCase   :					Dict				  =     None
 UpperCAmelCase   :					Optional[Any]				  =     None

 @property
 def       lowerCAmelCase_							(			self	:	int		):
       return self.feat_extract_tester.prepare_feat_extract_dict()

 def       lowerCAmelCase_							(			self	:	str		):
       _A        =       self.feature_extraction_class(**self.feat_extract_dict		)
       self.assertTrue(hasattr(_UpperCAmelCase   ,							'feature_size'		)		)
       self.assertTrue(hasattr(_UpperCAmelCase   ,							'sampling_rate'		)		)
       self.assertTrue(hasattr(_UpperCAmelCase   ,							'padding_value'		)		)

 def       lowerCAmelCase_							(			self	:	Optional[int]		):
       _A        =       self.feat_extract_tester.prepare_inputs_for_common()
       _A        =       self.feature_extraction_class(**self.feat_extract_dict		)
       _A        =       feat_extract.model_input_names[0]

       _A        =       BatchFeature({input_name: speech_inputs}		)

       self.assertTrue(all(len(_UpperCAmelCase		) == len(_UpperCAmelCase		) for x, y in zip(_UpperCAmelCase   ,							processed_features[input_name]		)		)		)

       _A        =       self.feat_extract_tester.prepare_inputs_for_common(equal_length=_UpperCAmelCase		)
       _A        =       BatchFeature({input_name: speech_inputs}   ,							tensor_type='np'		)

       _A        =       processed_features[input_name]

       if len(batch_features_input.shape		) < 3:
             _A        =       batch_features_input[:, :, None]

       self.assertTrue(
           batch_features_input.shape
           == (self.feat_extract_tester.batch_size, len(speech_inputs[0]		), self.feat_extract_tester.feature_size)		)

 @require_torch
 def       lowerCAmelCase_							(			self	:	Optional[Any]		):
       _A        =       self.feat_extract_tester.prepare_inputs_for_common(equal_length=_UpperCAmelCase		)
       _A        =       self.feature_extraction_class(**self.feat_extract_dict		)
       _A        =       feat_extract.model_input_names[0]

       _A        =       BatchFeature({input_name: speech_inputs}   ,							tensor_type='pt'		)

       _A        =       processed_features[input_name]

       if len(batch_features_input.shape		) < 3:
             _A        =       batch_features_input[:, :, None]

       self.assertTrue(
           batch_features_input.shape
           == (self.feat_extract_tester.batch_size, len(speech_inputs[0]		), self.feat_extract_tester.feature_size)		)

 @require_tf
 def       lowerCAmelCase_							(			self	:	Optional[int]		):
       _A        =       self.feat_extract_tester.prepare_inputs_for_common(equal_length=_UpperCAmelCase		)
       _A        =       self.feature_extraction_class(**self.feat_extract_dict		)
       _A        =       feat_extract.model_input_names[0]

       _A        =       BatchFeature({input_name: speech_inputs}   ,							tensor_type='tf'		)

       _A        =       processed_features[input_name]

       if len(batch_features_input.shape		) < 3:
             _A        =       batch_features_input[:, :, None]

       self.assertTrue(
           batch_features_input.shape
           == (self.feat_extract_tester.batch_size, len(speech_inputs[0]		), self.feat_extract_tester.feature_size)		)

 def       lowerCAmelCase_							(			self	:	str   ,							_UpperCAmelCase	:	Optional[Any]=False		):
       def _inputs_have_equal_length(_UpperCAmelCase	:	Optional[Any]		):
             _A        =       len(input[0]		)
             for input_slice in input[1:]:
                   if len(_UpperCAmelCase		) != length:
                         return False
             return True

       def _inputs_are_equal(_UpperCAmelCase	:	str   ,							_UpperCAmelCase	:	Dict		):
             if len(_UpperCAmelCase		) != len(_UpperCAmelCase		):
                   return False

             for input_slice_a, input_slice_a in zip(_UpperCAmelCase   ,							_UpperCAmelCase		):
                   if not np.allclose(np.asarray(_UpperCAmelCase		)   ,							np.asarray(_UpperCAmelCase		)   ,							atol=1E-3		):
                         return False
             return True

       _A        =       self.feature_extraction_class(**self.feat_extract_dict		)
       _A        =       self.feat_extract_tester.prepare_inputs_for_common(numpify=_UpperCAmelCase		)
       _A        =       feat_extract.model_input_names[0]

       _A        =       BatchFeature({input_name: speech_inputs}		)

       _A        =       self.feat_extract_tester.seq_length_diff
       _A        =       self.feat_extract_tester.max_seq_length + pad_diff
       _A        =       self.feat_extract_tester.min_seq_length
       _A        =       self.feat_extract_tester.batch_size
       _A        =       self.feat_extract_tester.feature_size

       # test padding for List[int] + numpy
       _A        =       feat_extract.pad(_UpperCAmelCase   ,							padding=_UpperCAmelCase		)
       _A        =       input_a[input_name]

       _A        =       feat_extract.pad(_UpperCAmelCase   ,							padding='longest'		)
       _A        =       input_a[input_name]

       _A        =       feat_extract.pad(_UpperCAmelCase   ,							padding='max_length'   ,							max_length=len(speech_inputs[-1]		)		)
       _A        =       input_a[input_name]

       _A        =       feat_extract.pad(_UpperCAmelCase   ,							padding='longest'   ,							return_tensors='np'		)
       _A        =       input_a[input_name]

       # max_length parameter has to be provided when setting `padding="max_length"`
       with self.assertRaises(_UpperCAmelCase		):
             feat_extract.pad(_UpperCAmelCase   ,							padding='max_length'		)[input_name]

       _A        =       feat_extract.pad(
           _UpperCAmelCase   ,							padding='max_length'   ,							max_length=_UpperCAmelCase   ,							return_tensors='np'		)
       _A        =       input_a[input_name]

       self.assertFalse(_inputs_have_equal_length(_UpperCAmelCase		)		)
       self.assertTrue(_inputs_have_equal_length(_UpperCAmelCase		)		)
       self.assertTrue(_inputs_have_equal_length(_UpperCAmelCase		)		)
       self.assertTrue(_inputs_are_equal(_UpperCAmelCase   ,							_UpperCAmelCase		)		)
       self.assertTrue(len(input_a[0]		) == pad_min_length		)
       self.assertTrue(len(input_a[1]		) == pad_min_length + pad_diff		)
       self.assertTrue(input_a.shape[:2] == (batch_size, len(input_a[0]		))		)
       self.assertTrue(input_a.shape[:2] == (batch_size, pad_max_length)		)

       if feature_size > 1:
             self.assertTrue(input_a.shape[2] == input_a.shape[2] == feature_size		)

       # test padding for `pad_to_multiple_of` for List[int] + numpy
       _A        =       feat_extract.pad(_UpperCAmelCase   ,							pad_to_multiple_of=10		)
       _A        =       input_a[input_name]

       _A        =       feat_extract.pad(_UpperCAmelCase   ,							padding='longest'   ,							pad_to_multiple_of=10		)
       _A        =       input_a[input_name]

       _A        =       feat_extract.pad(
           _UpperCAmelCase   ,							padding='max_length'   ,							pad_to_multiple_of=10   ,							max_length=_UpperCAmelCase		)
       _A        =       input_a[input_name]

       _A        =       feat_extract.pad(
           _UpperCAmelCase   ,							padding='max_length'   ,							pad_to_multiple_of=10   ,							max_length=_UpperCAmelCase   ,							return_tensors='np'   ,							)
       _A        =       input_a[input_name]

       self.assertTrue(all(len(_UpperCAmelCase		) % 10 == 0 for x in input_a		)		)
       self.assertTrue(_inputs_are_equal(_UpperCAmelCase   ,							_UpperCAmelCase		)		)

       _A        =       pad_max_length if pad_max_length % 10 == 0 else (pad_max_length // 10 + 1) * 10
       self.assertTrue(all(len(_UpperCAmelCase		) == expected_mult_pad_length for x in input_a		)		)
       self.assertEqual(input_a.shape[:2]   ,							(batch_size, expected_mult_pad_length)		)

       if feature_size > 1:
             self.assertTrue(input_a.shape[2] == feature_size		)

       # Check padding value is correct
       _A        =       (np.ones(self.feat_extract_tester.feature_size		) * feat_extract.padding_value).sum()
       self.assertTrue(
           abs(np.asarray(input_a[0]		)[pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)		)
           < 1E-3		)
       self.assertTrue(
           abs(
               np.asarray(input_a[1]		)[pad_min_length + pad_diff :].sum()
               - padding_vector_sum * (pad_max_length - pad_min_length - pad_diff)		)
           < 1E-3		)
       self.assertTrue(
           abs(
               np.asarray(input_a[2]		)[pad_min_length + 2 * pad_diff :].sum()
               - padding_vector_sum * (pad_max_length - pad_min_length - 2 * pad_diff)		)
           < 1E-3		)
       self.assertTrue(
           abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)		) < 1E-3		)
       self.assertTrue(
           abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (expected_mult_pad_length - pad_min_length)		)
           < 1E-3		)

 def       lowerCAmelCase_							(			self	:	List[str]   ,							_UpperCAmelCase	:	str=False		):
       def _inputs_have_equal_length(_UpperCAmelCase	:	Dict		):
             _A        =       len(input[0]		)
             for input_slice in input[1:]:
                   if len(_UpperCAmelCase		) != length:
                         return False
             return True

       def _inputs_are_equal(_UpperCAmelCase	:	List[Any]   ,							_UpperCAmelCase	:	str		):
             if len(_UpperCAmelCase		) != len(_UpperCAmelCase		):
                   return False

             for input_slice_a, input_slice_a in zip(_UpperCAmelCase   ,							_UpperCAmelCase		):
                   if not np.allclose(np.asarray(_UpperCAmelCase		)   ,							np.asarray(_UpperCAmelCase		)   ,							atol=1E-3		):
                         return False
             return True

       _A        =       self.feature_extraction_class(**self.feat_extract_dict		)
       _A        =       self.feat_extract_tester.prepare_inputs_for_common(numpify=_UpperCAmelCase		)
       _A        =       feat_extract.model_input_names[0]

       _A        =       BatchFeature({input_name: speech_inputs}		)

       # truncate to smallest
       _A        =       feat_extract.pad(
           _UpperCAmelCase   ,							padding='max_length'   ,							max_length=len(speech_inputs[0]		)   ,							truncation=_UpperCAmelCase		)
       _A        =       input_a[input_name]

       _A        =       feat_extract.pad(_UpperCAmelCase   ,							padding='max_length'   ,							max_length=len(speech_inputs[0]		)		)
       _A        =       input_a[input_name]

       self.assertTrue(_inputs_have_equal_length(_UpperCAmelCase		)		)
       self.assertFalse(_inputs_have_equal_length(_UpperCAmelCase		)		)

       # truncate to smallest with np
       _A        =       feat_extract.pad(
           _UpperCAmelCase   ,							padding='max_length'   ,							max_length=len(speech_inputs[0]		)   ,							return_tensors='np'   ,							truncation=_UpperCAmelCase   ,							)
       _A        =       input_a[input_name]

       _A        =       feat_extract.pad(
           _UpperCAmelCase   ,							padding='max_length'   ,							max_length=len(speech_inputs[0]		)   ,							return_tensors='np'		)
       _A        =       input_a[input_name]

       self.assertTrue(_inputs_have_equal_length(_UpperCAmelCase		)		)
       self.assertTrue(input_a.shape[1] == len(speech_inputs[0]		)		)

       # since truncation forces padding to be smaller than longest input
       # function can't return `np.ndarray`, but has to return list
       self.assertFalse(_inputs_have_equal_length(_UpperCAmelCase		)		)

       # truncate to middle
       _A        =       feat_extract.pad(
           _UpperCAmelCase   ,							padding='max_length'   ,							max_length=len(speech_inputs[1]		)   ,							truncation=_UpperCAmelCase   ,							return_tensors='np'   ,							)
       _A        =       input_a[input_name]

       _A        =       feat_extract.pad(
           _UpperCAmelCase   ,							padding='max_length'   ,							max_length=len(speech_inputs[1]		)   ,							truncation=_UpperCAmelCase		)
       _A        =       input_a[input_name]

       _A        =       feat_extract.pad(
           _UpperCAmelCase   ,							padding='max_length'   ,							max_length=len(speech_inputs[1]		)   ,							return_tensors='np'		)
       _A        =       input_a[input_name]

       self.assertTrue(input_a.shape[1] == len(speech_inputs[1]		)		)
       self.assertTrue(_inputs_have_equal_length(_UpperCAmelCase		)		)
       self.assertTrue(_inputs_have_equal_length(_UpperCAmelCase		)		)
       self.assertTrue(_inputs_are_equal(_UpperCAmelCase   ,							_UpperCAmelCase		)		)

       # since truncation forces padding to be smaller than longest input
       # function can't return `np.ndarray`, but has to return list
       self.assertFalse(_inputs_have_equal_length(_UpperCAmelCase		)		)
       self.assertTrue(len(input_a[-1]		) == len(speech_inputs[-1]		)		)

       # padding has to be max_length when setting `truncation=True`
       with self.assertRaises(_UpperCAmelCase		):
             feat_extract.pad(_UpperCAmelCase   ,							truncation=_UpperCAmelCase		)[input_name]

       # padding has to be max_length when setting `truncation=True`
       with self.assertRaises(_UpperCAmelCase		):
             feat_extract.pad(_UpperCAmelCase   ,							padding='longest'   ,							truncation=_UpperCAmelCase		)[input_name]

       # padding has to be max_length when setting `truncation=True`
       with self.assertRaises(_UpperCAmelCase		):
             feat_extract.pad(_UpperCAmelCase   ,							padding='longest'   ,							truncation=_UpperCAmelCase		)[input_name]

       # max_length parameter has to be provided when setting `truncation=True` and padding="max_length"
       with self.assertRaises(_UpperCAmelCase		):
             feat_extract.pad(_UpperCAmelCase   ,							padding='max_length'   ,							truncation=_UpperCAmelCase		)[input_name]

       # test truncation for `pad_to_multiple_of` for List[int] + numpy
       _A        =       12
       _A        =       feat_extract.pad(
           _UpperCAmelCase   ,							padding='max_length'   ,							max_length=len(speech_inputs[0]		)   ,							pad_to_multiple_of=_UpperCAmelCase   ,							truncation=_UpperCAmelCase   ,							)
       _A        =       input_a[input_name]

       _A        =       feat_extract.pad(
           _UpperCAmelCase   ,							padding='max_length'   ,							max_length=len(speech_inputs[0]		)   ,							pad_to_multiple_of=_UpperCAmelCase   ,							)
       _A        =       input_a[input_name]

       # retrieve expected_length as multiple of pad_to_multiple_of
       _A        =       len(speech_inputs[0]		)
       if expected_length % pad_to_multiple_of != 0:
             _A        =       ((len(speech_inputs[0]		) // pad_to_multiple_of) + 1) * pad_to_multiple_of

       self.assertTrue(len(input_a[0]		) == expected_length		)
       self.assertTrue(_inputs_have_equal_length(_UpperCAmelCase		)		)
       self.assertFalse(_inputs_have_equal_length(_UpperCAmelCase		)		)

 def       lowerCAmelCase_							(			self	:	Optional[Any]		):
       self._check_padding(numpify=_UpperCAmelCase		)

 def       lowerCAmelCase_							(			self	:	List[str]		):
       self._check_padding(numpify=_UpperCAmelCase		)

 def       lowerCAmelCase_							(			self	:	List[Any]		):
       self._check_truncation(numpify=_UpperCAmelCase		)

 def       lowerCAmelCase_							(			self	:	Optional[int]		):
       self._check_truncation(numpify=_UpperCAmelCase		)

 @require_torch
 def       lowerCAmelCase_							(			self	:	Optional[int]		):
       _A        =       self.feature_extraction_class(**self.feat_extract_dict		)
       _A        =       self.feat_extract_tester.prepare_inputs_for_common()
       _A        =       feat_extract.model_input_names[0]

       _A        =       BatchFeature({input_name: speech_inputs}		)

       _A        =       feat_extract.pad(_UpperCAmelCase   ,							padding='longest'   ,							return_tensors='np'		)[input_name]
       _A        =       feat_extract.pad(_UpperCAmelCase   ,							padding='longest'   ,							return_tensors='pt'		)[input_name]

       self.assertTrue(abs(input_np.astype(np.floataa		).sum() - input_pt.numpy().astype(np.floataa		).sum()		) < 1E-2		)

 @require_tf
 def       lowerCAmelCase_							(			self	:	List[str]		):
       _A        =       self.feature_extraction_class(**self.feat_extract_dict		)
       _A        =       self.feat_extract_tester.prepare_inputs_for_common()
       _A        =       feat_extract.model_input_names[0]

       _A        =       BatchFeature({input_name: speech_inputs}		)

       _A        =       feat_extract.pad(_UpperCAmelCase   ,							padding='longest'   ,							return_tensors='np'		)[input_name]
       _A        =       feat_extract.pad(_UpperCAmelCase   ,							padding='longest'   ,							return_tensors='tf'		)[input_name]

       self.assertTrue(abs(input_np.astype(np.floataa		).sum() - input_tf.numpy().astype(np.floataa		).sum()		) < 1E-2		)

 def       lowerCAmelCase_							(			self	:	str		):
       _A        =       self.feat_extract_dict
       _A        =       True
       _A        =       self.feature_extraction_class(**_UpperCAmelCase		)
       _A        =       self.feat_extract_tester.prepare_inputs_for_common()
       _A        =       [len(_UpperCAmelCase		) for x in speech_inputs]
       _A        =       feat_extract.model_input_names[0]

       _A        =       BatchFeature({input_name: speech_inputs}		)

       _A        =       feat_extract.pad(_UpperCAmelCase   ,							padding='longest'   ,							return_tensors='np'		)
       self.assertIn('attention_mask'   ,							_UpperCAmelCase		)
       self.assertListEqual(list(processed.attention_mask.shape		)   ,							list(processed[input_name].shape[:2]		)		)
       self.assertListEqual(processed.attention_mask.sum(-1		).tolist()   ,							_UpperCAmelCase		)



 def       lowerCAmelCase_							(			self	:	Dict		):
       _A        =       self.feat_extract_dict
       _A        =       True
       _A        =       self.feature_extraction_class(**_UpperCAmelCase		)
       _A        =       self.feat_extract_tester.prepare_inputs_for_common()
       _A        =       [len(_UpperCAmelCase		) for x in speech_inputs]
       _A        =       feat_extract.model_input_names[0]

       _A        =       BatchFeature({input_name: speech_inputs}		)
       _A        =       min(_UpperCAmelCase		)

       _A        =       feat_extract.pad(
           _UpperCAmelCase   ,							padding='max_length'   ,							max_length=_UpperCAmelCase   ,							truncation=_UpperCAmelCase   ,							return_tensors='np'		)
       self.assertIn('attention_mask'   ,							_UpperCAmelCase		)
       self.assertListEqual(
           list(processed_pad.attention_mask.shape		)   ,							[processed_pad[input_name].shape[0], max_length]		)
       self.assertListEqual(
           processed_pad.attention_mask[:, :max_length].sum(-1		).tolist()   ,							[max_length for x in speech_inputs]		)







"""simple docstring"""

import argparse
import json
from typing import List

from ltp import LTP

from transformers import BertTokenizer






def 						_snake_case					( _snake_case							: Dict							)   -> Any:






      '''simple docstring'''


      if (
          (cp >= 0X4e00 and cp <= 0X9fff)
          or (cp >= 0X3400 and cp <= 0X4dbf)  #
          or (cp >= 0X2_0000 and cp <= 0X2_a6df)  #
          or (cp >= 0X2_a700 and cp <= 0X2_b73f)  #
          or (cp >= 0X2_b740 and cp <= 0X2_b81f)  #
          or (cp >= 0X2_b820 and cp <= 0X2_ceaf)  #
          or (cp >= 0Xf900 and cp <= 0Xfaff)
          or (cp >= 0X2_f800 and cp <= 0X2_fa1f)  #
      ):  #
            return True

      return False






def 						_snake_case					( _snake_case							: str							)   -> Tuple:






      '''simple docstring'''


      for char in word:
            _A        =       ord(_snake_case							)
            if not _is_chinese_char(_snake_case							):
                  return 0
      return 1






def 						_snake_case					( _snake_case							: List[str]							)   -> Optional[Any]:






      '''simple docstring'''


      _A        =       set()

      for token in tokens:
            _A        =       len(_snake_case							) > 1 and is_chinese(_snake_case							)
            if chinese_word:
                  word_set.add(_snake_case							)
      _A        =       list(_snake_case							)
      return word_list






def 						_snake_case					( _snake_case							: List[str]    ,  _snake_case							: set()							)   -> Optional[Any]:






      '''simple docstring'''


      if not chinese_word_set:
            return bert_tokens
      _A        =       max([len(_snake_case							) for w in chinese_word_set]							)

      _A        =       bert_tokens
      _A       ,   _A        =       0, len(_snake_case							)
      while start < end:
            _A        =       True
            if is_chinese(bert_word[start]							):
                  _A        =       min(end - start    ,  _snake_case							)
                  for i in range(_snake_case    ,  1    ,  -1							):
                        _A        =       ''.join(bert_word[start : start + i]							)
                        if whole_word in chinese_word_set:
                              for j in range(start + 1    ,  start + i							):
                                    _A        =       '##' + bert_word[j]
                              _A        =       start + i
                              _A        =       False
                              break
            if single_word:
                  start += 1
      return bert_word






def 						_snake_case					( _snake_case							: List[str]    ,  _snake_case							: LTP    ,  _snake_case							: BertTokenizer							)   -> str:






      '''simple docstring'''


      _A        =       []

      for i in range(0    ,  len(_snake_case							)    ,  1_00							):
            _A        =       ltp_tokenizer.seg(lines[i : i + 1_00]							)[0]
            _A        =       [get_chinese_word(_snake_case							) for r in res]
            ltp_res.extend(_snake_case							)
      assert len(_snake_case							) == len(_snake_case							)

      _A        =       []
      for i in range(0    ,  len(_snake_case							)    ,  1_00							):
            _A        =       bert_tokenizer(lines[i : i + 1_00]    ,  add_special_tokens=_snake_case    ,  truncation=_snake_case    ,  max_length=5_12							)
            bert_res.extend(res['input_ids']							)
      assert len(_snake_case							) == len(_snake_case							)

      _A        =       []
      for input_ids, chinese_word in zip(_snake_case    ,  _snake_case							):
            _A        =       []
            for id in input_ids:
                  _A        =       bert_tokenizer._convert_id_to_token(_snake_case							)
                  input_tokens.append(_snake_case							)
            _A        =       add_sub_symbol(_snake_case    ,  _snake_case							)
            _A        =       []
            # We only save pos of chinese subwords start with ##, which mean is part of a whole word.
            for i, token in enumerate(_snake_case							):
                  if token[:2] == "##":
                        _A        =       token[2:]
                        # save chinese tokens' pos
                        if len(_snake_case							) == 1 and _is_chinese_char(ord(_snake_case							)							):
                              ref_id.append(_snake_case							)
            ref_ids.append(_snake_case							)

      assert len(_snake_case							) == len(_snake_case							)

      return ref_ids






def 						_snake_case					( _snake_case							: List[str]							)   -> Dict:






      '''simple docstring'''


      with open(args.file_name    ,  'r'    ,  encoding='utf-8'							) as f:
            _A        =       f.readlines()
      _A        =       [line.strip() for line in data if len(_snake_case							) > 0 and not line.isspace()]  # avoid delimiter like '\u2029'
      _A        =       LTP(args.ltp							)  # faster in GPU device
      _A        =       BertTokenizer.from_pretrained(args.bert							)

      _A        =       prepare_ref(_snake_case    ,  _snake_case    ,  _snake_case							)

      with open(args.save_path    ,  'w'    ,  encoding='utf-8'							) as f:
            _A        =       [json.dumps(_snake_case							) + '\n' for ref in ref_ids]
            f.writelines(_snake_case							)


if __name__ == "__main__":
       a         =      argparse.ArgumentParser(description='''prepare_chinese_ref''')
       parser.add_argument(
           '''--file_name''',
           type=str,
           default='''./resources/chinese-demo.txt''',
           help='''file need process, same as training data in lm''',
       )
       parser.add_argument(
           '''--ltp''', type=str, default='''./resources/ltp''', help='''resources for LTP tokenizer, usually a path'''
       )
       parser.add_argument('''--bert''', type=str, default='''./resources/robert''', help='''resources for Bert tokenizer''')
       parser.add_argument('''--save_path''', type=str, default='''./resources/ref.txt''', help='''path to save res''')

       a         =      parser.parse_args()
       main(args)

