



"""simple docstring"""





from datasets.utils.patching import _PatchedModuleObj, patch_submodule

from . import _test_patching




def 						UpperCAmelCase  (		):

		'''simple docstring'''


		import os as original_os
		from os import path as original_path
		from os import rename as original_rename
		from os.path import dirname as original_dirname
		from os.path import join as original_join

		assert _test_patching.os is original_os
		assert _test_patching.path is original_path
		assert _test_patching.join is original_join

		assert _test_patching.renamed_os is original_os
		assert _test_patching.renamed_path is original_path
		assert _test_patching.renamed_join is original_join

		_UpperCAmelCase							=		'__test_patch_submodule_mock__'
		with patch_submodule(_test_patching  ,					'os.path.join'  ,					__lowercase							):
				# Every way to access os.path.join must be patched, and the rest must stay untouched

				# check os.path.join
				assert isinstance(_test_patching.os  ,					_PatchedModuleObj							)
				assert isinstance(_test_patching.os.path  ,					_PatchedModuleObj							)
				assert _test_patching.os.path.join is mock

				# check path.join
				assert isinstance(_test_patching.path  ,					_PatchedModuleObj							)
				assert _test_patching.path.join is mock

				# check join
				assert _test_patching.join is mock

				# check that the other attributes are untouched
				assert _test_patching.os.rename is original_rename
				assert _test_patching.path.dirname is original_dirname
				assert _test_patching.os.path.dirname is original_dirname

				# Even renamed modules or objects must be patched

				# check renamed_os.path.join
				assert isinstance(_test_patching.renamed_os  ,					_PatchedModuleObj							)
				assert isinstance(_test_patching.renamed_os.path  ,					_PatchedModuleObj							)
				assert _test_patching.renamed_os.path.join is mock

				# check renamed_path.join
				assert isinstance(_test_patching.renamed_path  ,					_PatchedModuleObj							)
				assert _test_patching.renamed_path.join is mock

				# check renamed_join
				assert _test_patching.renamed_join is mock

				# check that the other attributes are untouched
				assert _test_patching.renamed_os.rename is original_rename
				assert _test_patching.renamed_path.dirname is original_dirname
				assert _test_patching.renamed_os.path.dirname is original_dirname

		# check that everthing is back to normal when the patch is over

		assert _test_patching.os is original_os
		assert _test_patching.path is original_path
		assert _test_patching.join is original_join

		assert _test_patching.renamed_os is original_os
		assert _test_patching.renamed_path is original_path
		assert _test_patching.renamed_join is original_join




def 						UpperCAmelCase  (		):

		'''simple docstring'''


		assert _test_patching.open is open

		_UpperCAmelCase							=		'__test_patch_submodule_builtin_mock__'
		# _test_patching has "open" in its globals
		assert _test_patching.open is open
		with patch_submodule(_test_patching  ,					'open'  ,					__lowercase							):
				assert _test_patching.open is mock

		# check that everthing is back to normal when the patch is over

		assert _test_patching.open is open




def 						UpperCAmelCase  (		):

		'''simple docstring'''


		_UpperCAmelCase							=		'__test_patch_submodule_missing_mock__'
		with patch_submodule(_test_patching  ,					'pandas.read_csv'  ,					__lowercase							):
				pass




def 						UpperCAmelCase  (		):

		'''simple docstring'''


		_UpperCAmelCase							=		'__test_patch_submodule_missing_builtin_mock__'
		# _test_patching doesn't have "len" in its globals
		assert getattr(_test_patching  ,					'len'  ,					__lowercase							) is None
		with patch_submodule(_test_patching  ,					'len'  ,					__lowercase							):
				assert _test_patching.len is mock
		assert _test_patching.len is len




def 						UpperCAmelCase  (		):

		'''simple docstring'''


		_UpperCAmelCase							=		'__test_patch_submodule_start_and_stop_mock__'
		_UpperCAmelCase							=		patch_submodule(_test_patching  ,					'open'  ,					__lowercase							)
		assert _test_patching.open is open
		patch.start()
		assert _test_patching.open is mock
		patch.stop()
		assert _test_patching.open is open




def 						UpperCAmelCase  (		):

		'''simple docstring'''


		from os import rename as original_rename
		from os.path import dirname as original_dirname
		from os.path import join as original_join

		_UpperCAmelCase							=		'__test_patch_submodule_successive_join__'
		_UpperCAmelCase							=		'__test_patch_submodule_successive_dirname__'
		_UpperCAmelCase							=		'__test_patch_submodule_successive_rename__'
		assert _test_patching.os.path.join is original_join
		assert _test_patching.os.path.dirname is original_dirname
		assert _test_patching.os.rename is original_rename

		with patch_submodule(_test_patching  ,					'os.path.join'  ,					__lowercase							):
				with patch_submodule(_test_patching  ,					'os.rename'  ,					__lowercase							):
						with patch_submodule(_test_patching  ,					'os.path.dirname'  ,					__lowercase							):
								assert _test_patching.os.path.join is mock_join
								assert _test_patching.os.path.dirname is mock_dirname
								assert _test_patching.os.rename is mock_rename

								# try another order
		with patch_submodule(_test_patching  ,					'os.rename'  ,					__lowercase							):
				with patch_submodule(_test_patching  ,					'os.path.join'  ,					__lowercase							):
						with patch_submodule(_test_patching  ,					'os.path.dirname'  ,					__lowercase							):
								assert _test_patching.os.path.join is mock_join
								assert _test_patching.os.path.dirname is mock_dirname
								assert _test_patching.os.rename is mock_rename

		assert _test_patching.os.path.join is original_join
		assert _test_patching.os.path.dirname is original_dirname
		assert _test_patching.os.rename is original_rename






def 						UpperCAmelCase  (		):

		'''simple docstring'''


		_UpperCAmelCase							=		'__test_patch_submodule_doesnt_exist_mock__'
		with patch_submodule(_test_patching  ,					'__module_that_doesn_exist__.__attribute_that_doesn_exist__'  ,					__lowercase							):
				pass
		with patch_submodule(_test_patching  ,					'os.__attribute_that_doesn_exist__'  ,					__lowercase							):
				pass




"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__SCREAMING_SNAKE_CASE							=     logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE							=     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__SCREAMING_SNAKE_CASE							=     {
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

__SCREAMING_SNAKE_CASE							=     {
    'junnyu/roformer_chinese_small': 1_536,
    'junnyu/roformer_chinese_base': 1_536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


__SCREAMING_SNAKE_CASE							=     {
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}





class 				a__					(						A__       ):
     UpperCAmelCase__         =      VOCAB_FILES_NAMES
     UpperCAmelCase__         =      PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__         =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__         =      PRETRAINED_INIT_CONFIGURATION
     UpperCAmelCase__         =      RoFormerTokenizer



     def __init__(							self :Tuple      ,  _lowerCamelCase :Dict=None      ,  _lowerCamelCase :Dict=None      ,  _lowerCamelCase :List[Any]=True      ,  _lowerCamelCase :Dict="[UNK]"      ,  _lowerCamelCase :List[str]="[SEP]"      ,  _lowerCamelCase :str="[PAD]"      ,  _lowerCamelCase :Optional[Any]="[CLS]"      ,  _lowerCamelCase :Optional[int]="[MASK]"      ,  _lowerCamelCase :str=True      ,  _lowerCamelCase :Tuple=None      ,  **_lowerCamelCase :Any      ,  ):







      '''simple docstring'''



      super().__init__(
          _lowerCamelCase      ,  tokenizer_file=_lowerCamelCase      ,  do_lower_case=_lowerCamelCase      ,  unk_token=_lowerCamelCase      ,  sep_token=_lowerCamelCase      ,  pad_token=_lowerCamelCase      ,  cls_token=_lowerCamelCase      ,  mask_token=_lowerCamelCase      ,  tokenize_chinese_chars=_lowerCamelCase      ,  strip_accents=_lowerCamelCase      ,  **_lowerCamelCase      ,  )

      UpperCamelCase_     :					Any   	=json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
      if (
          pre_tok_state.get('lowercase'      ,  _lowerCamelCase   ) != do_lower_case
          or pre_tok_state.get('strip_accents'      ,  _lowerCamelCase   ) != strip_accents
      ):
       UpperCamelCase_     :					Optional[Any]   	=getattr(_lowerCamelCase      ,  pre_tok_state.pop('type'   )   )
       UpperCamelCase_     :					Tuple   	=do_lower_case
       UpperCamelCase_     :					Union[str, Any]   	=strip_accents
       UpperCamelCase_     :					Union[str, Any]   	=pre_tok_class(**_lowerCamelCase   )

      UpperCamelCase_     :					List[str]   	=do_lower_case



     def __getstate__(							self :Any   ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=self.__dict__.copy()
      UpperCamelCase_     :					Union[str, Any]   	=BertPreTokenizer()
      return state



     def __setstate__(							self :str      ,  _lowerCamelCase :Optional[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=d
      UpperCamelCase_     :					Optional[int]   	=self.__dict__['_tokenizer'].get_vocab()
      UpperCamelCase_     :					Any   	=PreTokenizer.custom(JiebaPreTokenizer(_lowerCamelCase   )   )



     def  lowerCamelCase_						(							self :Dict      ,  _lowerCamelCase :Optional[Any]      ,  _lowerCamelCase :Any=None   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output



     def  lowerCamelCase_						(							self :Union[str, Any]      ,  _lowerCamelCase :List[int]      ,  _lowerCamelCase :Optional[List[int]] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[Any]   	=[self.sep_token_id]
      UpperCamelCase_     :					List[Any]   	=[self.cls_token_id]
      if token_ids_a is None:
       return len(cls + token_ids_a + sep   ) * [0]
      return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]



     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :Optional[str] = None   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[Any]   	=self._tokenizer.model.save(_lowerCamelCase      ,  name=_lowerCamelCase   )
      return tuple(_lowerCamelCase   )




     def  lowerCamelCase_						(							self :str      ,  _lowerCamelCase :List[str]      ,  _lowerCamelCase :List[Any]=None      ,  _lowerCamelCase :Optional[Any]=None      ,  _lowerCamelCase :int=False      ,  **_lowerCamelCase :Optional[int]      ,  ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=BertPreTokenizer()
      return super().save_pretrained(_lowerCamelCase      ,  _lowerCamelCase      ,  _lowerCamelCase      ,  _lowerCamelCase      ,  **_lowerCamelCase   )
