




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCamelCase__ :   Dict						=							logging.get_logger(__name__)

UpperCamelCase__ :   Optional[int]						=							{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase__ :   Union[str, Any]						=							{
    '''vocab_file''': {
        '''junnyu/roformer_chinese_small''': '''https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_base''': '''https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_char_small''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_chinese_char_base''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_discriminator''': (
            '''https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_generator''': (
            '''https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'''
        ),
    }
}

UpperCamelCase__ :   Tuple						=							{
    '''junnyu/roformer_chinese_small''': 15_36,
    '''junnyu/roformer_chinese_base''': 15_36,
    '''junnyu/roformer_chinese_char_small''': 5_12,
    '''junnyu/roformer_chinese_char_base''': 5_12,
    '''junnyu/roformer_small_discriminator''': 1_28,
    '''junnyu/roformer_small_generator''': 1_28,
}


UpperCamelCase__ :   List[Any]						=							{
    '''junnyu/roformer_chinese_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_discriminator''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_generator''': {'''do_lower_case''': True},
}







class 				lowerCAmelCase_       (					lowerCamelCase_							):
				__a     :       Dict						=     VOCAB_FILES_NAMES
				__a     :       List[str]						=     PRETRAINED_VOCAB_FILES_MAP
				__a     :       List[Any]						=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__a     :       str						=     PRETRAINED_INIT_CONFIGURATION
				__a     :       str						=     RoFormerTokenizer






				def __init__(			self      ,snake_case__=None      ,snake_case__=None      ,snake_case__=True      ,snake_case__="[UNK]"      ,snake_case__="[SEP]"      ,snake_case__="[PAD]"      ,snake_case__="[CLS]"      ,snake_case__="[MASK]"      ,snake_case__=True      ,snake_case__=None      ,**snake_case__      ,):
							super().__init__(
							    snake_case__      ,tokenizer_file=snake_case__      ,do_lower_case=snake_case__      ,unk_token=snake_case__      ,sep_token=snake_case__      ,pad_token=snake_case__      ,cls_token=snake_case__      ,mask_token=snake_case__      ,tokenize_chinese_chars=snake_case__      ,strip_accents=snake_case__      ,**snake_case__      ,)

							SCREAMING_SNAKE_CASE_   : Any					  =      json.loads(self.backend_tokenizer.normalizer.__getstate__() )
							if (
							    pre_tok_state.get('lowercase'      ,snake_case__ ) != do_lower_case
							    or pre_tok_state.get('strip_accents'      ,snake_case__ ) != strip_accents
							):
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      getattr(snake_case__      ,pre_tok_state.pop('type' ) )
										SCREAMING_SNAKE_CASE_   : Any					  =      do_lower_case
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      strip_accents
										SCREAMING_SNAKE_CASE_   : int					  =      pre_tok_class(**snake_case__ )

							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      do_lower_case






				def __getstate__(			self ):
							SCREAMING_SNAKE_CASE_   : Tuple					  =      self.__dict__.copy()
							SCREAMING_SNAKE_CASE_   : str					  =      BertPreTokenizer()
							return state






				def __setstate__(			self      ,snake_case__ ):
							SCREAMING_SNAKE_CASE_   : Any					  =      d
							SCREAMING_SNAKE_CASE_   : Any					  =      self.__dict__['_tokenizer'].get_vocab()
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      PreTokenizer.custom(JiebaPreTokenizer(snake_case__ ) )






				def 		snake_case	(			self      ,snake_case__      ,snake_case__=None ):
							SCREAMING_SNAKE_CASE_   : int					  =      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
										output += token_ids_a + [self.sep_token_id]

							return output






				def 		snake_case	(			self      ,snake_case__      ,snake_case__ = None ):
							SCREAMING_SNAKE_CASE_   : List[str]					  =      [self.sep_token_id]
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      [self.cls_token_id]
							if token_ids_a is None:
										return len(cls + token_ids_a + sep ) * [0]
							return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






				def 		snake_case	(			self      ,snake_case__      ,snake_case__ = None ):
							SCREAMING_SNAKE_CASE_   : int					  =      self._tokenizer.model.save(snake_case__      ,name=snake_case__ )
							return tuple(snake_case__ )






				def 		snake_case	(			self      ,snake_case__      ,snake_case__=None      ,snake_case__=None      ,snake_case__=False      ,**snake_case__      ,):
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      BertPreTokenizer()
							return super().save_pretrained(snake_case__      ,snake_case__      ,snake_case__      ,snake_case__      ,**snake_case__ )









import os
import sys
import unittest


UpperCamelCase__ :   Optional[Any]						=							os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, '''utils'''))

import check_dummies  # noqa: E402
from check_dummies import create_dummy_files, create_dummy_object, find_backend, read_init  # noqa: E402


# Align TRANSFORMERS_PATH in check_dummies with the current path
UpperCamelCase__ :   Tuple						=							os.path.join(git_repo_path, '''src''', '''transformers''')

UpperCamelCase__ :   List[Any]						=							'''
{0} = None
'''

UpperCamelCase__ :   str						=							'''
class {0}(metaclass=DummyObject):
    _backends = {1}

    def __init__(self, *args, **kwargs):
        requires_backends(self, {1})
'''


UpperCamelCase__ :   List[Any]						=							'''
def {0}(*args, **kwargs):
    requires_backends({0}, {1})
'''







class 				lowerCAmelCase_       (					unittest.TestCase							):






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : int					  =      find_backend('    _import_structure["models.albert"].append("AlbertTokenizerFast")' )
							self.assertIsNone(snake_case__ )

							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      find_backend('    if not is_tokenizers_available():' )
							self.assertEqual(snake_case__      ,'tokenizers' )

							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      find_backend('    if not is_tensorflow_text_available():' )
							self.assertEqual(snake_case__      ,'tensorflow_text' )

							SCREAMING_SNAKE_CASE_   : int					  =      find_backend('    if not (is_sentencepiece_available() and is_tokenizers_available()):' )
							self.assertEqual(snake_case__      ,'sentencepiece_and_tokenizers' )

							SCREAMING_SNAKE_CASE_   : str					  =      find_backend(
							    '    if not (is_sentencepiece_available() and is_tensorflow_text_available()):' )
							self.assertEqual(snake_case__      ,'sentencepiece_and_tensorflow_text' )

							SCREAMING_SNAKE_CASE_   : Tuple					  =      find_backend(
							    '    if not (is_sentencepiece_available() and is_tokenizers_available() and is_vision_available()):' )
							self.assertEqual(snake_case__      ,'sentencepiece_and_tokenizers_and_vision' )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : str					  =      read_init()
							# We don't assert on the exact list of keys to allow for smooth grow of backend-specific objects
							self.assertIn('torch'      ,snake_case__ )
							self.assertIn('tensorflow_text'      ,snake_case__ )
							self.assertIn('sentencepiece_and_tokenizers'      ,snake_case__ )

							# Likewise, we can't assert on the exact content of a key
							self.assertIn('BertModel'      ,objects['torch'] )
							self.assertIn('TFBertModel'      ,objects['tf'] )
							self.assertIn('FlaxBertModel'      ,objects['flax'] )
							self.assertIn('BertModel'      ,objects['torch'] )
							self.assertIn('TFBertTokenizer'      ,objects['tensorflow_text'] )
							self.assertIn('convert_slow_tokenizer'      ,objects['sentencepiece_and_tokenizers'] )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : str					  =      create_dummy_object('CONSTANT'      ,'\'torch\'' )
							self.assertEqual(snake_case__      ,'\nCONSTANT = None\n' )

							SCREAMING_SNAKE_CASE_   : Dict					  =      create_dummy_object('function'      ,'\'torch\'' )
							self.assertEqual(
							    snake_case__      ,'\ndef function(*args, **kwargs):\n    requires_backends(function, \'torch\')\n' )

							SCREAMING_SNAKE_CASE_   : List[str]					  =      '\nclass FakeClass(metaclass=DummyObject):\n    _backends = \'torch\'\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, \'torch\')\n'
							SCREAMING_SNAKE_CASE_   : Dict					  =      create_dummy_object('FakeClass'      ,'\'torch\'' )
							self.assertEqual(snake_case__      ,snake_case__ )






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      '# This file is autogenerated by the command `make fix-copies`, do not edit.\nfrom ..utils import DummyObject, requires_backends\n\n\nCONSTANT = None\n\n\ndef function(*args, **kwargs):\n    requires_backends(function, ["torch"])\n\n\nclass FakeClass(metaclass=DummyObject):\n    _backends = ["torch"]\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, ["torch"])\n'
							SCREAMING_SNAKE_CASE_   : Dict					  =      create_dummy_files({'torch': ['CONSTANT', 'function', 'FakeClass']} )
							self.assertEqual(dummy_files['torch']      ,snake_case__ )



