


'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCAmelCase__  			=				logging.get_logger(__name__)

UpperCAmelCase__  			=				{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase__  			=				{
    '''vocab_file''': {
        '''junnyu/roformer_chinese_small''': '''https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_base''': '''https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_char_small''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_chinese_char_base''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_discriminator''': (
            '''https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_generator''': (
            '''https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'''
        ),
    }
}

UpperCAmelCase__  			=				{
    '''junnyu/roformer_chinese_small''': 1_5_3_6,
    '''junnyu/roformer_chinese_base''': 1_5_3_6,
    '''junnyu/roformer_chinese_char_small''': 5_1_2,
    '''junnyu/roformer_chinese_char_base''': 5_1_2,
    '''junnyu/roformer_small_discriminator''': 1_2_8,
    '''junnyu/roformer_small_generator''': 1_2_8,
}


UpperCAmelCase__  			=				{
    '''junnyu/roformer_chinese_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_discriminator''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_generator''': {'''do_lower_case''': True},
}
class 						a__					(					a_   ):





			'''simple docstring'''

			A						:				int					      =			VOCAB_FILES_NAMES
			A						:				Union[str, Any]					      =			PRETRAINED_VOCAB_FILES_MAP
			A						:				str					      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			A						:				List[str]					      =			PRETRAINED_INIT_CONFIGURATION
			A						:				Tuple					      =			RoFormerTokenizer
			def __init__(  self							:			str			,    lowerCAmelCase_							:			Optional[int]=None			,    lowerCAmelCase_							:			Dict=None			,    lowerCAmelCase_							:			List[Any]=True			,    lowerCAmelCase_							:			Optional[int]="[UNK]"			,    lowerCAmelCase_							:			Optional[Any]="[SEP]"			,    lowerCAmelCase_							:			Optional[int]="[PAD]"			,    lowerCAmelCase_							:			int="[CLS]"			,    lowerCAmelCase_							:			Optional[int]="[MASK]"			,    lowerCAmelCase_							:			str=True			,    lowerCAmelCase_							:			Any=None			,    **lowerCAmelCase_							:			int			,    )      ->		Any:
						super().__init__(
						    lowerCAmelCase_			,    tokenizer_file=lowerCAmelCase_			,    do_lower_case=lowerCAmelCase_			,    unk_token=lowerCAmelCase_			,    sep_token=lowerCAmelCase_			,    pad_token=lowerCAmelCase_			,    cls_token=lowerCAmelCase_			,    mask_token=lowerCAmelCase_			,    tokenize_chinese_chars=lowerCAmelCase_			,    strip_accents=lowerCAmelCase_			,    **lowerCAmelCase_			,    )

						__A=	json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
						if (
						    pre_tok_state.get('lowercase'			,    lowerCAmelCase_	) != do_lower_case
						    or pre_tok_state.get('strip_accents'			,    lowerCAmelCase_	) != strip_accents
						):
									__A=	getattr(lowerCAmelCase_			,    pre_tok_state.pop('type'	)	)
									__A=	do_lower_case
									__A=	strip_accents
									__A=	pre_tok_class(**lowerCAmelCase_	)

						__A=	do_lower_case
			def __getstate__(  self							:			Dict	)      ->		int:
						__A=	self.__dict__.copy()
						__A=	BertPreTokenizer()
						return state
			def __setstate__(  self							:			str			,    lowerCAmelCase_							:			int	)      ->		Optional[Any]:
						__A=	d
						__A=	self.__dict__['_tokenizer'].get_vocab()
						__A=	PreTokenizer.custom(JiebaPreTokenizer(lowerCAmelCase_	)	)
			def 					lowerCAmelCase     (  self							:			Optional[Any]			,    lowerCAmelCase_							:			List[Any]			,    lowerCAmelCase_							:			str=None	)      ->		int:
						__A=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

						return output
			def 					lowerCAmelCase     (  self							:			int			,    lowerCAmelCase_							:			List[int]			,    lowerCAmelCase_							:			Optional[List[int]] = None	)      ->		List[int]:
						__A=	[self.sep_token_id]
						__A=	[self.cls_token_id]
						if token_ids_a is None:
									return len(cls + token_ids_a + sep	) * [0]
						return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]
			def 					lowerCAmelCase     (  self							:			Any			,    lowerCAmelCase_							:			str			,    lowerCAmelCase_							:			Optional[str] = None	)      ->		Tuple[str]:
						__A=	self._tokenizer.model.save(lowerCAmelCase_			,    name=lowerCAmelCase_	)
						return tuple(lowerCAmelCase_	)

			def 					lowerCAmelCase     (  self							:			List[Any]			,    lowerCAmelCase_							:			List[Any]			,    lowerCAmelCase_							:			List[str]=None			,    lowerCAmelCase_							:			Union[str, Any]=None			,    lowerCAmelCase_							:			Any=False			,    **lowerCAmelCase_							:			List[str]			,    )      ->		Optional[Any]:
						__A=	BertPreTokenizer()
						return super().save_pretrained(lowerCAmelCase_			,    lowerCAmelCase_			,    lowerCAmelCase_			,    lowerCAmelCase_			,    **lowerCAmelCase_	)







'''simple docstring'''





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


UpperCAmelCase__  			=				{}

try:
					if not is_sentencepiece_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCAmelCase__  			=				['''BartphoTokenizer''']

if TYPE_CHECKING:
					try:
										if not is_sentencepiece_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_bartpho import BartphoTokenizer

else:
					import sys

					UpperCAmelCase__  			=				_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



