
"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


UpperCAmelCase        =				logging.get_logger(__name__)

UpperCAmelCase        =				{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase        =				{
    """vocab_file""": {
        """junnyu/roformer_chinese_small""": """https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_base""": """https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt""",
        """junnyu/roformer_chinese_char_small""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_chinese_char_base""": (
            """https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_discriminator""": (
            """https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"""
        ),
        """junnyu/roformer_small_generator""": (
            """https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"""
        ),
    }
}

UpperCAmelCase        =				{
    """junnyu/roformer_chinese_small""": 1_5_3_6,
    """junnyu/roformer_chinese_base""": 1_5_3_6,
    """junnyu/roformer_chinese_char_small""": 5_1_2,
    """junnyu/roformer_chinese_char_base""": 5_1_2,
    """junnyu/roformer_small_discriminator""": 1_2_8,
    """junnyu/roformer_small_generator""": 1_2_8,
}


UpperCAmelCase        =				{
    """junnyu/roformer_chinese_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_base""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_small""": {"""do_lower_case""": True},
    """junnyu/roformer_chinese_char_base""": {"""do_lower_case""": True},
    """junnyu/roformer_small_discriminator""": {"""do_lower_case""": True},
    """junnyu/roformer_small_generator""": {"""do_lower_case""": True},
}



class 			lowercase					( lowercase__		):
			lowercase        =     VOCAB_FILES_NAMES
			lowercase        =     PRETRAINED_VOCAB_FILES_MAP
			lowercase        =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowercase        =     PRETRAINED_INIT_CONFIGURATION
			lowercase        =     RoFormerTokenizer






			def __init__(self  :	List[Any]      ,SCREAMING_SNAKE_CASE_  :	List[Any]=None      ,SCREAMING_SNAKE_CASE_  :	str=None      ,SCREAMING_SNAKE_CASE_  :	str=True      ,SCREAMING_SNAKE_CASE_  :	List[str]="[UNK]"      ,SCREAMING_SNAKE_CASE_  :	Optional[int]="[SEP]"      ,SCREAMING_SNAKE_CASE_  :	Any="[PAD]"      ,SCREAMING_SNAKE_CASE_  :	Tuple="[CLS]"      ,SCREAMING_SNAKE_CASE_  :	str="[MASK]"      ,SCREAMING_SNAKE_CASE_  :	str=True      ,SCREAMING_SNAKE_CASE_  :	Optional[Any]=None      ,**SCREAMING_SNAKE_CASE_  :	Dict      ,)       ->	Union[str, Any]:





				"""simple docstring"""


				super().__init__(
				    SCREAMING_SNAKE_CASE_      ,tokenizer_file=SCREAMING_SNAKE_CASE_      ,do_lower_case=SCREAMING_SNAKE_CASE_      ,unk_token=SCREAMING_SNAKE_CASE_      ,sep_token=SCREAMING_SNAKE_CASE_      ,pad_token=SCREAMING_SNAKE_CASE_      ,cls_token=SCREAMING_SNAKE_CASE_      ,mask_token=SCREAMING_SNAKE_CASE_      ,tokenize_chinese_chars=SCREAMING_SNAKE_CASE_      ,strip_accents=SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_      ,)

				lowerCAmelCase												=  json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
				if (
				    pre_tok_state.get('''lowercase'''      ,SCREAMING_SNAKE_CASE_			) != do_lower_case
				    or pre_tok_state.get('''strip_accents'''      ,SCREAMING_SNAKE_CASE_			) != strip_accents
				):
					lowerCAmelCase												=  getattr(SCREAMING_SNAKE_CASE_      ,pre_tok_state.pop('''type'''			)			)
					lowerCAmelCase												=  do_lower_case
					lowerCAmelCase												=  strip_accents
					lowerCAmelCase												=  pre_tok_class(**SCREAMING_SNAKE_CASE_			)

				lowerCAmelCase												=  do_lower_case






			def __getstate__(self  :	Optional[Any]			)       ->	int:





				"""simple docstring"""


				lowerCAmelCase												=  self.__dict__.copy()
				lowerCAmelCase												=  BertPreTokenizer()
				return state






			def __setstate__(self  :	Any      ,SCREAMING_SNAKE_CASE_  :	Dict			)       ->	int:





				"""simple docstring"""


				lowerCAmelCase												=  d
				lowerCAmelCase												=  self.__dict__['''_tokenizer'''].get_vocab()
				lowerCAmelCase												=  PreTokenizer.custom(JiebaPreTokenizer(SCREAMING_SNAKE_CASE_			)			)






			def 					UpperCAmelCase							(self  :	List[str]      ,SCREAMING_SNAKE_CASE_  :	Tuple      ,SCREAMING_SNAKE_CASE_  :	Optional[Any]=None			)       ->	Tuple:





				"""simple docstring"""


				lowerCAmelCase												=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
					output += token_ids_a + [self.sep_token_id]

				return output






			def 					UpperCAmelCase							(self  :	Optional[Any]      ,SCREAMING_SNAKE_CASE_  :	List[int]      ,SCREAMING_SNAKE_CASE_  :	Optional[List[int]] = None			)       ->	List[int]:





				"""simple docstring"""


				lowerCAmelCase												=  [self.sep_token_id]
				lowerCAmelCase												=  [self.cls_token_id]
				if token_ids_a is None:
					return len(cls + token_ids_a + sep			) * [0]
				return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]






			def 					UpperCAmelCase							(self  :	Any      ,SCREAMING_SNAKE_CASE_  :	str      ,SCREAMING_SNAKE_CASE_  :	Optional[str] = None			)       ->	Tuple[str]:





				"""simple docstring"""


				lowerCAmelCase												=  self._tokenizer.model.save(SCREAMING_SNAKE_CASE_      ,name=SCREAMING_SNAKE_CASE_			)
				return tuple(SCREAMING_SNAKE_CASE_			)






			def 					UpperCAmelCase							(self  :	Dict      ,SCREAMING_SNAKE_CASE_  :	Union[str, Any]      ,SCREAMING_SNAKE_CASE_  :	Any=None      ,SCREAMING_SNAKE_CASE_  :	Optional[Any]=None      ,SCREAMING_SNAKE_CASE_  :	Optional[Any]=False      ,**SCREAMING_SNAKE_CASE_  :	Optional[int]      ,)       ->	Optional[Any]:





				"""simple docstring"""


				lowerCAmelCase												=  BertPreTokenizer()
				return super().save_pretrained(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ,**SCREAMING_SNAKE_CASE_			)




"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase        =				logging.get_logger(__name__)

UpperCAmelCase        =				{"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/config.json"""}



class 			lowercase					( lowercase__		):
			lowercase        =     '''openai-gpt'''
			lowercase        =     {
			    '''max_position_embeddings''': '''n_positions''',
			    '''hidden_size''': '''n_embd''',
			    '''num_attention_heads''': '''n_head''',
			    '''num_hidden_layers''': '''n_layer''',
			}






			def __init__(self  :	List[Any]      ,SCREAMING_SNAKE_CASE_  :	str=40_478      ,SCREAMING_SNAKE_CASE_  :	Optional[int]=512      ,SCREAMING_SNAKE_CASE_  :	Union[str, Any]=768      ,SCREAMING_SNAKE_CASE_  :	Dict=12      ,SCREAMING_SNAKE_CASE_  :	Dict=12      ,SCREAMING_SNAKE_CASE_  :	int="gelu"      ,SCREAMING_SNAKE_CASE_  :	Dict=0.1      ,SCREAMING_SNAKE_CASE_  :	List[Any]=0.1      ,SCREAMING_SNAKE_CASE_  :	Dict=0.1      ,SCREAMING_SNAKE_CASE_  :	Optional[Any]=1e-5      ,SCREAMING_SNAKE_CASE_  :	List[str]=0.02      ,SCREAMING_SNAKE_CASE_  :	Union[str, Any]="cls_index"      ,SCREAMING_SNAKE_CASE_  :	str=True      ,SCREAMING_SNAKE_CASE_  :	Any=None      ,SCREAMING_SNAKE_CASE_  :	List[Any]=True      ,SCREAMING_SNAKE_CASE_  :	List[str]=0.1      ,**SCREAMING_SNAKE_CASE_  :	int      ,)       ->	int:





				"""simple docstring"""


				lowerCAmelCase												=  vocab_size
				lowerCAmelCase												=  n_positions
				lowerCAmelCase												=  n_embd
				lowerCAmelCase												=  n_layer
				lowerCAmelCase												=  n_head
				lowerCAmelCase												=  afn
				lowerCAmelCase												=  resid_pdrop
				lowerCAmelCase												=  embd_pdrop
				lowerCAmelCase												=  attn_pdrop
				lowerCAmelCase												=  layer_norm_epsilon
				lowerCAmelCase												=  initializer_range
				lowerCAmelCase												=  summary_type
				lowerCAmelCase												=  summary_use_proj
				lowerCAmelCase												=  summary_activation
				lowerCAmelCase												=  summary_first_dropout
				lowerCAmelCase												=  summary_proj_to_labels
				super().__init__(**SCREAMING_SNAKE_CASE_			)


