





import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         XLMForMultipleChoice,
         XLMForQuestionAnswering,
         XLMForQuestionAnsweringSimple,
         XLMForSequenceClassification,
         XLMForTokenClassification,
         XLMModel,
         XLMWithLMHeadModel,
     )
     from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST

class   A							:



      def __init__(						self:    Any		,	_lowerCAmelCase:    Tuple		,	_lowerCAmelCase:    Tuple=13		,	_lowerCAmelCase:    int=7		,	_lowerCAmelCase:    Tuple=True		,	_lowerCAmelCase:    Union[str, Any]=True		,	_lowerCAmelCase:    Union[str, Any]=True		,	_lowerCAmelCase:    Tuple=True		,	_lowerCAmelCase:    Optional[Any]=True		,	_lowerCAmelCase:    List[str]=False		,	_lowerCAmelCase:    int=False		,	_lowerCAmelCase:    Tuple=False		,	_lowerCAmelCase:    Union[str, Any]=2		,	_lowerCAmelCase:    Dict=99		,	_lowerCAmelCase:    List[str]=0		,	_lowerCAmelCase:    str=32		,	_lowerCAmelCase:    Dict=5		,	_lowerCAmelCase:    Tuple=4		,	_lowerCAmelCase:    Dict=0.1		,	_lowerCAmelCase:    str=0.1		,	_lowerCAmelCase:    str=512		,	_lowerCAmelCase:    Tuple=2		,	_lowerCAmelCase:    Optional[Any]=0.02		,	_lowerCAmelCase:    List[Any]=2		,	_lowerCAmelCase:    Any=4		,	_lowerCAmelCase:    Any="last"		,	_lowerCAmelCase:    Union[str, Any]=True		,	_lowerCAmelCase:    str=None		,	_lowerCAmelCase:    Any=0		,	)    ->     List[Any]:
         '''simple docstring'''

         UpperCAmelCase_           =parent
         UpperCAmelCase_           =batch_size
         UpperCAmelCase_           =seq_length
         UpperCAmelCase_           =is_training
         UpperCAmelCase_           =use_input_lengths
         UpperCAmelCase_           =use_token_type_ids
         UpperCAmelCase_           =use_labels
         UpperCAmelCase_           =gelu_activation
         UpperCAmelCase_           =sinusoidal_embeddings
         UpperCAmelCase_           =causal
         UpperCAmelCase_           =asm
         UpperCAmelCase_           =n_langs
         UpperCAmelCase_           =vocab_size
         UpperCAmelCase_           =n_special
         UpperCAmelCase_           =hidden_size
         UpperCAmelCase_           =num_hidden_layers
         UpperCAmelCase_           =num_attention_heads
         UpperCAmelCase_           =hidden_dropout_prob
         UpperCAmelCase_           =attention_probs_dropout_prob
         UpperCAmelCase_           =max_position_embeddings
         UpperCAmelCase_           =type_sequence_label_size
         UpperCAmelCase_           =initializer_range
         UpperCAmelCase_           =num_labels
         UpperCAmelCase_           =num_choices
         UpperCAmelCase_           =summary_type
         UpperCAmelCase_           =use_proj
         UpperCAmelCase_           =scope
         UpperCAmelCase_           =bos_token_id



      def 						lowerCAmelCase__     (						self:    Optional[Any]							)    ->     Optional[Any]:
         '''simple docstring'''

         UpperCAmelCase_           =ids_tensor([self.batch_size, self.seq_length]		,	self.vocab_size							)
         UpperCAmelCase_           =random_attention_mask([self.batch_size, self.seq_length]							)

         UpperCAmelCase_           =None
         if self.use_input_lengths:
            UpperCAmelCase_           =(
                ids_tensor([self.batch_size]		,	vocab_size=2							) + self.seq_length - 2
            )  # small variation of seq_length

         UpperCAmelCase_           =None
         if self.use_token_type_ids:
            UpperCAmelCase_           =ids_tensor([self.batch_size, self.seq_length]		,	self.n_langs							)

         UpperCAmelCase_           =None
         UpperCAmelCase_           =None
         UpperCAmelCase_           =None
         if self.use_labels:
            UpperCAmelCase_           =ids_tensor([self.batch_size]		,	self.type_sequence_label_size							)
            UpperCAmelCase_           =ids_tensor([self.batch_size, self.seq_length]		,	self.num_labels							)
            UpperCAmelCase_           =ids_tensor([self.batch_size]		,	2							).float()
            UpperCAmelCase_           =ids_tensor([self.batch_size]		,	self.num_choices							)

         UpperCAmelCase_           =self.get_config()

         return (
             config,
             input_ids,
             token_type_ids,
             input_lengths,
             sequence_labels,
             token_labels,
             is_impossible_labels,
             choice_labels,
             input_mask,
         )



      def 						lowerCAmelCase__     (						self:    Optional[int]							)    ->     List[str]:
         '''simple docstring'''

         return XLMConfig(
             vocab_size=self.vocab_size		,	n_special=self.n_special		,	emb_dim=self.hidden_size		,	n_layers=self.num_hidden_layers		,	n_heads=self.num_attention_heads		,	dropout=self.hidden_dropout_prob		,	attention_dropout=self.attention_probs_dropout_prob		,	gelu_activation=self.gelu_activation		,	sinusoidal_embeddings=self.sinusoidal_embeddings		,	asm=self.asm		,	causal=self.causal		,	n_langs=self.n_langs		,	max_position_embeddings=self.max_position_embeddings		,	initializer_range=self.initializer_range		,	summary_type=self.summary_type		,	use_proj=self.use_proj		,	num_labels=self.num_labels		,	bos_token_id=self.bos_token_id		,	)



      def 						lowerCAmelCase__     (						self:    Optional[int]		,	_lowerCAmelCase:    Union[str, Any]		,	_lowerCAmelCase:    Optional[Any]		,	_lowerCAmelCase:    int		,	_lowerCAmelCase:    Optional[int]		,	_lowerCAmelCase:    Tuple		,	_lowerCAmelCase:    int		,	_lowerCAmelCase:    Any		,	_lowerCAmelCase:    Dict		,	_lowerCAmelCase:    Dict		,	)    ->     int:
         '''simple docstring'''

         UpperCAmelCase_           =XLMModel(config=__a							)
         model.to(__a							)
         model.eval()
         UpperCAmelCase_           =model(__a		,	lengths=__a		,	langs=__a							)
         UpperCAmelCase_           =model(__a		,	langs=__a							)
         UpperCAmelCase_           =model(__a							)
         self.parent.assertEqual(result.last_hidden_state.shape		,	(self.batch_size, self.seq_length, self.hidden_size)							)



      def 						lowerCAmelCase__     (						self:    List[str]		,	_lowerCAmelCase:    Any		,	_lowerCAmelCase:    Dict		,	_lowerCAmelCase:    int		,	_lowerCAmelCase:    str		,	_lowerCAmelCase:    Dict		,	_lowerCAmelCase:    Optional[int]		,	_lowerCAmelCase:    int		,	_lowerCAmelCase:    Tuple		,	_lowerCAmelCase:    Optional[int]		,	)    ->     List[Any]:
         '''simple docstring'''

         UpperCAmelCase_           =XLMWithLMHeadModel(__a							)
         model.to(__a							)
         model.eval()

         UpperCAmelCase_           =model(__a		,	token_type_ids=__a		,	labels=__a							)
         self.parent.assertEqual(result.loss.shape		,	()							)
         self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length, self.vocab_size)							)



      def 						lowerCAmelCase__     (						self:    Dict		,	_lowerCAmelCase:    Tuple		,	_lowerCAmelCase:    Union[str, Any]		,	_lowerCAmelCase:    List[str]		,	_lowerCAmelCase:    Dict		,	_lowerCAmelCase:    Any		,	_lowerCAmelCase:    Optional[Any]		,	_lowerCAmelCase:    Union[str, Any]		,	_lowerCAmelCase:    Dict		,	_lowerCAmelCase:    List[Any]		,	)    ->     Dict:
         '''simple docstring'''

         UpperCAmelCase_           =XLMForQuestionAnsweringSimple(__a							)
         model.to(__a							)
         model.eval()

         UpperCAmelCase_           =model(__a							)

         UpperCAmelCase_           =model(__a		,	start_positions=__a		,	end_positions=__a							)
         UpperCAmelCase_           =outputs
         self.parent.assertEqual(result.start_logits.shape		,	(self.batch_size, self.seq_length)							)
         self.parent.assertEqual(result.end_logits.shape		,	(self.batch_size, self.seq_length)							)



      def 						lowerCAmelCase__     (						self:    int		,	_lowerCAmelCase:    Optional[Any]		,	_lowerCAmelCase:    List[Any]		,	_lowerCAmelCase:    str		,	_lowerCAmelCase:    Dict		,	_lowerCAmelCase:    Optional[Any]		,	_lowerCAmelCase:    List[Any]		,	_lowerCAmelCase:    Union[str, Any]		,	_lowerCAmelCase:    Union[str, Any]		,	_lowerCAmelCase:    int		,	)    ->     List[Any]:
         '''simple docstring'''

         UpperCAmelCase_           =XLMForQuestionAnswering(__a							)
         model.to(__a							)
         model.eval()

         UpperCAmelCase_           =model(__a							)

         UpperCAmelCase_           =model(
             __a		,	start_positions=__a		,	end_positions=__a		,	cls_index=__a		,	is_impossible=__a		,	p_mask=__a		,	)

         UpperCAmelCase_           =model(
             __a		,	start_positions=__a		,	end_positions=__a		,	cls_index=__a		,	is_impossible=__a		,	)

         (UpperCAmelCase_		)           =result_with_labels.to_tuple()

         UpperCAmelCase_           =model(__a		,	start_positions=__a		,	end_positions=__a							)

         (UpperCAmelCase_		)           =result_with_labels.to_tuple()

         self.parent.assertEqual(result_with_labels.loss.shape		,	()							)
         self.parent.assertEqual(result.start_top_log_probs.shape		,	(self.batch_size, model.config.start_n_top)							)
         self.parent.assertEqual(result.start_top_index.shape		,	(self.batch_size, model.config.start_n_top)							)
         self.parent.assertEqual(
             result.end_top_log_probs.shape		,	(self.batch_size, model.config.start_n_top * model.config.end_n_top)							)
         self.parent.assertEqual(
             result.end_top_index.shape		,	(self.batch_size, model.config.start_n_top * model.config.end_n_top)							)
         self.parent.assertEqual(result.cls_logits.shape		,	(self.batch_size,)							)



      def 						lowerCAmelCase__     (						self:    Optional[int]		,	_lowerCAmelCase:    List[str]		,	_lowerCAmelCase:    Optional[Any]		,	_lowerCAmelCase:    Optional[Any]		,	_lowerCAmelCase:    List[Any]		,	_lowerCAmelCase:    str		,	_lowerCAmelCase:    Optional[Any]		,	_lowerCAmelCase:    Any		,	_lowerCAmelCase:    Tuple		,	_lowerCAmelCase:    Dict		,	)    ->     Tuple:
         '''simple docstring'''

         UpperCAmelCase_           =XLMForSequenceClassification(__a							)
         model.to(__a							)
         model.eval()

         UpperCAmelCase_           =model(__a							)
         UpperCAmelCase_           =model(__a		,	labels=__a							)
         self.parent.assertEqual(result.loss.shape		,	()							)
         self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.type_sequence_label_size)							)



      def 						lowerCAmelCase__     (						self:    List[Any]		,	_lowerCAmelCase:    str		,	_lowerCAmelCase:    List[str]		,	_lowerCAmelCase:    Tuple		,	_lowerCAmelCase:    Optional[int]		,	_lowerCAmelCase:    Optional[Any]		,	_lowerCAmelCase:    Optional[Any]		,	_lowerCAmelCase:    Any		,	_lowerCAmelCase:    Optional[Any]		,	_lowerCAmelCase:    int		,	)    ->     Optional[Any]:
         '''simple docstring'''

         UpperCAmelCase_           =self.num_labels
         UpperCAmelCase_           =XLMForTokenClassification(__a							)
         model.to(__a							)
         model.eval()

         UpperCAmelCase_           =model(__a		,	attention_mask=__a		,	labels=__a							)
         self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.seq_length, self.num_labels)							)



      def 						lowerCAmelCase__     (						self:    Dict		,	_lowerCAmelCase:    List[Any]		,	_lowerCAmelCase:    List[str]		,	_lowerCAmelCase:    List[Any]		,	_lowerCAmelCase:    Optional[int]		,	_lowerCAmelCase:    Dict		,	_lowerCAmelCase:    Tuple		,	_lowerCAmelCase:    int		,	_lowerCAmelCase:    str		,	_lowerCAmelCase:    Any		,	)    ->     Any:
         '''simple docstring'''

         UpperCAmelCase_           =self.num_choices
         UpperCAmelCase_           =XLMForMultipleChoice(config=__a							)
         model.to(__a							)
         model.eval()
         UpperCAmelCase_           =input_ids.unsqueeze(1							).expand(-1		,	self.num_choices		,	-1							).contiguous()
         UpperCAmelCase_           =token_type_ids.unsqueeze(1							).expand(-1		,	self.num_choices		,	-1							).contiguous()
         UpperCAmelCase_           =input_mask.unsqueeze(1							).expand(-1		,	self.num_choices		,	-1							).contiguous()
         UpperCAmelCase_           =model(
             __a		,	attention_mask=__a		,	token_type_ids=__a		,	labels=__a		,	)
         self.parent.assertEqual(result.logits.shape		,	(self.batch_size, self.num_choices)							)






      def 						lowerCAmelCase__     (						self:    List[Any]							)    ->     Union[str, Any]:
         '''simple docstring'''

         UpperCAmelCase_           =self.prepare_config_and_inputs()
         (
             UpperCAmelCase_		
         )           =config_and_inputs
         UpperCAmelCase_           ={"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """lengths""": input_lengths}
         return config, inputs_dict

@require_torch
class   A							(						a__		,   a__		,   a__		,   unittest.TestCase    ):
      _snake_case				  =(
          (
              XLMModel,
              XLMWithLMHeadModel,
              XLMForQuestionAnswering,
              XLMForSequenceClassification,
              XLMForQuestionAnsweringSimple,
              XLMForTokenClassification,
              XLMForMultipleChoice,
          )
          if is_torch_available()
          else ()
      )
      _snake_case				  =(
          (XLMWithLMHeadModel,) if is_torch_available() else ()
      )  # TODO (PVP): Check other models whether language generation is also applicable
      _snake_case				  =(
          {
              """feature-extraction""": XLMModel,
              """fill-mask""": XLMWithLMHeadModel,
              """question-answering""": XLMForQuestionAnsweringSimple,
              """text-classification""": XLMForSequenceClassification,
              """text-generation""": XLMWithLMHeadModel,
              """token-classification""": XLMForTokenClassification,
              """zero-shot""": XLMForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )



      def 						lowerCAmelCase__     (						self:    Any		,	_lowerCAmelCase:    Union[str, Any]		,	_lowerCAmelCase:    Union[str, Any]		,	_lowerCAmelCase:    int		,	_lowerCAmelCase:    Dict		,	_lowerCAmelCase:    Dict							)    ->     int:
         '''simple docstring'''

         if (
             pipeline_test_casse_name == "QAPipelineTests"
             and tokenizer_name is not None
             and not tokenizer_name.endswith("Fast"							)
         ):
            # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
            # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
            # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
            return True

         return False



      def 						lowerCAmelCase__     (						self:    Optional[int]		,	_lowerCAmelCase:    Dict		,	_lowerCAmelCase:    Any		,	_lowerCAmelCase:    Optional[int]=False							)    ->     Tuple:
         '''simple docstring'''

         UpperCAmelCase_           =super()._prepare_for_class(__a		,	__a		,	return_labels=__a							)

         if return_labels:
            if model_class.__name__ == "XLMForQuestionAnswering":
               UpperCAmelCase_           =torch.zeros(
                   self.model_tester.batch_size		,	dtype=torch.long		,	device=__a							)
               UpperCAmelCase_           =torch.zeros(
                   self.model_tester.batch_size		,	dtype=torch.long		,	device=__a							)

         return inputs_dict



      def 						lowerCAmelCase__     (						self:    Tuple							)    ->     List[Any]:
         '''simple docstring'''

         UpperCAmelCase_           =XLMModelTester(self							)
         UpperCAmelCase_           =ConfigTester(self		,	config_class=__a		,	emb_dim=37							)



      def 						lowerCAmelCase__     (						self:    Dict							)    ->     int:
         '''simple docstring'''

         self.config_tester.run_common_tests()



      def 						lowerCAmelCase__     (						self:    Union[str, Any]							)    ->     Any:
         '''simple docstring'''

         UpperCAmelCase_           =self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_model(*__a							)



      def 						lowerCAmelCase__     (						self:    Any							)    ->     Any:
         '''simple docstring'''

         UpperCAmelCase_           =self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_lm_head(*__a							)



      def 						lowerCAmelCase__     (						self:    int							)    ->     str:
         '''simple docstring'''

         UpperCAmelCase_           =self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_simple_qa(*__a							)



      def 						lowerCAmelCase__     (						self:    Tuple							)    ->     Dict:
         '''simple docstring'''

         UpperCAmelCase_           =self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_qa(*__a							)



      def 						lowerCAmelCase__     (						self:    int							)    ->     Union[str, Any]:
         '''simple docstring'''

         UpperCAmelCase_           =self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_sequence_classif(*__a							)



      def 						lowerCAmelCase__     (						self:    List[Any]							)    ->     int:
         '''simple docstring'''

         UpperCAmelCase_           =self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_token_classif(*__a							)



      def 						lowerCAmelCase__     (						self:    Any							)    ->     Any:
         '''simple docstring'''

         UpperCAmelCase_           =self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_xlm_for_multiple_choice(*__a							)



      def 						lowerCAmelCase__     (						self:    List[Any]		,	_lowerCAmelCase:    int		,	_lowerCAmelCase:    Tuple		,	_lowerCAmelCase:    Dict		,	_lowerCAmelCase:    Tuple		,	_lowerCAmelCase:    int		,	_lowerCAmelCase:    Dict=False		,	_lowerCAmelCase:    int=1							)    ->     int:
         '''simple docstring'''

         self.assertIsInstance(__a		,	__a							)
         self.assertListEqual(
             [isinstance(__a		,	__a							) for iter_attentions in attentions]		,	[True] * len(__a							)							)
         self.assertEqual(len(__a							)		,	(max_length - min_length) * num_beam_groups							)

         for idx, iter_attentions in enumerate(__a							):
            # adds PAD dummy token
            UpperCAmelCase_           =min_length + idx + 1
            UpperCAmelCase_           =min_length + idx + 1

            UpperCAmelCase_           =(
                batch_size * num_beam_groups,
                config.num_attention_heads,
                tgt_len,
                src_len,
            )
            # check attn size
            self.assertListEqual(
                [layer_attention.shape for layer_attention in iter_attentions]		,	[expected_shape] * len(__a							)							)



      def 						lowerCAmelCase__     (						self:    List[str]		,	_lowerCAmelCase:    List[str]		,	_lowerCAmelCase:    str		,	_lowerCAmelCase:    Union[str, Any]		,	_lowerCAmelCase:    List[str]		,	_lowerCAmelCase:    Dict		,	_lowerCAmelCase:    Union[str, Any]=False		,	_lowerCAmelCase:    Any=1							)    ->     str:
         '''simple docstring'''

         self.assertIsInstance(__a		,	__a							)
         self.assertListEqual(
             [isinstance(__a		,	__a							) for iter_hidden_states in hidden_states]		,	[True] * len(__a							)		,	)
         self.assertEqual(len(__a							)		,	(max_length - min_length) * num_beam_groups							)

         for idx, iter_hidden_states in enumerate(__a							):
            # adds PAD dummy token
            UpperCAmelCase_           =min_length + idx + 1
            UpperCAmelCase_           =(batch_size * num_beam_groups, seq_len, config.hidden_size)
            # check hidden size
            self.assertListEqual(
                [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states]		,	[expected_shape] * len(__a							)		,	)
         pass






      @slow
      def 						lowerCAmelCase__     (						self:    List[Any]							)    ->     Union[str, Any]:
         '''simple docstring'''

         for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            UpperCAmelCase_           =XLMModel.from_pretrained(__a							)
            self.assertIsNotNone(__a							)

@require_torch
class   A							(						unittest.TestCase    ):



      @slow
      def 						lowerCAmelCase__     (						self:    List[Any]							)    ->     List[str]:
         '''simple docstring'''

         UpperCAmelCase_           =XLMWithLMHeadModel.from_pretrained("xlm-mlm-en-2048"							)
         model.to(__a							)
         UpperCAmelCase_           =torch.tensor([[14, 447]]		,	dtype=torch.long		,	device=__a							)  # the president
         UpperCAmelCase_           =[
             14,
             447,
             14,
             447,
             14,
             447,
             14,
             447,
             14,
             447,
             14,
             447,
             14,
             447,
             14,
             447,
             14,
             447,
             14,
             447,
         ]  # the president the president the president the president the president the president the president the president the president the president
         # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
         UpperCAmelCase_           =model.generate(__a		,	do_sample=__a							)
         self.assertListEqual(output_ids[0].cpu().numpy().tolist()		,	__a							)










'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


lowercase      :   Tuple      =       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase      :   Dict      =       {
    """vocab_file""": {
        """unc-nlp/lxmert-base-uncased""": """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """unc-nlp/lxmert-base-uncased""": (
            """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

lowercase      :   List[Any]      =       {
    """unc-nlp/lxmert-base-uncased""": 512,
}

lowercase      :   Tuple      =       {
    """unc-nlp/lxmert-base-uncased""": {"""do_lower_case""": True},
}







class 					_a (a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Optional[int]          =					VOCAB_FILES_NAMES
     lowerCAmelCase_  :     Optional[int]          =					PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase_  :     Any          =					PRETRAINED_INIT_CONFIGURATION
     lowerCAmelCase_  :     Optional[int]          =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase_  :     Optional[int]          =					LxmertTokenizer







     def __init__(				self							,__a=None							,__a=None							,__a=True							,__a="[UNK]"							,__a="[SEP]"							,__a="[PAD]"							,__a="[CLS]"							,__a="[MASK]"							,__a=True							,__a=None							,**__a							,)      ->				str:
          super().__init__(
              __a							,tokenizer_file=__a							,do_lower_case=__a							,unk_token=__a							,sep_token=__a							,pad_token=__a							,cls_token=__a							,mask_token=__a							,tokenize_chinese_chars=__a							,strip_accents=__a							,**__a							,)

          snake_case			:      List[Any]        =			json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
          if (
              normalizer_state.get("""lowercase"""							,__a							) != do_lower_case
              or normalizer_state.get("""strip_accents"""							,__a							) != strip_accents
              or normalizer_state.get("""handle_chinese_chars"""							,__a							) != tokenize_chinese_chars
          ):
               snake_case			:      Union[str, Any]        =			getattr(__a							,normalizer_state.pop("""type"""							)							)
               snake_case			:      Optional[int]        =			do_lower_case
               snake_case			:      Optional[int]        =			strip_accents
               snake_case			:      List[Any]        =			tokenize_chinese_chars
               snake_case			:      Union[str, Any]        =			normalizer_class(**__a							)

          snake_case			:      str        =			do_lower_case







     def 			snake_case_      (				self							,__a							,__a=None							)      ->				Optional[int]:
          snake_case			:      Tuple        =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				List[int]:
          snake_case			:      int        =			[self.sep_token_id]
          snake_case			:      Any        =			[self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep							) * [0]
          return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				Tuple[str]:
          snake_case			:      Optional[int]        =			self._tokenizer.model.save(__a							,name=__a							)
          return tuple(__a							)



