






"""simple docstring"""







import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
  from transformers.models.electra.modeling_flax_electra import (
      FlaxElectraForCausalLM,
      FlaxElectraForMaskedLM,
      FlaxElectraForMultipleChoice,
      FlaxElectraForPreTraining,
      FlaxElectraForQuestionAnswering,
      FlaxElectraForSequenceClassification,
      FlaxElectraForTokenClassification,
      FlaxElectraModel,
  )







class 					lowerCamelCase__					(       unittest.TestCase ):






    def __init__( self	,A	,A=13	,A=7	,A=True	,A=True	,A=True	,A=True	,A=99	,A=24	,A=32	,A=5	,A=4	,A=37	,A="gelu"	,A=0.1	,A=0.1	,A=512	,A=16	,A=2	,A=0.02	,A=4	,):
          UpperCAmelCase							=       parent
          UpperCAmelCase							=       batch_size
          UpperCAmelCase							=       seq_length
          UpperCAmelCase							=       is_training
          UpperCAmelCase							=       use_attention_mask
          UpperCAmelCase							=       use_token_type_ids
          UpperCAmelCase							=       use_labels
          UpperCAmelCase							=       vocab_size
          UpperCAmelCase							=       hidden_size
          UpperCAmelCase							=       embedding_size
          UpperCAmelCase							=       num_hidden_layers
          UpperCAmelCase							=       num_attention_heads
          UpperCAmelCase							=       intermediate_size
          UpperCAmelCase							=       hidden_act
          UpperCAmelCase							=       hidden_dropout_prob
          UpperCAmelCase							=       attention_probs_dropout_prob
          UpperCAmelCase							=       max_position_embeddings
          UpperCAmelCase							=       type_vocab_size
          UpperCAmelCase							=       type_sequence_label_size
          UpperCAmelCase							=       initializer_range
          UpperCAmelCase							=       num_choices






    def 				_UpperCamelCase							( self					):
          UpperCAmelCase							=       ids_tensor([self.batch_size, self.seq_length]	,self.vocab_size					)

          UpperCAmelCase							=       None
          if self.use_attention_mask:
                UpperCAmelCase							=       random_attention_mask([self.batch_size, self.seq_length]					)

          UpperCAmelCase							=       None
          if self.use_token_type_ids:
                UpperCAmelCase							=       ids_tensor([self.batch_size, self.seq_length]	,self.type_vocab_size					)

          UpperCAmelCase							=       ElectraConfig(
              vocab_size=self.vocab_size	,hidden_size=self.hidden_size	,embedding_size=self.embedding_size	,num_hidden_layers=self.num_hidden_layers	,num_attention_heads=self.num_attention_heads	,intermediate_size=self.intermediate_size	,hidden_act=self.hidden_act	,hidden_dropout_prob=self.hidden_dropout_prob	,attention_probs_dropout_prob=self.attention_probs_dropout_prob	,max_position_embeddings=self.max_position_embeddings	,type_vocab_size=self.type_vocab_size	,initializer_range=self.initializer_range	,)

          return config, input_ids, token_type_ids, attention_mask






    def 				_UpperCamelCase							( self					):
          UpperCAmelCase							=       self.prepare_config_and_inputs()
          UpperCAmelCase							=       config_and_inputs
          UpperCAmelCase							=       {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": attention_mask}
          return config, inputs_dict







@require_flax
class 					lowerCamelCase__					(       _A    ,     unittest.TestCase ):
    SCREAMING_SNAKE_CASE		=			True

    SCREAMING_SNAKE_CASE		=			(
        (
            FlaxElectraModel,
            FlaxElectraForCausalLM,
            FlaxElectraForMaskedLM,
            FlaxElectraForPreTraining,
            FlaxElectraForTokenClassification,
            FlaxElectraForQuestionAnswering,
            FlaxElectraForMultipleChoice,
            FlaxElectraForSequenceClassification,
        )
        if is_flax_available()
        else ()
    )






    def 				_UpperCamelCase							( self					):
          UpperCAmelCase							=       FlaxElectraModelTester(self					)






    @slow
    def 				_UpperCamelCase							( self					):
          for model_class_name in self.all_model_classes:
                if model_class_name == FlaxElectraForMaskedLM:
                      UpperCAmelCase							=       model_class_name.from_pretrained("""google/electra-small-generator"""					)
                else:
                      UpperCAmelCase							=       model_class_name.from_pretrained("""google/electra-small-discriminator"""					)
                UpperCAmelCase							=       model(np.ones((1, 1)					)					)
                self.assertIsNotNone(snake_case_					)





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_xglm import XGLMTokenizer
else:
							lowerCamelCase_  :       Optional[Any]					      =     None


lowerCamelCase_  :       str					      =     logging.get_logger(__name__)

lowerCamelCase_  :       List[str]					      =     {"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase_  :       List[str]					      =     {
    """vocab_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json""",
    },
}

lowerCamelCase_  :       Dict					      =     {
    """facebook/xglm-564M""": 20_48,
}

class 				_UpperCamelCase (    _A     ):






			'''simple docstring'''

			__UpperCamelCase					:  Optional[Any]    					=       VOCAB_FILES_NAMES
			__UpperCamelCase					:  int    					=       PRETRAINED_VOCAB_FILES_MAP
			__UpperCamelCase					:  List[str]    					=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__UpperCamelCase					:  Dict    					=       ["""input_ids""", """attention_mask"""]
			__UpperCamelCase					:  Union[str, Any]    					=       XGLMTokenizer

			def __init__(				self       :				Optional[Any]  ,		snake_case_       :				Optional[Any]=None  ,		snake_case_       :				Any=None  ,		snake_case_       :				Optional[Any]="<s>"  ,		snake_case_       :				Union[str, Any]="</s>"  ,		snake_case_       :				Optional[Any]="</s>"  ,		snake_case_       :				Dict="<s>"  ,		snake_case_       :				int="<unk>"  ,		snake_case_       :				Dict="<pad>"  ,		**snake_case_       :				List[str]  ,		):
							# Compatibility with the original tokenizer
							UpperCamelCase_:							List[Any]		=  7
							UpperCamelCase_:							int		=  [f'''<madeupword{i}>''' for i in range(self.num_madeup_words					)]

							UpperCamelCase_:							Any		=  kwargs.get("""additional_special_tokens"""  ,		[]					)
							kwargs["additional_special_tokens"] += [
							    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    snake_case_  ,		tokenizer_file=snake_case_  ,		bos_token=snake_case_  ,		eos_token=snake_case_  ,		sep_token=snake_case_  ,		cls_token=snake_case_  ,		unk_token=snake_case_  ,		pad_token=snake_case_  ,		**snake_case_  ,		)

							UpperCamelCase_:							Dict		=  vocab_file
							UpperCamelCase_:							Union[str, Any]		=  False if not self.vocab_file else True

			def 					lowerCAmelCase__		(				self       :				Optional[Any]  ,		snake_case_       :				List[int]  ,		snake_case_       :				Optional[List[int]] = None					):

							if token_ids_a is None:
											return [self.sep_token_id] + token_ids_a
							UpperCamelCase_:							Optional[Any]		=  [self.sep_token_id]
							return sep + token_ids_a + sep + sep + token_ids_a

			def 					lowerCAmelCase__		(				self       :				Optional[Any]  ,		snake_case_       :				List[int]  ,		snake_case_       :				Optional[List[int]] = None					):
							UpperCamelCase_:							Any		=  [self.sep_token_id]

							if token_ids_a is None:
											return len(sep + token_ids_a					) * [0]
							return len(sep + token_ids_a + sep + sep + token_ids_a					) * [0]




			def 					lowerCAmelCase__		(				self       :				Optional[Any]  ,		snake_case_       :				str  ,		snake_case_       :				Optional[str] = None					):
							if not self.can_save_slow_tokenizer:
											raise ValueError(
											    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
											    """tokenizer."""					)

							if not os.path.isdir(snake_case_					):
											logger.error(f'''Vocabulary path ({save_directory}) should be a directory.'''					)
											return
							UpperCamelCase_:							Optional[int]		=  os.path.join(
							    snake_case_  ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)

							if os.path.abspath(self.vocab_file					) != os.path.abspath(snake_case_					):
											copyfile(self.vocab_file  ,		snake_case_					)

							return (out_vocab_file,)






