
import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
							from transformers.models.electra.modeling_flax_electra import (
							    FlaxElectraForCausalLM,
							    FlaxElectraForMaskedLM,
							    FlaxElectraForMultipleChoice,
							    FlaxElectraForPreTraining,
							    FlaxElectraForQuestionAnswering,
							    FlaxElectraForSequenceClassification,
							    FlaxElectraForTokenClassification,
							    FlaxElectraModel,
							)

class 				_UpperCamelCase (    unittest.TestCase     ):






			'''simple docstring'''

			def __init__(				self       :				Union[str, Any]  ,		snake_case_       :				Optional[Any]  ,		snake_case_       :				Any=13  ,		snake_case_       :				str=7  ,		snake_case_       :				Any=True  ,		snake_case_       :				Any=True  ,		snake_case_       :				List[str]=True  ,		snake_case_       :				Optional[int]=True  ,		snake_case_       :				Optional[Any]=99  ,		snake_case_       :				Dict=24  ,		snake_case_       :				str=32  ,		snake_case_       :				List[Any]=5  ,		snake_case_       :				Any=4  ,		snake_case_       :				Tuple=37  ,		snake_case_       :				Optional[int]="gelu"  ,		snake_case_       :				Dict=0.1  ,		snake_case_       :				Optional[int]=0.1  ,		snake_case_       :				List[str]=512  ,		snake_case_       :				Optional[Any]=16  ,		snake_case_       :				Optional[int]=2  ,		snake_case_       :				int=0.02  ,		snake_case_       :				Dict=4  ,		):
							UpperCamelCase_:							Any		=  parent
							UpperCamelCase_:							List[str]		=  batch_size
							UpperCamelCase_:							List[Any]		=  seq_length
							UpperCamelCase_:							Tuple		=  is_training
							UpperCamelCase_:							Optional[int]		=  use_attention_mask
							UpperCamelCase_:							Union[str, Any]		=  use_token_type_ids
							UpperCamelCase_:							int		=  use_labels
							UpperCamelCase_:							List[Any]		=  vocab_size
							UpperCamelCase_:							Tuple		=  hidden_size
							UpperCamelCase_:							Union[str, Any]		=  embedding_size
							UpperCamelCase_:							Any		=  num_hidden_layers
							UpperCamelCase_:							List[Any]		=  num_attention_heads
							UpperCamelCase_:							int		=  intermediate_size
							UpperCamelCase_:							List[str]		=  hidden_act
							UpperCamelCase_:							List[str]		=  hidden_dropout_prob
							UpperCamelCase_:							List[Any]		=  attention_probs_dropout_prob
							UpperCamelCase_:							Union[str, Any]		=  max_position_embeddings
							UpperCamelCase_:							List[Any]		=  type_vocab_size
							UpperCamelCase_:							int		=  type_sequence_label_size
							UpperCamelCase_:							Any		=  initializer_range
							UpperCamelCase_:							Dict		=  num_choices

			def 					lowerCAmelCase__		(				self       :				int					):
							UpperCamelCase_:							Optional[Any]		=  ids_tensor([self.batch_size, self.seq_length]  ,		self.vocab_size					)

							UpperCamelCase_:							Optional[Any]		=  None
							if self.use_attention_mask:
											UpperCamelCase_:							str		=  random_attention_mask([self.batch_size, self.seq_length]					)

							UpperCamelCase_:							List[Any]		=  None
							if self.use_token_type_ids:
											UpperCamelCase_:							int		=  ids_tensor([self.batch_size, self.seq_length]  ,		self.type_vocab_size					)

							UpperCamelCase_:							Dict		=  ElectraConfig(
							    vocab_size=self.vocab_size  ,		hidden_size=self.hidden_size  ,		embedding_size=self.embedding_size  ,		num_hidden_layers=self.num_hidden_layers  ,		num_attention_heads=self.num_attention_heads  ,		intermediate_size=self.intermediate_size  ,		hidden_act=self.hidden_act  ,		hidden_dropout_prob=self.hidden_dropout_prob  ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,		max_position_embeddings=self.max_position_embeddings  ,		type_vocab_size=self.type_vocab_size  ,		initializer_range=self.initializer_range  ,		)

							return config, input_ids, token_type_ids, attention_mask




			def 					lowerCAmelCase__		(				self       :				Optional[Any]					):
							UpperCamelCase_:							Optional[Any]		=  self.prepare_config_and_inputs()
							UpperCamelCase_,      UpperCamelCase_,      UpperCamelCase_,      UpperCamelCase_:							Any		=  config_and_inputs
							UpperCamelCase_:							Tuple		=  {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": attention_mask}
							return config, inputs_dict




@require_flax
class 				_UpperCamelCase (    _A   ,				unittest.TestCase     ):






			'''simple docstring'''

			__UpperCamelCase					:  Tuple    					=       True

			__UpperCamelCase					:  List[str]    					=       (
			    (
			        FlaxElectraModel,
			        FlaxElectraForCausalLM,
			        FlaxElectraForMaskedLM,
			        FlaxElectraForPreTraining,
			        FlaxElectraForTokenClassification,
			        FlaxElectraForQuestionAnswering,
			        FlaxElectraForMultipleChoice,
			        FlaxElectraForSequenceClassification,
			    )
			    if is_flax_available()
			    else ()
			)

			def 					lowerCAmelCase__		(				self       :				List[str]					):
							UpperCamelCase_:							Any		=  FlaxElectraModelTester(self					)




			@slow
			def 					lowerCAmelCase__		(				self       :				Tuple					):
							for model_class_name in self.all_model_classes:
											if model_class_name == FlaxElectraForMaskedLM:
															UpperCamelCase_:							Any		=  model_class_name.from_pretrained("""google/electra-small-generator"""					)
											else:
															UpperCamelCase_:							Any		=  model_class_name.from_pretrained("""google/electra-small-discriminator"""					)
											UpperCamelCase_:							Optional[Any]		=  model(np.ones((1, 1)					)					)
											self.assertIsNotNone(snake_case_					)








import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_xglm import XGLMTokenizer
else:
							lowerCamelCase_  :       Optional[Any]					      =     None


lowerCamelCase_  :       str					      =     logging.get_logger(__name__)

lowerCamelCase_  :       List[str]					      =     {"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase_  :       List[str]					      =     {
    """vocab_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json""",
    },
}

lowerCamelCase_  :       Dict					      =     {
    """facebook/xglm-564M""": 20_48,
}

class 				_UpperCamelCase (    _A     ):






			'''simple docstring'''

			__UpperCamelCase					:  Optional[Any]    					=       VOCAB_FILES_NAMES
			__UpperCamelCase					:  int    					=       PRETRAINED_VOCAB_FILES_MAP
			__UpperCamelCase					:  List[str]    					=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__UpperCamelCase					:  Dict    					=       ["""input_ids""", """attention_mask"""]
			__UpperCamelCase					:  Union[str, Any]    					=       XGLMTokenizer

			def __init__(				self       :				Optional[Any]  ,		snake_case_       :				Optional[Any]=None  ,		snake_case_       :				Any=None  ,		snake_case_       :				Optional[Any]="<s>"  ,		snake_case_       :				Union[str, Any]="</s>"  ,		snake_case_       :				Optional[Any]="</s>"  ,		snake_case_       :				Dict="<s>"  ,		snake_case_       :				int="<unk>"  ,		snake_case_       :				Dict="<pad>"  ,		**snake_case_       :				List[str]  ,		):
							# Compatibility with the original tokenizer
							UpperCamelCase_:							List[Any]		=  7
							UpperCamelCase_:							int		=  [f'''<madeupword{i}>''' for i in range(self.num_madeup_words					)]

							UpperCamelCase_:							Any		=  kwargs.get("""additional_special_tokens"""  ,		[]					)
							kwargs["additional_special_tokens"] += [
							    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    snake_case_  ,		tokenizer_file=snake_case_  ,		bos_token=snake_case_  ,		eos_token=snake_case_  ,		sep_token=snake_case_  ,		cls_token=snake_case_  ,		unk_token=snake_case_  ,		pad_token=snake_case_  ,		**snake_case_  ,		)

							UpperCamelCase_:							Dict		=  vocab_file
							UpperCamelCase_:							Union[str, Any]		=  False if not self.vocab_file else True

			def 					lowerCAmelCase__		(				self       :				Optional[Any]  ,		snake_case_       :				List[int]  ,		snake_case_       :				Optional[List[int]] = None					):

							if token_ids_a is None:
											return [self.sep_token_id] + token_ids_a
							UpperCamelCase_:							Optional[Any]		=  [self.sep_token_id]
							return sep + token_ids_a + sep + sep + token_ids_a

			def 					lowerCAmelCase__		(				self       :				Optional[Any]  ,		snake_case_       :				List[int]  ,		snake_case_       :				Optional[List[int]] = None					):
							UpperCamelCase_:							Any		=  [self.sep_token_id]

							if token_ids_a is None:
											return len(sep + token_ids_a					) * [0]
							return len(sep + token_ids_a + sep + sep + token_ids_a					) * [0]




			def 					lowerCAmelCase__		(				self       :				Optional[Any]  ,		snake_case_       :				str  ,		snake_case_       :				Optional[str] = None					):
							if not self.can_save_slow_tokenizer:
											raise ValueError(
											    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
											    """tokenizer."""					)

							if not os.path.isdir(snake_case_					):
											logger.error(f'''Vocabulary path ({save_directory}) should be a directory.'''					)
											return
							UpperCamelCase_:							Optional[int]		=  os.path.join(
							    snake_case_  ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)

							if os.path.abspath(self.vocab_file					) != os.path.abspath(snake_case_					):
											copyfile(self.vocab_file  ,		snake_case_					)

							return (out_vocab_file,)






