






"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


lowerCAmelCase_		:			Dict				   =					{
    '''auto_factory''': ['''get_values'''],
    '''configuration_auto''': ['''ALL_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''CONFIG_MAPPING''', '''MODEL_NAMES_MAPPING''', '''AutoConfig'''],
    '''feature_extraction_auto''': ['''FEATURE_EXTRACTOR_MAPPING''', '''AutoFeatureExtractor'''],
    '''image_processing_auto''': ['''IMAGE_PROCESSOR_MAPPING''', '''AutoImageProcessor'''],
    '''processing_auto''': ['''PROCESSOR_MAPPING''', '''AutoProcessor'''],
    '''tokenization_auto''': ['''TOKENIZER_MAPPING''', '''AutoTokenizer'''],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCAmelCase_		:			Any				   =					[
     '''MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING''',
     '''MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING''',
     '''MODEL_FOR_AUDIO_XVECTOR_MAPPING''',
     '''MODEL_FOR_BACKBONE_MAPPING''',
     '''MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING''',
     '''MODEL_FOR_CAUSAL_LM_MAPPING''',
     '''MODEL_FOR_CTC_MAPPING''',
     '''MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING''',
     '''MODEL_FOR_DEPTH_ESTIMATION_MAPPING''',
     '''MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING''',
     '''MODEL_FOR_IMAGE_SEGMENTATION_MAPPING''',
     '''MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING''',
     '''MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING''',
     '''MODEL_FOR_MASKED_LM_MAPPING''',
     '''MODEL_FOR_MASK_GENERATION_MAPPING''',
     '''MODEL_FOR_MULTIPLE_CHOICE_MAPPING''',
     '''MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING''',
     '''MODEL_FOR_OBJECT_DETECTION_MAPPING''',
     '''MODEL_FOR_PRETRAINING_MAPPING''',
     '''MODEL_FOR_QUESTION_ANSWERING_MAPPING''',
     '''MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING''',
     '''MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING''',
     '''MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING''',
     '''MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING''',
     '''MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING''',
     '''MODEL_FOR_TEXT_ENCODING_MAPPING''',
     '''MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING''',
     '''MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING''',
     '''MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING''',
     '''MODEL_FOR_VISION_2_SEQ_MAPPING''',
     '''MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING''',
     '''MODEL_MAPPING''',
     '''MODEL_WITH_LM_HEAD_MAPPING''',
     '''MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING''',
     '''MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING''',
     '''AutoModel''',
     '''AutoBackbone''',
     '''AutoModelForAudioClassification''',
     '''AutoModelForAudioFrameClassification''',
     '''AutoModelForAudioXVector''',
     '''AutoModelForCausalLM''',
     '''AutoModelForCTC''',
     '''AutoModelForDepthEstimation''',
     '''AutoModelForImageClassification''',
     '''AutoModelForImageSegmentation''',
     '''AutoModelForInstanceSegmentation''',
     '''AutoModelForMaskGeneration''',
     '''AutoModelForTextEncoding''',
     '''AutoModelForMaskedImageModeling''',
     '''AutoModelForMaskedLM''',
     '''AutoModelForMultipleChoice''',
     '''AutoModelForNextSentencePrediction''',
     '''AutoModelForObjectDetection''',
     '''AutoModelForPreTraining''',
     '''AutoModelForQuestionAnswering''',
     '''AutoModelForSemanticSegmentation''',
     '''AutoModelForSeq2SeqLM''',
     '''AutoModelForSequenceClassification''',
     '''AutoModelForSpeechSeq2Seq''',
     '''AutoModelForTableQuestionAnswering''',
     '''AutoModelForTokenClassification''',
     '''AutoModelForUniversalSegmentation''',
     '''AutoModelForVideoClassification''',
     '''AutoModelForVision2Seq''',
     '''AutoModelForVisualQuestionAnswering''',
     '''AutoModelForDocumentQuestionAnswering''',
     '''AutoModelWithLMHead''',
     '''AutoModelForZeroShotImageClassification''',
     '''AutoModelForZeroShotObjectDetection''',
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCAmelCase_		:			List[Any]				   =					[
     '''TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING''',
     '''TF_MODEL_FOR_CAUSAL_LM_MAPPING''',
     '''TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING''',
     '''TF_MODEL_FOR_MASK_GENERATION_MAPPING''',
     '''TF_MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING''',
     '''TF_MODEL_FOR_MASKED_LM_MAPPING''',
     '''TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING''',
     '''TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING''',
     '''TF_MODEL_FOR_PRETRAINING_MAPPING''',
     '''TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING''',
     '''TF_MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING''',
     '''TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING''',
     '''TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING''',
     '''TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING''',
     '''TF_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING''',
     '''TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING''',
     '''TF_MODEL_FOR_TEXT_ENCODING_MAPPING''',
     '''TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING''',
     '''TF_MODEL_FOR_VISION_2_SEQ_MAPPING''',
     '''TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING''',
     '''TF_MODEL_MAPPING''',
     '''TF_MODEL_WITH_LM_HEAD_MAPPING''',
     '''TFAutoModel''',
     '''TFAutoModelForAudioClassification''',
     '''TFAutoModelForCausalLM''',
     '''TFAutoModelForImageClassification''',
     '''TFAutoModelForMaskedImageModeling''',
     '''TFAutoModelForMaskedLM''',
     '''TFAutoModelForMaskGeneration''',
     '''TFAutoModelForMultipleChoice''',
     '''TFAutoModelForNextSentencePrediction''',
     '''TFAutoModelForPreTraining''',
     '''TFAutoModelForDocumentQuestionAnswering''',
     '''TFAutoModelForQuestionAnswering''',
     '''TFAutoModelForSemanticSegmentation''',
     '''TFAutoModelForSeq2SeqLM''',
     '''TFAutoModelForSequenceClassification''',
     '''TFAutoModelForSpeechSeq2Seq''',
     '''TFAutoModelForTableQuestionAnswering''',
     '''TFAutoModelForTextEncoding''',
     '''TFAutoModelForTokenClassification''',
     '''TFAutoModelForVision2Seq''',
     '''TFAutoModelForZeroShotImageClassification''',
     '''TFAutoModelWithLMHead''',
 ]

try:
 if not is_flax_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCAmelCase_		:			Any				   =					[
     '''FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING''',
     '''FLAX_MODEL_FOR_CAUSAL_LM_MAPPING''',
     '''FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING''',
     '''FLAX_MODEL_FOR_MASKED_LM_MAPPING''',
     '''FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING''',
     '''FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING''',
     '''FLAX_MODEL_FOR_PRETRAINING_MAPPING''',
     '''FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING''',
     '''FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING''',
     '''FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING''',
     '''FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING''',
     '''FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING''',
     '''FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING''',
     '''FLAX_MODEL_MAPPING''',
     '''FlaxAutoModel''',
     '''FlaxAutoModelForCausalLM''',
     '''FlaxAutoModelForImageClassification''',
     '''FlaxAutoModelForMaskedLM''',
     '''FlaxAutoModelForMultipleChoice''',
     '''FlaxAutoModelForNextSentencePrediction''',
     '''FlaxAutoModelForPreTraining''',
     '''FlaxAutoModelForQuestionAnswering''',
     '''FlaxAutoModelForSeq2SeqLM''',
     '''FlaxAutoModelForSequenceClassification''',
     '''FlaxAutoModelForSpeechSeq2Seq''',
     '''FlaxAutoModelForTokenClassification''',
     '''FlaxAutoModelForVision2Seq''',
 ]


if TYPE_CHECKING:
 from .auto_factory import get_values
 from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP, CONFIG_MAPPING, MODEL_NAMES_MAPPING, AutoConfig
 from .feature_extraction_auto import FEATURE_EXTRACTOR_MAPPING, AutoFeatureExtractor
 from .image_processing_auto import IMAGE_PROCESSOR_MAPPING, AutoImageProcessor
 from .processing_auto import PROCESSOR_MAPPING, AutoProcessor
 from .tokenization_auto import TOKENIZER_MAPPING, AutoTokenizer

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_auto import (
      MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
      MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING,
      MODEL_FOR_AUDIO_XVECTOR_MAPPING,
      MODEL_FOR_BACKBONE_MAPPING,
      MODEL_FOR_CAUSAL_IMAGE_MODELING_MAPPING,
      MODEL_FOR_CAUSAL_LM_MAPPING,
      MODEL_FOR_CTC_MAPPING,
      MODEL_FOR_DEPTH_ESTIMATION_MAPPING,
      MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
      MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
      MODEL_FOR_IMAGE_SEGMENTATION_MAPPING,
      MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING,
      MODEL_FOR_MASK_GENERATION_MAPPING,
      MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING,
      MODEL_FOR_MASKED_LM_MAPPING,
      MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
      MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
      MODEL_FOR_OBJECT_DETECTION_MAPPING,
      MODEL_FOR_PRETRAINING_MAPPING,
      MODEL_FOR_QUESTION_ANSWERING_MAPPING,
      MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
      MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
      MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
      MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
      MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
      MODEL_FOR_TEXT_ENCODING_MAPPING,
      MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
      MODEL_FOR_UNIVERSAL_SEGMENTATION_MAPPING,
      MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
      MODEL_FOR_VISION_2_SEQ_MAPPING,
      MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING,
      MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
      MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING,
      MODEL_MAPPING,
      MODEL_WITH_LM_HEAD_MAPPING,
      AutoBackbone,
      AutoModel,
      AutoModelForAudioClassification,
      AutoModelForAudioFrameClassification,
      AutoModelForAudioXVector,
      AutoModelForCausalLM,
      AutoModelForCTC,
      AutoModelForDepthEstimation,
      AutoModelForDocumentQuestionAnswering,
      AutoModelForImageClassification,
      AutoModelForImageSegmentation,
      AutoModelForInstanceSegmentation,
      AutoModelForMaskedImageModeling,
      AutoModelForMaskedLM,
      AutoModelForMaskGeneration,
      AutoModelForMultipleChoice,
      AutoModelForNextSentencePrediction,
      AutoModelForObjectDetection,
      AutoModelForPreTraining,
      AutoModelForQuestionAnswering,
      AutoModelForSemanticSegmentation,
      AutoModelForSeqaSeqLM,
      AutoModelForSequenceClassification,
      AutoModelForSpeechSeqaSeq,
      AutoModelForTableQuestionAnswering,
      AutoModelForTextEncoding,
      AutoModelForTokenClassification,
      AutoModelForUniversalSegmentation,
      AutoModelForVideoClassification,
      AutoModelForVisionaSeq,
      AutoModelForVisualQuestionAnswering,
      AutoModelForZeroShotImageClassification,
      AutoModelForZeroShotObjectDetection,
      AutoModelWithLMHead,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_auto import (
      TF_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
      TF_MODEL_FOR_CAUSAL_LM_MAPPING,
      TF_MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,
      TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
      TF_MODEL_FOR_MASK_GENERATION_MAPPING,
      TF_MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING,
      TF_MODEL_FOR_MASKED_LM_MAPPING,
      TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
      TF_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
      TF_MODEL_FOR_PRETRAINING_MAPPING,
      TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
      TF_MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
      TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
      TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
      TF_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
      TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
      TF_MODEL_FOR_TEXT_ENCODING_MAPPING,
      TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
      TF_MODEL_FOR_VISION_2_SEQ_MAPPING,
      TF_MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
      TF_MODEL_MAPPING,
      TF_MODEL_WITH_LM_HEAD_MAPPING,
      TFAutoModel,
      TFAutoModelForAudioClassification,
      TFAutoModelForCausalLM,
      TFAutoModelForDocumentQuestionAnswering,
      TFAutoModelForImageClassification,
      TFAutoModelForMaskedImageModeling,
      TFAutoModelForMaskedLM,
      TFAutoModelForMaskGeneration,
      TFAutoModelForMultipleChoice,
      TFAutoModelForNextSentencePrediction,
      TFAutoModelForPreTraining,
      TFAutoModelForQuestionAnswering,
      TFAutoModelForSemanticSegmentation,
      TFAutoModelForSeqaSeqLM,
      TFAutoModelForSequenceClassification,
      TFAutoModelForSpeechSeqaSeq,
      TFAutoModelForTableQuestionAnswering,
      TFAutoModelForTextEncoding,
      TFAutoModelForTokenClassification,
      TFAutoModelForVisionaSeq,
      TFAutoModelForZeroShotImageClassification,
      TFAutoModelWithLMHead,
  )

 try:
  if not is_flax_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_flax_auto import (
      FLAX_MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING,
      FLAX_MODEL_FOR_CAUSAL_LM_MAPPING,
      FLAX_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
      FLAX_MODEL_FOR_MASKED_LM_MAPPING,
      FLAX_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
      FLAX_MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING,
      FLAX_MODEL_FOR_PRETRAINING_MAPPING,
      FLAX_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
      FLAX_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
      FLAX_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
      FLAX_MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING,
      FLAX_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
      FLAX_MODEL_FOR_VISION_2_SEQ_MAPPING,
      FLAX_MODEL_MAPPING,
      FlaxAutoModel,
      FlaxAutoModelForCausalLM,
      FlaxAutoModelForImageClassification,
      FlaxAutoModelForMaskedLM,
      FlaxAutoModelForMultipleChoice,
      FlaxAutoModelForNextSentencePrediction,
      FlaxAutoModelForPreTraining,
      FlaxAutoModelForQuestionAnswering,
      FlaxAutoModelForSeqaSeqLM,
      FlaxAutoModelForSequenceClassification,
      FlaxAutoModelForSpeechSeqaSeq,
      FlaxAutoModelForTokenClassification,
      FlaxAutoModelForVisionaSeq,
  )

else:
 import sys

 lowerCAmelCase_		:			List[Any]				   =					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__       :	Any     				=logging.get_logger(__name__)

lowerCAmelCase__       :	Union[str, Any]     				={
    'google/reformer-crime-and-punishment': (
        'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/config.json'
    ),
    'google/reformer-enwik8': 'https://huggingface.co/google/reformer-enwik8/resolve/main/config.json',
}







class      __lowercase  (__SCREAMING_SNAKE_CASE   ):





							"""simple docstring"""







							_UpperCAmelCase     =		"""reformer"""
							_UpperCAmelCase     =		["""past_buckets_states"""]
							_UpperCAmelCase     =		{}






							def __init__(	self       ,		lowerCAmelCase__=6_4       ,		lowerCAmelCase__=["local", "lsh", "local", "lsh", "local", "lsh"]       ,		lowerCAmelCase__=1.0       ,		lowerCAmelCase__=True       ,		lowerCAmelCase__=[6_4, 6_4]       ,		lowerCAmelCase__=[6_4, 1_9_2]       ,		lowerCAmelCase__=0       ,		lowerCAmelCase__=2       ,		lowerCAmelCase__=5_1_2       ,		lowerCAmelCase__=None       ,		lowerCAmelCase__="relu"       ,		lowerCAmelCase__=0.05       ,		lowerCAmelCase__=2_5_6       ,		lowerCAmelCase__=0.02       ,		lowerCAmelCase__=False       ,		lowerCAmelCase__=1E-12       ,		lowerCAmelCase__=1       ,		lowerCAmelCase__=0       ,		lowerCAmelCase__=0.05       ,		lowerCAmelCase__=6_4       ,		lowerCAmelCase__=6_4       ,		lowerCAmelCase__=0.0       ,		lowerCAmelCase__=1       ,		lowerCAmelCase__=0       ,		lowerCAmelCase__=4_0_9_6       ,		lowerCAmelCase__=1_2       ,		lowerCAmelCase__=None       ,		lowerCAmelCase__=1       ,		lowerCAmelCase__=0       ,		lowerCAmelCase__=3_2_0       ,		lowerCAmelCase__=False       ,		lowerCAmelCase__=True       ,		lowerCAmelCase__=None       ,		**lowerCAmelCase__       ,		):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_						:			str       						= hash_seed
										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= vocab_size
										SCREAMING_SNAKE_CASE_						:			Any       						= attention_head_size
										SCREAMING_SNAKE_CASE_						:			List[Any]       						= hidden_size
										SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= num_attention_heads
										SCREAMING_SNAKE_CASE_						:			Any       						= num_hashes
										SCREAMING_SNAKE_CASE_						:			int       						= len(lowerCAmelCase__    )
										SCREAMING_SNAKE_CASE_						:			str       						= tuple(lowerCAmelCase__    ) if isinstance(lowerCAmelCase__       ,		lowerCAmelCase__    ) else num_buckets
										SCREAMING_SNAKE_CASE_						:			int       						= lsh_attn_chunk_length
										SCREAMING_SNAKE_CASE_						:			List[str]       						= local_attn_chunk_length
										SCREAMING_SNAKE_CASE_						:			Tuple       						= lsh_num_chunks_after
										SCREAMING_SNAKE_CASE_						:			str       						= lsh_num_chunks_before
										SCREAMING_SNAKE_CASE_						:			List[Any]       						= local_num_chunks_after
										SCREAMING_SNAKE_CASE_						:			Any       						= local_num_chunks_before
										SCREAMING_SNAKE_CASE_						:			Any       						= hidden_act
										SCREAMING_SNAKE_CASE_						:			Dict       						= feed_forward_size
										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= hidden_dropout_prob
										SCREAMING_SNAKE_CASE_						:			Tuple       						= lsh_attention_probs_dropout_prob
										SCREAMING_SNAKE_CASE_						:			Any       						= local_attention_probs_dropout_prob
										SCREAMING_SNAKE_CASE_						:			str       						= max_position_embeddings
										SCREAMING_SNAKE_CASE_						:			Optional[int]       						= initializer_range
										SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= layer_norm_eps
										SCREAMING_SNAKE_CASE_						:			List[str]       						= axial_pos_embds
										SCREAMING_SNAKE_CASE_						:			List[str]       						= tuple(lowerCAmelCase__    )
										SCREAMING_SNAKE_CASE_						:			Tuple       						= tuple(lowerCAmelCase__    )
										SCREAMING_SNAKE_CASE_						:			Dict       						= axial_norm_std
										SCREAMING_SNAKE_CASE_						:			Any       						= chunk_size_lm_head
										SCREAMING_SNAKE_CASE_						:			int       						= attn_layers
										SCREAMING_SNAKE_CASE_						:			Union[str, Any]       						= use_cache
										SCREAMING_SNAKE_CASE_						:			Tuple       						= classifier_dropout
										super().__init__(
										    pad_token_id=lowerCAmelCase__       ,		eos_token_id=lowerCAmelCase__       ,		is_decoder=lowerCAmelCase__       ,		tie_word_embeddings=lowerCAmelCase__       ,		**lowerCAmelCase__       ,		)





