



"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase:   Dict       			=							logging.get_logger(__name__)

UpperCAmelCase:   Optional[Any]       			=							{
    """facebook/opt-125m""": """https://huggingface.co/facebook/opt-125m/blob/main/config.json""",
    """facebook/opt-350m""": """https://huggingface.co/facebook/opt-350m/blob/main/config.json""",
    """facebook/opt-1.3b""": """https://huggingface.co/facebook/opt-1.3b/blob/main/config.json""",
    """facebook/opt-2.7b""": """https://huggingface.co/facebook/opt-2.7b/blob/main/config.json""",
    """facebook/opt-6.7b""": """https://huggingface.co/facebook/opt-6.7b/blob/main/config.json""",
    """facebook/opt-13b""": """https://huggingface.co/facebook/opt-13b/blob/main/config.json""",
}





class    UpperCamelCase     (			snake_case    ):




	"""simple docstring"""
	SCREAMING_SNAKE_CASE_				:    Dict		    =   "opt"
	SCREAMING_SNAKE_CASE_				:    List[str]		    =   ["past_key_values"]






	def __init__(			self							,UpperCAmelCase_=5_02_72							,UpperCAmelCase_=7_68							,UpperCAmelCase_=12							,UpperCAmelCase_=30_72							,UpperCAmelCase_=20_48							,UpperCAmelCase_=True							,UpperCAmelCase_=False							,UpperCAmelCase_=None							,UpperCAmelCase_=0.1							,UpperCAmelCase_=0.0							,UpperCAmelCase_=12							,UpperCAmelCase_="relu"							,UpperCAmelCase_=0.0							,UpperCAmelCase_=0.02							,UpperCAmelCase_=True							,UpperCAmelCase_=1							,UpperCAmelCase_=2							,UpperCAmelCase_=2							,UpperCAmelCase_=True							,UpperCAmelCase_=True							,**UpperCAmelCase_							,):
			super().__init__(
			    pad_token_id=UpperCAmelCase_							,bos_token_id=UpperCAmelCase_							,eos_token_id=UpperCAmelCase_							,**UpperCAmelCase_							,)
			_lowercase							:      Optional[Any]			=					vocab_size
			_lowercase							:      Tuple			=					max_position_embeddings
			_lowercase							:      Union[str, Any]			=					num_attention_heads
			_lowercase							:      List[str]			=					word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
			_lowercase							:      List[str]			=					ffn_dim
			_lowercase							:      List[str]			=					hidden_size
			_lowercase							:      List[Any]			=					num_hidden_layers
			_lowercase							:      List[Any]			=					dropout
			_lowercase							:      str			=					attention_dropout
			_lowercase							:      List[Any]			=					activation_function
			_lowercase							:      int			=					init_std
			_lowercase							:      List[Any]			=					layerdrop
			_lowercase							:      Tuple			=					use_cache
			_lowercase							:      Any			=					do_layer_norm_before
			# We keep these variables at `True` for backward compatibility.
			_lowercase							:      int			=					enable_bias
			_lowercase							:      str			=					layer_norm_elementwise_affine

			# Note that the only purpose of `_remove_final_layer_norm` is to keep backward compatibility
			# with checkpoints that have been fine-tuned before transformers v4.20.1
			# see https://github.com/facebookresearch/metaseq/pull/164
			_lowercase							:      Optional[Any]			=					_remove_final_layer_norm







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase			=    {
    """configuration_big_bird""": ["""BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP""", """BigBirdConfig""", """BigBirdOnnxConfig"""],
}

try:
   if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    ["""BigBirdTokenizer"""]

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    ["""BigBirdTokenizerFast"""]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    [
       """BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """BigBirdForCausalLM""",
       """BigBirdForMaskedLM""",
       """BigBirdForMultipleChoice""",
       """BigBirdForPreTraining""",
       """BigBirdForQuestionAnswering""",
       """BigBirdForSequenceClassification""",
       """BigBirdForTokenClassification""",
       """BigBirdLayer""",
       """BigBirdModel""",
       """BigBirdPreTrainedModel""",
       """load_tf_weights_in_big_bird""",
   ]

try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase			=    [
       """FlaxBigBirdForCausalLM""",
       """FlaxBigBirdForMaskedLM""",
       """FlaxBigBirdForMultipleChoice""",
       """FlaxBigBirdForPreTraining""",
       """FlaxBigBirdForQuestionAnswering""",
       """FlaxBigBirdForSequenceClassification""",
       """FlaxBigBirdForTokenClassification""",
       """FlaxBigBirdModel""",
       """FlaxBigBirdPreTrainedModel""",
   ]

if TYPE_CHECKING:
   from .configuration_big_bird import BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP, BigBirdConfig, BigBirdOnnxConfig

   try:
      if not is_sentencepiece_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_big_bird import BigBirdTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_big_bird_fast import BigBirdTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_big_bird import (
          BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST,
          BigBirdForCausalLM,
          BigBirdForMaskedLM,
          BigBirdForMultipleChoice,
          BigBirdForPreTraining,
          BigBirdForQuestionAnswering,
          BigBirdForSequenceClassification,
          BigBirdForTokenClassification,
          BigBirdLayer,
          BigBirdModel,
          BigBirdPreTrainedModel,
          load_tf_weights_in_big_bird,
      )

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_flax_big_bird import (
          FlaxBigBirdForCausalLM,
          FlaxBigBirdForMaskedLM,
          FlaxBigBirdForMultipleChoice,
          FlaxBigBirdForPreTraining,
          FlaxBigBirdForQuestionAnswering,
          FlaxBigBirdForSequenceClassification,
          FlaxBigBirdForTokenClassification,
          FlaxBigBirdModel,
          FlaxBigBirdPreTrainedModel,
      )

else:
   import sys

   UpperCAmelCase			=    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)