

"""simple docstring"""




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_torch_available


__UpperCAmelCase			=					{
    'configuration_longt5': ['LONGT5_PRETRAINED_CONFIG_ARCHIVE_MAP', 'LongT5Config', 'LongT5OnnxConfig'],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase			=					[
          'LONGT5_PRETRAINED_MODEL_ARCHIVE_LIST',
          'LongT5EncoderModel',
          'LongT5ForConditionalGeneration',
          'LongT5Model',
          'LongT5PreTrainedModel',
      ]

try:
      if not is_flax_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase			=					[
          'FlaxLongT5ForConditionalGeneration',
          'FlaxLongT5Model',
          'FlaxLongT5PreTrainedModel',
      ]


if TYPE_CHECKING:
      from .configuration_longta import LONGT5_PRETRAINED_CONFIG_ARCHIVE_MAP, LongTaConfig, LongTaOnnxConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_longta import (
                LONGT5_PRETRAINED_MODEL_ARCHIVE_LIST,
                LongTaEncoderModel,
                LongTaForConditionalGeneration,
                LongTaModel,
                LongTaPreTrainedModel,
            )

      try:
            if not is_flax_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_flax_longta import (
                FlaxLongTaForConditionalGeneration,
                FlaxLongTaModel,
                FlaxLongTaPreTrainedModel,
            )


else:
      import sys

      __UpperCAmelCase			=					_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)







"""simple docstring"""

from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_            =    logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_            =    {
    '''tiiuae/falcon-40b''': '''https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json''',
    '''tiiuae/falcon-7b''': '''https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json''',
}
class   _UpperCAmelCase				(		SCREAMING_SNAKE_CASE_				):
      __SCREAMING_SNAKE_CASE :				Optional[Any]				   =						"falcon"
      __SCREAMING_SNAKE_CASE :				List[Any]				   =						["past_key_values"]




      def __init__(				self ,							lowercase_=6_5_0_2_4 ,							lowercase_=4_5_4_4 ,							lowercase_=3_2 ,							lowercase_=7_1 ,							lowercase_=1E-5 ,							lowercase_=0.0_2 ,							lowercase_=True ,							lowercase_=0.0 ,							lowercase_=0.0 ,							lowercase_=None ,							lowercase_=False ,							lowercase_=False ,							lowercase_=True ,							lowercase_=True ,							lowercase_=False ,							lowercase_=1_1 ,							lowercase_=1_1 ,							**lowercase_ ,							)    ->       Any:
          UpperCAmelCase          =       vocab_size
          # Backward compatibility with n_embed kwarg
          UpperCAmelCase          =       kwargs.pop('n_embed' ,							lowercase_				)
          UpperCAmelCase          =       hidden_size if n_embed is None else n_embed
          UpperCAmelCase          =       num_hidden_layers
          UpperCAmelCase          =       num_attention_heads
          UpperCAmelCase          =       layer_norm_epsilon
          UpperCAmelCase          =       initializer_range
          UpperCAmelCase          =       use_cache
          UpperCAmelCase          =       hidden_dropout
          UpperCAmelCase          =       attention_dropout

          UpperCAmelCase          =       bos_token_id
          UpperCAmelCase          =       eos_token_id
          UpperCAmelCase          =       num_attention_heads if num_kv_heads is None else num_kv_heads
          UpperCAmelCase          =       alibi
          UpperCAmelCase          =       new_decoder_architecture
          UpperCAmelCase          =       multi_query  # Ignored when new_decoder_architecture is True
          UpperCAmelCase          =       parallel_attn
          UpperCAmelCase          =       bias

          super().__init__(bos_token_id=lowercase_ ,							eos_token_id=lowercase_ ,							**lowercase_				)




      @property
      def      a_   (				self				)    ->       List[Any]:
          return self.hidden_size // self.num_attention_heads




      @property
      def      a_   (				self				)    ->       List[str]:
          return not self.alibi



