



'''simple docstring'''






import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin

class 				a__(   lowerCamelCase__						,					lowerCamelCase__						,					lowerCamelCase__      ):







    @register_to_config
    def __init__( self					:    int		,     __snake_case					:    int		,     __snake_case					:    int		,     __snake_case					:    int		,     __snake_case					:    float		,     __snake_case					:    int		,     __snake_case					:    int		,     __snake_case					:    int		,     __snake_case					:    int		,     __snake_case					:    str		,     __snake_case					:    bool = False		,     ):
      super().__init__()

      a		:   Tuple	  =	nn.Embedding(__snake_case		,     __snake_case      )

      a		:   Dict	  =	nn.Embedding(__snake_case		,     __snake_case      )
      a		:   Tuple	  =	False

      a		:   Optional[Any]	  =	nn.Dropout(p=__snake_case      )

      a		:   int	  =	TaConfig(
          vocab_size=__snake_case		,     d_model=__snake_case		,     num_heads=__snake_case		,     d_kv=__snake_case		,     d_ff=__snake_case		,     dropout_rate=__snake_case		,     feed_forward_proj=__snake_case		,     is_decoder=__snake_case		,     is_encoder_decoder=__snake_case		,     )

      a		:   Optional[int]	  =	nn.ModuleList()
      for lyr_num in range(__snake_case      ):
        a		:   List[Any]	  =	TaBlock(__snake_case      )
        self.encoders.append(__snake_case      )

      a		:   int	  =	TaLayerNorm(__snake_case      )
      a		:   Tuple	  =	nn.Dropout(p=__snake_case      )







    def      lowercase_			( self					:    Optional[Any]		,     __snake_case					:    Tuple		,     __snake_case					:    str      ):
      a		:   Optional[Any]	  =	self.token_embedder(__snake_case      )

      a		:   List[Any]	  =	encoder_input_tokens.shape[1]
      a		:   Tuple	  =	torch.arange(__snake_case		,     device=encoder_input_tokens.device      )
      x += self.position_encoding(__snake_case      )

      a		:   Optional[Any]	  =	self.dropout_pre(__snake_case      )

      # inverted the attention mask
      a		:   Optional[int]	  =	encoder_input_tokens.size()
      a		:   Optional[int]	  =	self.get_extended_attention_mask(__snake_case		,     __snake_case      )

      for lyr in self.encoders:
        a		:   Any	  =	lyr(__snake_case		,     __snake_case      )[0]
      a		:   Optional[Any]	  =	self.layer_norm(__snake_case      )

      return self.dropout_post(__snake_case      ), encoder_inputs_mask




'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


lowerCAmelCase:   List[str]					=		{
    'configuration_falcon': ['FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP', 'FalconConfig'],
}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCAmelCase:   str					=		[
         'FALCON_PRETRAINED_MODEL_ARCHIVE_LIST',
         'FalconForCausalLM',
         'FalconModel',
         'FalconPreTrainedModel',
         'FalconForSequenceClassification',
         'FalconForTokenClassification',
         'FalconForQuestionAnswering',
     ]


if TYPE_CHECKING:
     from .configuration_falcon import FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP, FalconConfig

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_falcon import (
              FALCON_PRETRAINED_MODEL_ARCHIVE_LIST,
              FalconForCausalLM,
              FalconForQuestionAnswering,
              FalconForSequenceClassification,
              FalconForTokenClassification,
              FalconModel,
              FalconPreTrainedModel,
          )


else:
     import sys

     lowerCAmelCase:   Optional[int]					=		_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)