





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tensorflow_text_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase_           =	{
    "configuration_bert": ["BERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "BertConfig", "BertOnnxConfig"],
    "tokenization_bert": ["BasicTokenizer", "BertTokenizer", "WordpieceTokenizer"],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase_           =	["BertTokenizerFast"]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase_           =	[
       "BERT_PRETRAINED_MODEL_ARCHIVE_LIST",
       "BertForMaskedLM",
       "BertForMultipleChoice",
       "BertForNextSentencePrediction",
       "BertForPreTraining",
       "BertForQuestionAnswering",
       "BertForSequenceClassification",
       "BertForTokenClassification",
       "BertLayer",
       "BertLMHeadModel",
       "BertModel",
       "BertPreTrainedModel",
       "load_tf_weights_in_bert",
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase_           =	[
       "TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST",
       "TFBertEmbeddings",
       "TFBertForMaskedLM",
       "TFBertForMultipleChoice",
       "TFBertForNextSentencePrediction",
       "TFBertForPreTraining",
       "TFBertForQuestionAnswering",
       "TFBertForSequenceClassification",
       "TFBertForTokenClassification",
       "TFBertLMHeadModel",
       "TFBertMainLayer",
       "TFBertModel",
       "TFBertPreTrainedModel",
   ]
try:
   if not is_tensorflow_text_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase_           =	["TFBertTokenizer"]

try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase_           =	[
       "FlaxBertForCausalLM",
       "FlaxBertForMaskedLM",
       "FlaxBertForMultipleChoice",
       "FlaxBertForNextSentencePrediction",
       "FlaxBertForPreTraining",
       "FlaxBertForQuestionAnswering",
       "FlaxBertForSequenceClassification",
       "FlaxBertForTokenClassification",
       "FlaxBertModel",
       "FlaxBertPreTrainedModel",
   ]

if TYPE_CHECKING:
   from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig, BertOnnxConfig
   from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_bert_fast import BertTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_bert import (
          BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          BertForMaskedLM,
          BertForMultipleChoice,
          BertForNextSentencePrediction,
          BertForPreTraining,
          BertForQuestionAnswering,
          BertForSequenceClassification,
          BertForTokenClassification,
          BertLayer,
          BertLMHeadModel,
          BertModel,
          BertPreTrainedModel,
          load_tf_weights_in_bert,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_bert import (
          TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFBertEmbeddings,
          TFBertForMaskedLM,
          TFBertForMultipleChoice,
          TFBertForNextSentencePrediction,
          TFBertForPreTraining,
          TFBertForQuestionAnswering,
          TFBertForSequenceClassification,
          TFBertForTokenClassification,
          TFBertLMHeadModel,
          TFBertMainLayer,
          TFBertModel,
          TFBertPreTrainedModel,
      )

   try:
      if not is_tensorflow_text_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_bert_tf import TFBertTokenizer

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_flax_bert import (
          FlaxBertForCausalLM,
          FlaxBertForMaskedLM,
          FlaxBertForMultipleChoice,
          FlaxBertForNextSentencePrediction,
          FlaxBertForPreTraining,
          FlaxBertForQuestionAnswering,
          FlaxBertForSequenceClassification,
          FlaxBertForTokenClassification,
          FlaxBertModel,
          FlaxBertPreTrainedModel,
      )

else:
   import sys

   UpperCAmelCase_           =	_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_           =	logging.get_logger(__name__)

UpperCAmelCase_           =	{
    "RWKV/rwkv-4-169m-pile": "https://huggingface.co/RWKV/rwkv-4-169m-pile/resolve/main/config.json",
    "RWKV/rwkv-4-430m-pile": "https://huggingface.co/RWKV/rwkv-4-430m-pile/resolve/main/config.json",
    "RWKV/rwkv-4-1b5-pile": "https://huggingface.co/RWKV/rwkv-4-1b5-pile/resolve/main/config.json",
    "RWKV/rwkv-4-3b-pile": "https://huggingface.co/RWKV/rwkv-4-3b-pile/resolve/main/config.json",
    "RWKV/rwkv-4-7b-pile": "https://huggingface.co/RWKV/rwkv-4-7b-pile/resolve/main/config.json",
    "RWKV/rwkv-4-14b-pile": "https://huggingface.co/RWKV/rwkv-4-14b-pile/resolve/main/config.json",
    "RWKV/rwkv-raven-1b5": "https://huggingface.co/RWKV/rwkv-raven-1b5/resolve/main/config.json",
    "RWKV/rwkv-raven-3b": "https://huggingface.co/RWKV/rwkv-raven-3b/resolve/main/config.json",
    "RWKV/rwkv-raven-7b": "https://huggingface.co/RWKV/rwkv-raven-7b/resolve/main/config.json",
    "RWKV/rwkv-raven-14b": "https://huggingface.co/RWKV/rwkv-raven-14b/resolve/main/config.json",
}




class 			__UpperCamelCase    (							A__    ):
     __A	:  Tuple							  =   """rwkv"""
     __A	:  Any							  =   {"""max_position_embeddings""": """context_length"""}





     def __init__(     self				,							_UpperCamelCase=50277				,							_UpperCamelCase=1024				,							_UpperCamelCase=4096				,							_UpperCamelCase=32				,							_UpperCamelCase=None				,							_UpperCamelCase=None				,							_UpperCamelCase=1e-5				,							_UpperCamelCase=0				,							_UpperCamelCase=0				,							_UpperCamelCase=6				,							_UpperCamelCase=False				,							_UpperCamelCase=True				,							**_UpperCamelCase				,							):
        _UpperCAmelCase           =   vocab_size
        _UpperCAmelCase           =   context_length
        _UpperCAmelCase           =   hidden_size
        _UpperCAmelCase           =   num_hidden_layers
        _UpperCAmelCase           =   attention_hidden_size if attention_hidden_size is not None else hidden_size
        _UpperCAmelCase           =   intermediate_size if intermediate_size is not None else 4 * hidden_size
        _UpperCAmelCase           =   layer_norm_epsilon
        _UpperCAmelCase           =   rescale_every
        _UpperCAmelCase           =   use_cache

        _UpperCAmelCase           =   bos_token_id
        _UpperCAmelCase           =   eos_token_id

        super().__init__(
            tie_word_embeddings=_UpperCamelCase				,							bos_token_id=_UpperCamelCase				,							eos_token_id=_UpperCamelCase				,							**_UpperCamelCase					)