




'''simple docstring'''





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_		:     List[Any]          =		logging.get_logger(__name__)

UpperCamelCase_		:     Tuple          =		{
    '''RWKV/rwkv-4-169m-pile''': '''https://huggingface.co/RWKV/rwkv-4-169m-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-430m-pile''': '''https://huggingface.co/RWKV/rwkv-4-430m-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-1b5-pile''': '''https://huggingface.co/RWKV/rwkv-4-1b5-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-3b-pile''': '''https://huggingface.co/RWKV/rwkv-4-3b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-7b-pile''': '''https://huggingface.co/RWKV/rwkv-4-7b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-4-14b-pile''': '''https://huggingface.co/RWKV/rwkv-4-14b-pile/resolve/main/config.json''',
    '''RWKV/rwkv-raven-1b5''': '''https://huggingface.co/RWKV/rwkv-raven-1b5/resolve/main/config.json''',
    '''RWKV/rwkv-raven-3b''': '''https://huggingface.co/RWKV/rwkv-raven-3b/resolve/main/config.json''',
    '''RWKV/rwkv-raven-7b''': '''https://huggingface.co/RWKV/rwkv-raven-7b/resolve/main/config.json''',
    '''RWKV/rwkv-raven-14b''': '''https://huggingface.co/RWKV/rwkv-raven-14b/resolve/main/config.json''',
}

class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       List[str]       							=		"""rwkv"""
     SCREAMING_SNAKE_CASE_				:       Any       							=		{"""max_position_embeddings""": """context_length"""}






     def __init__(					self   ,_SCREAMING_SNAKE_CASE=50_277   ,_SCREAMING_SNAKE_CASE=1_024   ,_SCREAMING_SNAKE_CASE=4_096   ,_SCREAMING_SNAKE_CASE=32   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=1e-5   ,_SCREAMING_SNAKE_CASE=0   ,_SCREAMING_SNAKE_CASE=0   ,_SCREAMING_SNAKE_CASE=6   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=True   ,**_SCREAMING_SNAKE_CASE   ,)				->     List[str]:
          _snake_case				= vocab_size
          _snake_case				= context_length
          _snake_case				= hidden_size
          _snake_case				= num_hidden_layers
          _snake_case				= attention_hidden_size if attention_hidden_size is not None else hidden_size
          _snake_case				= intermediate_size if intermediate_size is not None else 4 * hidden_size
          _snake_case				= layer_norm_epsilon
          _snake_case				= rescale_every
          _snake_case				= use_cache

          _snake_case				= bos_token_id
          _snake_case				= eos_token_id

          super().__init__(
              tie_word_embeddings=_SCREAMING_SNAKE_CASE   ,bos_token_id=_SCREAMING_SNAKE_CASE   ,eos_token_id=_SCREAMING_SNAKE_CASE   ,**_SCREAMING_SNAKE_CASE       )








'''simple docstring'''





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


UpperCamelCase_		:     Optional[Any]          =		{
    '''configuration_roc_bert''': ['''ROC_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''RoCBertConfig'''],
    '''tokenization_roc_bert''': ['''RoCBertTokenizer'''],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   pass

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCamelCase_		:     Union[str, Any]          =		[
       '''ROC_BERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''RoCBertForCausalLM''',
       '''RoCBertForMaskedLM''',
       '''RoCBertForMultipleChoice''',
       '''RoCBertForPreTraining''',
       '''RoCBertForQuestionAnswering''',
       '''RoCBertForSequenceClassification''',
       '''RoCBertForTokenClassification''',
       '''RoCBertLayer''',
       '''RoCBertModel''',
       '''RoCBertPreTrainedModel''',
       '''load_tf_weights_in_roc_bert''',
   ]

if TYPE_CHECKING:
   from .configuration_roc_bert import ROC_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RoCBertConfig
   from .tokenization_roc_bert import RoCBertTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      raise OptionalDependencyNotAvailable()

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_roc_bert import (
          ROC_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          RoCBertForCausalLM,
          RoCBertForMaskedLM,
          RoCBertForMultipleChoice,
          RoCBertForPreTraining,
          RoCBertForQuestionAnswering,
          RoCBertForSequenceClassification,
          RoCBertForTokenClassification,
          RoCBertLayer,
          RoCBertModel,
          RoCBertPreTrainedModel,
          load_tf_weights_in_roc_bert,
      )


else:
   import sys

   UpperCamelCase_		:     int          =		_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


