





import re

import jax.numpy as jnp
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.random import PRNGKey

from ..utils import logging


UpperCAmelCase_           =	logging.get_logger(__name__)
def      A__       (    SCREAMING_SNAKE_CASE_   :      int						)  ->  int:






   """simple docstring"""







   _UpperCAmelCase           =   R'''\w+[.]\d+'''
   _UpperCAmelCase           =   re.findall(SCREAMING_SNAKE_CASE_				,	SCREAMING_SNAKE_CASE_						)
   for pat in pats:
      _UpperCAmelCase           =   key.replace(SCREAMING_SNAKE_CASE_				,	'''_'''.join(pat.split('''.'''						)						)						)
   return key
def      A__       (    SCREAMING_SNAKE_CASE_   :      Any				,	SCREAMING_SNAKE_CASE_   :      Optional[Any]				,	SCREAMING_SNAKE_CASE_   :      Union[str, Any]						)  ->  Tuple:






   """simple docstring"""







   _UpperCAmelCase           =   pt_tuple_key[:-1] + ('''scale''',)
   if (
       any('''norm''' in str_ for str_ in pt_tuple_key						)
       and (pt_tuple_key[-1] == "bias")
       and (pt_tuple_key[:-1] + ("bias",) not in random_flax_state_dict)
       and (pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict)
   ):
      _UpperCAmelCase           =   pt_tuple_key[:-1] + ('''scale''',)
      return renamed_pt_tuple_key, pt_tensor
   elif pt_tuple_key[-1] in ["weight", "gamma"] and pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict:
      _UpperCAmelCase           =   pt_tuple_key[:-1] + ('''scale''',)
      return renamed_pt_tuple_key, pt_tensor

   # embedding
   if pt_tuple_key[-1] == "weight" and pt_tuple_key[:-1] + ("embedding",) in random_flax_state_dict:
      _UpperCAmelCase           =   pt_tuple_key[:-1] + ('''embedding''',)
      return renamed_pt_tuple_key, pt_tensor

   # conv layer
   _UpperCAmelCase           =   pt_tuple_key[:-1] + ('''kernel''',)
   if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4:
      _UpperCAmelCase           =   pt_tensor.transpose(2				,	3				,	1				,	0						)
      return renamed_pt_tuple_key, pt_tensor

   # linear layer
   _UpperCAmelCase           =   pt_tuple_key[:-1] + ('''kernel''',)
   if pt_tuple_key[-1] == "weight":
      _UpperCAmelCase           =   pt_tensor.T
      return renamed_pt_tuple_key, pt_tensor

   # old PyTorch layer norm weight
   _UpperCAmelCase           =   pt_tuple_key[:-1] + ('''weight''',)
   if pt_tuple_key[-1] == "gamma":
      return renamed_pt_tuple_key, pt_tensor

   # old PyTorch layer norm bias
   _UpperCAmelCase           =   pt_tuple_key[:-1] + ('''bias''',)
   if pt_tuple_key[-1] == "beta":
      return renamed_pt_tuple_key, pt_tensor

   return pt_tuple_key, pt_tensor
def      A__       (    SCREAMING_SNAKE_CASE_   :      Any				,	SCREAMING_SNAKE_CASE_   :      List[str]				,	SCREAMING_SNAKE_CASE_   :      Dict=42						)  ->  List[Any]:






   """simple docstring"""







   _UpperCAmelCase           =   {k: v.numpy() for k, v in pt_state_dict.items()}

   # Step 2: Since the model is stateless, get random Flax params
   _UpperCAmelCase           =   flax_model.init_weights(PRNGKey(SCREAMING_SNAKE_CASE_						)						)

   _UpperCAmelCase           =   flatten_dict(SCREAMING_SNAKE_CASE_						)
   _UpperCAmelCase           =   {}

   # Need to change some parameters name to match Flax names
   for pt_key, pt_tensor in pt_state_dict.items():
      _UpperCAmelCase           =   rename_key(SCREAMING_SNAKE_CASE_						)
      _UpperCAmelCase           =   tuple(renamed_pt_key.split('''.'''						)						)

      # Correctly rename weight parameters
      _UpperCAmelCase       ,      _UpperCAmelCase           =   rename_key_and_reshape_tensor(SCREAMING_SNAKE_CASE_				,	SCREAMING_SNAKE_CASE_				,	SCREAMING_SNAKE_CASE_						)

      if flax_key in random_flax_state_dict:
         if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
            raise ValueError(
                F'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
                F'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''						)

        # also add unexpected weight so that warning is thrown
      _UpperCAmelCase           =   jnp.asarray(SCREAMING_SNAKE_CASE_						)

   return unflatten_dict(SCREAMING_SNAKE_CASE_						)






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_           =	logging.get_logger(__name__)

UpperCAmelCase_           =	{
    "tiiuae/falcon-40b": "https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json",
    "tiiuae/falcon-7b": "https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json",
}




class 			__UpperCamelCase    (							A__    ):
     __A	:  Dict							  =   """falcon"""
     __A	:  Any							  =   ["""past_key_values"""]





     def __init__(     self				,							_UpperCamelCase=65024				,							_UpperCamelCase=4544				,							_UpperCamelCase=32				,							_UpperCamelCase=71				,							_UpperCamelCase=1e-5				,							_UpperCamelCase=0.02				,							_UpperCamelCase=True				,							_UpperCamelCase=0.0				,							_UpperCamelCase=0.0				,							_UpperCamelCase=None				,							_UpperCamelCase=False				,							_UpperCamelCase=False				,							_UpperCamelCase=True				,							_UpperCamelCase=True				,							_UpperCamelCase=False				,							_UpperCamelCase=11				,							_UpperCamelCase=11				,							**_UpperCamelCase				,							):
        _UpperCAmelCase           =   vocab_size
        # Backward compatibility with n_embed kwarg
        _UpperCAmelCase           =   kwargs.pop('''n_embed'''				,							_UpperCamelCase					)
        _UpperCAmelCase           =   hidden_size if n_embed is None else n_embed
        _UpperCAmelCase           =   num_hidden_layers
        _UpperCAmelCase           =   num_attention_heads
        _UpperCAmelCase           =   layer_norm_epsilon
        _UpperCAmelCase           =   initializer_range
        _UpperCAmelCase           =   use_cache
        _UpperCAmelCase           =   hidden_dropout
        _UpperCAmelCase           =   attention_dropout

        _UpperCAmelCase           =   bos_token_id
        _UpperCAmelCase           =   eos_token_id
        _UpperCAmelCase           =   num_attention_heads if num_kv_heads is None else num_kv_heads
        _UpperCAmelCase           =   alibi
        _UpperCAmelCase           =   new_decoder_architecture
        _UpperCAmelCase           =   multi_query  # Ignored when new_decoder_architecture is True
        _UpperCAmelCase           =   parallel_attn
        _UpperCAmelCase           =   bias

        super().__init__(bos_token_id=_UpperCamelCase				,							eos_token_id=_UpperCamelCase				,							**_UpperCamelCase					)





     @property
     def 		UpperCamelCase(     self					):
        return self.hidden_size // self.num_attention_heads





     @property
     def 		UpperCamelCase(     self					):
        return not self.alibi