

'''simple docstring'''





import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


A_	= get_tests_dir("fixtures/test_sentencepiece.model")
A_	= get_tests_dir("fixtures/test_sentencepiece_bpe.model")

A_	= "pt" if is_torch_available() else "tf"


@require_sentencepiece
@require_tokenizers
class  UpperCAmelCase (							UpperCAmelCase__  , unittest.TestCase    ):

		'''simple docstring'''






		SCREAMING_SNAKE_CASE_           =					CamembertTokenizer
		SCREAMING_SNAKE_CASE_           =					CamembertTokenizerFast
		SCREAMING_SNAKE_CASE_           =					True
		SCREAMING_SNAKE_CASE_           =					True
		def 			UpperCamelCase(					self		)  ->     Tuple:






							'''simple docstring'''



							super().setUp()

							# We have a SentencePiece fixture for testing
							lowerCamelCase_  				=      CamembertTokenizer(SCREAMING_SNAKE_CASE_		)
							tokenizer.save_pretrained(self.tmpdirname		)
		def 			UpperCamelCase(					self		)  ->     List[str]:






							'''simple docstring'''



							lowerCamelCase_  				=      '<pad>'
							lowerCamelCase_  				=      1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE_		)   ,			SCREAMING_SNAKE_CASE_		)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE_		)   ,			SCREAMING_SNAKE_CASE_		)
		def 			UpperCamelCase(					self		)  ->     int:






							'''simple docstring'''



							lowerCamelCase_  				=      list(self.get_tokenizer().get_vocab().keys()		)

							self.assertEqual(vocab_keys[0]   ,			'<s>NOTUSED'		)
							self.assertEqual(vocab_keys[1]   ,			'<pad>'		)
							self.assertEqual(vocab_keys[-1]   ,			'<mask>'		)
							self.assertEqual(len(SCREAMING_SNAKE_CASE_		)   ,			1004		)
		def 			UpperCamelCase(					self		)  ->     Any:






							'''simple docstring'''



							self.assertEqual(self.get_tokenizer().vocab_size   ,			1005		)
		def 			UpperCamelCase(					self		)  ->     int:






							'''simple docstring'''



							lowerCamelCase_  				=      CamembertTokenizer(SCREAMING_SNAKE_CASE_		)
							tokenizer.save_pretrained(self.tmpdirname		)
							lowerCamelCase_  				=      CamembertTokenizerFast.from_pretrained(self.tmpdirname		)

							lowerCamelCase_  				=      'I was born in 92000, and this is falsé.'

							lowerCamelCase_  				=      tokenizer.encode(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      rust_tokenizer.encode(SCREAMING_SNAKE_CASE_		)
							self.assertListEqual(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)

							lowerCamelCase_  				=      tokenizer.encode(SCREAMING_SNAKE_CASE_   ,			add_special_tokens=SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      rust_tokenizer.encode(SCREAMING_SNAKE_CASE_   ,			add_special_tokens=SCREAMING_SNAKE_CASE_		)
							self.assertListEqual(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)

							# <unk> tokens are not the same for `rust` than for `slow`.
							# Because spm gives back raw token instead of `unk` in EncodeAsPieces
							# tokens = tokenizer.tokenize(sequence)
							lowerCamelCase_  				=      tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_		)
							self.assertListEqual(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)
		def 			UpperCamelCase(					self		)  ->     List[str]:






							'''simple docstring'''



							if not self.test_rust_tokenizer:
												return

							lowerCamelCase_  				=      self.get_tokenizer()
							lowerCamelCase_  				=      self.get_rust_tokenizer()

							lowerCamelCase_  				=      'I was born in 92000, and this is falsé.'

							lowerCamelCase_  				=      tokenizer.tokenize(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_		)
							self.assertListEqual(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)

							lowerCamelCase_  				=      tokenizer.encode(SCREAMING_SNAKE_CASE_   ,			add_special_tokens=SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      rust_tokenizer.encode(SCREAMING_SNAKE_CASE_   ,			add_special_tokens=SCREAMING_SNAKE_CASE_		)
							self.assertListEqual(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)

							lowerCamelCase_  				=      self.get_rust_tokenizer()
							lowerCamelCase_  				=      tokenizer.encode(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      rust_tokenizer.encode(SCREAMING_SNAKE_CASE_		)
							self.assertListEqual(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)






		@slow
		def 			UpperCamelCase(					self		)  ->     Any:






							'''simple docstring'''



							lowerCamelCase_  				=      {'input_ids': [[5, 54, 7196, 297, 30, 23, 776, 18, 11, 3215, 3705, 8252, 22, 3164, 1181, 2116, 29, 16, 813, 25, 791, 3314, 20, 3446, 38, 27575, 120, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 468, 17, 11, 9088, 20, 1517, 8, 22804, 18818, 10, 38, 629, 607, 607, 142, 19, 7196, 867, 56, 10326, 24, 2267, 20, 416, 5072, 15612, 233, 734, 7, 2399, 27, 16, 3015, 1649, 7, 24, 20, 4338, 2399, 27, 13, 3400, 14, 13, 6189, 8, 930, 9, 6]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
							# fmt: on

							# camembert is a french model. So we also use french texts.
							lowerCamelCase_  				=      [
							    'Le transformeur est un modèle d\'apprentissage profond introduit en 2017, '
							    'utilisé principalement dans le domaine du traitement automatique des langues (TAL).',
							    'À l\'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus '
							    'pour gérer des données séquentielles, telles que le langage naturel, pour des tâches '
							    'telles que la traduction et la synthèse de texte.',
							]

							self.tokenizer_integration_test_util(
							    expected_encoding=SCREAMING_SNAKE_CASE_   ,			model_name='camembert-base'   ,			revision='3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf'   ,			sequences=SCREAMING_SNAKE_CASE_   ,			)










'''simple docstring'''


from ...configuration_utils import PretrainedConfig
from ...utils import logging


_SCREAMING_SNAKE_CASE									=				logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE									=				{
    "tiiuae/falcon-40b": "https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json",
    "tiiuae/falcon-7b": "https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json",
}

class  lowerCAmelCase_		(		__magic_name__		):
 __lowerCamelCase    :				Any								=       "falcon"
 __lowerCamelCase    :				List[str]								=       ["past_key_values"]




 def __init__(						self      ,  _lowerCAmelCase=65024      ,  _lowerCAmelCase=4544      ,  _lowerCAmelCase=32      ,  _lowerCAmelCase=71      ,  _lowerCAmelCase=1E-5      ,  _lowerCAmelCase=0.02      ,  _lowerCAmelCase=True      ,  _lowerCAmelCase=0.0      ,  _lowerCAmelCase=0.0      ,  _lowerCAmelCase=None      ,  _lowerCAmelCase=False      ,  _lowerCAmelCase=False      ,  _lowerCAmelCase=True      ,  _lowerCAmelCase=True      ,  _lowerCAmelCase=False      ,  _lowerCAmelCase=11      ,  _lowerCAmelCase=11      ,  **_lowerCAmelCase      ,  )     ->				Union[str, Any]:
     _lowerCAmelCase						      =   vocab_size
     # Backward compatibility with n_embed kwarg
     _lowerCAmelCase						      =   kwargs.pop("n_embed"      ,  _lowerCAmelCase		)
     _lowerCAmelCase						      =   hidden_size if n_embed is None else n_embed
     _lowerCAmelCase						      =   num_hidden_layers
     _lowerCAmelCase						      =   num_attention_heads
     _lowerCAmelCase						      =   layer_norm_epsilon
     _lowerCAmelCase						      =   initializer_range
     _lowerCAmelCase						      =   use_cache
     _lowerCAmelCase						      =   hidden_dropout
     _lowerCAmelCase						      =   attention_dropout

     _lowerCAmelCase						      =   bos_token_id
     _lowerCAmelCase						      =   eos_token_id
     _lowerCAmelCase						      =   num_attention_heads if num_kv_heads is None else num_kv_heads
     _lowerCAmelCase						      =   alibi
     _lowerCAmelCase						      =   new_decoder_architecture
     _lowerCAmelCase						      =   multi_query  # Ignored when new_decoder_architecture is True
     _lowerCAmelCase						      =   parallel_attn
     _lowerCAmelCase						      =   bias

     super().__init__(bos_token_id=_lowerCAmelCase      ,  eos_token_id=_lowerCAmelCase      ,  **_lowerCAmelCase		)




 @property
 def 				_snake_case (						self		)     ->				Optional[Any]:
     return self.hidden_size // self.num_attention_heads





 @property
 def 				_snake_case (						self		)     ->				Optional[Any]:
     return not self.alibi


