

import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


__A      		=     get_tests_dir('''fixtures/test_sentencepiece.model''')
__A      		=     get_tests_dir('''fixtures/test_sentencepiece_bpe.model''')

__A      		=     'pt' if is_torch_available() else 'tf'



@require_sentencepiece
@require_tokenizers
class   lowercase_   (  __SCREAMING_SNAKE_CASE       ,							unittest.TestCase					):
						UpperCamelCase_      :		Optional[Any]										=    CamembertTokenizer
						UpperCamelCase_      :		Any										=    CamembertTokenizerFast
						UpperCamelCase_      :		int										=    True
						UpperCamelCase_      :		int										=    True



						def 						UpperCamelCase_		(      self				:    List[str]		)   ->					List[str]:
								super().setUp()

								# We have a SentencePiece fixture for testing
								_snake_case						=							CamembertTokenizer(_SCREAMING_SNAKE_CASE		)
								tokenizer.save_pretrained(self.tmpdirname		)



						def 						UpperCamelCase_		(      self				:    Dict		)   ->					Tuple:
								_snake_case						=							"<pad>"
								_snake_case						=							1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE		)      , _SCREAMING_SNAKE_CASE		)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE		)      , _SCREAMING_SNAKE_CASE		)



						def 						UpperCamelCase_		(      self				:    Optional[Any]		)   ->					Tuple:
								_snake_case						=							list(self.get_tokenizer().get_vocab().keys()		)

								self.assertEqual(vocab_keys[0]      , '''<s>NOTUSED'''		)
								self.assertEqual(vocab_keys[1]      , '''<pad>'''		)
								self.assertEqual(vocab_keys[-1]      , '''<mask>'''		)
								self.assertEqual(len(_SCREAMING_SNAKE_CASE		)      , 1004		)



						def 						UpperCamelCase_		(      self				:    Optional[Any]		)   ->					Dict:
								self.assertEqual(self.get_tokenizer().vocab_size      , 1005		)



						def 						UpperCamelCase_		(      self				:    Union[str, Any]		)   ->					Dict:
								_snake_case						=							CamembertTokenizer(_SCREAMING_SNAKE_CASE		)
								tokenizer.save_pretrained(self.tmpdirname		)
								_snake_case						=							CamembertTokenizerFast.from_pretrained(self.tmpdirname		)

								_snake_case						=							"I was born in 92000, and this is falsé."

								_snake_case						=							tokenizer.encode(_SCREAMING_SNAKE_CASE		)
								_snake_case						=							rust_tokenizer.encode(_SCREAMING_SNAKE_CASE		)
								self.assertListEqual(_SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE		)

								_snake_case						=							tokenizer.encode(_SCREAMING_SNAKE_CASE      , add_special_tokens=_SCREAMING_SNAKE_CASE		)
								_snake_case						=							rust_tokenizer.encode(_SCREAMING_SNAKE_CASE      , add_special_tokens=_SCREAMING_SNAKE_CASE		)
								self.assertListEqual(_SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE		)

								# <unk> tokens are not the same for `rust` than for `slow`.
								# Because spm gives back raw token instead of `unk` in EncodeAsPieces
								# tokens = tokenizer.tokenize(sequence)
								_snake_case						=							tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE		)
								_snake_case						=							rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE		)
								self.assertListEqual(_SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE		)



						def 						UpperCamelCase_		(      self				:    Any		)   ->					int:

								if not self.test_rust_tokenizer:
										return

								_snake_case						=							self.get_tokenizer()
								_snake_case						=							self.get_rust_tokenizer()

								_snake_case						=							"I was born in 92000, and this is falsé."

								_snake_case						=							tokenizer.tokenize(_SCREAMING_SNAKE_CASE		)
								_snake_case						=							rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE		)
								self.assertListEqual(_SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE		)

								_snake_case						=							tokenizer.encode(_SCREAMING_SNAKE_CASE      , add_special_tokens=_SCREAMING_SNAKE_CASE		)
								_snake_case						=							rust_tokenizer.encode(_SCREAMING_SNAKE_CASE      , add_special_tokens=_SCREAMING_SNAKE_CASE		)
								self.assertListEqual(_SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE		)

								_snake_case						=							self.get_rust_tokenizer()
								_snake_case						=							tokenizer.encode(_SCREAMING_SNAKE_CASE		)
								_snake_case						=							rust_tokenizer.encode(_SCREAMING_SNAKE_CASE		)
								self.assertListEqual(_SCREAMING_SNAKE_CASE      , _SCREAMING_SNAKE_CASE		)







						@slow
						def 						UpperCamelCase_		(      self				:    Union[str, Any]		)   ->					List[str]:
								_snake_case						=							{"input_ids": [[5, 54, 7196, 297, 30, 23, 776, 18, 11, 3215, 3705, 8252, 22, 3164, 1181, 2116, 29, 16, 813, 25, 791, 3314, 20, 3446, 38, 27575, 120, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 468, 17, 11, 9088, 20, 1517, 8, 22804, 18818, 10, 38, 629, 607, 607, 142, 19, 7196, 867, 56, 10326, 24, 2267, 20, 416, 5072, 15612, 233, 734, 7, 2399, 27, 16, 3015, 1649, 7, 24, 20, 4338, 2399, 27, 13, 3400, 14, 13, 6189, 8, 930, 9, 6]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
								# fmt: on

								# camembert is a french model. So we also use french texts.
								_snake_case						=							[
								    "Le transformeur est un modèle d'apprentissage profond introduit en 2017, "
								    "utilisé principalement dans le domaine du traitement automatique des langues (TAL).",
								    "À l'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus "
								    "pour gérer des données séquentielles, telles que le langage naturel, pour des tâches "
								    "telles que la traduction et la synthèse de texte.",
								]

								self.tokenizer_integration_test_util(
								    expected_encoding=_SCREAMING_SNAKE_CASE      , model_name='''camembert-base'''      , revision='''3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf'''      , sequences=_SCREAMING_SNAKE_CASE      , )




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


__A      		=     {
    '''configuration_falcon''': ['''FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''FalconConfig'''],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __A      		=     [
     '''FALCON_PRETRAINED_MODEL_ARCHIVE_LIST''',
     '''FalconForCausalLM''',
     '''FalconModel''',
     '''FalconPreTrainedModel''',
     '''FalconForSequenceClassification''',
     '''FalconForTokenClassification''',
     '''FalconForQuestionAnswering''',
 ]


if TYPE_CHECKING:
 from .configuration_falcon import FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP, FalconConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_falcon import (
      FALCON_PRETRAINED_MODEL_ARCHIVE_LIST,
      FalconForCausalLM,
      FalconForQuestionAnswering,
      FalconForSequenceClassification,
      FalconForTokenClassification,
      FalconModel,
      FalconPreTrainedModel,
  )


else:
 import sys

 __A      		=     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

