






import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE 							=						'▁'

SCREAMING_SNAKE_CASE 							=						get_tests_dir('fixtures/test_sentencepiece.model')

@require_sentencepiece
@require_tokenizers
class 							A_    (						__UpperCAmelCase ,   unittest.TestCase			):






 '''simple docstring'''




 _SCREAMING_SNAKE_CASE							:			Union[str, Any] =					BigBirdTokenizer
 _SCREAMING_SNAKE_CASE							:			Any =					BigBirdTokenizerFast
 _SCREAMING_SNAKE_CASE							:			str =					True
 _SCREAMING_SNAKE_CASE							:			Optional[Any] =					True


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     super().setUp()

     _UpperCAmelCase				:      Dict			  =				self.tokenizer_class(UpperCAmelCase_							,  keep_accents=UpperCAmelCase_)
     tokenizer.save_pretrained(self.tmpdirname)


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     _UpperCAmelCase				:      Union[str, Any]			  =				'''<s>'''
     _UpperCAmelCase				:      Dict			  =				1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase_)							,  UpperCAmelCase_)
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase_)							,  UpperCAmelCase_)


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     _UpperCAmelCase				:      Union[str, Any]			  =				list(self.get_tokenizer().get_vocab().keys())

     self.assertEqual(vocab_keys[0]							,  '''<unk>''')
     self.assertEqual(vocab_keys[1]							,  '''<s>''')
     self.assertEqual(vocab_keys[-1]							,  '''[MASK]''')
     self.assertEqual(len(UpperCAmelCase_)							,  1004)


 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     self.assertEqual(self.get_tokenizer().vocab_size							,  1000)


 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     if not self.test_rust_tokenizer:
         return

     _UpperCAmelCase				:      str			  =				self.get_tokenizer()
     _UpperCAmelCase				:      Union[str, Any]			  =				self.get_rust_tokenizer()

     _UpperCAmelCase				:      Optional[Any]			  =				'''I was born in 92000, and this is falsé.'''

     _UpperCAmelCase				:      List[Any]			  =				tokenizer.tokenize(UpperCAmelCase_)
     _UpperCAmelCase				:      str			  =				rust_tokenizer.tokenize(UpperCAmelCase_)
     self.assertListEqual(UpperCAmelCase_							,  UpperCAmelCase_)

     _UpperCAmelCase				:      Dict			  =				tokenizer.encode(UpperCAmelCase_							,  add_special_tokens=UpperCAmelCase_)
     _UpperCAmelCase				:      Dict			  =				rust_tokenizer.encode(UpperCAmelCase_							,  add_special_tokens=UpperCAmelCase_)
     self.assertListEqual(UpperCAmelCase_							,  UpperCAmelCase_)

     _UpperCAmelCase				:      str			  =				self.get_rust_tokenizer()
     _UpperCAmelCase				:      List[str]			  =				tokenizer.encode(UpperCAmelCase_)
     _UpperCAmelCase				:      Optional[Any]			  =				rust_tokenizer.encode(UpperCAmelCase_)
     self.assertListEqual(UpperCAmelCase_							,  UpperCAmelCase_)


 def  snake_case__				(	self) ->		Optional[int]:

     """simple docstring"""



     _UpperCAmelCase				:      str			  =				BigBirdTokenizer(UpperCAmelCase_							,  keep_accents=UpperCAmelCase_)

     _UpperCAmelCase				:      int			  =				tokenizer.tokenize('''This is a test''')
     self.assertListEqual(UpperCAmelCase_							,  ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(UpperCAmelCase_)							,  [285, 46, 10, 170, 382]							,  )

     _UpperCAmelCase				:      int			  =				tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
     self.assertListEqual(
         UpperCAmelCase_							,  [
             SPIECE_UNDERLINE + '''I''',
             SPIECE_UNDERLINE + '''was''',
             SPIECE_UNDERLINE + '''b''',
             '''or''',
             '''n''',
             SPIECE_UNDERLINE + '''in''',
             SPIECE_UNDERLINE + '''''',
             '''9''',
             '''2''',
             '''0''',
             '''0''',
             '''0''',
             ''',''',
             SPIECE_UNDERLINE + '''and''',
             SPIECE_UNDERLINE + '''this''',
             SPIECE_UNDERLINE + '''is''',
             SPIECE_UNDERLINE + '''f''',
             '''al''',
             '''s''',
             '''é''',
             '''.''',
         ]							,  )
     _UpperCAmelCase				:      Optional[int]			  =				tokenizer.convert_tokens_to_ids(UpperCAmelCase_)
     self.assertListEqual(
         UpperCAmelCase_							,  [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]							,  )

     _UpperCAmelCase				:      str			  =				tokenizer.convert_ids_to_tokens(UpperCAmelCase_)
     self.assertListEqual(
         UpperCAmelCase_							,  [
             SPIECE_UNDERLINE + '''I''',
             SPIECE_UNDERLINE + '''was''',
             SPIECE_UNDERLINE + '''b''',
             '''or''',
             '''n''',
             SPIECE_UNDERLINE + '''in''',
             SPIECE_UNDERLINE + '''''',
             '''<unk>''',
             '''2''',
             '''0''',
             '''0''',
             '''0''',
             ''',''',
             SPIECE_UNDERLINE + '''and''',
             SPIECE_UNDERLINE + '''this''',
             SPIECE_UNDERLINE + '''is''',
             SPIECE_UNDERLINE + '''f''',
             '''al''',
             '''s''',
             '''<unk>''',
             '''.''',
         ]							,  )


 @cached_property
 def  snake_case__				(	self) ->		Any:

     """simple docstring"""



     return BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base''')


 @slow
 def  snake_case__				(	self) ->		List[Any]:

     """simple docstring"""



     _UpperCAmelCase				:      Dict			  =				'''Hello World!'''
     _UpperCAmelCase				:      Dict			  =				[65, 18536, 2260, 101, 66]

     self.assertListEqual(UpperCAmelCase_							,  self.big_tokenizer.encode(UpperCAmelCase_))


 @slow
 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     _UpperCAmelCase				:      str			  =				(
         '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
         ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
     )
     # fmt: off
     _UpperCAmelCase				:      Any			  =				[65, 871, 419, 358, 946, 991, 2521, 452, 358, 1357, 387, 7751, 3536, 112, 985, 456, 126, 865, 938, 5400, 5734, 458, 1368, 467, 786, 2462, 5246, 1159, 633, 865, 4519, 457, 582, 852, 2557, 427, 916, 508, 405, 34324, 497, 391, 408, 11342, 1244, 385, 100, 938, 985, 456, 574, 362, 12597, 3200, 3129, 1172, 66]  # noqa: E231
     # fmt: on
     self.assertListEqual(UpperCAmelCase_							,  self.big_tokenizer.encode(UpperCAmelCase_))


 @require_torch
 @slow
 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     import torch

     from transformers import BigBirdConfig, BigBirdModel

     # Build sequence
     _UpperCAmelCase				:      Union[str, Any]			  =				list(self.big_tokenizer.get_vocab().keys())[:10]
     _UpperCAmelCase				:      Optional[Any]			  =				''' '''.join(UpperCAmelCase_)
     _UpperCAmelCase				:      Tuple			  =				self.big_tokenizer.encode_plus(UpperCAmelCase_							,  return_tensors='''pt'''							,  return_token_type_ids=UpperCAmelCase_)
     _UpperCAmelCase				:      Optional[Any]			  =				self.big_tokenizer.batch_encode_plus(
         [sequence + ''' ''' + sequence]							,  return_tensors='''pt'''							,  return_token_type_ids=UpperCAmelCase_)

     _UpperCAmelCase				:      Optional[Any]			  =				BigBirdConfig(attention_type='''original_full''')
     _UpperCAmelCase				:      Any			  =				BigBirdModel(UpperCAmelCase_)

     assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

     with torch.no_grad():
         model(**UpperCAmelCase_)
         model(**UpperCAmelCase_)


 @slow
 def  snake_case__				(	self) ->		str:

     """simple docstring"""



     _UpperCAmelCase				:      int			  =				BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base''')
     _UpperCAmelCase				:      Optional[Any]			  =				tokenizer.decode(tokenizer('''Paris is the [MASK].''').input_ids)

     self.assertTrue(decoded_text == '''[CLS] Paris is the[MASK].[SEP]''')


 @slow
 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     _UpperCAmelCase				:      List[str]			  =				{'''input_ids''': [[65, 39286, 458, 36335, 2001, 456, 13073, 13266, 455, 113, 7746, 1741, 11157, 391, 13073, 13266, 455, 113, 3967, 35412, 113, 4936, 109, 3870, 2377, 113, 30084, 45720, 458, 134, 17496, 112, 503, 11672, 113, 118, 112, 5665, 13347, 38687, 112, 1496, 31389, 112, 3268, 47264, 134, 962, 112, 16377, 8035, 23130, 430, 12169, 15518, 28592, 458, 146, 41697, 109, 391, 12169, 15518, 16689, 458, 146, 41358, 109, 452, 726, 4034, 111, 763, 35412, 5082, 388, 1903, 111, 9051, 391, 2870, 48918, 1900, 1123, 550, 998, 112, 9586, 15985, 455, 391, 410, 22955, 37636, 114, 66], [65, 448, 17496, 419, 3663, 385, 763, 113, 27533, 2870, 3283, 13043, 1639, 24713, 523, 656, 24013, 18550, 2521, 517, 27014, 21244, 420, 1212, 1465, 391, 927, 4833, 388, 578, 11786, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 484, 2169, 7687, 21932, 18146, 726, 363, 17032, 3391, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=UpperCAmelCase_							,  model_name='''google/bigbird-roberta-base'''							,  revision='''215c99f1600e06f83acce68422f2035b2b5c3510'''							,  )












import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 							A_    (						__lowercase ,   unittest.TestCase			):






 '''simple docstring'''




 _SCREAMING_SNAKE_CASE							:			int =					None
 _SCREAMING_SNAKE_CASE							:			str =					BloomTokenizerFast
 _SCREAMING_SNAKE_CASE							:			str =					BloomTokenizerFast
 _SCREAMING_SNAKE_CASE							:			Any =					True
 _SCREAMING_SNAKE_CASE							:			Optional[int] =					False
 _SCREAMING_SNAKE_CASE							:			Optional[int] =					"tokenizer_file"
 _SCREAMING_SNAKE_CASE							:			Optional[Any] =					{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}


 def  snake_case__				(	self) ->		Dict:

     """simple docstring"""



     super().setUp()
     _UpperCAmelCase				:      List[Any]			  =				BloomTokenizerFast.from_pretrained('''bigscience/tokenizer''')
     tokenizer.save_pretrained(self.tmpdirname)


 def  snake_case__				(	self							,  **_A) ->		List[Any]:

     """simple docstring"""



     kwargs.update(self.special_tokens_map)
     return BloomTokenizerFast.from_pretrained(self.tmpdirname							,  **_A)


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				self.get_rust_tokenizer()

     _UpperCAmelCase				:      int			  =				['''The quick brown fox</s>''', '''jumps over the lazy dog</s>''']
     _UpperCAmelCase				:      List[Any]			  =				[[2175, 23714, 73173, 144252, 2], [77, 132619, 3478, 368, 109586, 35433, 2]]

     _UpperCAmelCase				:      Any			  =				tokenizer.batch_encode_plus(_A)['''input_ids''']
     self.assertListEqual(_A							,  _A)

     _UpperCAmelCase				:      Any			  =				tokenizer.batch_decode(_A)
     self.assertListEqual(_A							,  _A)


 def  snake_case__				(	self							,  _A=6) ->		Optional[Any]:

     """simple docstring"""



     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
             _UpperCAmelCase				:      Optional[int]			  =				self.rust_tokenizer_class.from_pretrained(_A							,  **_A)
             # tokenizer_r.pad_token = None # Hotfixing padding = None
             # Simple input
             _UpperCAmelCase				:      List[Any]			  =				'''This is a simple input'''
             _UpperCAmelCase				:      Any			  =				['''This is a simple input 1''', '''This is a simple input 2''']
             _UpperCAmelCase				:      int			  =				('''This is a simple input''', '''This is a pair''')
             _UpperCAmelCase				:      Optional[int]			  =				[
                 ('''This is a simple input 1''', '''This is a simple input 2'''),
                 ('''This is a simple pair 1''', '''This is a simple pair 2'''),
             ]

             # Simple input tests
             try:
                 tokenizer_r.encode(_A							,  max_length=_A)
                 tokenizer_r.encode_plus(_A							,  max_length=_A)

                 tokenizer_r.batch_encode_plus(_A							,  max_length=_A)
                 tokenizer_r.encode(_A							,  max_length=_A)
                 tokenizer_r.batch_encode_plus(_A							,  max_length=_A)
             except ValueError:
                 self.fail('''Bloom Tokenizer should be able to deal with padding''')

             _UpperCAmelCase				:      Tuple			  =				None  # Hotfixing padding = None
             self.assertRaises(_A							,  tokenizer_r.encode							,  _A							,  max_length=_A							,  padding='''max_length''')

             # Simple input
             self.assertRaises(_A							,  tokenizer_r.encode_plus							,  _A							,  max_length=_A							,  padding='''max_length''')

             # Simple input
             self.assertRaises(
                 _A							,  tokenizer_r.batch_encode_plus							,  _A							,  max_length=_A							,  padding='''max_length'''							,  )

             # Pair input
             self.assertRaises(_A							,  tokenizer_r.encode							,  _A							,  max_length=_A							,  padding='''max_length''')

             # Pair input
             self.assertRaises(_A							,  tokenizer_r.encode_plus							,  _A							,  max_length=_A							,  padding='''max_length''')

             # Pair input
             self.assertRaises(
                 _A							,  tokenizer_r.batch_encode_plus							,  _A							,  max_length=_A							,  padding='''max_length'''							,  )


 def  snake_case__				(	self) ->		Any:

     """simple docstring"""



     _UpperCAmelCase				:      Dict			  =				self.get_rust_tokenizer()
     _UpperCAmelCase				:      int			  =				load_dataset('''xnli'''							,  '''all_languages'''							,  split='''test'''							,  streaming=_A)

     _UpperCAmelCase				:      Tuple			  =				next(iter(_A))['''premise''']  # pick up one data
     _UpperCAmelCase				:      List[Any]			  =				list(sample_data.values())

     _UpperCAmelCase				:      Any			  =				list(map(tokenizer.encode							,  _A))
     _UpperCAmelCase				:      List[str]			  =				[tokenizer.decode(_A							,  clean_up_tokenization_spaces=_A) for x in output_tokens]
     self.assertListEqual(_A							,  _A)


 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map)							,  1)
     self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0])							,  1)




