




'''simple docstring'''


import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_snake_case   :   Optional[Any]     							=	'▁'

_snake_case   :   int     							=	get_tests_dir('fixtures/test_sentencepiece.model')

@require_sentencepiece
@require_tokenizers
class     A					(	_a  ,unittest.TestCase					):
							lowercase_           =       BigBirdTokenizer
							lowercase_           =       BigBirdTokenizerFast
							lowercase_           =       True
							lowercase_           =       True







							def      __lowerCAmelCase    (						self :						int      )    ->							Any:


														"""simple docstring"""


														super().setUp()

														_a			  =       self.tokenizer_class(lowerCAmelCase_							,    keep_accents=lowerCAmelCase_      )
														tokenizer.save_pretrained(self.tmpdirname      )







							def      __lowerCAmelCase    (						self :						List[Any]      )    ->							List[str]:


														"""simple docstring"""


														_a			  =       '''<s>'''
														_a			  =       1

														self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase_      )							,    lowerCAmelCase_      )
														self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase_      )							,    lowerCAmelCase_      )







							def      __lowerCAmelCase    (						self :						Dict      )    ->							int:


														"""simple docstring"""


														_a			  =       list(self.get_tokenizer().get_vocab().keys()      )

														self.assertEqual(vocab_keys[0]							,    '''<unk>'''      )
														self.assertEqual(vocab_keys[1]							,    '''<s>'''      )
														self.assertEqual(vocab_keys[-1]							,    '''[MASK]'''      )
														self.assertEqual(len(lowerCAmelCase_      )							,    10_04      )







							def      __lowerCAmelCase    (						self :						List[str]      )    ->							str:


														"""simple docstring"""


														self.assertEqual(self.get_tokenizer().vocab_size							,    10_00      )







							def      __lowerCAmelCase    (						self :						List[str]      )    ->							Any:


														"""simple docstring"""


														if not self.test_rust_tokenizer:
																					return

														_a			  =       self.get_tokenizer()
														_a			  =       self.get_rust_tokenizer()

														_a			  =       '''I was born in 92000, and this is falsé.'''

														_a			  =       tokenizer.tokenize(lowerCAmelCase_      )
														_a			  =       rust_tokenizer.tokenize(lowerCAmelCase_      )
														self.assertListEqual(lowerCAmelCase_							,    lowerCAmelCase_      )

														_a			  =       tokenizer.encode(lowerCAmelCase_							,    add_special_tokens=lowerCAmelCase_      )
														_a			  =       rust_tokenizer.encode(lowerCAmelCase_							,    add_special_tokens=lowerCAmelCase_      )
														self.assertListEqual(lowerCAmelCase_							,    lowerCAmelCase_      )

														_a			  =       self.get_rust_tokenizer()
														_a			  =       tokenizer.encode(lowerCAmelCase_      )
														_a			  =       rust_tokenizer.encode(lowerCAmelCase_      )
														self.assertListEqual(lowerCAmelCase_							,    lowerCAmelCase_      )







							def      __lowerCAmelCase    (						self :						Optional[Any]      )    ->							int:


														"""simple docstring"""


														_a			  =       BigBirdTokenizer(lowerCAmelCase_							,    keep_accents=lowerCAmelCase_      )

														_a			  =       tokenizer.tokenize('''This is a test'''      )
														self.assertListEqual(lowerCAmelCase_							,    ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

														self.assertListEqual(
														    tokenizer.convert_tokens_to_ids(lowerCAmelCase_      )							,    [2_85, 46, 10, 1_70, 3_82]							,    )

														_a			  =       tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
														self.assertListEqual(
														    lowerCAmelCase_							,    [
														        SPIECE_UNDERLINE + '''I''',
														        SPIECE_UNDERLINE + '''was''',
														        SPIECE_UNDERLINE + '''b''',
														        '''or''',
														        '''n''',
														        SPIECE_UNDERLINE + '''in''',
														        SPIECE_UNDERLINE + '''''',
														        '''9''',
														        '''2''',
														        '''0''',
														        '''0''',
														        '''0''',
														        ''',''',
														        SPIECE_UNDERLINE + '''and''',
														        SPIECE_UNDERLINE + '''this''',
														        SPIECE_UNDERLINE + '''is''',
														        SPIECE_UNDERLINE + '''f''',
														        '''al''',
														        '''s''',
														        '''é''',
														        '''.''',
														    ]							,    )
														_a			  =       tokenizer.convert_tokens_to_ids(lowerCAmelCase_      )
														self.assertListEqual(
														    lowerCAmelCase_							,    [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]							,    )

														_a			  =       tokenizer.convert_ids_to_tokens(lowerCAmelCase_      )
														self.assertListEqual(
														    lowerCAmelCase_							,    [
														        SPIECE_UNDERLINE + '''I''',
														        SPIECE_UNDERLINE + '''was''',
														        SPIECE_UNDERLINE + '''b''',
														        '''or''',
														        '''n''',
														        SPIECE_UNDERLINE + '''in''',
														        SPIECE_UNDERLINE + '''''',
														        '''<unk>''',
														        '''2''',
														        '''0''',
														        '''0''',
														        '''0''',
														        ''',''',
														        SPIECE_UNDERLINE + '''and''',
														        SPIECE_UNDERLINE + '''this''',
														        SPIECE_UNDERLINE + '''is''',
														        SPIECE_UNDERLINE + '''f''',
														        '''al''',
														        '''s''',
														        '''<unk>''',
														        '''.''',
														    ]							,    )







							@cached_property
							def      __lowerCAmelCase    (						self :						List[str]      )    ->							int:


														"""simple docstring"""


														return BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base'''      )







							@slow
							def      __lowerCAmelCase    (						self :						List[Any]      )    ->							str:


														"""simple docstring"""


														_a			  =       '''Hello World!'''
														_a			  =       [65, 1_85_36, 22_60, 1_01, 66]

														self.assertListEqual(lowerCAmelCase_							,    self.big_tokenizer.encode(lowerCAmelCase_      )      )







							@slow
							def      __lowerCAmelCase    (						self :						str      )    ->							Dict:


														"""simple docstring"""


														_a			  =       (
														    '''This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'''
														    ''' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'''
														)
														# fmt: off
														_a			  =       [65, 8_71, 4_19, 3_58, 9_46, 9_91, 25_21, 4_52, 3_58, 13_57, 3_87, 77_51, 35_36, 1_12, 9_85, 4_56, 1_26, 8_65, 9_38, 54_00, 57_34, 4_58, 13_68, 4_67, 7_86, 24_62, 52_46, 11_59, 6_33, 8_65, 45_19, 4_57, 5_82, 8_52, 25_57, 4_27, 9_16, 5_08, 4_05, 3_43_24, 4_97, 3_91, 4_08, 1_13_42, 12_44, 3_85, 1_00, 9_38, 9_85, 4_56, 5_74, 3_62, 1_25_97, 32_00, 31_29, 11_72, 66]  # noqa: E231
														# fmt: on
														self.assertListEqual(lowerCAmelCase_							,    self.big_tokenizer.encode(lowerCAmelCase_      )      )







							@require_torch
							@slow
							def      __lowerCAmelCase    (						self :						str      )    ->							Union[str, Any]:


														"""simple docstring"""


														import torch

														from transformers import BigBirdConfig, BigBirdModel

														# Build sequence
														_a			  =       list(self.big_tokenizer.get_vocab().keys()      )[:10]
														_a			  =       ''' '''.join(lowerCAmelCase_      )
														_a			  =       self.big_tokenizer.encode_plus(lowerCAmelCase_							,    return_tensors='''pt'''							,    return_token_type_ids=lowerCAmelCase_      )
														_a			  =       self.big_tokenizer.batch_encode_plus(
														    [sequence + ''' ''' + sequence]							,    return_tensors='''pt'''							,    return_token_type_ids=lowerCAmelCase_      )

														_a			  =       BigBirdConfig(attention_type='''original_full'''      )
														_a			  =       BigBirdModel(lowerCAmelCase_      )

														assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

														with torch.no_grad():
																					model(**lowerCAmelCase_      )
																					model(**lowerCAmelCase_      )







							@slow
							def      __lowerCAmelCase    (						self :						Any      )    ->							Optional[int]:


														"""simple docstring"""


														_a			  =       BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base'''      )
														_a			  =       tokenizer.decode(tokenizer('''Paris is the [MASK].'''      ).input_ids      )

														self.assertTrue(decoded_text == '''[CLS] Paris is the[MASK].[SEP]'''      )







							@slow
							def      __lowerCAmelCase    (						self :						Any      )    ->							Optional[Any]:


														"""simple docstring"""


														_a			  =       {'''input_ids''': [[65, 3_92_86, 4_58, 3_63_35, 20_01, 4_56, 1_30_73, 1_32_66, 4_55, 1_13, 77_46, 17_41, 1_11_57, 3_91, 1_30_73, 1_32_66, 4_55, 1_13, 39_67, 3_54_12, 1_13, 49_36, 1_09, 38_70, 23_77, 1_13, 3_00_84, 4_57_20, 4_58, 1_34, 1_74_96, 1_12, 5_03, 1_16_72, 1_13, 1_18, 1_12, 56_65, 1_33_47, 3_86_87, 1_12, 14_96, 3_13_89, 1_12, 32_68, 4_72_64, 1_34, 9_62, 1_12, 1_63_77, 80_35, 2_31_30, 4_30, 1_21_69, 1_55_18, 2_85_92, 4_58, 1_46, 4_16_97, 1_09, 3_91, 1_21_69, 1_55_18, 1_66_89, 4_58, 1_46, 4_13_58, 1_09, 4_52, 7_26, 40_34, 1_11, 7_63, 3_54_12, 50_82, 3_88, 19_03, 1_11, 90_51, 3_91, 28_70, 4_89_18, 19_00, 11_23, 5_50, 9_98, 1_12, 95_86, 1_59_85, 4_55, 3_91, 4_10, 2_29_55, 3_76_36, 1_14, 66], [65, 4_48, 1_74_96, 4_19, 36_63, 3_85, 7_63, 1_13, 2_75_33, 28_70, 32_83, 1_30_43, 16_39, 2_47_13, 5_23, 6_56, 2_40_13, 1_85_50, 25_21, 5_17, 2_70_14, 2_12_44, 4_20, 12_12, 14_65, 3_91, 9_27, 48_33, 3_88, 5_78, 1_17_86, 1_14, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 4_84, 21_69, 76_87, 2_19_32, 1_81_46, 7_26, 3_63, 1_70_32, 33_91, 1_14, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
														# fmt: on

														self.tokenizer_integration_test_util(
														    expected_encoding=lowerCAmelCase_							,    model_name='''google/bigbird-roberta-base'''							,    revision='''215c99f1600e06f83acce68422f2035b2b5c3510'''							,    )







'''simple docstring'''


_snake_case   :   Any     							=	tuple[float, float, float]
_snake_case   :   Optional[int]     							=	tuple[float, float, float]
def 	snake_case_					(UpperCamelCase :						Pointad				,		UpperCamelCase :						Pointad   ):





							'''simple docstring'''


							_a			  =       end_pointa[0] - end_pointa[0]
							_a			  =       end_pointa[1] - end_pointa[1]
							_a			  =       end_pointa[2] - end_pointa[2]
							return (x, y, z)
def 	snake_case_					(UpperCamelCase :						Vectorad				,		UpperCamelCase :						Vectorad   ):





							'''simple docstring'''


							_a			  =       ab[1] * ac[2] - ab[2] * ac[1]  # *i
							_a			  =       (ab[0] * ac[2] - ab[2] * ac[0]) * -1  # *j
							_a			  =       ab[0] * ac[1] - ab[1] * ac[0]  # *k
							return (x, y, z)
def 	snake_case_					(UpperCamelCase :						Vectorad				,		UpperCamelCase :						int   ):





							'''simple docstring'''


							return tuple(round(UpperCamelCase				,		UpperCamelCase   ) for x in vector   ) == (0, 0, 0)

def 	snake_case_					(UpperCamelCase :						Pointad				,		UpperCamelCase :						Pointad				,		UpperCamelCase :						Pointad				,		UpperCamelCase :						int = 10   ):





							'''simple docstring'''


							_a			  =       create_vector(UpperCamelCase				,		UpperCamelCase   )
							_a			  =       create_vector(UpperCamelCase				,		UpperCamelCase   )
							return is_zero_vector(get_ad_vectors_cross(UpperCamelCase				,		UpperCamelCase   )				,		UpperCamelCase   )

