





"""simple docstring"""



import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
       a_					=     """pt"""
elif is_tf_available():
       a_					=     """tf"""
else:
       a_					=     """jax"""






class 						__snake_case   (						SCREAMING_SNAKE_CASE__				,				unittest.TestCase	):





 """simple docstring"""


 _lowerCamelCase								=	PerceiverTokenizer
 _lowerCamelCase								=	False

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     super().setUp()
     __A       :		int							=	PerceiverTokenizer()
     tokenizer.save_pretrained(self.tmpdirname    )

 @cached_property
 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     return PerceiverTokenizer.from_pretrained('''deepmind/language-perceiver'''    )

 def    UpperCamelCase__(       self   ,		**__lowerCamelCase    ):



     '''simple docstring'''




     return self.tokenizer_class.from_pretrained(self.tmpdirname   ,		**__lowerCamelCase    )

 def    UpperCamelCase__(       self   ,		__lowerCamelCase   ,		__lowerCamelCase=False   ,		__lowerCamelCase=20   ,		__lowerCamelCase=5    ):



     '''simple docstring'''




     __A       :		List[str]							=	[]
     for i in range(len(__lowerCamelCase    )    ):
         try:
             __A       :		Optional[Any]							=	tokenizer.decode([i]   ,		clean_up_tokenization_spaces=__lowerCamelCase    )
         except UnicodeDecodeError:
             pass
         toks.append((i, tok)    )

     __A       :		Dict							=	list(filter(lambda __lowerCamelCase    : re.match(r'''^[ a-zA-Z]+$'''   ,		t[1]    )   ,		__lowerCamelCase    )    )
     __A       :		Optional[int]							=	list(filter(lambda __lowerCamelCase    : [t[0]] == tokenizer.encode(t[1]   ,		add_special_tokens=__lowerCamelCase    )   ,		__lowerCamelCase    )    )
     if max_length is not None and len(__lowerCamelCase    ) > max_length:
         __A       :		Any							=	toks[:max_length]
     if min_length is not None and len(__lowerCamelCase    ) < min_length and len(__lowerCamelCase    ) > 0:
         while len(__lowerCamelCase    ) < min_length:
             __A       :		Dict							=	toks + toks
        # toks_str = [t[1] for t in toks]
     __A       :		List[Any]							=	[t[0] for t in toks]

     # Ensure consistency
     __A       :		Optional[int]							=	tokenizer.decode(__lowerCamelCase   ,		clean_up_tokenization_spaces=__lowerCamelCase    )
     if " " not in output_txt and len(__lowerCamelCase    ) > 1:
         __A       :		List[str]							=	(
             tokenizer.decode([toks_ids[0]]   ,		clean_up_tokenization_spaces=__lowerCamelCase    )
             + ''' '''
             + tokenizer.decode(toks_ids[1:]   ,		clean_up_tokenization_spaces=__lowerCamelCase    )
         )
     if with_prefix_space:
         __A       :		int							=	''' ''' + output_txt
     __A       :		Optional[int]							=	tokenizer.encode(__lowerCamelCase   ,		add_special_tokens=__lowerCamelCase    )
     return output_txt, output_ids

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		List[str]							=	self.perceiver_tokenizer
     __A       :		List[str]							=	'''Unicode €.'''
     __A       :		str							=	tokenizer(__lowerCamelCase    )
     __A       :		Tuple							=	[4, 91, 116, 111, 105, 117, 106, 107, 38, 232, 136, 178, 52, 5]
     self.assertEqual(encoded['''input_ids''']   ,		__lowerCamelCase    )

     # decoding
     __A       :		Tuple							=	tokenizer.decode(__lowerCamelCase    )
     self.assertEqual(__lowerCamelCase   ,		'''[CLS]Unicode €.[SEP]'''    )

     __A       :		Union[str, Any]							=	tokenizer('''e è é ê ë'''    )
     __A       :		Union[str, Any]							=	[4, 107, 38, 201, 174, 38, 201, 175, 38, 201, 176, 38, 201, 177, 5]
     self.assertEqual(encoded['''input_ids''']   ,		__lowerCamelCase    )
     # decoding
     __A       :		Tuple							=	tokenizer.decode(__lowerCamelCase    )
     self.assertEqual(__lowerCamelCase   ,		'''[CLS]e è é ê ë[SEP]'''    )

     # encode/decode, but with `encode` instead of `__call__`
     self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''    )    )   ,		'''[CLS]e è é ê ë[SEP]'''    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Optional[int]							=	self.perceiver_tokenizer
     __A       :		int							=	['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
     # fmt: off
     __A       :		Any							=	[4, 71, 38, 114, 117, 116, 109, 38, 118, 103, 120, 103, 109, 120, 103, 118, 110, 38, 108, 117, 120, 38, 121, 123, 115, 115, 103, 120, 111, 128, 103, 122, 111, 117, 116, 52, 5, 0]
     # fmt: on
     __A       :		List[str]							=	tokenizer(__lowerCamelCase   ,		padding=__lowerCamelCase   ,		return_tensors=__lowerCamelCase    )
     self.assertIsInstance(__lowerCamelCase   ,		__lowerCamelCase    )

     if FRAMEWORK != "jax":
         __A       :		List[Any]							=	list(batch.input_ids.numpy()[0]    )
     else:
         __A       :		Any							=	list(batch.input_ids.tolist()[0]    )

     self.assertListEqual(__lowerCamelCase   ,		__lowerCamelCase    )

     self.assertEqual((2, 38)   ,		batch.input_ids.shape    )
     self.assertEqual((2, 38)   ,		batch.attention_mask.shape    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		List[str]							=	self.perceiver_tokenizer
     __A       :		str							=	['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
     __A       :		Optional[Any]							=	tokenizer(__lowerCamelCase   ,		padding=__lowerCamelCase   ,		return_tensors=__lowerCamelCase    )
     # check if input_ids are returned and no decoder_input_ids
     self.assertIn('''input_ids'''   ,		__lowerCamelCase    )
     self.assertIn('''attention_mask'''   ,		__lowerCamelCase    )
     self.assertNotIn('''decoder_input_ids'''   ,		__lowerCamelCase    )
     self.assertNotIn('''decoder_attention_mask'''   ,		__lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Optional[Any]							=	self.perceiver_tokenizer
     __A       :		Tuple							=	[
         '''Summary of the text.''',
         '''Another summary.''',
     ]
     __A       :		Optional[Any]							=	tokenizer(
         text_target=__lowerCamelCase   ,		max_length=32   ,		padding='''max_length'''   ,		truncation=__lowerCamelCase   ,		return_tensors=__lowerCamelCase    )
     self.assertEqual(32   ,		targets['''input_ids'''].shape[1]    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Any							=	self.get_tokenizers()
     for tokenizer in tokenizers:
         with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
             self.assertNotEqual(tokenizer.model_max_length   ,		42    )

        # Now let's start the test
     __A       :		Optional[int]							=	self.get_tokenizers()
     for tokenizer in tokenizers:
         with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
             # Isolate this from the other tests because we save additional tokens/etc
             __A       :		Tuple							=	tempfile.mkdtemp()

             __A       :		List[str]							=	''' He is very happy, UNwant\u00E9d,running'''
             __A       :		int							=	tokenizer.encode(__lowerCamelCase   ,		add_special_tokens=__lowerCamelCase    )
             tokenizer.save_pretrained(__lowerCamelCase    )

             __A       :		Optional[Any]							=	tokenizer.__class__.from_pretrained(__lowerCamelCase    )
             __A       :		Optional[int]							=	after_tokenizer.encode(__lowerCamelCase   ,		add_special_tokens=__lowerCamelCase    )
             self.assertListEqual(__lowerCamelCase   ,		__lowerCamelCase    )

             shutil.rmtree(__lowerCamelCase    )

     __A       :		Union[str, Any]							=	self.get_tokenizers(model_max_length=42    )
     for tokenizer in tokenizers:
         with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
             # Isolate this from the other tests because we save additional tokens/etc
             __A       :		Optional[int]							=	tempfile.mkdtemp()

             __A       :		Any							=	''' He is very happy, UNwant\u00E9d,running'''
             tokenizer.add_tokens(['''bim''', '''bambam''']    )
             __A       :		Dict							=	tokenizer.additional_special_tokens
             additional_special_tokens.append('''new_additional_special_token'''    )
             tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}    )
             __A       :		List[Any]							=	tokenizer.encode(__lowerCamelCase   ,		add_special_tokens=__lowerCamelCase    )
             tokenizer.save_pretrained(__lowerCamelCase    )

             __A       :		List[str]							=	tokenizer.__class__.from_pretrained(__lowerCamelCase    )
             __A       :		Dict							=	after_tokenizer.encode(__lowerCamelCase   ,		add_special_tokens=__lowerCamelCase    )
             self.assertListEqual(__lowerCamelCase   ,		__lowerCamelCase    )
             self.assertIn('''new_additional_special_token'''   ,		after_tokenizer.additional_special_tokens    )
             self.assertEqual(after_tokenizer.model_max_length   ,		42    )

             __A       :		Any							=	tokenizer.__class__.from_pretrained(__lowerCamelCase   ,		model_max_length=43    )
             self.assertEqual(tokenizer.model_max_length   ,		43    )

             shutil.rmtree(__lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		List[Any]							=	[]
     if self.test_slow_tokenizer:
         tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())    )

     if self.test_rust_tokenizer:
         tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())    )

     for tokenizer_class, tokenizer_utils in tokenizer_list:
         with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer_utils.save_pretrained(__lowerCamelCase    )

             with open(os.path.join(__lowerCamelCase   ,		'''special_tokens_map.json'''    )   ,		encoding='''utf-8'''    ) as json_file:
                 __A       :		Optional[Any]							=	json.load(__lowerCamelCase    )

             with open(os.path.join(__lowerCamelCase   ,		'''tokenizer_config.json'''    )   ,		encoding='''utf-8'''    ) as json_file:
                 __A       :		List[str]							=	json.load(__lowerCamelCase    )

             __A       :		Tuple							=	[F"""<extra_id_{i}>""" for i in range(125    )]

             __A       :		List[Any]							=	added_tokens_extra_ids + [
                 '''an_additional_special_token'''
             ]
             __A       :		List[Any]							=	added_tokens_extra_ids + [
                 '''an_additional_special_token'''
             ]

             with open(os.path.join(__lowerCamelCase   ,		'''special_tokens_map.json'''    )   ,		'''w'''   ,		encoding='''utf-8'''    ) as outfile:
                 json.dump(__lowerCamelCase   ,		__lowerCamelCase    )
             with open(os.path.join(__lowerCamelCase   ,		'''tokenizer_config.json'''    )   ,		'''w'''   ,		encoding='''utf-8'''    ) as outfile:
                 json.dump(__lowerCamelCase   ,		__lowerCamelCase    )

             # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
             # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
             # "special_tokens_map.json" files
             __A       :		Tuple							=	tokenizer_class.from_pretrained(
                 __lowerCamelCase   ,		)
             self.assertIn(
                 '''an_additional_special_token'''   ,		tokenizer_without_change_in_init.additional_special_tokens    )
             self.assertEqual(
                 ['''an_additional_special_token''']   ,		tokenizer_without_change_in_init.convert_ids_to_tokens(
                     tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']    )    )   ,		)

             # Now we test that we can change the value of additional_special_tokens in the from_pretrained
             __A       :		Tuple							=	added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''   ,		lstrip=__lowerCamelCase    )]
             __A       :		Tuple							=	tokenizer_class.from_pretrained(
                 __lowerCamelCase   ,		additional_special_tokens=__lowerCamelCase   ,		)

             self.assertIn('''a_new_additional_special_token'''   ,		tokenizer.additional_special_tokens    )
             self.assertEqual(
                 ['''a_new_additional_special_token''']   ,		tokenizer.convert_ids_to_tokens(
                     tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']    )    )   ,		)

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Tuple							=	self.perceiver_tokenizer
     self.assertEqual(tokenizer.decode([178]    )   ,		'''�'''    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     pass

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     pass

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     pass

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     pass







 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		str							=	self.get_tokenizers(fast=__lowerCamelCase   ,		do_lower_case=__lowerCamelCase    )
     for tokenizer in tokenizers:
         with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
             __A       :		List[Any]							=	['''[CLS]''', '''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''s''', '''t''', '''[SEP]''']
             __A       :		Any							=	tokenizer.convert_tokens_to_string(__lowerCamelCase    )

             self.assertIsInstance(__lowerCamelCase   ,		__lowerCamelCase    )











"""simple docstring"""



import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a_					=     get_tests_dir("""fixtures/spiece.model""")






@require_sentencepiece
@require_tokenizers
class 						__snake_case   (						SCREAMING_SNAKE_CASE__				,				unittest.TestCase	):





 """simple docstring"""


 _lowerCamelCase								=	AlbertTokenizer
 _lowerCamelCase								=	AlbertTokenizerFast
 _lowerCamelCase								=	True
 _lowerCamelCase								=	True
 _lowerCamelCase								=	True

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     super().setUp()

     # We have a SentencePiece fixture for testing
     __A       :		str							=	AlbertTokenizer(__lowerCamelCase    )
     tokenizer.save_pretrained(self.tmpdirname    )

 def    UpperCamelCase__(       self   ,		__lowerCamelCase    ):



     '''simple docstring'''




     __A       :		Tuple							=	'''this is a test'''
     __A       :		Union[str, Any]							=	'''this is a test'''
     return input_text, output_text

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		int							=	'''<pad>'''
     __A       :		Optional[int]							=	0

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase    )   ,		__lowerCamelCase    )
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase    )   ,		__lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		str							=	list(self.get_tokenizer().get_vocab().keys()    )

     self.assertEqual(vocab_keys[0]   ,		'''<pad>'''    )
     self.assertEqual(vocab_keys[1]   ,		'''<unk>'''    )
     self.assertEqual(vocab_keys[-1]   ,		'''▁eloquent'''    )
     self.assertEqual(len(__lowerCamelCase    )   ,		3_0000    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     self.assertEqual(self.get_tokenizer().vocab_size   ,		3_0000    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     if not self.test_rust_tokenizer:
         return

     __A       :		Dict							=	self.get_tokenizer()
     __A       :		List[Any]							=	self.get_rust_tokenizer()

     __A       :		int							=	'''I was born in 92000, and this is falsé.'''

     __A       :		Any							=	tokenizer.tokenize(__lowerCamelCase    )
     __A       :		Dict							=	rust_tokenizer.tokenize(__lowerCamelCase    )
     self.assertListEqual(__lowerCamelCase   ,		__lowerCamelCase    )

     __A       :		List[Any]							=	tokenizer.encode(__lowerCamelCase   ,		add_special_tokens=__lowerCamelCase    )
     __A       :		List[str]							=	rust_tokenizer.encode(__lowerCamelCase   ,		add_special_tokens=__lowerCamelCase    )
     self.assertListEqual(__lowerCamelCase   ,		__lowerCamelCase    )

     __A       :		List[Any]							=	self.get_rust_tokenizer()
     __A       :		Union[str, Any]							=	tokenizer.encode(__lowerCamelCase    )
     __A       :		List[Any]							=	rust_tokenizer.encode(__lowerCamelCase    )
     self.assertListEqual(__lowerCamelCase   ,		__lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		List[str]							=	AlbertTokenizer(__lowerCamelCase   ,		keep_accents=__lowerCamelCase    )

     __A       :		Optional[int]							=	tokenizer.tokenize('''This is a test'''    )
     self.assertListEqual(__lowerCamelCase   ,		['''▁this''', '''▁is''', '''▁a''', '''▁test''']    )

     self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase    )   ,		[48, 25, 21, 1289]    )

     __A       :		Tuple							=	tokenizer.tokenize('''I was born in 92000, and this is falsé.'''    )
     self.assertListEqual(
         __lowerCamelCase   ,		['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''']    )
     __A       :		Optional[Any]							=	tokenizer.convert_tokens_to_ids(__lowerCamelCase    )
     self.assertListEqual(__lowerCamelCase   ,		[31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]    )

     __A       :		Optional[Any]							=	tokenizer.convert_ids_to_tokens(__lowerCamelCase    )
     self.assertListEqual(
         __lowerCamelCase   ,		['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''']   ,		)

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Optional[Any]							=	AlbertTokenizer(__lowerCamelCase    )

     __A       :		List[Any]							=	tokenizer.encode('''sequence builders'''    )
     __A       :		str							=	tokenizer.encode('''multi-sequence build'''    )

     __A       :		Union[str, Any]							=	tokenizer.build_inputs_with_special_tokens(__lowerCamelCase    )
     __A       :		Optional[int]							=	tokenizer.build_inputs_with_special_tokens(__lowerCamelCase   ,		__lowerCamelCase    )

     assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
     assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
         tokenizer.sep_token_id
     ]







 @slow
 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Dict							=	{'''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''input_ids''': [[2, 2_1970, 13, 5, 6092, 167, 28, 7103, 2153, 673, 8, 7028, 1_2051, 18, 17, 7103, 2153, 673, 8, 3515, 1_8684, 8, 4461, 6, 1927, 297, 8, 1_2060, 2607, 18, 13, 5, 4461, 15, 1_0538, 38, 8, 135, 15, 822, 58, 15, 993, 1_0363, 15, 1460, 8005, 4461, 15, 993, 255, 2328, 9, 9, 9, 6, 26, 1112, 816, 3260, 13, 5, 103, 2377, 6, 17, 1112, 816, 2782, 13, 5, 103, 1_0641, 6, 29, 84, 2512, 2430, 782, 1_8684, 2761, 19, 808, 2430, 2556, 17, 855, 1480, 9477, 4091, 128, 1_1712, 15, 7103, 2153, 673, 17, 2_4883, 9990, 9, 3], [2, 1_1502, 25, 1006, 20, 782, 8, 1_1809, 855, 1732, 1_9393, 1_8667, 37, 367, 2_1018, 69, 1854, 34, 1_1860, 1_9124, 27, 156, 225, 17, 193, 4141, 19, 65, 9124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2231, 886, 2385, 1_7659, 84, 14, 1_6792, 1952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=__lowerCamelCase   ,		model_name='''albert-base-v2'''   ,		revision='''6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'''   ,		)




