


import glob
import os
import random
from string import ascii_lowercase, digits

import cva
import numpy as np

# Parrameters
lowerCAmelCase       		=    (720, 1_280)  # Height, Width
lowerCAmelCase       		=    (0.4, 0.6)  # if height or width lower than this scale, drop it.
lowerCAmelCase       		=    1 / 100
lowerCAmelCase       		=    """"""
lowerCAmelCase       		=    """"""
lowerCAmelCase       		=    """"""
lowerCAmelCase       		=    250




def     __SCREAMING_SNAKE_CASE							(       )		->     None:




						'''simple docstring'''







						__UpperCAmelCase    ,      __UpperCAmelCase			:							Dict	    =	get_dataset(lowercase_      ,       lowercase_      )
						for index in range(lowercase_      ):
												__UpperCAmelCase			:							Optional[int]	    =	random.sample(range(len(lowercase_      )      )      ,       4      )
												__UpperCAmelCase    ,      __UpperCAmelCase    ,      __UpperCAmelCase			:							Union[str, Any]	    =	update_image_and_anno(
												    lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_      ,       filter_scale=lowercase_      ,       )

												# Get random string code: '7b7ad245cdff75241935e4dd860f3bad'
												__UpperCAmelCase			:							Optional[Any]	    =	random_chars(32      )
												__UpperCAmelCase			:							List[Any]	    =	path.split(os.sep      )[-1].rsplit('''.'''      ,       1      )[0]
												__UpperCAmelCase			:							Dict	    =	f"{OUTPUT_DIR}/{file_name}_MOSAIC_{letter_code}"
												cva.imwrite(f"{file_root}.jpg"      ,       lowercase_      ,       [cva.IMWRITE_JPEG_QUALITY, 85]      )
												print(f"Succeeded {index+1}/{NUMBER_IMAGES} with {file_name}"      )
												__UpperCAmelCase			:							Dict	    =	[]
												for anno in new_annos:
																		__UpperCAmelCase			:							str	    =	anno[3] - anno[1]
																		__UpperCAmelCase			:							str	    =	anno[4] - anno[2]
																		__UpperCAmelCase			:							Optional[int]	    =	anno[1] + width / 2
																		__UpperCAmelCase			:							int	    =	anno[2] + height / 2
																		__UpperCAmelCase			:							List[Any]	    =	f"{anno[0]} {x_center} {y_center} {width} {height}"
																		annos_list.append(lowercase_      )
												with open(f"{file_root}.txt"      ,       '''w'''      ) as outfile:
																		outfile.write('''\n'''.join(line for line in annos_list      )      )




def     __SCREAMING_SNAKE_CASE							(       lowercase_      ,       lowercase_      )		->     tuple[list, list]:




						'''simple docstring'''







						__UpperCAmelCase			:							str	    =	[]
						__UpperCAmelCase			:							List[str]	    =	[]
						for label_file in glob.glob(os.path.join(lowercase_      ,       '''*.txt'''      )      ):
												__UpperCAmelCase			:							Tuple	    =	label_file.split(os.sep      )[-1].rsplit('''.'''      ,       1      )[0]
												with open(lowercase_      ) as in_file:
																		__UpperCAmelCase			:							str	    =	in_file.readlines()
												__UpperCAmelCase			:							Tuple	    =	os.path.join(lowercase_      ,       f"{label_name}.jpg"      )

												__UpperCAmelCase			:							Optional[int]	    =	[]
												for obj_list in obj_lists:
																		__UpperCAmelCase			:							Dict	    =	obj_list.rstrip('''\n'''      ).split(''' '''      )
																		__UpperCAmelCase			:							Union[str, Any]	    =	float(obj[1]      ) - float(obj[3]      ) / 2
																		__UpperCAmelCase			:							Union[str, Any]	    =	float(obj[2]      ) - float(obj[4]      ) / 2
																		__UpperCAmelCase			:							str	    =	float(obj[1]      ) + float(obj[3]      ) / 2
																		__UpperCAmelCase			:							str	    =	float(obj[2]      ) + float(obj[4]      ) / 2

																		boxes.append([int(obj[0]      ), xmin, ymin, xmax, ymax]      )
												if not boxes:
																		continue
												img_paths.append(lowercase_      )
												labels.append(lowercase_      )
						return img_paths, labels




def     __SCREAMING_SNAKE_CASE							(       lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_ = 0.0      ,       )		->     tuple[list, list, str]:




						'''simple docstring'''







						__UpperCAmelCase			:							Tuple	    =	np.zeros([output_size[0], output_size[1], 3]      ,       dtype=np.uinta      )
						__UpperCAmelCase			:							Union[str, Any]	    =	scale_range[0] + random.random() * (scale_range[1] - scale_range[0])
						__UpperCAmelCase			:							Union[str, Any]	    =	scale_range[0] + random.random() * (scale_range[1] - scale_range[0])
						__UpperCAmelCase			:							List[Any]	    =	int(scale_x * output_size[1]      )
						__UpperCAmelCase			:							Optional[int]	    =	int(scale_y * output_size[0]      )

						__UpperCAmelCase			:							List[str]	    =	[]
						__UpperCAmelCase			:							int	    =	[]
						for i, index in enumerate(lowercase_      ):
												__UpperCAmelCase			:							Optional[Any]	    =	all_img_list[index]
												path_list.append(lowercase_      )
												__UpperCAmelCase			:							Any	    =	all_annos[index]
												__UpperCAmelCase			:							List[str]	    =	cva.imread(lowercase_      )
												if i == 0:  # top-left
																		__UpperCAmelCase			:							Tuple	    =	cva.resize(lowercase_      ,       (divid_point_x, divid_point_y)      )
																		__UpperCAmelCase			:							Tuple	    =	img
																		for bbox in img_annos:
																								__UpperCAmelCase			:							Tuple	    =	bbox[1] * scale_x
																								__UpperCAmelCase			:							Tuple	    =	bbox[2] * scale_y
																								__UpperCAmelCase			:							List[Any]	    =	bbox[3] * scale_x
																								__UpperCAmelCase			:							Union[str, Any]	    =	bbox[4] * scale_y
																								new_anno.append([bbox[0], xmin, ymin, xmax, ymax]      )
												elif i == 1:  # top-right
																		__UpperCAmelCase			:							Union[str, Any]	    =	cva.resize(lowercase_      ,       (output_size[1] - divid_point_x, divid_point_y)      )
																		__UpperCAmelCase			:							List[str]	    =	img
																		for bbox in img_annos:
																								__UpperCAmelCase			:							List[Any]	    =	scale_x + bbox[1] * (1 - scale_x)
																								__UpperCAmelCase			:							Dict	    =	bbox[2] * scale_y
																								__UpperCAmelCase			:							List[Any]	    =	scale_x + bbox[3] * (1 - scale_x)
																								__UpperCAmelCase			:							Union[str, Any]	    =	bbox[4] * scale_y
																								new_anno.append([bbox[0], xmin, ymin, xmax, ymax]      )
												elif i == 2:  # bottom-left
																		__UpperCAmelCase			:							int	    =	cva.resize(lowercase_      ,       (divid_point_x, output_size[0] - divid_point_y)      )
																		__UpperCAmelCase			:							List[Any]	    =	img
																		for bbox in img_annos:
																								__UpperCAmelCase			:							Optional[Any]	    =	bbox[1] * scale_x
																								__UpperCAmelCase			:							str	    =	scale_y + bbox[2] * (1 - scale_y)
																								__UpperCAmelCase			:							Optional[int]	    =	bbox[3] * scale_x
																								__UpperCAmelCase			:							Any	    =	scale_y + bbox[4] * (1 - scale_y)
																								new_anno.append([bbox[0], xmin, ymin, xmax, ymax]      )
												else:  # bottom-right
																		__UpperCAmelCase			:							Dict	    =	cva.resize(
																		    lowercase_      ,       (output_size[1] - divid_point_x, output_size[0] - divid_point_y)      )
																		__UpperCAmelCase			:							Dict	    =	img
																		for bbox in img_annos:
																								__UpperCAmelCase			:							Any	    =	scale_x + bbox[1] * (1 - scale_x)
																								__UpperCAmelCase			:							Optional[int]	    =	scale_y + bbox[2] * (1 - scale_y)
																								__UpperCAmelCase			:							Optional[int]	    =	scale_x + bbox[3] * (1 - scale_x)
																								__UpperCAmelCase			:							Tuple	    =	scale_y + bbox[4] * (1 - scale_y)
																								new_anno.append([bbox[0], xmin, ymin, xmax, ymax]      )

    # Remove bounding box small than scale of filter
						if filter_scale > 0:
												__UpperCAmelCase			:							List[str]	    =	[
												    anno
												    for anno in new_anno
												    if filter_scale < (anno[3] - anno[1]) and filter_scale < (anno[4] - anno[2])
												]

						return output_img, new_anno, path_list[0]




def     __SCREAMING_SNAKE_CASE							(       lowercase_      )		->     str:




						'''simple docstring'''







						assert number_char > 1, "The number of character should greater than 1"
						__UpperCAmelCase			:							int	    =	ascii_lowercase + digits
						return "".join(random.choice(lowercase_      ) for _ in range(lowercase_      )      )


if __name__ == "__main__":
				main()
				print("""DONE ✅""")





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase       		=    get_tests_dir("""fixtures/test_sentencepiece.model""")




@require_sentencepiece
@require_tokenizers
class 							lowerCamelCase   (  _UpperCamelCase							,						unittest.TestCase ):
	_lowerCAmelCase     :		Union[str, Any]										=						XLNetTokenizer
	_lowerCAmelCase     :		int										=						XLNetTokenizerFast
	_lowerCAmelCase     :		Optional[Any]										=						True
	_lowerCAmelCase     :		Optional[Any]										=						True


	def  A(    self):
							super().setUp()

							# We have a SentencePiece fixture for testing
							__UpperCAmelCase			:							Optional[Any]	    =	XLNetTokenizer(lowercase__				,     keep_accents=lowercase__)
							tokenizer.sanitize_special_tokens()
							tokenizer.save_pretrained(self.tmpdirname)


	def  A(    self):
							__UpperCAmelCase			:							Tuple	    =	'''<s>'''
							__UpperCAmelCase			:							List[str]	    =	1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase__)				,     lowercase__)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase__)				,     lowercase__)


	def  A(    self):
							__UpperCAmelCase			:							Union[str, Any]	    =	list(self.get_tokenizer().get_vocab().keys())

							self.assertEqual(vocab_keys[0]				,     '''<unk>''')
							self.assertEqual(vocab_keys[1]				,     '''<s>''')
							self.assertEqual(vocab_keys[-1]				,     '''<eod>''')
							self.assertEqual(len(lowercase__)				,     1_0_0_6)


	def  A(    self):
							self.assertEqual(self.get_tokenizer().vocab_size				,     1_0_0_0)


	def  A(    self):
							__UpperCAmelCase			:							List[str]	    =	XLNetTokenizer(lowercase__				,     keep_accents=lowercase__)

							__UpperCAmelCase			:							Optional[Any]	    =	tokenizer.tokenize('''This is a test''')
							self.assertListEqual(lowercase__				,     ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__)				,     [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2])

							__UpperCAmelCase			:							Tuple	    =	tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
							self.assertListEqual(
							    lowercase__				,     [
							        SPIECE_UNDERLINE + '''I''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''9''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''s''',
							        '''é''',
							        '''.''',
							    ]				,     )
							__UpperCAmelCase			:							Union[str, Any]	    =	tokenizer.convert_tokens_to_ids(lowercase__)
							self.assertListEqual(lowercase__				,     [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4])

							__UpperCAmelCase			:							int	    =	tokenizer.convert_ids_to_tokens(lowercase__)
							self.assertListEqual(
							    lowercase__				,     [
							        SPIECE_UNDERLINE + '''I''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''<unk>''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''s''',
							        '''<unk>''',
							        '''.''',
							    ]				,     )


	def  A(    self):
							__UpperCAmelCase			:							str	    =	XLNetTokenizer(lowercase__				,     do_lower_case=lowercase__)
							__UpperCAmelCase			:							int	    =	tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
							self.assertListEqual(
							    lowercase__				,     [
							        SPIECE_UNDERLINE + '''''',
							        '''i''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''9''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''se''',
							        '''.''',
							    ]				,     )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')				,     ['''▁he''', '''ll''', '''o'''])


	def  A(    self):
							__UpperCAmelCase			:							Optional[Any]	    =	XLNetTokenizer(lowercase__				,     do_lower_case=lowercase__)
							__UpperCAmelCase			:							Union[str, Any]	    =	tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
							self.assertListEqual(
							    lowercase__				,     [
							        SPIECE_UNDERLINE + '''I''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''9''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''se''',
							        '''.''',
							    ]				,     )


	@slow
	def  A(    self):
							__UpperCAmelCase			:							Tuple	    =	XLNetTokenizer.from_pretrained('''xlnet-base-cased''')

							__UpperCAmelCase			:							Optional[int]	    =	tokenizer.encode('''sequence builders'''				,     add_special_tokens=lowercase__)
							__UpperCAmelCase			:							Optional[int]	    =	tokenizer.encode('''multi-sequence build'''				,     add_special_tokens=lowercase__)

							__UpperCAmelCase			:							Union[str, Any]	    =	tokenizer.build_inputs_with_special_tokens(lowercase__)
							__UpperCAmelCase			:							Optional[Any]	    =	tokenizer.build_inputs_with_special_tokens(lowercase__				,     lowercase__)

							assert encoded_sentence == text + [4, 3]
							assert encoded_pair == text + [4] + text_a + [4, 3]


	@slow
	def  A(    self):
							# fmt: off
							__UpperCAmelCase			:							Optional[Any]	    =	{'''input_ids''': [[1_7, 2_1_4_4_2, 2_7_0, 1_7, 1_0, 1_4_6_4_5, 3_1_8, 3_4, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 7_7_5_2, 2_2_0_1_8, 2_3, 2_1, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 3_3_5_2, 1_4_4_3_1, 1_3, 5_5_0_0, 1_1, 1_1_7_6, 5_8_0, 1_3, 1_6_8_1_9, 4_7_9_7, 2_3, 1_7, 1_0, 1_7_1_3_5, 6_5_8, 1_9, 4_5_7, 7_9_3_2, 1_3, 1_8_4, 1_9, 3_1_5_4, 1_7_1_3_5, 6_4_6_8, 1_9, 1_4_0_4, 1_2_2_6_9, 1_9, 4_2_2_9, 5_3_5_6, 1_6_2_6_4, 4_6, 1_9, 1_7, 2_0_5_4_5, 1_0_3_9_5, 9, 9, 9, 1_1, 2_8, 6_4_2_1, 9_5_3_1, 2_0_7_2_9, 1_7, 1_0, 3_5_3, 1_7_0_2_2, 1_1, 2_1, 6_4_2_1, 9_5_3_1, 1_6_9_4_9, 1_7, 1_0, 1_1_5_0_9, 7_5_3, 1_1, 3_3, 9_5, 2_4_2_1, 7_3_8_5, 9_5_6, 1_4_4_3_1, 2_6_2_6, 2_5, 8_4_2, 7_3_8_5, 4_8_3_6, 2_1, 1_4_2_9, 2_2_7_2, 9_8_5_5, 3_1_2_0, 1_6_1, 2_4_7_3_8, 1_9, 1_3_2_0_3, 6_5_8, 2_1_8, 7_8_7, 2_1, 4_3_0, 1_8_4_8_2, 8_4_7, 2_6_3_7, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2_2, 2_2_1_7_8, 2_7, 1_0_6_4, 2_2, 9_5_6, 1_3, 1_1_1_0_1, 1_4_2_9, 5_8_5_4, 2_4_3_1_3, 1_8_9_5_3, 4_0, 4_2_2, 2_4_3_6_6, 6_8, 1_7_5_8, 3_7, 1_0_4_8_3, 1_4_2_5_7, 3_1, 2_0_7, 2_6_3, 2_1, 2_0_3, 3_7_7_3, 2_5, 7_1, 9_7_3_5, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2, 2_0_4_9, 3_4_4_2, 1_7, 1_3_8_9_4, 3_3_8_0, 2_3, 9_5, 1_8, 1_7_6_3_4, 2_2_8_8, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=lowercase__				,     model_name='''xlnet-base-cased'''				,     revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''				,     )

