


import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

lowerCAmelCase       		=    """bert-base-cased"""
lowerCAmelCase       		=    """fp16"""
lowerCAmelCase       		=    """bf16"""
lowerCAmelCase       		=    [FPaa, BFaa]




@require_fsdp
@require_cuda
class 							lowerCamelCase   (  _UpperCamelCase ):


	def  A(    self):
							super().setUp()

							__UpperCAmelCase			:							List[str]	    =	dict(
							    ACCELERATE_USE_FSDP='''true'''				,     MASTER_ADDR='''localhost'''				,     MASTER_PORT='''10999'''				,     RANK='''0'''				,     LOCAL_RANK='''0'''				,     WORLD_SIZE='''1'''				,     )


	def  A(    self):
							from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

							for i, strategy in enumerate(lowercase__):
													__UpperCAmelCase			:							Dict	    =	self.dist_env.copy()
													__UpperCAmelCase			:							str	    =	F"{i + 1}"
													__UpperCAmelCase			:							Union[str, Any]	    =	strategy
													with mockenv_context(**lowercase__):
																			__UpperCAmelCase			:							List[str]	    =	FullyShardedDataParallelPlugin()
																			self.assertEqual(fsdp_plugin.sharding_strategy				,     ShardingStrategy(i + 1))


	def  A(    self):
							from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

							for i, prefetch_policy in enumerate(lowercase__):
													__UpperCAmelCase			:							int	    =	self.dist_env.copy()
													__UpperCAmelCase			:							Tuple	    =	prefetch_policy
													with mockenv_context(**lowercase__):
																			__UpperCAmelCase			:							Tuple	    =	FullyShardedDataParallelPlugin()
																			if prefetch_policy == "NO_PREFETCH":
																									self.assertIsNone(fsdp_plugin.backward_prefetch)
																			else:
																									self.assertEqual(fsdp_plugin.backward_prefetch				,     BackwardPrefetch(i + 1))


	def  A(    self):
							from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

							for i, state_dict_type in enumerate(lowercase__):
													__UpperCAmelCase			:							Optional[Any]	    =	self.dist_env.copy()
													__UpperCAmelCase			:							List[str]	    =	state_dict_type
													with mockenv_context(**lowercase__):
																			__UpperCAmelCase			:							List[Any]	    =	FullyShardedDataParallelPlugin()
																			self.assertEqual(fsdp_plugin.state_dict_type				,     StateDictType(i + 1))
																			if state_dict_type == "FULL_STATE_DICT":
																									self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu)
																									self.assertTrue(fsdp_plugin.state_dict_config.ranka_only)


	def  A(    self):
							__UpperCAmelCase			:							Any	    =	AutoModel.from_pretrained(lowercase__)
							for policy in FSDP_AUTO_WRAP_POLICY:
													__UpperCAmelCase			:							Optional[int]	    =	self.dist_env.copy()
													__UpperCAmelCase			:							Optional[Any]	    =	policy
													if policy == "TRANSFORMER_BASED_WRAP":
																			__UpperCAmelCase			:							Dict	    =	'''BertLayer'''
													elif policy == "SIZE_BASED_WRAP":
																			__UpperCAmelCase			:							List[Any]	    =	'''2000'''
													with mockenv_context(**lowercase__):
																			__UpperCAmelCase			:							Dict	    =	FullyShardedDataParallelPlugin()
																			fsdp_plugin.set_auto_wrap_policy(lowercase__)
																			if policy == "NO_WRAP":
																									self.assertIsNone(fsdp_plugin.auto_wrap_policy)
																			else:
																									self.assertIsNotNone(fsdp_plugin.auto_wrap_policy)

							__UpperCAmelCase			:							Dict	    =	self.dist_env.copy()
							__UpperCAmelCase			:							Optional[Any]	    =	'''TRANSFORMER_BASED_WRAP'''
							__UpperCAmelCase			:							Tuple	    =	'''T5Layer'''
							with mockenv_context(**lowercase__):
													__UpperCAmelCase			:							Any	    =	FullyShardedDataParallelPlugin()
													with self.assertRaises(lowercase__) as cm:
																			fsdp_plugin.set_auto_wrap_policy(lowercase__)
													self.assertTrue('''Could not find the transformer layer class to wrap in the model.''' in str(cm.exception))

							__UpperCAmelCase			:							Dict	    =	self.dist_env.copy()
							__UpperCAmelCase			:							Optional[Any]	    =	'''SIZE_BASED_WRAP'''
							__UpperCAmelCase			:							int	    =	'''0'''
							with mockenv_context(**lowercase__):
													__UpperCAmelCase			:							List[Any]	    =	FullyShardedDataParallelPlugin()
													fsdp_plugin.set_auto_wrap_policy(lowercase__)
													self.assertIsNone(fsdp_plugin.auto_wrap_policy)


	def  A(    self):
							from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
							from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

							for mp_dtype in dtypes:
													__UpperCAmelCase			:							Any	    =	self.dist_env.copy()
													__UpperCAmelCase			:							List[Any]	    =	mp_dtype
													with mockenv_context(**lowercase__):
																			__UpperCAmelCase			:							Any	    =	Accelerator()
																			if mp_dtype == "fp16":
																									__UpperCAmelCase			:							Optional[int]	    =	torch.floataa
																			elif mp_dtype == "bf16":
																									__UpperCAmelCase			:							str	    =	torch.bfloataa
																			__UpperCAmelCase			:							Optional[Any]	    =	MixedPrecision(param_dtype=lowercase__				,     reduce_dtype=lowercase__				,     buffer_dtype=lowercase__)
																			self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy				,     lowercase__)
																			if mp_dtype == FPaa:
																									self.assertTrue(isinstance(accelerator.scaler				,     lowercase__))
																			elif mp_dtype == BFaa:
																									self.assertIsNone(accelerator.scaler)
																			AcceleratorState._reset_state(lowercase__)


	def  A(    self):
							from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

							for flag in [True, False]:
													__UpperCAmelCase			:							Tuple	    =	self.dist_env.copy()
													__UpperCAmelCase			:							int	    =	str(lowercase__).lower()
													with mockenv_context(**lowercase__):
																			__UpperCAmelCase			:							Optional[Any]	    =	FullyShardedDataParallelPlugin()
																			self.assertEqual(fsdp_plugin.cpu_offload				,     CPUOffload(offload_params=lowercase__))






@require_fsdp
@require_multi_gpu
@slow
class 							lowerCamelCase   (  _UpperCamelCase ):


	def  A(    self):
							super().setUp()
							__UpperCAmelCase			:							List[Any]	    =	0.8_2
							__UpperCAmelCase			:							int	    =	[
							    '''fsdp_shard_grad_op_transformer_based_wrap''',
							    '''fsdp_full_shard_transformer_based_wrap''',
							]
							__UpperCAmelCase			:							Any	    =	{
							    '''multi_gpu_fp16''': 3_2_0_0,
							    '''fsdp_shard_grad_op_transformer_based_wrap_fp16''': 2_0_0_0,
							    '''fsdp_full_shard_transformer_based_wrap_fp16''': 1_9_0_0,
							    # Disabling below test as it overwhelms the RAM memory usage
							    # on CI self-hosted runner leading to tests getting killed.
							    # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
							}
							__UpperCAmelCase			:							Tuple	    =	1_6_0
							__UpperCAmelCase			:							int	    =	1_6_0

							__UpperCAmelCase			:							str	    =	inspect.getfile(accelerate.test_utils)
							__UpperCAmelCase			:							int	    =	os.path.sep.join(mod_file.split(os.path.sep)[:-1] + ['''scripts''', '''external_deps'''])


	def  A(    self):
							__UpperCAmelCase			:							str	    =	os.path.join(self.test_scripts_folder				,     '''test_performance.py''')
							__UpperCAmelCase			:							int	    =	['''accelerate''', '''launch''', '''--num_processes=2''', '''--num_machines=1''', '''--machine_rank=0''', '''--use_fsdp''']
							for config in self.performance_configs:
													__UpperCAmelCase			:							Optional[int]	    =	cmd.copy()
													for i, strategy in enumerate(lowercase__):
																			if strategy.lower() in config:
																									cmd_config.append(F"--fsdp_sharding_strategy={i+1}")
																									break

													if "fp32" in config:
																			cmd_config.append('''--mixed_precision=no''')
													else:
																			cmd_config.append('''--mixed_precision=fp16''')

													if "cpu_offload" in config:
																			cmd_config.append('''--fsdp_offload_params=True''')

													for policy in FSDP_AUTO_WRAP_POLICY:
																			if policy.lower() in config:
																									cmd_config.append(F"--fsdp_auto_wrap_policy={policy}")
																									break

													if policy == "TRANSFORMER_BASED_WRAP":
																			cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer''')
													elif policy == "SIZE_BASED_WRAP":
																			cmd_config.append('''--fsdp_min_num_params=2000''')

													cmd_config.extend(
													    [
													        self.test_file_path,
													        F"--output_dir={self.tmpdir}",
													        F"--performance_lower_bound={self.performance_lower_bound}",
													    ])
													with patch_environment(omp_num_threads=1):
																			execute_subprocess_async(lowercase__				,     env=os.environ.copy())


	def  A(    self):
							__UpperCAmelCase			:							Optional[int]	    =	os.path.join(self.test_scripts_folder				,     '''test_checkpointing.py''')
							__UpperCAmelCase			:							int	    =	[
							    '''accelerate''',
							    '''launch''',
							    '''--num_processes=2''',
							    '''--num_machines=1''',
							    '''--machine_rank=0''',
							    '''--use_fsdp''',
							    '''--mixed_precision=fp16''',
							    '''--fsdp_transformer_layer_cls_to_wrap=BertLayer''',
							]

							for i, strategy in enumerate(lowercase__):
													__UpperCAmelCase			:							int	    =	cmd.copy()
													cmd_config.append(F"--fsdp_sharding_strategy={i+1}")
													if strategy != "FULL_SHARD":
																			continue
													__UpperCAmelCase			:							Optional[Any]	    =	len(lowercase__)
													for state_dict_type in FSDP_STATE_DICT_TYPE:
																			__UpperCAmelCase			:							Tuple	    =	cmd_config[:state_dict_config_index]
																			cmd_config.append(F"--fsdp_state_dict_type={state_dict_type}")
																			cmd_config.extend(
																			    [
																			        self.test_file_path,
																			        F"--output_dir={self.tmpdir}",
																			        '''--partial_train_epoch=1''',
																			    ])
																			with patch_environment(omp_num_threads=1):
																									execute_subprocess_async(lowercase__				,     env=os.environ.copy())

																			__UpperCAmelCase			:							str	    =	cmd_config[:-1]
																			__UpperCAmelCase			:							Any	    =	os.path.join(self.tmpdir				,     '''epoch_0''')
																			cmd_config.extend(
																			    [
																			        F"--resume_from_checkpoint={resume_from_checkpoint}",
																			    ])
																			with patch_environment(omp_num_threads=1):
																									execute_subprocess_async(lowercase__				,     env=os.environ.copy())


	def  A(    self):
							__UpperCAmelCase			:							Union[str, Any]	    =	os.path.join(self.test_scripts_folder				,     '''test_peak_memory_usage.py''')
							__UpperCAmelCase			:							int	    =	[
							    '''accelerate''',
							    '''launch''',
							    '''--num_processes=2''',
							    '''--num_machines=1''',
							    '''--machine_rank=0''',
							]
							for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
													__UpperCAmelCase			:							List[str]	    =	cmd.copy()
													if "fp16" in spec:
																			cmd_config.extend(['''--mixed_precision=fp16'''])
													else:
																			cmd_config.extend(['''--mixed_precision=no'''])

													if "multi_gpu" in spec:
																			continue
													else:
																			cmd_config.extend(['''--use_fsdp'''])
																			for i, strategy in enumerate(lowercase__):
																									if strategy.lower() in spec:
																															cmd_config.append(F"--fsdp_sharding_strategy={i+1}")
																															break

																			if "cpu_offload" in spec:
																									cmd_config.append('''--fsdp_offload_params=True''')

																			for policy in FSDP_AUTO_WRAP_POLICY:
																									if policy.lower() in spec:
																															cmd_config.append(F"--fsdp_auto_wrap_policy={policy}")
																															break

																			if policy == "TRANSFORMER_BASED_WRAP":
																									cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer''')
																			elif policy == "SIZE_BASED_WRAP":
																									cmd_config.append('''--fsdp_min_num_params=2000''')

													cmd_config.extend(
													    [
													        self.test_file_path,
													        F"--output_dir={self.tmpdir}",
													        F"--peak_memory_upper_bound={peak_mem_upper_bound}",
													        F"--n_train={self.n_train}",
													        F"--n_val={self.n_val}",
													    ])
													with patch_environment(omp_num_threads=1):
																			execute_subprocess_async(lowercase__				,     env=os.environ.copy())





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase       		=    get_tests_dir("""fixtures/test_sentencepiece.model""")




@require_sentencepiece
@require_tokenizers
class 							lowerCamelCase   (  _UpperCamelCase							,						unittest.TestCase ):
	_lowerCAmelCase     :		Union[str, Any]										=						XLNetTokenizer
	_lowerCAmelCase     :		int										=						XLNetTokenizerFast
	_lowerCAmelCase     :		Optional[Any]										=						True
	_lowerCAmelCase     :		Optional[Any]										=						True


	def  A(    self):
							super().setUp()

							# We have a SentencePiece fixture for testing
							__UpperCAmelCase			:							Optional[Any]	    =	XLNetTokenizer(lowercase__				,     keep_accents=lowercase__)
							tokenizer.sanitize_special_tokens()
							tokenizer.save_pretrained(self.tmpdirname)


	def  A(    self):
							__UpperCAmelCase			:							Tuple	    =	'''<s>'''
							__UpperCAmelCase			:							List[str]	    =	1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase__)				,     lowercase__)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase__)				,     lowercase__)


	def  A(    self):
							__UpperCAmelCase			:							Union[str, Any]	    =	list(self.get_tokenizer().get_vocab().keys())

							self.assertEqual(vocab_keys[0]				,     '''<unk>''')
							self.assertEqual(vocab_keys[1]				,     '''<s>''')
							self.assertEqual(vocab_keys[-1]				,     '''<eod>''')
							self.assertEqual(len(lowercase__)				,     1_0_0_6)


	def  A(    self):
							self.assertEqual(self.get_tokenizer().vocab_size				,     1_0_0_0)


	def  A(    self):
							__UpperCAmelCase			:							List[str]	    =	XLNetTokenizer(lowercase__				,     keep_accents=lowercase__)

							__UpperCAmelCase			:							Optional[Any]	    =	tokenizer.tokenize('''This is a test''')
							self.assertListEqual(lowercase__				,     ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__)				,     [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2])

							__UpperCAmelCase			:							Tuple	    =	tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
							self.assertListEqual(
							    lowercase__				,     [
							        SPIECE_UNDERLINE + '''I''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''9''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''s''',
							        '''é''',
							        '''.''',
							    ]				,     )
							__UpperCAmelCase			:							Union[str, Any]	    =	tokenizer.convert_tokens_to_ids(lowercase__)
							self.assertListEqual(lowercase__				,     [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4])

							__UpperCAmelCase			:							int	    =	tokenizer.convert_ids_to_tokens(lowercase__)
							self.assertListEqual(
							    lowercase__				,     [
							        SPIECE_UNDERLINE + '''I''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''<unk>''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''s''',
							        '''<unk>''',
							        '''.''',
							    ]				,     )


	def  A(    self):
							__UpperCAmelCase			:							str	    =	XLNetTokenizer(lowercase__				,     do_lower_case=lowercase__)
							__UpperCAmelCase			:							int	    =	tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
							self.assertListEqual(
							    lowercase__				,     [
							        SPIECE_UNDERLINE + '''''',
							        '''i''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''9''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''se''',
							        '''.''',
							    ]				,     )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')				,     ['''▁he''', '''ll''', '''o'''])


	def  A(    self):
							__UpperCAmelCase			:							Optional[Any]	    =	XLNetTokenizer(lowercase__				,     do_lower_case=lowercase__)
							__UpperCAmelCase			:							Union[str, Any]	    =	tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
							self.assertListEqual(
							    lowercase__				,     [
							        SPIECE_UNDERLINE + '''I''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''9''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''se''',
							        '''.''',
							    ]				,     )


	@slow
	def  A(    self):
							__UpperCAmelCase			:							Tuple	    =	XLNetTokenizer.from_pretrained('''xlnet-base-cased''')

							__UpperCAmelCase			:							Optional[int]	    =	tokenizer.encode('''sequence builders'''				,     add_special_tokens=lowercase__)
							__UpperCAmelCase			:							Optional[int]	    =	tokenizer.encode('''multi-sequence build'''				,     add_special_tokens=lowercase__)

							__UpperCAmelCase			:							Union[str, Any]	    =	tokenizer.build_inputs_with_special_tokens(lowercase__)
							__UpperCAmelCase			:							Optional[Any]	    =	tokenizer.build_inputs_with_special_tokens(lowercase__				,     lowercase__)

							assert encoded_sentence == text + [4, 3]
							assert encoded_pair == text + [4] + text_a + [4, 3]


	@slow
	def  A(    self):
							# fmt: off
							__UpperCAmelCase			:							Optional[Any]	    =	{'''input_ids''': [[1_7, 2_1_4_4_2, 2_7_0, 1_7, 1_0, 1_4_6_4_5, 3_1_8, 3_4, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 7_7_5_2, 2_2_0_1_8, 2_3, 2_1, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 3_3_5_2, 1_4_4_3_1, 1_3, 5_5_0_0, 1_1, 1_1_7_6, 5_8_0, 1_3, 1_6_8_1_9, 4_7_9_7, 2_3, 1_7, 1_0, 1_7_1_3_5, 6_5_8, 1_9, 4_5_7, 7_9_3_2, 1_3, 1_8_4, 1_9, 3_1_5_4, 1_7_1_3_5, 6_4_6_8, 1_9, 1_4_0_4, 1_2_2_6_9, 1_9, 4_2_2_9, 5_3_5_6, 1_6_2_6_4, 4_6, 1_9, 1_7, 2_0_5_4_5, 1_0_3_9_5, 9, 9, 9, 1_1, 2_8, 6_4_2_1, 9_5_3_1, 2_0_7_2_9, 1_7, 1_0, 3_5_3, 1_7_0_2_2, 1_1, 2_1, 6_4_2_1, 9_5_3_1, 1_6_9_4_9, 1_7, 1_0, 1_1_5_0_9, 7_5_3, 1_1, 3_3, 9_5, 2_4_2_1, 7_3_8_5, 9_5_6, 1_4_4_3_1, 2_6_2_6, 2_5, 8_4_2, 7_3_8_5, 4_8_3_6, 2_1, 1_4_2_9, 2_2_7_2, 9_8_5_5, 3_1_2_0, 1_6_1, 2_4_7_3_8, 1_9, 1_3_2_0_3, 6_5_8, 2_1_8, 7_8_7, 2_1, 4_3_0, 1_8_4_8_2, 8_4_7, 2_6_3_7, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2_2, 2_2_1_7_8, 2_7, 1_0_6_4, 2_2, 9_5_6, 1_3, 1_1_1_0_1, 1_4_2_9, 5_8_5_4, 2_4_3_1_3, 1_8_9_5_3, 4_0, 4_2_2, 2_4_3_6_6, 6_8, 1_7_5_8, 3_7, 1_0_4_8_3, 1_4_2_5_7, 3_1, 2_0_7, 2_6_3, 2_1, 2_0_3, 3_7_7_3, 2_5, 7_1, 9_7_3_5, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2, 2_0_4_9, 3_4_4_2, 1_7, 1_3_8_9_4, 3_3_8_0, 2_3, 9_5, 1_8, 1_7_6_3_4, 2_2_8_8, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=lowercase__				,     model_name='''xlnet-base-cased'''				,     revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''				,     )

