


import os
from pickle import UnpicklingError
from typing import Dict, Tuple

import jax
import jax.numpy as jnp
import numpy as np
from flax.serialization import from_bytes
from flax.traverse_util import flatten_dict, unflatten_dict

import transformers

from .utils import logging


lowerCAmelCase       		=    logging.get_logger(__name__)




def     __SCREAMING_SNAKE_CASE							(       lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_=False      )		->     str:




						'''simple docstring'''







						try:
												import torch  # noqa: F401
						except ImportError:
												logger.error(
												    '''Loading a PyTorch model in Flax, requires both PyTorch and Flax to be installed. Please see'''
												    ''' https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation'''
												    ''' instructions.'''      )
												raise

						if not is_sharded:
												__UpperCAmelCase			:							Any	    =	os.path.abspath(lowercase_      )
												logger.info(f"Loading PyTorch weights from {pt_path}"      )

												__UpperCAmelCase			:							Dict	    =	torch.load(lowercase_      ,       map_location='''cpu'''      )
												logger.info(f"PyTorch checkpoint contains {sum(t.numel() for t in pt_state_dict.values()      ):,} parameters."      )

												__UpperCAmelCase			:							Any	    =	convert_pytorch_state_dict_to_flax(lowercase_      ,       lowercase_      )
						else:
												# model is sharded and pytorch_checkpoint_path already contains the list of .pt shard files
												__UpperCAmelCase			:							List[str]	    =	convert_pytorch_sharded_state_dict_to_flax(lowercase_      ,       lowercase_      )
						return flax_state_dict




def     __SCREAMING_SNAKE_CASE							(       lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_      ,       )		->     (Tuple[str], np.ndarray):




						'''simple docstring'''







						def is_key_or_prefix_key_in_dict(lowercase_      ) -> bool:
												return len(set(lowercase_      ) & {key, (model_prefix,) + key}      ) > 0

						# layer norm
						__UpperCAmelCase			:							Union[str, Any]	    =	pt_tuple_key[:-1] + ('''scale''',)
						if pt_tuple_key[-1] in ["weight", "gamma"] and is_key_or_prefix_key_in_dict(lowercase_      ):
												return renamed_pt_tuple_key, pt_tensor

						# batch norm layer mean
						__UpperCAmelCase			:							str	    =	pt_tuple_key[:-1] + ('''mean''',)
						if pt_tuple_key[-1] == "running_mean" and not is_key_or_prefix_key_in_dict(lowercase_      ):
												return renamed_pt_tuple_key, pt_tensor

						# batch norm layer var
						__UpperCAmelCase			:							Tuple	    =	pt_tuple_key[:-1] + ('''var''',)
						if pt_tuple_key[-1] == "running_var" and not is_key_or_prefix_key_in_dict(lowercase_      ):
												return renamed_pt_tuple_key, pt_tensor

						# embedding
						__UpperCAmelCase			:							Dict	    =	pt_tuple_key[:-1] + ('''embedding''',)
						if pt_tuple_key[-1] == "weight" and is_key_or_prefix_key_in_dict(lowercase_      ):
												return renamed_pt_tuple_key, pt_tensor

						# conv layer
						__UpperCAmelCase			:							Dict	    =	pt_tuple_key[:-1] + ('''kernel''',)
						if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4 and not is_key_or_prefix_key_in_dict(lowercase_      ):
												__UpperCAmelCase			:							int	    =	pt_tensor.transpose(2      ,       3      ,       1      ,       0      )
												return renamed_pt_tuple_key, pt_tensor

						# linear layer
						__UpperCAmelCase			:							Tuple	    =	pt_tuple_key[:-1] + ('''kernel''',)
						if pt_tuple_key[-1] == "weight" and not is_key_or_prefix_key_in_dict(lowercase_      ):
												__UpperCAmelCase			:							Tuple	    =	pt_tensor.T
												return renamed_pt_tuple_key, pt_tensor

						# old PyTorch layer norm weight
						__UpperCAmelCase			:							Tuple	    =	pt_tuple_key[:-1] + ('''weight''',)
						if pt_tuple_key[-1] == "gamma":
												return renamed_pt_tuple_key, pt_tensor

						# old PyTorch layer norm bias
						__UpperCAmelCase			:							Dict	    =	pt_tuple_key[:-1] + ('''bias''',)
						if pt_tuple_key[-1] == "beta":
												return renamed_pt_tuple_key, pt_tensor

						# New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
						__UpperCAmelCase			:							str	    =	None
						if pt_tuple_key[-3::2] == ("parametrizations", "original0"):
												__UpperCAmelCase			:							int	    =	pt_tuple_key[-2] + '''_g'''
						elif pt_tuple_key[-3::2] == ("parametrizations", "original1"):
												__UpperCAmelCase			:							Dict	    =	pt_tuple_key[-2] + '''_v'''
						if name is not None:
												__UpperCAmelCase			:							Optional[Any]	    =	pt_tuple_key[:-3] + (name,)
												return renamed_pt_tuple_key, pt_tensor

						return pt_tuple_key, pt_tensor




def     __SCREAMING_SNAKE_CASE							(       lowercase_      ,       lowercase_      )		->     Tuple:




						'''simple docstring'''







						__UpperCAmelCase			:							Optional[int]	    =	{k: v.numpy() for k, v in pt_state_dict.items()}

						__UpperCAmelCase			:							Union[str, Any]	    =	flax_model.base_model_prefix

						# use params dict if the model contains batch norm layers
						if "params" in flax_model.params:
												__UpperCAmelCase			:							str	    =	flax_model.params['''params''']
						else:
												__UpperCAmelCase			:							str	    =	flax_model.params
						__UpperCAmelCase			:							List[str]	    =	flatten_dict(lowercase_      )

						# add batch_stats keys,values to dict
						if "batch_stats" in flax_model.params:
												__UpperCAmelCase			:							Optional[int]	    =	flatten_dict(flax_model.params['''batch_stats''']      )
												random_flax_state_dict.update(lowercase_      )

						__UpperCAmelCase			:							Optional[int]	    =	{}

						__UpperCAmelCase			:							Dict	    =	(model_prefix not in flax_model_params) and (
						    model_prefix in {k.split('''.'''      )[0] for k in pt_state_dict.keys()}
						)
						__UpperCAmelCase			:							Tuple	    =	(model_prefix in flax_model_params) and (
						    model_prefix not in {k.split('''.'''      )[0] for k in pt_state_dict.keys()}
						)

						# Need to change some parameters name to match Flax names
						for pt_key, pt_tensor in pt_state_dict.items():
												__UpperCAmelCase			:							Optional[int]	    =	tuple(pt_key.split('''.'''      )      )

												# remove base model prefix if necessary
												__UpperCAmelCase			:							Optional[int]	    =	pt_tuple_key[0] == model_prefix
												if load_model_with_head_into_base_model and has_base_model_prefix:
																		__UpperCAmelCase			:							Any	    =	pt_tuple_key[1:]

												# Correctly rename weight parameters
												__UpperCAmelCase    ,      __UpperCAmelCase			:							Tuple	    =	rename_key_and_reshape_tensor(
												    lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_      )

												# add model prefix if necessary
												__UpperCAmelCase			:							List[str]	    =	(model_prefix,) + flax_key in random_flax_state_dict
												if load_base_model_into_model_with_head and require_base_model_prefix:
																		__UpperCAmelCase			:							Optional[int]	    =	(model_prefix,) + flax_key

												if flax_key in random_flax_state_dict:
																		if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
																								raise ValueError(
																								    f"PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape "
																								    f"{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}."      )

        # add batch stats if the model contains batchnorm layers
												if "batch_stats" in flax_model.params:
																		if "mean" in flax_key[-1] or "var" in flax_key[-1]:
																								__UpperCAmelCase			:							Dict	    =	jnp.asarray(lowercase_      )
																								continue
																		# remove num_batches_tracked key
																		if "num_batches_tracked" in flax_key[-1]:
																								flax_state_dict.pop(lowercase_      ,       lowercase_      )
																								continue

																		# also add unexpected weight so that warning is thrown
																		__UpperCAmelCase			:							Union[str, Any]	    =	jnp.asarray(lowercase_      )

												else:
																		# also add unexpected weight so that warning is thrown
																		__UpperCAmelCase			:							Tuple	    =	jnp.asarray(lowercase_      )

						return unflatten_dict(lowercase_      )




def     __SCREAMING_SNAKE_CASE							(       lowercase_      ,       lowercase_      )		->     str:




						'''simple docstring'''







						import torch

						# Load the index
						__UpperCAmelCase			:							int	    =	{}
						for shard_file in shard_filenames:
												# load using msgpack utils
												__UpperCAmelCase			:							List[str]	    =	torch.load(lowercase_      )
												__UpperCAmelCase			:							Optional[int]	    =	{k: v.numpy() for k, v in pt_state_dict.items()}

												__UpperCAmelCase			:							Any	    =	flax_model.base_model_prefix

												# use params dict if the model contains batch norm layers and then add batch_stats keys,values to dict
												if "batch_stats" in flax_model.params:
																		__UpperCAmelCase			:							int	    =	flax_model.params['''params''']

																		__UpperCAmelCase			:							Optional[int]	    =	flatten_dict(lowercase_      )
																		random_flax_state_dict.update(flatten_dict(flax_model.params['''batch_stats''']      )      )
												else:
																		__UpperCAmelCase			:							Dict	    =	flax_model.params
																		__UpperCAmelCase			:							Optional[Any]	    =	flatten_dict(lowercase_      )

												__UpperCAmelCase			:							int	    =	(model_prefix not in flax_model_params) and (
												    model_prefix in {k.split('''.'''      )[0] for k in pt_state_dict.keys()}
												)
												__UpperCAmelCase			:							Optional[Any]	    =	(model_prefix in flax_model_params) and (
												    model_prefix not in {k.split('''.'''      )[0] for k in pt_state_dict.keys()}
												)
												# Need to change some parameters name to match Flax names
												for pt_key, pt_tensor in pt_state_dict.items():
																		__UpperCAmelCase			:							Tuple	    =	tuple(pt_key.split('''.'''      )      )

																		# remove base model prefix if necessary
																		__UpperCAmelCase			:							Union[str, Any]	    =	pt_tuple_key[0] == model_prefix
																		if load_model_with_head_into_base_model and has_base_model_prefix:
																								__UpperCAmelCase			:							Optional[int]	    =	pt_tuple_key[1:]

																		# Correctly rename weight parameters
																		__UpperCAmelCase    ,      __UpperCAmelCase			:							Tuple	    =	rename_key_and_reshape_tensor(
																		    lowercase_      ,       lowercase_      ,       lowercase_      ,       lowercase_      )
																		# add model prefix if necessary
																		__UpperCAmelCase			:							Any	    =	(model_prefix,) + flax_key in random_flax_state_dict
																		if load_base_model_into_model_with_head and require_base_model_prefix:
																								__UpperCAmelCase			:							str	    =	(model_prefix,) + flax_key

																		if flax_key in random_flax_state_dict:
																								if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
																														raise ValueError(
																														    f"PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape "
																														    f"{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}."      )

            # add batch stats if the model contains batchnorm layers
																		if "batch_stats" in flax_model.params:
																								if "mean" in flax_key[-1]:
																														__UpperCAmelCase			:							List[Any]	    =	jnp.asarray(lowercase_      )
																														continue
																								if "var" in flax_key[-1]:
																														__UpperCAmelCase			:							int	    =	jnp.asarray(lowercase_      )
																														continue
																								# remove num_batches_tracked key
																								if "num_batches_tracked" in flax_key[-1]:
																														flax_state_dict.pop(lowercase_      ,       lowercase_      )
																														continue

																								# also add unexpected weight so that warning is thrown
																								__UpperCAmelCase			:							List[Any]	    =	jnp.asarray(lowercase_      )

																		else:
																								# also add unexpected weight so that warning is thrown
																								__UpperCAmelCase			:							List[Any]	    =	jnp.asarray(lowercase_      )
						return unflatten_dict(lowercase_      )




def     __SCREAMING_SNAKE_CASE							(       lowercase_      ,       lowercase_      )		->     Any:




						'''simple docstring'''







						__UpperCAmelCase			:							List[str]	    =	os.path.abspath(lowercase_      )
						logger.info(f"Loading Flax weights from {flax_checkpoint_path}"      )

						# import correct flax class
						__UpperCAmelCase			:							str	    =	getattr(lowercase_      ,       '''Flax''' + model.__class__.__name__      )

						# load flax weight dict
						with open(lowercase_      ,       '''rb'''      ) as state_f:
												try:
																		__UpperCAmelCase			:							Optional[Any]	    =	from_bytes(lowercase_      ,       state_f.read()      )
												except UnpicklingError:
																		raise EnvironmentError(f"Unable to convert {flax_checkpoint_path} to Flax deserializable object. "      )

						return load_flax_weights_in_pytorch_model(lowercase_      ,       lowercase_      )





def     __SCREAMING_SNAKE_CASE							(       lowercase_      ,       lowercase_      )		->     List[str]:




						'''simple docstring'''







						try:
												import torch  # noqa: F401
						except ImportError:
												logger.error(
												    '''Loading a Flax weights in PyTorch, requires both PyTorch and Flax to be installed. Please see'''
												    ''' https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation'''
												    ''' instructions.'''      )
												raise

						# check if we have bf16 weights
						__UpperCAmelCase			:							str	    =	flatten_dict(jax.tree_util.tree_map(lambda lowercase_      : x.dtype == jnp.bfloataa      ,       lowercase_      )      ).values()
						if any(lowercase_      ):
												# convert all weights to fp32 if the are bf16 since torch.from_numpy can-not handle bf16
												# and bf16 is not fully supported in PT yet.
												logger.warning(
												    '''Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` '''
												    '''before loading those in PyTorch model.'''      )
												__UpperCAmelCase			:							Any	    =	jax.tree_util.tree_map(
												    lambda lowercase_      : params.astype(np.floataa      ) if params.dtype == jnp.bfloataa else params      ,       lowercase_      )

						__UpperCAmelCase			:							Tuple	    =	flatten_dict(lowercase_      )
						__UpperCAmelCase			:							Dict	    =	pt_model.state_dict()

						__UpperCAmelCase			:							int	    =	(pt_model.base_model_prefix in flax_state) and (
						    pt_model.base_model_prefix not in {k.split('''.'''      )[0] for k in pt_model_dict.keys()}
						)
						__UpperCAmelCase			:							Tuple	    =	(pt_model.base_model_prefix not in flax_state) and (
						    pt_model.base_model_prefix in {k.split('''.'''      )[0] for k in pt_model_dict.keys()}
						)

						# keep track of unexpected & missing keys
						__UpperCAmelCase			:							Optional[Any]	    =	[]
						__UpperCAmelCase			:							int	    =	set(pt_model_dict.keys()      )

						for flax_key_tuple, flax_tensor in flax_state_dict.items():
												__UpperCAmelCase			:							str	    =	flax_key_tuple[0] == pt_model.base_model_prefix
												__UpperCAmelCase			:							List[Any]	    =	'''.'''.join((pt_model.base_model_prefix,) + flax_key_tuple      ) in pt_model_dict

												# adapt flax_key to prepare for loading from/to base model only
												if load_model_with_head_into_base_model and has_base_model_prefix:
																		__UpperCAmelCase			:							List[str]	    =	flax_key_tuple[1:]
												elif load_base_model_into_model_with_head and require_base_model_prefix:
																		__UpperCAmelCase			:							List[str]	    =	(pt_model.base_model_prefix,) + flax_key_tuple

												# rename flax weights to PyTorch format
												if flax_key_tuple[-1] == "kernel" and flax_tensor.ndim == 4 and ".".join(lowercase_      ) not in pt_model_dict:
																		# conv layer
																		__UpperCAmelCase			:							Dict	    =	flax_key_tuple[:-1] + ('''weight''',)
																		__UpperCAmelCase			:							str	    =	jnp.transpose(lowercase_      ,       (3, 2, 0, 1)      )
												elif flax_key_tuple[-1] == "kernel" and ".".join(lowercase_      ) not in pt_model_dict:
																		# linear layer
																		__UpperCAmelCase			:							Optional[int]	    =	flax_key_tuple[:-1] + ('''weight''',)
																		__UpperCAmelCase			:							Dict	    =	flax_tensor.T
												elif flax_key_tuple[-1] in ["scale", "embedding"]:
																		__UpperCAmelCase			:							str	    =	flax_key_tuple[:-1] + ('''weight''',)

												# adding batch stats from flax batch norm to pt
												elif "mean" in flax_key_tuple[-1]:
																		__UpperCAmelCase			:							Optional[Any]	    =	flax_key_tuple[:-1] + ('''running_mean''',)
												elif "var" in flax_key_tuple[-1]:
																		__UpperCAmelCase			:							Optional[Any]	    =	flax_key_tuple[:-1] + ('''running_var''',)

												if "batch_stats" in flax_state:
																		__UpperCAmelCase			:							List[Any]	    =	'''.'''.join(flax_key_tuple[1:]      )  # Remove the params/batch_stats header
												else:
																		__UpperCAmelCase			:							Optional[Any]	    =	'''.'''.join(lowercase_      )

												# We also need to look at `pt_model_dict` and see if there are keys requiring further transformation.
												__UpperCAmelCase			:							int	    =	{}
												# New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
												for key in pt_model_dict:
																		__UpperCAmelCase			:							Union[str, Any]	    =	key.split('''.'''      )
																		__UpperCAmelCase			:							Optional[int]	    =	None
																		if key_components[-3::2] == ["parametrizations", "original0"]:
																								__UpperCAmelCase			:							Optional[int]	    =	key_components[-2] + '''_g'''
																		elif key_components[-3::2] == ["parametrizations", "original1"]:
																								__UpperCAmelCase			:							int	    =	key_components[-2] + '''_v'''
																		if name is not None:
																								__UpperCAmelCase			:							Dict	    =	key_components[:-3] + [name]
																								__UpperCAmelCase			:							Dict	    =	'''.'''.join(lowercase_      )
																								__UpperCAmelCase			:							int	    =	key

												if flax_key in special_pt_names:
																		__UpperCAmelCase			:							Optional[Any]	    =	special_pt_names[flax_key]

												if flax_key in pt_model_dict:
																		if flax_tensor.shape != pt_model_dict[flax_key].shape:
																								raise ValueError(
																								    f"Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected "
																								    f"to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}."      )
																		else:
																								# add weight to pytorch dict
																								__UpperCAmelCase			:							List[Any]	    =	np.asarray(lowercase_      ) if not isinstance(lowercase_      ,       np.ndarray      ) else flax_tensor
																								__UpperCAmelCase			:							int	    =	torch.from_numpy(lowercase_      )
																								# remove from missing keys
																								missing_keys.remove(lowercase_      )
												else:
																		# weight is not expected by PyTorch model
																		unexpected_keys.append(lowercase_      )

						pt_model.load_state_dict(lowercase_      )

						# re-transform missing_keys to list
						__UpperCAmelCase			:							Optional[Any]	    =	list(lowercase_      )

						if len(lowercase_      ) > 0:
												logger.warning(
												    '''Some weights of the Flax model were not used when initializing the PyTorch model'''
												    f" {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing"
												    f" {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture"
												    ''' (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This'''
												    f" IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect"
												    ''' to be exactly identical (e.g. initializing a BertForSequenceClassification model from a'''
												    ''' FlaxBertForSequenceClassification model).'''      )
						else:
												logger.warning(f"All Flax model weights were used when initializing {pt_model.__class__.__name__}.\n"      )
						if len(lowercase_      ) > 0:
												logger.warning(
												    f"Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly"
												    f" initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to"
												    ''' use it for predictions and inference.'''      )
						else:
												logger.warning(
												    f"All the weights of {pt_model.__class__.__name__} were initialized from the Flax model.\n"
												    '''If your task is similar to the task the model of the checkpoint was trained on, '''
												    f"you can already use {pt_model.__class__.__name__} for predictions without further training."      )

						return pt_model





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase       		=    get_tests_dir("""fixtures/test_sentencepiece.model""")




@require_sentencepiece
@require_tokenizers
class 							lowerCamelCase   (  _UpperCamelCase							,						unittest.TestCase ):
	_lowerCAmelCase     :		Union[str, Any]										=						XLNetTokenizer
	_lowerCAmelCase     :		int										=						XLNetTokenizerFast
	_lowerCAmelCase     :		Optional[Any]										=						True
	_lowerCAmelCase     :		Optional[Any]										=						True


	def  A(    self):
							super().setUp()

							# We have a SentencePiece fixture for testing
							__UpperCAmelCase			:							Optional[Any]	    =	XLNetTokenizer(lowercase__				,     keep_accents=lowercase__)
							tokenizer.sanitize_special_tokens()
							tokenizer.save_pretrained(self.tmpdirname)


	def  A(    self):
							__UpperCAmelCase			:							Tuple	    =	'''<s>'''
							__UpperCAmelCase			:							List[str]	    =	1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase__)				,     lowercase__)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase__)				,     lowercase__)


	def  A(    self):
							__UpperCAmelCase			:							Union[str, Any]	    =	list(self.get_tokenizer().get_vocab().keys())

							self.assertEqual(vocab_keys[0]				,     '''<unk>''')
							self.assertEqual(vocab_keys[1]				,     '''<s>''')
							self.assertEqual(vocab_keys[-1]				,     '''<eod>''')
							self.assertEqual(len(lowercase__)				,     1_0_0_6)


	def  A(    self):
							self.assertEqual(self.get_tokenizer().vocab_size				,     1_0_0_0)


	def  A(    self):
							__UpperCAmelCase			:							List[str]	    =	XLNetTokenizer(lowercase__				,     keep_accents=lowercase__)

							__UpperCAmelCase			:							Optional[Any]	    =	tokenizer.tokenize('''This is a test''')
							self.assertListEqual(lowercase__				,     ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

							self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__)				,     [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2])

							__UpperCAmelCase			:							Tuple	    =	tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
							self.assertListEqual(
							    lowercase__				,     [
							        SPIECE_UNDERLINE + '''I''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''9''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''s''',
							        '''é''',
							        '''.''',
							    ]				,     )
							__UpperCAmelCase			:							Union[str, Any]	    =	tokenizer.convert_tokens_to_ids(lowercase__)
							self.assertListEqual(lowercase__				,     [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4])

							__UpperCAmelCase			:							int	    =	tokenizer.convert_ids_to_tokens(lowercase__)
							self.assertListEqual(
							    lowercase__				,     [
							        SPIECE_UNDERLINE + '''I''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''<unk>''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''s''',
							        '''<unk>''',
							        '''.''',
							    ]				,     )


	def  A(    self):
							__UpperCAmelCase			:							str	    =	XLNetTokenizer(lowercase__				,     do_lower_case=lowercase__)
							__UpperCAmelCase			:							int	    =	tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
							self.assertListEqual(
							    lowercase__				,     [
							        SPIECE_UNDERLINE + '''''',
							        '''i''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''9''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''se''',
							        '''.''',
							    ]				,     )
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')				,     ['''▁he''', '''ll''', '''o'''])


	def  A(    self):
							__UpperCAmelCase			:							Optional[Any]	    =	XLNetTokenizer(lowercase__				,     do_lower_case=lowercase__)
							__UpperCAmelCase			:							Union[str, Any]	    =	tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
							self.assertListEqual(
							    lowercase__				,     [
							        SPIECE_UNDERLINE + '''I''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''9''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''se''',
							        '''.''',
							    ]				,     )


	@slow
	def  A(    self):
							__UpperCAmelCase			:							Tuple	    =	XLNetTokenizer.from_pretrained('''xlnet-base-cased''')

							__UpperCAmelCase			:							Optional[int]	    =	tokenizer.encode('''sequence builders'''				,     add_special_tokens=lowercase__)
							__UpperCAmelCase			:							Optional[int]	    =	tokenizer.encode('''multi-sequence build'''				,     add_special_tokens=lowercase__)

							__UpperCAmelCase			:							Union[str, Any]	    =	tokenizer.build_inputs_with_special_tokens(lowercase__)
							__UpperCAmelCase			:							Optional[Any]	    =	tokenizer.build_inputs_with_special_tokens(lowercase__				,     lowercase__)

							assert encoded_sentence == text + [4, 3]
							assert encoded_pair == text + [4] + text_a + [4, 3]


	@slow
	def  A(    self):
							# fmt: off
							__UpperCAmelCase			:							Optional[Any]	    =	{'''input_ids''': [[1_7, 2_1_4_4_2, 2_7_0, 1_7, 1_0, 1_4_6_4_5, 3_1_8, 3_4, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 7_7_5_2, 2_2_0_1_8, 2_3, 2_1, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 3_3_5_2, 1_4_4_3_1, 1_3, 5_5_0_0, 1_1, 1_1_7_6, 5_8_0, 1_3, 1_6_8_1_9, 4_7_9_7, 2_3, 1_7, 1_0, 1_7_1_3_5, 6_5_8, 1_9, 4_5_7, 7_9_3_2, 1_3, 1_8_4, 1_9, 3_1_5_4, 1_7_1_3_5, 6_4_6_8, 1_9, 1_4_0_4, 1_2_2_6_9, 1_9, 4_2_2_9, 5_3_5_6, 1_6_2_6_4, 4_6, 1_9, 1_7, 2_0_5_4_5, 1_0_3_9_5, 9, 9, 9, 1_1, 2_8, 6_4_2_1, 9_5_3_1, 2_0_7_2_9, 1_7, 1_0, 3_5_3, 1_7_0_2_2, 1_1, 2_1, 6_4_2_1, 9_5_3_1, 1_6_9_4_9, 1_7, 1_0, 1_1_5_0_9, 7_5_3, 1_1, 3_3, 9_5, 2_4_2_1, 7_3_8_5, 9_5_6, 1_4_4_3_1, 2_6_2_6, 2_5, 8_4_2, 7_3_8_5, 4_8_3_6, 2_1, 1_4_2_9, 2_2_7_2, 9_8_5_5, 3_1_2_0, 1_6_1, 2_4_7_3_8, 1_9, 1_3_2_0_3, 6_5_8, 2_1_8, 7_8_7, 2_1, 4_3_0, 1_8_4_8_2, 8_4_7, 2_6_3_7, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2_2, 2_2_1_7_8, 2_7, 1_0_6_4, 2_2, 9_5_6, 1_3, 1_1_1_0_1, 1_4_2_9, 5_8_5_4, 2_4_3_1_3, 1_8_9_5_3, 4_0, 4_2_2, 2_4_3_6_6, 6_8, 1_7_5_8, 3_7, 1_0_4_8_3, 1_4_2_5_7, 3_1, 2_0_7, 2_6_3, 2_1, 2_0_3, 3_7_7_3, 2_5, 7_1, 9_7_3_5, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2, 2_0_4_9, 3_4_4_2, 1_7, 1_3_8_9_4, 3_3_8_0, 2_3, 9_5, 1_8, 1_7_6_3_4, 2_2_8_8, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=lowercase__				,     model_name='''xlnet-base-cased'''				,     revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''				,     )

