




"""simple docstring"""

import re
import string
from collections import Counter

import sacrebleu
import sacremoses
from packaging import version

import datasets


_lowerCamelCase  :			List[Any]        =       '\n@inproceedings{xu-etal-2016-optimizing,\n    title = {Optimizing Statistical Machine Translation for Text Simplification},\n    authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris},\n    journal = {Transactions of the Association for Computational Linguistics},\n    volume = {4},\n    year={2016},\n    url = {https://www.aclweb.org/anthology/Q16-1029},\n    pages = {401--415\n},\n@inproceedings{post-2018-call,\n    title = \"A Call for Clarity in Reporting {BLEU} Scores\",\n    author = \"Post, Matt\",\n    booktitle = \"Proceedings of the Third Conference on Machine Translation: Research Papers\",\n    month = oct,\n    year = \"2018\",\n    address = \"Belgium, Brussels\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://www.aclweb.org/anthology/W18-6319\",\n    pages = \"186--191\",\n}\n'

_lowerCamelCase  :			Union[str, Any]        =       '\\nWIKI_SPLIT is the combination of three metrics SARI, EXACT and SACREBLEU\nIt can be used to evaluate the quality of machine-generated texts.\n'


_lowerCamelCase  :			Union[str, Any]        =       '\nCalculates sari score (between 0 and 100) given a list of source and predicted\nsentences, and a list of lists of reference sentences. It also computes the BLEU score as well as the exact match score.\nArgs:\n    sources: list of source sentences where each sentence should be a string.\n    predictions: list of predicted sentences where each sentence should be a string.\n    references: list of lists of reference sentences where each sentence should be a string.\nReturns:\n    sari: sari score\n    sacrebleu: sacrebleu score\n    exact: exact score\n\nExamples:\n    >>> sources=[\"About 95 species are currently accepted .\"]\n    >>> predictions=[\"About 95 you now get in .\"]\n    >>> references=[[\"About 95 species are currently known .\"]]\n    >>> wiki_split = datasets.load_metric(\"wiki_split\")\n    >>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references)\n    >>> print(results)\n    {\'sari\': 21.805555555555557, \'sacrebleu\': 14.535768424205482, \'exact\': 0.0}\n'

def 		lowercase_      (		_UpperCAmelCase	):




	"""simple docstring"""






	def remove_articles(_UpperCAmelCase	):
		A_  :				Union[str, Any]	   =		re.compile(r'''\b(a|an|the)\b'''  ,      re.UNICODE	)
		return re.sub(_lowerCAmelCase  ,      ''' '''  ,      _lowerCAmelCase	)

	def white_space_fix(_UpperCAmelCase	):
		return " ".join(text.split()	)

	def remove_punc(_UpperCAmelCase	):
		A_  :				List[Any]	   =		set(string.punctuation	)
		return "".join(ch for ch in text if ch not in exclude	)

	def lower(_UpperCAmelCase	):
		return text.lower()

	return white_space_fix(remove_articles(remove_punc(lower(_lowerCAmelCase	)	)	)	)

def 		lowercase_      (		_UpperCAmelCase  ,      _UpperCAmelCase	):




	"""simple docstring"""






	return int(normalize_answer(_lowerCAmelCase	) == normalize_answer(_lowerCAmelCase	)	)

def 		lowercase_      (		_UpperCAmelCase  ,      _UpperCAmelCase	):




	"""simple docstring"""






	A_  :				Union[str, Any]	   =		[any(compute_exact(_lowerCAmelCase  ,      _lowerCAmelCase	) for ref in refs	) for pred, refs in zip(_lowerCAmelCase  ,      _lowerCAmelCase	)]
	return (sum(_lowerCAmelCase	) / len(_lowerCAmelCase	)) * 100

def 		lowercase_      (		_UpperCAmelCase  ,      _UpperCAmelCase  ,      _UpperCAmelCase  ,      _UpperCAmelCase	):




	"""simple docstring"""






	A_  :				Union[str, Any]	   =		[rgram for rgrams in rgramslist for rgram in rgrams]
	A_  :				Tuple	   =		Counter(_lowerCAmelCase	)

	A_  :				str	   =		Counter(_lowerCAmelCase	)
	A_  :				List[Any]	   =		Counter()
	for sgram, scount in sgramcounter.items():
		A_  :				Optional[Any]	   =		scount * numref

	A_  :				Optional[Any]	   =		Counter(_lowerCAmelCase	)
	A_  :				Union[str, Any]	   =		Counter()
	for cgram, ccount in cgramcounter.items():
		A_  :				Union[str, Any]	   =		ccount * numref

	# KEEP
	A_  :				Dict	   =		sgramcounter_rep & cgramcounter_rep
	A_  :				Tuple	   =		keepgramcounter_rep & rgramcounter
	A_  :				Dict	   =		sgramcounter_rep & rgramcounter

	A_  :				Tuple	   =		0
	A_  :				Optional[Any]	   =		0
	for keepgram in keepgramcountergood_rep:
		keeptmpscorea += keepgramcountergood_rep[keepgram] / keepgramcounter_rep[keepgram]
		# Fix an alleged bug [2] in the keep score computation.
		# keeptmpscore2 += keepgramcountergood_rep[keepgram] / keepgramcounterall_rep[keepgram]
		keeptmpscorea += keepgramcountergood_rep[keepgram]
	# Define 0/0=1 instead of 0 to give higher scores for predictions that match
	#      a target exactly.
	A_  :				int	   =		1
	A_  :				Tuple	   =		1
	if len(_lowerCAmelCase	) > 0:
		A_  :				Any	   =		keeptmpscorea / len(_lowerCAmelCase	)
	if len(_lowerCAmelCase	) > 0:
		# Fix an alleged bug [2] in the keep score computation.
		# keepscore_recall = keeptmpscore2 / len(keepgramcounterall_rep)
		A_  :				str	   =		keeptmpscorea / sum(keepgramcounterall_rep.values()	)
	A_  :				List[str]	   =		0
	if keepscore_precision > 0 or keepscore_recall > 0:
		A_  :				Any	   =		2 * keepscore_precision * keepscore_recall / (keepscore_precision + keepscore_recall)

	# DELETION
	A_  :				Optional[Any]	   =		sgramcounter_rep - cgramcounter_rep
	A_  :				Tuple	   =		delgramcounter_rep - rgramcounter
	A_  :				Tuple	   =		sgramcounter_rep - rgramcounter
	A_  :				Any	   =		0
	A_  :				Dict	   =		0
	for delgram in delgramcountergood_rep:
		deltmpscorea += delgramcountergood_rep[delgram] / delgramcounter_rep[delgram]
		deltmpscorea += delgramcountergood_rep[delgram] / delgramcounterall_rep[delgram]
	# Define 0/0=1 instead of 0 to give higher scores for predictions that match
	# a target exactly.
	A_  :				List[Any]	   =		1
	if len(_lowerCAmelCase	) > 0:
		A_  :				Any	   =		deltmpscorea / len(_lowerCAmelCase	)

	# ADDITION
	A_  :				int	   =		set(_lowerCAmelCase	) - set(_lowerCAmelCase	)
	A_  :				Union[str, Any]	   =		set(_lowerCAmelCase	) & set(_lowerCAmelCase	)
	A_  :				int	   =		set(_lowerCAmelCase	) - set(_lowerCAmelCase	)

	A_  :				Any	   =		0
	for addgram in addgramcountergood:
		addtmpscore += 1

	# Define 0/0=1 instead of 0 to give higher scores for predictions that match
	# a target exactly.
	A_  :				int	   =		1
	A_  :				Optional[Any]	   =		1
	if len(_lowerCAmelCase	) > 0:
		A_  :				List[str]	   =		addtmpscore / len(_lowerCAmelCase	)
	if len(_lowerCAmelCase	) > 0:
		A_  :				Tuple	   =		addtmpscore / len(_lowerCAmelCase	)
	A_  :				int	   =		0
	if addscore_precision > 0 or addscore_recall > 0:
		A_  :				List[Any]	   =		2 * addscore_precision * addscore_recall / (addscore_precision + addscore_recall)

	return (keepscore, delscore_precision, addscore)

def 		lowercase_      (		_UpperCAmelCase  ,      _UpperCAmelCase  ,      _UpperCAmelCase	):




	"""simple docstring"""






	A_  :				Union[str, Any]	   =		len(_lowerCAmelCase	)

	A_  :				List[Any]	   =		ssent.split(''' '''	)
	A_  :				Union[str, Any]	   =		csent.split(''' '''	)
	A_  :				Dict	   =		[]
	A_  :				Dict	   =		[]
	A_  :				List[str]	   =		[]
	A_  :				List[Any]	   =		[]
	A_  :				int	   =		[]
	A_  :				Optional[int]	   =		[]

	A_  :				Union[str, Any]	   =		[]
	A_  :				Any	   =		[]
	A_  :				List[Any]	   =		[]
	A_  :				List[str]	   =		[]
	for rsent in rsents:
		A_  :				Any	   =		rsent.split(''' '''	)
		A_  :				Union[str, Any]	   =		[]
		A_  :				Any	   =		[]
		A_  :				Any	   =		[]
		ragramslist.append(_lowerCAmelCase	)
		for i in range(0  ,      len(_lowerCAmelCase	) - 1	):
			if i < len(_lowerCAmelCase	) - 1:
				A_  :				int	   =		ragrams[i] + ''' ''' + ragrams[i + 1]
				ragrams.append(_lowerCAmelCase	)
			if i < len(_lowerCAmelCase	) - 2:
				A_  :				Tuple	   =		ragrams[i] + ''' ''' + ragrams[i + 1] + ''' ''' + ragrams[i + 2]
				ragrams.append(_lowerCAmelCase	)
			if i < len(_lowerCAmelCase	) - 3:
				A_  :				List[Any]	   =		ragrams[i] + ''' ''' + ragrams[i + 1] + ''' ''' + ragrams[i + 2] + ''' ''' + ragrams[i + 3]
				ragrams.append(_lowerCAmelCase	)
		ragramslist.append(_lowerCAmelCase	)
		ragramslist.append(_lowerCAmelCase	)
		ragramslist.append(_lowerCAmelCase	)

	for i in range(0  ,      len(_lowerCAmelCase	) - 1	):
		if i < len(_lowerCAmelCase	) - 1:
			A_  :				Dict	   =		sagrams[i] + ''' ''' + sagrams[i + 1]
			sagrams.append(_lowerCAmelCase	)
		if i < len(_lowerCAmelCase	) - 2:
			A_  :				Any	   =		sagrams[i] + ''' ''' + sagrams[i + 1] + ''' ''' + sagrams[i + 2]
			sagrams.append(_lowerCAmelCase	)
		if i < len(_lowerCAmelCase	) - 3:
			A_  :				List[str]	   =		sagrams[i] + ''' ''' + sagrams[i + 1] + ''' ''' + sagrams[i + 2] + ''' ''' + sagrams[i + 3]
			sagrams.append(_lowerCAmelCase	)

	for i in range(0  ,      len(_lowerCAmelCase	) - 1	):
		if i < len(_lowerCAmelCase	) - 1:
			A_  :				int	   =		cagrams[i] + ''' ''' + cagrams[i + 1]
			cagrams.append(_lowerCAmelCase	)
		if i < len(_lowerCAmelCase	) - 2:
			A_  :				Any	   =		cagrams[i] + ''' ''' + cagrams[i + 1] + ''' ''' + cagrams[i + 2]
			cagrams.append(_lowerCAmelCase	)
		if i < len(_lowerCAmelCase	) - 3:
			A_  :				Dict	   =		cagrams[i] + ''' ''' + cagrams[i + 1] + ''' ''' + cagrams[i + 2] + ''' ''' + cagrams[i + 3]
			cagrams.append(_lowerCAmelCase	)

	((A_)   , (A_)   , (A_))  :				Optional[Any]	   =		SARIngram(_lowerCAmelCase  ,      _lowerCAmelCase  ,      _lowerCAmelCase  ,      _lowerCAmelCase	)
	((A_)   , (A_)   , (A_))  :				Optional[Any]	   =		SARIngram(_lowerCAmelCase  ,      _lowerCAmelCase  ,      _lowerCAmelCase  ,      _lowerCAmelCase	)
	((A_)   , (A_)   , (A_))  :				str	   =		SARIngram(_lowerCAmelCase  ,      _lowerCAmelCase  ,      _lowerCAmelCase  ,      _lowerCAmelCase	)
	((A_)   , (A_)   , (A_))  :				str	   =		SARIngram(_lowerCAmelCase  ,      _lowerCAmelCase  ,      _lowerCAmelCase  ,      _lowerCAmelCase	)
	A_  :				Any	   =		sum([keepascore, keepascore, keepascore, keepascore]	) / 4
	A_  :				str	   =		sum([delascore, delascore, delascore, delascore]	) / 4
	A_  :				List[str]	   =		sum([addascore, addascore, addascore, addascore]	) / 4
	A_  :				int	   =		(avgkeepscore + avgdelscore + avgaddscore) / 3
	return finalscore

def 		lowercase_      (		_UpperCAmelCase  ,      _UpperCAmelCase = True  ,      _UpperCAmelCase = "13a"  ,      _UpperCAmelCase = True	):




	"""simple docstring"""






	if lowercase:
		A_  :				Any	   =		sentence.lower()

	if tokenizer in ["13a", "intl"]:
		if version.parse(sacrebleu.__version__	).major >= 2:
			A_  :				Optional[Any]	   =		sacrebleu.metrics.bleu._get_tokenizer(_lowerCAmelCase	)()(_lowerCAmelCase	)
		else:
			A_  :				List[Any]	   =		sacrebleu.TOKENIZERS[tokenizer]()(_lowerCAmelCase	)
	elif tokenizer == "moses":
		A_  :				Union[str, Any]	   =		sacremoses.MosesTokenizer().tokenize(_lowerCAmelCase  ,      return_str=_lowerCAmelCase  ,      escape=_lowerCAmelCase	)
	elif tokenizer == "penn":
		A_  :				Tuple	   =		sacremoses.MosesTokenizer().penn_tokenize(_lowerCAmelCase  ,      return_str=_lowerCAmelCase	)
	else:
		A_  :				Dict	   =		sentence

	if not return_str:
		A_  :				Optional[int]	   =		normalized_sent.split()

	return normalized_sent

def 		lowercase_      (		_UpperCAmelCase  ,      _UpperCAmelCase  ,      _UpperCAmelCase	):




	"""simple docstring"""






	if not (len(_lowerCAmelCase	) == len(_lowerCAmelCase	) == len(_lowerCAmelCase	)):
		raise ValueError('''Sources length must match predictions and references lengths.'''	)
	A_  :				List[str]	   =		0
	for src, pred, refs in zip(_lowerCAmelCase  ,      _lowerCAmelCase  ,      _lowerCAmelCase	):
		sari_score += SARIsent(normalize(_lowerCAmelCase	)  ,      normalize(_lowerCAmelCase	)  ,      [normalize(_lowerCAmelCase	) for sent in refs]	)
	A_  :				Tuple	   =		sari_score / len(_lowerCAmelCase	)
	return 100 * sari_score

def 		lowercase_      (		_UpperCAmelCase  ,      _UpperCAmelCase  ,      _UpperCAmelCase="exp"  ,      _UpperCAmelCase=None  ,      _UpperCAmelCase=False  ,      _UpperCAmelCase=False  ,      _UpperCAmelCase=False  ,      ):




	"""simple docstring"""






	A_  :				str	   =		len(references[0]	)
	if any(len(_lowerCAmelCase	) != references_per_prediction for refs in references	):
		raise ValueError('''Sacrebleu requires the same number of references for each prediction'''	)
	A_  :				Optional[int]	   =		[[refs[i] for refs in references] for i in range(_lowerCAmelCase	)]
	A_  :				List[Any]	   =		sacrebleu.corpus_bleu(
	    _lowerCAmelCase  ,      _lowerCAmelCase  ,      smooth_method=_lowerCAmelCase  ,      smooth_value=_lowerCAmelCase  ,      force=_lowerCAmelCase  ,      lowercase=_lowerCAmelCase  ,      use_effective_order=_lowerCAmelCase  ,      )
	return output.score

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,	_KWARGS_DESCRIPTION		)
class        lowercase      (    datasets.Metric		):







				def 					a_  (							self   :					Tuple       ):



					"""simple docstring"""

					return datasets.MetricInfo(
					    description=_DESCRIPTION      ,     citation=_CITATION      ,     inputs_description=_KWARGS_DESCRIPTION      ,     features=datasets.Features(
					        {
					            '''predictions''': datasets.Value('''string'''      ,     id='''sequence'''       ),
					            '''references''': datasets.Sequence(datasets.Value('''string'''      ,     id='''sequence'''       )      ,     id='''references'''       ),
					        }       )      ,     codebase_urls=[
					        '''https://github.com/huggingface/transformers/blob/master/src/transformers/data/metrics/squad_metrics.py''',
					        '''https://github.com/cocoxu/simplification/blob/master/SARI.py''',
					        '''https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py''',
					        '''https://github.com/mjpost/sacreBLEU''',
					    ]      ,     reference_urls=[
					        '''https://www.aclweb.org/anthology/Q16-1029.pdf''',
					        '''https://github.com/mjpost/sacreBLEU''',
					        '''https://en.wikipedia.org/wiki/BLEU''',
					        '''https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213''',
					    ]      ,     )







				def 					a_  (							self   :					Tuple      ,     _lowerCamelCase   :					List[str]      ,     _lowerCamelCase   :					List[Any]      ,     _lowerCamelCase   :					Any       ):



					"""simple docstring"""

					A_  :				Optional[Any]	   =		{}
					result.update({'''sari''': compute_sari(sources=_lowerCamelCase      ,     predictions=_lowerCamelCase      ,     references=_lowerCamelCase       )}       )
					result.update({'''sacrebleu''': compute_sacrebleu(predictions=_lowerCamelCase      ,     references=_lowerCamelCase       )}       )
					result.update({'''exact''': compute_em(predictions=_lowerCamelCase      ,     references=_lowerCamelCase       )}       )
					return result











"""simple docstring"""

import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCamelCase  :			Tuple        =       get_tests_dir('fixtures/test_sentencepiece.model')

@require_sentencepiece
@require_tokenizers
class        lowercase      (    __UpperCAmelCase     ,	unittest.TestCase):
				__lowerCAmelCase    :    Tuple         =   XLNetTokenizer
				__lowerCAmelCase    :    Optional[int]         =   XLNetTokenizerFast
				__lowerCAmelCase    :    Union[str, Any]         =   True
				__lowerCAmelCase    :    Any         =   True







				def 					a_  (							self   :					List[str]       ):



					"""simple docstring"""

					super().setUp()

					# We have a SentencePiece fixture for testing
					A_  :				Optional[Any]	   =		XLNetTokenizer(_lowerCamelCase      ,     keep_accents=_lowerCamelCase       )
					tokenizer.sanitize_special_tokens()
					tokenizer.save_pretrained(self.tmpdirname       )







				def 					a_  (							self   :					List[str]       ):



					"""simple docstring"""

					A_  :				Optional[int]	   =		'''<s>'''
					A_  :				int	   =		1

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCamelCase       )      ,     _lowerCamelCase       )
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCamelCase       )      ,     _lowerCamelCase       )







				def 					a_  (							self   :					str       ):



					"""simple docstring"""

					A_  :				Any	   =		list(self.get_tokenizer().get_vocab().keys()       )

					self.assertEqual(vocab_keys[0]      ,     '''<unk>'''       )
					self.assertEqual(vocab_keys[1]      ,     '''<s>'''       )
					self.assertEqual(vocab_keys[-1]      ,     '''<eod>'''       )
					self.assertEqual(len(_lowerCamelCase       )      ,     10_06       )







				def 					a_  (							self   :					Optional[int]       ):



					"""simple docstring"""

					self.assertEqual(self.get_tokenizer().vocab_size      ,     10_00       )







				def 					a_  (							self   :					Optional[Any]       ):



					"""simple docstring"""

					A_  :				Optional[int]	   =		XLNetTokenizer(_lowerCamelCase      ,     keep_accents=_lowerCamelCase       )

					A_  :				Union[str, Any]	   =		tokenizer.tokenize('''This is a test'''       )
					self.assertListEqual(_lowerCamelCase      ,     ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']       )

					self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase       )      ,     [2_85, 46, 10, 1_70, 3_82]       )

					A_  :				List[str]	   =		tokenizer.tokenize('''I was born in 92000, and this is falsé.'''       )
					self.assertListEqual(
					    _lowerCamelCase      ,     [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''9''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''s''',
					        '''é''',
					        '''.''',
					    ]      ,     )
					A_  :				int	   =		tokenizer.convert_tokens_to_ids(_lowerCamelCase       )
					self.assertListEqual(_lowerCamelCase      ,     [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]       )

					A_  :				Union[str, Any]	   =		tokenizer.convert_ids_to_tokens(_lowerCamelCase       )
					self.assertListEqual(
					    _lowerCamelCase      ,     [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''<unk>''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''s''',
					        '''<unk>''',
					        '''.''',
					    ]      ,     )







				def 					a_  (							self   :					List[str]       ):



					"""simple docstring"""

					A_  :				Optional[Any]	   =		XLNetTokenizer(_lowerCamelCase      ,     do_lower_case=_lowerCamelCase       )
					A_  :				str	   =		tokenizer.tokenize('''I was born in 92000, and this is falsé.'''       )
					self.assertListEqual(
					    _lowerCamelCase      ,     [
					        SPIECE_UNDERLINE + '''''',
					        '''i''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''9''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''se''',
					        '''.''',
					    ]      ,     )
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )      ,     ['''▁he''', '''ll''', '''o''']       )







				def 					a_  (							self   :					Optional[Any]       ):



					"""simple docstring"""

					A_  :				Optional[int]	   =		XLNetTokenizer(_lowerCamelCase      ,     do_lower_case=_lowerCamelCase       )
					A_  :				Tuple	   =		tokenizer.tokenize('''I was born in 92000, and this is falsé.'''       )
					self.assertListEqual(
					    _lowerCamelCase      ,     [
					        SPIECE_UNDERLINE + '''I''',
					        SPIECE_UNDERLINE + '''was''',
					        SPIECE_UNDERLINE + '''b''',
					        '''or''',
					        '''n''',
					        SPIECE_UNDERLINE + '''in''',
					        SPIECE_UNDERLINE + '''''',
					        '''9''',
					        '''2''',
					        '''0''',
					        '''0''',
					        '''0''',
					        ''',''',
					        SPIECE_UNDERLINE + '''and''',
					        SPIECE_UNDERLINE + '''this''',
					        SPIECE_UNDERLINE + '''is''',
					        SPIECE_UNDERLINE + '''f''',
					        '''al''',
					        '''se''',
					        '''.''',
					    ]      ,     )







				@slow
				def 					a_  (							self   :					Dict       ):



					"""simple docstring"""

					A_  :				int	   =		XLNetTokenizer.from_pretrained('''xlnet-base-cased'''       )

					A_  :				Optional[int]	   =		tokenizer.encode('''sequence builders'''      ,     add_special_tokens=_lowerCamelCase       )
					A_  :				Optional[int]	   =		tokenizer.encode('''multi-sequence build'''      ,     add_special_tokens=_lowerCamelCase       )

					A_  :				Tuple	   =		tokenizer.build_inputs_with_special_tokens(_lowerCamelCase       )
					A_  :				Union[str, Any]	   =		tokenizer.build_inputs_with_special_tokens(_lowerCamelCase      ,     _lowerCamelCase       )

					assert encoded_sentence == text + [4, 3]
					assert encoded_pair == text + [4] + text_a + [4, 3]







				@slow
				def 					a_  (							self   :					List[str]       ):



					"""simple docstring"""

					A_  :				Dict	   =		{'''input_ids''': [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
					# fmt: on

					self.tokenizer_integration_test_util(
					    expected_encoding=_lowerCamelCase      ,     model_name='''xlnet-base-cased'''      ,     revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''      ,     )





