


'''simple docstring'''



import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin





class 							__UpperCamelCase    (		lowerCAmelCase_		,				unittest.TestCase ):
	A_           =   XLMTokenizer
	A_           =   False




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						__a      :	Optional[int]    =  [
						    'l',
						    'o',
						    'w',
						    'e',
						    'r',
						    's',
						    't',
						    'i',
						    'd',
						    'n',
						    'w</w>',
						    'r</w>',
						    't</w>',
						    'lo',
						    'low',
						    'er</w>',
						    'low</w>',
						    'lowest</w>',
						    'newer</w>',
						    'wider</w>',
						    '<unk>',
						]
						__a      :	Optional[int]    =  dict(zip(__a						,    range(len(__a							)							)							)							)
						__a      :	List[Any]    =  ['l o 123', 'lo w 1456', 'e r</w> 1789', '']

						__a      :	Optional[Any]    =  os.path.join(self.tmpdirname						,    VOCAB_FILES_NAMES['vocab_file']							)
						__a      :	Any    =  os.path.join(self.tmpdirname						,    VOCAB_FILES_NAMES['merges_file']							)
						with open(self.vocab_file						,    'w'							) as fp:
											fp.write(json.dumps(__a							)							)
						with open(self.merges_file						,    'w'							) as fp:
											fp.write('\n'.join(__a							)							)




	def 				__UpperCAmelCase							(       self						,    __a							):

						'''simple docstring'''





						__a      :	str    =  'lower newer'
						__a      :	Union[str, Any]    =  'lower newer'
						return input_text, output_text




	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Optional[Any]    =  XLMTokenizer(self.vocab_file						,    self.merges_file							)

						__a      :	List[Any]    =  'lower'
						__a      :	Any    =  ['low', 'er</w>']
						__a      :	Tuple    =  tokenizer.tokenize(__a							)
						self.assertListEqual(__a						,    __a							)

						__a      :	Optional[int]    =  tokens + ['<unk>']
						__a      :	int    =  [14, 15, 20]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__a							)						,    __a							)





	@slow
	def 				__UpperCAmelCase							(       self							):

						'''simple docstring'''





						__a      :	Tuple    =  XLMTokenizer.from_pretrained('xlm-mlm-en-2048'							)

						__a      :	str    =  tokenizer.encode('sequence builders'						,    add_special_tokens=__a							)
						__a      :	Dict    =  tokenizer.encode('multi-sequence build'						,    add_special_tokens=__a							)

						__a      :	int    =  tokenizer.build_inputs_with_special_tokens(__a							)
						__a      :	Optional[int]    =  tokenizer.build_inputs_with_special_tokens(__a						,    __a							)

						assert encoded_sentence == [0] + text + [1]
						assert encoded_pair == [0] + text + [1] + text_a + [1]







'''simple docstring'''



from .imports import is_rich_available


if is_rich_available():
	from rich.traceback import install

	install(show_locals=False)

else:
	raise ModuleNotFoundError('To use the rich extension, install rich with `pip install rich`')



