


"""simple docstring"""


import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class 			lowerCamelCase__						(   UpperCAmelCase_							,    unittest.TestCase			):
						lowerCAmelCase       =     BertTokenizer
						lowerCAmelCase       =     BertTokenizerFast
						lowerCAmelCase       =     True
						lowerCAmelCase       =     True
						lowerCAmelCase       =     filter_non_english







						def __a						(							self   :    Optional[Any]		):
								super().setUp()

								A					       = [
								    '[UNK]',
								    '[CLS]',
								    '[SEP]',
								    '[PAD]',
								    '[MASK]',
								    'want',
								    '##want',
								    '##ed',
								    'wa',
								    'un',
								    'runn',
								    '##ing',
								    ',',
								    'low',
								    'lowest',
								]
								A					       = os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES['vocab_file']		)
								with open(self.vocab_file			,			'w'			,			encoding='utf-8'		) as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]		)		)







						def __a						(							self   :    int			,			_lowercase   :    Any		):
								A					       = 'UNwant\u00E9d,running'
								A					       = 'unwanted, running'
								return input_text, output_text







						def __a						(							self   :    int		):
								A					       = self.tokenizer_class(self.vocab_file		)

								A					       = tokenizer.tokenize('UNwant\u00E9d,running'		)
								self.assertListEqual(_lowercase			,			['un', '##want', '##ed', ',', 'runn', '##ing']		)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase		)			,			[9, 6, 7, 12, 10, 11]		)







						def __a						(							self   :    Union[str, Any]		):
								if not self.test_rust_tokenizer:
										return

								A					       = self.get_tokenizer()
								A					       = self.get_rust_tokenizer()

								A					       = 'UNwant\u00E9d,running'

								A					       = tokenizer.tokenize(_lowercase		)
								A					       = rust_tokenizer.tokenize(_lowercase		)
								self.assertListEqual(_lowercase			,			_lowercase		)

								A					       = tokenizer.encode(_lowercase			,			add_special_tokens=_lowercase		)
								A					       = rust_tokenizer.encode(_lowercase			,			add_special_tokens=_lowercase		)
								self.assertListEqual(_lowercase			,			_lowercase		)

								A					       = self.get_rust_tokenizer()
								A					       = tokenizer.encode(_lowercase		)
								A					       = rust_tokenizer.encode(_lowercase		)
								self.assertListEqual(_lowercase			,			_lowercase		)

								# With lower casing
								A					       = self.get_tokenizer(do_lower_case=_lowercase		)
								A					       = self.get_rust_tokenizer(do_lower_case=_lowercase		)

								A					       = 'UNwant\u00E9d,running'

								A					       = tokenizer.tokenize(_lowercase		)
								A					       = rust_tokenizer.tokenize(_lowercase		)
								self.assertListEqual(_lowercase			,			_lowercase		)

								A					       = tokenizer.encode(_lowercase			,			add_special_tokens=_lowercase		)
								A					       = rust_tokenizer.encode(_lowercase			,			add_special_tokens=_lowercase		)
								self.assertListEqual(_lowercase			,			_lowercase		)

								A					       = self.get_rust_tokenizer()
								A					       = tokenizer.encode(_lowercase		)
								A					       = rust_tokenizer.encode(_lowercase		)
								self.assertListEqual(_lowercase			,			_lowercase		)







						def __a						(							self   :    List[Any]		):
								A					       = BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'		)			,			['ah', '\u535A', '\u63A8', 'zz']		)







						def __a						(							self   :    int		):
								A					       = BasicTokenizer(do_lower_case=_lowercase		)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '		)			,			['hello', '!', 'how', 'are', 'you', '?']		)
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)			,			['hello']		)







						def __a						(							self   :    List[Any]		):
								A					       = BasicTokenizer(do_lower_case=_lowercase			,			strip_accents=_lowercase		)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,			['hällo', '!', 'how', 'are', 'you', '?']		)
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)			,			['h\u00E9llo']		)







						def __a						(							self   :    Any		):
								A					       = BasicTokenizer(do_lower_case=_lowercase			,			strip_accents=_lowercase		)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,			['hallo', '!', 'how', 'are', 'you', '?']		)
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)			,			['hello']		)







						def __a						(							self   :    Tuple		):
								A					       = BasicTokenizer(do_lower_case=_lowercase		)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,			['hallo', '!', 'how', 'are', 'you', '?']		)
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)			,			['hello']		)







						def __a						(							self   :    int		):
								A					       = BasicTokenizer(do_lower_case=_lowercase		)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '		)			,			['HeLLo', '!', 'how', 'Are', 'yoU', '?']		)







						def __a						(							self   :    Tuple		):
								A					       = BasicTokenizer(do_lower_case=_lowercase			,			strip_accents=_lowercase		)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,			['HäLLo', '!', 'how', 'Are', 'yoU', '?']		)







						def __a						(							self   :    Any		):
								A					       = BasicTokenizer(do_lower_case=_lowercase			,			strip_accents=_lowercase		)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,			['HaLLo', '!', 'how', 'Are', 'yoU', '?']		)







						def __a						(							self   :    List[Any]		):
								A					       = BasicTokenizer(do_lower_case=_lowercase			,			never_split=['[UNK]']		)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'		)			,			['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']		)







						def __a						(							self   :    List[Any]		):
								A					       = BasicTokenizer()
								A					       = 'a\n\'ll !!to?\'d of, can\'t.'
								A					       = ['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
								self.assertListEqual(tokenizer.tokenize(_lowercase		)			,			_lowercase		)







						def __a						(							self   :    List[Any]		):
								A					       = ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

								A					       = {}
								for i, token in enumerate(_lowercase		):
										A					       = i
								A					       = WordpieceTokenizer(vocab=_lowercase			,			unk_token='[UNK]'		)

								self.assertListEqual(tokenizer.tokenize(''		)			,			[]		)

								self.assertListEqual(tokenizer.tokenize('unwanted running'		)			,			['un', '##want', '##ed', 'runn', '##ing']		)

								self.assertListEqual(tokenizer.tokenize('unwantedX running'		)			,			['[UNK]', 'runn', '##ing']		)







						def __a						(							self   :    int		):
								self.assertTrue(_is_whitespace(' '		)		)
								self.assertTrue(_is_whitespace('\t'		)		)
								self.assertTrue(_is_whitespace('\r'		)		)
								self.assertTrue(_is_whitespace('\n'		)		)
								self.assertTrue(_is_whitespace('\u00A0'		)		)

								self.assertFalse(_is_whitespace('A'		)		)
								self.assertFalse(_is_whitespace('-'		)		)







						def __a						(							self   :    Optional[int]		):
								self.assertTrue(_is_control('\u0005'		)		)

								self.assertFalse(_is_control('A'		)		)
								self.assertFalse(_is_control(' '		)		)
								self.assertFalse(_is_control('\t'		)		)
								self.assertFalse(_is_control('\r'		)		)







						def __a						(							self   :    Optional[int]		):
								self.assertTrue(_is_punctuation('-'		)		)
								self.assertTrue(_is_punctuation('$'		)		)
								self.assertTrue(_is_punctuation('`'		)		)
								self.assertTrue(_is_punctuation('.'		)		)

								self.assertFalse(_is_punctuation('A'		)		)
								self.assertFalse(_is_punctuation(' '		)		)







						def __a						(							self   :    Union[str, Any]		):
								A					       = self.get_tokenizer()
								A					       = self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(_lowercase		) for t in ['Test', '\xad', 'test']]			,			[['[UNK]'], [], ['[UNK]']]		)

								self.assertListEqual(
								    [rust_tokenizer.tokenize(_lowercase		) for t in ['Test', '\xad', 'test']]			,			[['[UNK]'], [], ['[UNK]']]		)







						@slow
						def __a						(							self   :    List[str]		):
								A					       = self.tokenizer_class.from_pretrained('bert-base-uncased'		)

								A					       = tokenizer.encode('sequence builders'			,			add_special_tokens=_lowercase		)
								A					       = tokenizer.encode('multi-sequence build'			,			add_special_tokens=_lowercase		)

								A					       = tokenizer.build_inputs_with_special_tokens(_lowercase		)
								A					       = tokenizer.build_inputs_with_special_tokens(_lowercase			,			_lowercase		)

								assert encoded_sentence == [101] + text + [102]
								assert encoded_pair == [101] + text + [102] + text_a + [102]







						def __a						(							self   :    Any		):
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'		):
												A					       = self.rust_tokenizer_class.from_pretrained(_lowercase			,			**_lowercase		)

												A					       = f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
												A					       = tokenizer_r.encode_plus(
												    _lowercase			,			return_attention_mask=_lowercase			,			return_token_type_ids=_lowercase			,			return_offsets_mapping=_lowercase			,			add_special_tokens=_lowercase			,			)

												A					       = tokenizer_r.do_lower_case if hasattr(_lowercase			,			'do_lower_case'		) else False
												A					       = (
												    [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), 'A'),
												        ((1, 2), ','),
												        ((3, 5), 'na'),
												        ((5, 6), '##ï'),
												        ((6, 8), '##ve'),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), 'Allen'),
												        ((21, 23), '##NL'),
												        ((23, 24), '##P'),
												        ((25, 33), 'sentence'),
												        ((33, 34), '.'),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												    if not do_lower_case
												    else [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), 'a'),
												        ((1, 2), ','),
												        ((3, 8), 'naive'),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), 'allen'),
												        ((21, 23), '##nl'),
												        ((23, 24), '##p'),
												        ((25, 33), 'sentence'),
												        ((33, 34), '.'),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												)

												self.assertEqual(
												    [e[1] for e in expected_results]			,			tokenizer_r.convert_ids_to_tokens(tokens['input_ids']		)		)
												self.assertEqual([e[0] for e in expected_results]			,			tokens['offset_mapping']		)







						def __a						(							self   :    List[str]		):
								A					       = ['的', '人', '有']
								A					       = ''.join(_lowercase		)
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'		):
												A					       = True
												A					       = self.tokenizer_class.from_pretrained(_lowercase			,			**_lowercase		)
												A					       = self.rust_tokenizer_class.from_pretrained(_lowercase			,			**_lowercase		)

												A					       = tokenizer_p.encode(_lowercase			,			add_special_tokens=_lowercase		)
												A					       = tokenizer_r.encode(_lowercase			,			add_special_tokens=_lowercase		)

												A					       = tokenizer_r.convert_ids_to_tokens(_lowercase		)
												A					       = tokenizer_p.convert_ids_to_tokens(_lowercase		)

												# it is expected that each Chinese character is not preceded by "##"
												self.assertListEqual(_lowercase			,			_lowercase		)
												self.assertListEqual(_lowercase			,			_lowercase		)

												A					       = False
												A					       = self.rust_tokenizer_class.from_pretrained(_lowercase			,			**_lowercase		)
												A					       = self.tokenizer_class.from_pretrained(_lowercase			,			**_lowercase		)

												A					       = tokenizer_r.encode(_lowercase			,			add_special_tokens=_lowercase		)
												A					       = tokenizer_p.encode(_lowercase			,			add_special_tokens=_lowercase		)

												A					       = tokenizer_r.convert_ids_to_tokens(_lowercase		)
												A					       = tokenizer_p.convert_ids_to_tokens(_lowercase		)

												# it is expected that only the first Chinese character is not preceded by "##".
												A					       = [
												    f'##{token}' if idx != 0 else token for idx, token in enumerate(_lowercase		)
												]
												self.assertListEqual(_lowercase			,			_lowercase		)
												self.assertListEqual(_lowercase			,			_lowercase		)









"""simple docstring"""


import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class 			lowerCamelCase__						(   UpperCAmelCase_							,    unittest.TestCase			):
						lowerCAmelCase       =     TransfoXLTokenizer
						lowerCAmelCase       =     False
						lowerCAmelCase       =     False







						def __a						(							self   :    Union[str, Any]		):
								super().setUp()

								A					       = [
								    '<unk>',
								    '[CLS]',
								    '[SEP]',
								    'want',
								    'unwanted',
								    'wa',
								    'un',
								    'running',
								    ',',
								    'low',
								    'l',
								]
								A					       = os.path.join(self.tmpdirname			,			VOCAB_FILES_NAMES['vocab_file']		)
								with open(self.vocab_file			,			'w'			,			encoding='utf-8'		) as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]		)		)







						def __a						(							self   :    int			,			**_lowercase   :    List[str]		):
								A					       = True
								return TransfoXLTokenizer.from_pretrained(self.tmpdirname			,			**_lowercase		)







						def __a						(							self   :    int			,			_lowercase   :    List[Any]		):
								A					       = '<unk> UNwanted , running'
								A					       = '<unk> unwanted, running'
								return input_text, output_text







						def __a						(							self   :    List[str]		):
								A					       = TransfoXLTokenizer(vocab_file=self.vocab_file			,			lower_case=_lowercase		)

								A					       = tokenizer.tokenize('<unk> UNwanted , running'		)
								self.assertListEqual(_lowercase			,			['<unk>', 'unwanted', ',', 'running']		)

								self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase		)			,			[0, 4, 8, 7]		)







						def __a						(							self   :    int		):
								A					       = TransfoXLTokenizer(lower_case=_lowercase		)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '		)			,			['hello', '!', 'how', 'are', 'you', '?']		)







						def __a						(							self   :    Optional[Any]		):
								A					       = TransfoXLTokenizer(lower_case=_lowercase		)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo ! how  \n Are yoU ?  '		)			,			['HeLLo', '!', 'how', 'Are', 'yoU', '?']		)







						def __a						(							self   :    str		):
								A					       = TransfoXLTokenizer(lower_case=_lowercase		)
								A					       = 'Hello (bracket) and side-scrolled [and] Henry\'s $5,000 with 3.34 m. What\'s up!?'
								A					       = [
								    'Hello',
								    '(',
								    'bracket',
								    ')',
								    'and',
								    'side',
								    '@-@',
								    'scrolled',
								    '[',
								    'and',
								    ']',
								    'Henry',
								    '\'s',
								    '$',
								    '5',
								    '@,@',
								    '000',
								    'with',
								    '3',
								    '@.@',
								    '34',
								    'm',
								    '.',
								    'What',
								    '\'s',
								    'up',
								    '!',
								    '?',
								]

								self.assertListEqual(tokenizer.tokenize(_lowercase		)			,			_lowercase		)

								self.assertEqual(tokenizer.convert_tokens_to_string(_lowercase		)			,			_lowercase		)







						def __a						(							self   :    Dict		):
								A					       = self.get_tokenizer()
								A					       = len(_lowercase		)

								tokenizer.add_tokens(['new1', 'new2']		)
								tokenizer.move_added_token('new1'			,			1		)

								# Check that moved token is not copied (duplicate)
								self.assertEqual(len(_lowercase		)			,			original_len + 2		)
								# Check that token is moved to specified id
								self.assertEqual(tokenizer.encode('new1'		)			,			[1]		)
								self.assertEqual(tokenizer.decode([1]		)			,			'new1'		)





