





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class       lowercase					(				lowerCamelCase__		,      unittest.TestCase  ):



						"""simple docstring"""

						_UpperCamelCase	:						Tuple    		=						MobileBertTokenizer
						_UpperCamelCase	:						List[Any]    		=						MobileBertTokenizerFast
						_UpperCamelCase	:						List[Any]    		=						True
						_UpperCamelCase	:						Optional[int]    		=						True
						_UpperCamelCase	:						List[str]    		=						filter_non_english
						_UpperCamelCase	:						Any    		=						"""google/mobilebert-uncased"""







						def 		__UpperCAmelCase							(     self      :		Optional[Any]       ):





										'''simple docstring'''







										super().setUp()

										_snake_case			:      Tuple	    = [
										    """[UNK]""",
										    """[CLS]""",
										    """[SEP]""",
										    """[PAD]""",
										    """[MASK]""",
										    """want""",
										    """##want""",
										    """##ed""",
										    """wa""",
										    """un""",
										    """runn""",
										    """##ing""",
										    """,""",
										    """low""",
										    """lowest""",
										]
										_snake_case			:      int	    = os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['vocab_file']       )
										with open(self.vocab_file			,					'w'			,					encoding='utf-8'       ) as vocab_writer:
														vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]       )       )

										_snake_case			:      Optional[Any]	    = [
										    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
										    for tokenizer_def in self.tokenizers_list
										]







						def 		__UpperCAmelCase							(     self      :		str			,					lowerCamelCase_      :		Dict       ):





										'''simple docstring'''







										_snake_case			:      Any	    = """UNwant\u00E9d,running"""
										_snake_case			:      Optional[Any]	    = """unwanted, running"""
										return input_text, output_text







						def 		__UpperCAmelCase							(     self      :		Any       ):





										'''simple docstring'''







										_snake_case			:      List[str]	    = self.tokenizer_class(self.vocab_file       )

										_snake_case			:      Union[str, Any]	    = tokenizer.tokenize('UNwant\u00E9d,running'       )
										self.assertListEqual(snake_case__			,					['un', '##want', '##ed', ',', 'runn', '##ing']       )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__       )			,					[9, 6, 7, 12, 10, 11]       )







						def 		__UpperCAmelCase							(     self      :		str       ):





										'''simple docstring'''







										if not self.test_rust_tokenizer:
														return

										_snake_case			:      int	    = self.get_tokenizer()
										_snake_case			:      int	    = self.get_rust_tokenizer()

										_snake_case			:      List[Any]	    = """UNwant\u00E9d,running"""

										_snake_case			:      Tuple	    = tokenizer.tokenize(snake_case__       )
										_snake_case			:      str	    = rust_tokenizer.tokenize(snake_case__       )
										self.assertListEqual(snake_case__			,					snake_case__       )

										_snake_case			:      Union[str, Any]	    = tokenizer.encode(snake_case__			,					add_special_tokens=snake_case__       )
										_snake_case			:      Any	    = rust_tokenizer.encode(snake_case__			,					add_special_tokens=snake_case__       )
										self.assertListEqual(snake_case__			,					snake_case__       )

										_snake_case			:      int	    = self.get_rust_tokenizer()
										_snake_case			:      Any	    = tokenizer.encode(snake_case__       )
										_snake_case			:      Tuple	    = rust_tokenizer.encode(snake_case__       )
										self.assertListEqual(snake_case__			,					snake_case__       )

										# With lower casing
										_snake_case			:      List[Any]	    = self.get_tokenizer(do_lower_case=snake_case__       )
										_snake_case			:      Any	    = self.get_rust_tokenizer(do_lower_case=snake_case__       )

										_snake_case			:      Any	    = """UNwant\u00E9d,running"""

										_snake_case			:      List[str]	    = tokenizer.tokenize(snake_case__       )
										_snake_case			:      List[str]	    = rust_tokenizer.tokenize(snake_case__       )
										self.assertListEqual(snake_case__			,					snake_case__       )

										_snake_case			:      Any	    = tokenizer.encode(snake_case__			,					add_special_tokens=snake_case__       )
										_snake_case			:      str	    = rust_tokenizer.encode(snake_case__			,					add_special_tokens=snake_case__       )
										self.assertListEqual(snake_case__			,					snake_case__       )

										_snake_case			:      List[Any]	    = self.get_rust_tokenizer()
										_snake_case			:      Tuple	    = tokenizer.encode(snake_case__       )
										_snake_case			:      str	    = rust_tokenizer.encode(snake_case__       )
										self.assertListEqual(snake_case__			,					snake_case__       )







						def 		__UpperCAmelCase							(     self      :		List[str]       ):





										'''simple docstring'''







										_snake_case			:      str	    = BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'       )			,					['ah', '\u535A', '\u63A8', 'zz']       )







						def 		__UpperCAmelCase							(     self      :		Tuple       ):





										'''simple docstring'''







										_snake_case			:      List[Any]	    = BasicTokenizer(do_lower_case=snake_case__       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '       )			,					['hello', '!', 'how', 'are', 'you', '?']       )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )			,					['hello']       )







						def 		__UpperCAmelCase							(     self      :		Optional[Any]       ):





										'''simple docstring'''







										_snake_case			:      List[str]	    = BasicTokenizer(do_lower_case=snake_case__			,					strip_accents=snake_case__       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )			,					['hällo', '!', 'how', 'are', 'you', '?']       )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )			,					['h\u00E9llo']       )







						def 		__UpperCAmelCase							(     self      :		int       ):





										'''simple docstring'''







										_snake_case			:      Optional[int]	    = BasicTokenizer(do_lower_case=snake_case__			,					strip_accents=snake_case__       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )			,					['hallo', '!', 'how', 'are', 'you', '?']       )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )			,					['hello']       )







						def 		__UpperCAmelCase							(     self      :		List[str]       ):





										'''simple docstring'''







										_snake_case			:      str	    = BasicTokenizer(do_lower_case=snake_case__       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )			,					['hallo', '!', 'how', 'are', 'you', '?']       )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'       )			,					['hello']       )







						def 		__UpperCAmelCase							(     self      :		Union[str, Any]       ):





										'''simple docstring'''







										_snake_case			:      Union[str, Any]	    = BasicTokenizer(do_lower_case=snake_case__       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '       )			,					['HeLLo', '!', 'how', 'Are', 'yoU', '?']       )







						def 		__UpperCAmelCase							(     self      :		int       ):





										'''simple docstring'''







										_snake_case			:      List[Any]	    = BasicTokenizer(do_lower_case=snake_case__			,					strip_accents=snake_case__       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )			,					['HäLLo', '!', 'how', 'Are', 'yoU', '?']       )







						def 		__UpperCAmelCase							(     self      :		Optional[Any]       ):





										'''simple docstring'''







										_snake_case			:      Tuple	    = BasicTokenizer(do_lower_case=snake_case__			,					strip_accents=snake_case__       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '       )			,					['HaLLo', '!', 'how', 'Are', 'yoU', '?']       )







						def 		__UpperCAmelCase							(     self      :		Optional[Any]       ):





										'''simple docstring'''







										_snake_case			:      Optional[int]	    = BasicTokenizer(do_lower_case=snake_case__			,					never_split=['[UNK]']       )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'       )			,					['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']       )







						def 		__UpperCAmelCase							(     self      :		Union[str, Any]       ):





										'''simple docstring'''







										_snake_case			:      int	    = ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

										_snake_case			:      Union[str, Any]	    = {}
										for i, token in enumerate(snake_case__       ):
														_snake_case			:      int	    = i
										_snake_case			:      Union[str, Any]	    = WordpieceTokenizer(vocab=snake_case__			,					unk_token='[UNK]'       )

										self.assertListEqual(tokenizer.tokenize(''       )			,					[]       )

										self.assertListEqual(tokenizer.tokenize('unwanted running'       )			,					['un', '##want', '##ed', 'runn', '##ing']       )

										self.assertListEqual(tokenizer.tokenize('unwantedX running'       )			,					['[UNK]', 'runn', '##ing']       )







						def 		__UpperCAmelCase							(     self      :		Union[str, Any]       ):





										'''simple docstring'''







										self.assertTrue(_is_whitespace(' '       )       )
										self.assertTrue(_is_whitespace('\t'       )       )
										self.assertTrue(_is_whitespace('\r'       )       )
										self.assertTrue(_is_whitespace('\n'       )       )
										self.assertTrue(_is_whitespace('\u00A0'       )       )

										self.assertFalse(_is_whitespace('A'       )       )
										self.assertFalse(_is_whitespace('-'       )       )







						def 		__UpperCAmelCase							(     self      :		int       ):





										'''simple docstring'''







										self.assertTrue(_is_control('\u0005'       )       )

										self.assertFalse(_is_control('A'       )       )
										self.assertFalse(_is_control(' '       )       )
										self.assertFalse(_is_control('\t'       )       )
										self.assertFalse(_is_control('\r'       )       )







						def 		__UpperCAmelCase							(     self      :		Union[str, Any]       ):





										'''simple docstring'''







										self.assertTrue(_is_punctuation('-'       )       )
										self.assertTrue(_is_punctuation('$'       )       )
										self.assertTrue(_is_punctuation('`'       )       )
										self.assertTrue(_is_punctuation('.'       )       )

										self.assertFalse(_is_punctuation('A'       )       )
										self.assertFalse(_is_punctuation(' '       )       )







						def 		__UpperCAmelCase							(     self      :		List[str]       ):





										'''simple docstring'''







										_snake_case			:      Tuple	    = self.get_tokenizer()
										_snake_case			:      int	    = self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(snake_case__       ) for t in ['Test', '\xad', 'test']]			,					[['[UNK]'], [], ['[UNK]']]       )

										self.assertListEqual(
										    [rust_tokenizer.tokenize(snake_case__       ) for t in ['Test', '\xad', 'test']]			,					[['[UNK]'], [], ['[UNK]']]       )







						@slow
						def 		__UpperCAmelCase							(     self      :		int       ):





										'''simple docstring'''







										_snake_case			:      List[str]	    = self.tokenizer_class.from_pretrained('google/mobilebert-uncased'       )

										_snake_case			:      Optional[int]	    = tokenizer.encode('sequence builders'			,					add_special_tokens=snake_case__       )
										_snake_case			:      Optional[Any]	    = tokenizer.encode('multi-sequence build'			,					add_special_tokens=snake_case__       )

										_snake_case			:      int	    = tokenizer.build_inputs_with_special_tokens(snake_case__       )
										_snake_case			:      Any	    = tokenizer.build_inputs_with_special_tokens(snake_case__			,					snake_case__       )

										assert encoded_sentence == [1_01] + text + [1_02]
										assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]







						def 		__UpperCAmelCase							(     self      :		Tuple       ):





										'''simple docstring'''







										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
																		_snake_case			:      Any	    = self.rust_tokenizer_class.from_pretrained(snake_case__			,					**snake_case__       )

																		_snake_case			:      Any	    = f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																		_snake_case			:      Dict	    = tokenizer_r.encode_plus(
																		    snake_case__			,					return_attention_mask=snake_case__			,					return_token_type_ids=snake_case__			,					return_offsets_mapping=snake_case__			,					add_special_tokens=snake_case__			,					)

																		_snake_case			:      str	    = tokenizer_r.do_lower_case if hasattr(snake_case__			,					'do_lower_case'       ) else False
																		_snake_case			:      Any	    = (
																		    [
																		        ((0, 0), tokenizer_r.cls_token),
																		        ((0, 1), """A"""),
																		        ((1, 2), ""","""),
																		        ((3, 5), """na"""),
																		        ((5, 6), """##ï"""),
																		        ((6, 8), """##ve"""),
																		        ((9, 15), tokenizer_r.mask_token),
																		        ((16, 21), """Allen"""),
																		        ((21, 23), """##NL"""),
																		        ((23, 24), """##P"""),
																		        ((25, 33), """sentence"""),
																		        ((33, 34), """."""),
																		        ((0, 0), tokenizer_r.sep_token),
																		    ]
																		    if not do_lower_case
																		    else [
																		        ((0, 0), tokenizer_r.cls_token),
																		        ((0, 1), """a"""),
																		        ((1, 2), ""","""),
																		        ((3, 8), """naive"""),
																		        ((9, 15), tokenizer_r.mask_token),
																		        ((16, 21), """allen"""),
																		        ((21, 23), """##nl"""),
																		        ((23, 24), """##p"""),
																		        ((25, 33), """sentence"""),
																		        ((33, 34), """."""),
																		        ((0, 0), tokenizer_r.sep_token),
																		    ]
																		)

																		self.assertEqual(
																		    [e[1] for e in expected_results]			,					tokenizer_r.convert_ids_to_tokens(tokens['input_ids']       )       )
																		self.assertEqual([e[0] for e in expected_results]			,					tokens['offset_mapping']       )







						def 		__UpperCAmelCase							(     self      :		List[Any]       ):





										'''simple docstring'''







										_snake_case			:      str	    = ["""的""", """人""", """有"""]
										_snake_case			:      Optional[int]	    = """""".join(snake_case__       )
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
																		_snake_case			:      Tuple	    = True
																		_snake_case			:      List[str]	    = self.tokenizer_class.from_pretrained(snake_case__			,					**snake_case__       )
																		_snake_case			:      int	    = self.rust_tokenizer_class.from_pretrained(snake_case__			,					**snake_case__       )

																		_snake_case			:      Dict	    = tokenizer_p.encode(snake_case__			,					add_special_tokens=snake_case__       )
																		_snake_case			:      int	    = tokenizer_r.encode(snake_case__			,					add_special_tokens=snake_case__       )

																		_snake_case			:      Tuple	    = tokenizer_r.convert_ids_to_tokens(snake_case__       )
																		_snake_case			:      Any	    = tokenizer_p.convert_ids_to_tokens(snake_case__       )

																		# it is expected that each Chinese character is not preceded by "##"
																		self.assertListEqual(snake_case__			,					snake_case__       )
																		self.assertListEqual(snake_case__			,					snake_case__       )

																		_snake_case			:      Union[str, Any]	    = False
																		_snake_case			:      Dict	    = self.rust_tokenizer_class.from_pretrained(snake_case__			,					**snake_case__       )
																		_snake_case			:      str	    = self.tokenizer_class.from_pretrained(snake_case__			,					**snake_case__       )

																		_snake_case			:      int	    = tokenizer_r.encode(snake_case__			,					add_special_tokens=snake_case__       )
																		_snake_case			:      Tuple	    = tokenizer_p.encode(snake_case__			,					add_special_tokens=snake_case__       )

																		_snake_case			:      Optional[Any]	    = tokenizer_r.convert_ids_to_tokens(snake_case__       )
																		_snake_case			:      Optional[int]	    = tokenizer_p.convert_ids_to_tokens(snake_case__       )

																		# it is expected that only the first Chinese character is not preceded by "##".
																		_snake_case			:      Dict	    = [
																		    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(snake_case__       )
																		]
																		self.assertListEqual(snake_case__			,					snake_case__       )
																		self.assertListEqual(snake_case__			,					snake_case__       )













import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin
class     UpperCamelCase__					(     lowerCamelCase__				, lowerCamelCase__				, lowerCamelCase__				):






		'''simple docstring'''






		@register_to_config
		def __init__(							self,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__,   snake_case__ = False,   )       -> Any:






									"""simple docstring"""
									super().__init__()

									lowercase_	:   str  				=     nn.Embedding(snake_case__,   snake_case__  )

									lowercase_	:   Dict  				=     nn.Embedding(snake_case__,   snake_case__  )
									lowercase_	:   List[str]  				=     False

									lowercase_	:   Optional[int]  				=     nn.Dropout(p=snake_case__  )

									lowercase_	:   Any  				=     TaConfig(
									    vocab_size=snake_case__,   d_model=snake_case__,   num_heads=snake_case__,   d_kv=snake_case__,   d_ff=snake_case__,   dropout_rate=snake_case__,   feed_forward_proj=snake_case__,   is_decoder=snake_case__,   is_encoder_decoder=snake_case__,   )

									lowercase_	:   List[Any]  				=     nn.ModuleList()
									for lyr_num in range(snake_case__  ):
																lowercase_	:   List[str]  				=     TaBlock(snake_case__  )
																self.encoders.append(snake_case__  )

									lowercase_	:   Tuple  				=     TaLayerNorm(snake_case__  )
									lowercase_	:   Dict  				=     nn.Dropout(p=snake_case__  )






		def 						snake_case__				(							self,   snake_case__,   snake_case__  )       -> Union[str, Any]:






									"""simple docstring"""
									lowercase_	:   Union[str, Any]  				=     self.token_embedder(snake_case__  )

									lowercase_	:   Tuple  				=     encoder_input_tokens.shape[1]
									lowercase_	:   List[str]  				=     torch.arange(snake_case__,   device=encoder_input_tokens.device  )
									x += self.position_encoding(snake_case__  )

									lowercase_	:   Union[str, Any]  				=     self.dropout_pre(snake_case__  )

									# inverted the attention mask
									lowercase_	:   List[str]  				=     encoder_input_tokens.size()
									lowercase_	:   Union[str, Any]  				=     self.get_extended_attention_mask(snake_case__,   snake_case__  )

									for lyr in self.encoders:
																lowercase_	:   Optional[Any]  				=     lyr(snake_case__,   snake_case__  )[0]
									lowercase_	:   Dict  				=     self.layer_norm(snake_case__  )

									return self.dropout_post(snake_case__  ), encoder_inputs_mask