



'''simple docstring'''





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class __A    (						_lowerCamelCase     ,     unittest.TestCase		):



 '''simple docstring'''







 __lowerCamelCase					: Union[str, Any]    = MobileBertTokenizer
 __lowerCamelCase					: int    = MobileBertTokenizerFast
 __lowerCamelCase					: Union[str, Any]    = True
 __lowerCamelCase					: Tuple    = True
 __lowerCamelCase					: Optional[int]    = filter_non_english
 __lowerCamelCase					: Tuple    = 'google/mobilebert-uncased'



 def       a__						(self )					->  Any:


      """simple docstring"""



      super().setUp()

      _a											=						[
          '''[UNK]''',
          '''[CLS]''',
          '''[SEP]''',
          '''[PAD]''',
          '''[MASK]''',
          '''want''',
          '''##want''',
          '''##ed''',
          '''wa''',
          '''un''',
          '''runn''',
          '''##ing''',
          ''',''',
          '''low''',
          '''lowest''',
      ]
      _a											=						os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES['''vocab_file'''] )
      with open(self.vocab_file				,     '''w'''				,     encoding='''utf-8''' ) as vocab_writer:
           vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )

      _a											=						[
          (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
          for tokenizer_def in self.tokenizers_list
      ]



 def       a__						(self				,     A )					->  List[str]:


      """simple docstring"""



      _a											=						'''UNwant\u00E9d,running'''
      _a											=						'''unwanted, running'''
      return input_text, output_text



 def       a__						(self )					->  Any:


      """simple docstring"""



      _a											=						self.tokenizer_class(self.vocab_file )

      _a											=						tokenizer.tokenize('''UNwant\u00E9d,running''' )
      self.assertListEqual(A__				,     ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''] )
      self.assertListEqual(tokenizer.convert_tokens_to_ids(A__ )				,     [9, 6, 7, 12, 10, 11] )



 def       a__						(self )					->  Optional[int]:


      """simple docstring"""



      if not self.test_rust_tokenizer:
           return

      _a											=						self.get_tokenizer()
      _a											=						self.get_rust_tokenizer()

      _a											=						'''UNwant\u00E9d,running'''

      _a											=						tokenizer.tokenize(A__ )
      _a											=						rust_tokenizer.tokenize(A__ )
      self.assertListEqual(A__				,     A__ )

      _a											=						tokenizer.encode(A__				,     add_special_tokens=A__ )
      _a											=						rust_tokenizer.encode(A__				,     add_special_tokens=A__ )
      self.assertListEqual(A__				,     A__ )

      _a											=						self.get_rust_tokenizer()
      _a											=						tokenizer.encode(A__ )
      _a											=						rust_tokenizer.encode(A__ )
      self.assertListEqual(A__				,     A__ )

      # With lower casing
      _a											=						self.get_tokenizer(do_lower_case=A__ )
      _a											=						self.get_rust_tokenizer(do_lower_case=A__ )

      _a											=						'''UNwant\u00E9d,running'''

      _a											=						tokenizer.tokenize(A__ )
      _a											=						rust_tokenizer.tokenize(A__ )
      self.assertListEqual(A__				,     A__ )

      _a											=						tokenizer.encode(A__				,     add_special_tokens=A__ )
      _a											=						rust_tokenizer.encode(A__				,     add_special_tokens=A__ )
      self.assertListEqual(A__				,     A__ )

      _a											=						self.get_rust_tokenizer()
      _a											=						tokenizer.encode(A__ )
      _a											=						rust_tokenizer.encode(A__ )
      self.assertListEqual(A__				,     A__ )



 def       a__						(self )					->  Tuple:


      """simple docstring"""



      _a											=						BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''' )				,     ['''ah''', '''\u535A''', '''\u63A8''', '''zz'''] )



 def       a__						(self )					->  List[Any]:


      """simple docstring"""



      _a											=						BasicTokenizer(do_lower_case=A__ )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' )				,     ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )				,     ['''hello'''] )



 def       a__						(self )					->  Optional[Any]:


      """simple docstring"""



      _a											=						BasicTokenizer(do_lower_case=A__				,     strip_accents=A__ )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )				,     ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )				,     ['''h\u00E9llo'''] )



 def       a__						(self )					->  Any:


      """simple docstring"""



      _a											=						BasicTokenizer(do_lower_case=A__				,     strip_accents=A__ )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )				,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )				,     ['''hello'''] )



 def       a__						(self )					->  Any:


      """simple docstring"""



      _a											=						BasicTokenizer(do_lower_case=A__ )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )				,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
      self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )				,     ['''hello'''] )



 def       a__						(self )					->  str:


      """simple docstring"""



      _a											=						BasicTokenizer(do_lower_case=A__ )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' )				,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )



 def       a__						(self )					->  Dict:


      """simple docstring"""



      _a											=						BasicTokenizer(do_lower_case=A__				,     strip_accents=A__ )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )				,     ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )



 def       a__						(self )					->  Union[str, Any]:


      """simple docstring"""



      _a											=						BasicTokenizer(do_lower_case=A__				,     strip_accents=A__ )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )				,     ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )



 def       a__						(self )					->  str:


      """simple docstring"""



      _a											=						BasicTokenizer(do_lower_case=A__				,     never_split=['''[UNK]'''] )

      self.assertListEqual(
          tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''' )				,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''] )



 def       a__						(self )					->  Tuple:


      """simple docstring"""



      _a											=						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

      _a											=						{}
      for i, token in enumerate(A__ ):
           _a											=						i
      _a											=						WordpieceTokenizer(vocab=A__				,     unk_token='''[UNK]''' )

      self.assertListEqual(tokenizer.tokenize('''''' )				,     [] )

      self.assertListEqual(tokenizer.tokenize('''unwanted running''' )				,     ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''] )

      self.assertListEqual(tokenizer.tokenize('''unwantedX running''' )				,     ['''[UNK]''', '''runn''', '''##ing'''] )



 def       a__						(self )					->  str:


      """simple docstring"""



      self.assertTrue(_is_whitespace(''' ''' ) )
      self.assertTrue(_is_whitespace('''\t''' ) )
      self.assertTrue(_is_whitespace('''\r''' ) )
      self.assertTrue(_is_whitespace('''\n''' ) )
      self.assertTrue(_is_whitespace('''\u00A0''' ) )

      self.assertFalse(_is_whitespace('''A''' ) )
      self.assertFalse(_is_whitespace('''-''' ) )



 def       a__						(self )					->  Union[str, Any]:


      """simple docstring"""



      self.assertTrue(_is_control('''\u0005''' ) )

      self.assertFalse(_is_control('''A''' ) )
      self.assertFalse(_is_control(''' ''' ) )
      self.assertFalse(_is_control('''\t''' ) )
      self.assertFalse(_is_control('''\r''' ) )



 def       a__						(self )					->  Dict:


      """simple docstring"""



      self.assertTrue(_is_punctuation('''-''' ) )
      self.assertTrue(_is_punctuation('''$''' ) )
      self.assertTrue(_is_punctuation('''`''' ) )
      self.assertTrue(_is_punctuation('''.''' ) )

      self.assertFalse(_is_punctuation('''A''' ) )
      self.assertFalse(_is_punctuation(''' ''' ) )



 def       a__						(self )					->  str:


      """simple docstring"""



      _a											=						self.get_tokenizer()
      _a											=						self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(A__ ) for t in ['''Test''', '''\xad''', '''test''']]				,     [['''[UNK]'''], [], ['''[UNK]''']] )

      self.assertListEqual(
          [rust_tokenizer.tokenize(A__ ) for t in ['''Test''', '''\xad''', '''test''']]				,     [['''[UNK]'''], [], ['''[UNK]''']] )



 @slow
 def       a__						(self )					->  Any:


      """simple docstring"""



      _a											=						self.tokenizer_class.from_pretrained('''google/mobilebert-uncased''' )

      _a											=						tokenizer.encode('''sequence builders'''				,     add_special_tokens=A__ )
      _a											=						tokenizer.encode('''multi-sequence build'''				,     add_special_tokens=A__ )

      _a											=						tokenizer.build_inputs_with_special_tokens(A__ )
      _a											=						tokenizer.build_inputs_with_special_tokens(A__				,     A__ )

      assert encoded_sentence == [101] + text + [102]
      assert encoded_pair == [101] + text + [102] + text_a + [102]



 def       a__						(self )					->  List[str]:


      """simple docstring"""



      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
                _a											=						self.rust_tokenizer_class.from_pretrained(A__				,     **A__ )

                _a											=						f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                _a											=						tokenizer_r.encode_plus(
                    A__				,     return_attention_mask=A__				,     return_token_type_ids=A__				,     return_offsets_mapping=A__				,     add_special_tokens=A__				,     )

                _a											=						tokenizer_r.do_lower_case if hasattr(A__				,     '''do_lower_case''' ) else False
                _a											=						(
                    [
                        ((0, 0), tokenizer_r.cls_token),
                        ((0, 1), '''A'''),
                        ((1, 2), ''','''),
                        ((3, 5), '''na'''),
                        ((5, 6), '''##ï'''),
                        ((6, 8), '''##ve'''),
                        ((9, 15), tokenizer_r.mask_token),
                        ((16, 21), '''Allen'''),
                        ((21, 23), '''##NL'''),
                        ((23, 24), '''##P'''),
                        ((25, 33), '''sentence'''),
                        ((33, 34), '''.'''),
                        ((0, 0), tokenizer_r.sep_token),
                    ]
                    if not do_lower_case
                    else [
                        ((0, 0), tokenizer_r.cls_token),
                        ((0, 1), '''a'''),
                        ((1, 2), ''','''),
                        ((3, 8), '''naive'''),
                        ((9, 15), tokenizer_r.mask_token),
                        ((16, 21), '''allen'''),
                        ((21, 23), '''##nl'''),
                        ((23, 24), '''##p'''),
                        ((25, 33), '''sentence'''),
                        ((33, 34), '''.'''),
                        ((0, 0), tokenizer_r.sep_token),
                    ]
                )

                self.assertEqual(
                    [e[1] for e in expected_results]				,     tokenizer_r.convert_ids_to_tokens(tokens['''input_ids'''] ) )
                self.assertEqual([e[0] for e in expected_results]				,     tokens['''offset_mapping'''] )



 def       a__						(self )					->  Any:


      """simple docstring"""



      _a											=						['''的''', '''人''', '''有''']
      _a											=						''''''.join(A__ )
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
                _a											=						True
                _a											=						self.tokenizer_class.from_pretrained(A__				,     **A__ )
                _a											=						self.rust_tokenizer_class.from_pretrained(A__				,     **A__ )

                _a											=						tokenizer_p.encode(A__				,     add_special_tokens=A__ )
                _a											=						tokenizer_r.encode(A__				,     add_special_tokens=A__ )

                _a											=						tokenizer_r.convert_ids_to_tokens(A__ )
                _a											=						tokenizer_p.convert_ids_to_tokens(A__ )

                # it is expected that each Chinese character is not preceded by "##"
                self.assertListEqual(A__				,     A__ )
                self.assertListEqual(A__				,     A__ )

                _a											=						False
                _a											=						self.rust_tokenizer_class.from_pretrained(A__				,     **A__ )
                _a											=						self.tokenizer_class.from_pretrained(A__				,     **A__ )

                _a											=						tokenizer_r.encode(A__				,     add_special_tokens=A__ )
                _a											=						tokenizer_p.encode(A__				,     add_special_tokens=A__ )

                _a											=						tokenizer_r.convert_ids_to_tokens(A__ )
                _a											=						tokenizer_p.convert_ids_to_tokens(A__ )

                # it is expected that only the first Chinese character is not preceded by "##".
                _a											=						[
                    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(A__ )
                ]
                self.assertListEqual(A__				,     A__ )
                self.assertListEqual(A__				,     A__ )








'''simple docstring'''





import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
  import torch

if is_vision_available():
  from PIL import Image

  from transformers import MobileNetVaImageProcessor






class __A    (						unittest.TestCase		):



 '''simple docstring'''







 def __init__(self				,     A				,     A=7				,     A=3				,     A=18				,     A=30				,     A=400				,     A=True				,     A=None				,     A=True				,     A=None				,     )					->  Any:


      """simple docstring"""



      _a											=						size if size is not None else {'''shortest_edge''': 20}
      _a											=						crop_size if crop_size is not None else {'''height''': 18, '''width''': 18}
      _a											=						parent
      _a											=						batch_size
      _a											=						num_channels
      _a											=						image_size
      _a											=						min_resolution
      _a											=						max_resolution
      _a											=						do_resize
      _a											=						size
      _a											=						do_center_crop
      _a											=						crop_size



 def       a__						(self )					->  Optional[Any]:


      """simple docstring"""



      return {
          "do_resize": self.do_resize,
          "size": self.size,
          "do_center_crop": self.do_center_crop,
          "crop_size": self.crop_size,
      }






@require_torch
@require_vision
class __A    (						A     ,     unittest.TestCase		):



 '''simple docstring'''







 __lowerCamelCase					: Tuple    = MobileNetVaImageProcessor if is_vision_available() else None



 def       a__						(self )					->  int:


      """simple docstring"""



      _a											=						MobileNetVaImageProcessingTester(self )



 @property
 def       a__						(self )					->  Optional[Any]:


      """simple docstring"""



      return self.image_processor_tester.prepare_image_processor_dict()



 def       a__						(self )					->  str:


      """simple docstring"""



      _a											=						self.image_processing_class(**self.image_processor_dict )
      self.assertTrue(hasattr(A				,     '''do_resize''' ) )
      self.assertTrue(hasattr(A				,     '''size''' ) )
      self.assertTrue(hasattr(A				,     '''do_center_crop''' ) )
      self.assertTrue(hasattr(A				,     '''crop_size''' ) )



 def       a__						(self )					->  Optional[int]:


      """simple docstring"""



      _a											=						self.image_processing_class.from_dict(self.image_processor_dict )
      self.assertEqual(image_processor.size				,     {'''shortest_edge''': 20} )
      self.assertEqual(image_processor.crop_size				,     {'''height''': 18, '''width''': 18} )

      _a											=						self.image_processing_class.from_dict(self.image_processor_dict				,     size=42				,     crop_size=84 )
      self.assertEqual(image_processor.size				,     {'''shortest_edge''': 42} )
      self.assertEqual(image_processor.crop_size				,     {'''height''': 84, '''width''': 84} )



 def       a__						(self )					->  Union[str, Any]:


      """simple docstring"""



      pass



 def       a__						(self )					->  Any:


      """simple docstring"""



      _a											=						self.image_processing_class(**self.image_processor_dict )
      # create random PIL images
      _a											=						prepare_image_inputs(self.image_processor_tester				,     equal_resolution=A )
      for image in image_inputs:
           self.assertIsInstance(A				,     Image.Image )

      # Test not batched input
      _a											=						image_processing(image_inputs[0]				,     return_tensors='''pt''' ).pixel_values
      self.assertEqual(
          encoded_images.shape				,     (
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )				,     )

      # Test batched
      _a											=						image_processing(A				,     return_tensors='''pt''' ).pixel_values
      self.assertEqual(
          encoded_images.shape				,     (
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )				,     )



 def       a__						(self )					->  Optional[int]:


      """simple docstring"""



      _a											=						self.image_processing_class(**self.image_processor_dict )
      # create random numpy tensors
      _a											=						prepare_image_inputs(self.image_processor_tester				,     equal_resolution=A				,     numpify=A )
      for image in image_inputs:
           self.assertIsInstance(A				,     np.ndarray )

      # Test not batched input
      _a											=						image_processing(image_inputs[0]				,     return_tensors='''pt''' ).pixel_values
      self.assertEqual(
          encoded_images.shape				,     (
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )				,     )

      # Test batched
      _a											=						image_processing(A				,     return_tensors='''pt''' ).pixel_values
      self.assertEqual(
          encoded_images.shape				,     (
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )				,     )



 def       a__						(self )					->  Optional[Any]:


      """simple docstring"""



      _a											=						self.image_processing_class(**self.image_processor_dict )
      # create random PyTorch tensors
      _a											=						prepare_image_inputs(self.image_processor_tester				,     equal_resolution=A				,     torchify=A )
      for image in image_inputs:
           self.assertIsInstance(A				,     torch.Tensor )

      # Test not batched input
      _a											=						image_processing(image_inputs[0]				,     return_tensors='''pt''' ).pixel_values
      self.assertEqual(
          encoded_images.shape				,     (
              1,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )				,     )

      # Test batched
      _a											=						image_processing(A				,     return_tensors='''pt''' ).pixel_values
      self.assertEqual(
          encoded_images.shape				,     (
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              self.image_processor_tester.crop_size['''height'''],
              self.image_processor_tester.crop_size['''width'''],
          )				,     )



