


import unittest

import numpy as np
from datasets import load_dataset

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
    import torch

if is_vision_available():
    from PIL import Image

    from transformers import BeitImageProcessor

class 							lowercase  (						unittest.TestCase			):







     def __init__(						self :						Union[str, Any]   ,   _lowercase :						Tuple   ,   _lowercase :						Dict=7   ,   _lowercase :						List[Any]=3   ,   _lowercase :						str=18   ,   _lowercase :						Optional[int]=30   ,   _lowercase :						List[Any]=4_00   ,   _lowercase :						Any=True   ,   _lowercase :						Optional[int]=None   ,   _lowercase :						int=True   ,   _lowercase :						List[str]=None   ,   _lowercase :						int=True   ,   _lowercase :						List[str]=[0.5, 0.5, 0.5]   ,   _lowercase :						Dict=[0.5, 0.5, 0.5]   ,   _lowercase :						Union[str, Any]=False   ,   ):
           SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		size if size is not None else {'''height''': 20, '''width''': 20}
           SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		crop_size if crop_size is not None else {'''height''': 18, '''width''': 18}
           SCREAMING_SNAKE_CASE__       :  str						     =		parent
           SCREAMING_SNAKE_CASE__       :  Dict						     =		batch_size
           SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		num_channels
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		image_size
           SCREAMING_SNAKE_CASE__       :  List[str]						     =		min_resolution
           SCREAMING_SNAKE_CASE__       :  Dict						     =		max_resolution
           SCREAMING_SNAKE_CASE__       :  int						     =		do_resize
           SCREAMING_SNAKE_CASE__       :  List[Any]						     =		size
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		do_center_crop
           SCREAMING_SNAKE_CASE__       :  Tuple						     =		crop_size
           SCREAMING_SNAKE_CASE__       :  List[Any]						     =		do_normalize
           SCREAMING_SNAKE_CASE__       :  str						     =		image_mean
           SCREAMING_SNAKE_CASE__       :  int						     =		image_std
           SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		do_reduce_labels







     def 							lowercase__       (						self :						Tuple      ):
           return {
               "do_resize": self.do_resize,
               "size": self.size,
               "do_center_crop": self.do_center_crop,
               "crop_size": self.crop_size,
               "do_normalize": self.do_normalize,
               "image_mean": self.image_mean,
               "image_std": self.image_std,
               "do_reduce_labels": self.do_reduce_labels,
           }




def    a   (							)	-> Any:




      '''simple docstring'''

      SCREAMING_SNAKE_CASE__       :  Any						     =		load_dataset('''hf-internal-testing/fixtures_ade20k'''      ,    split='''test'''							)

      SCREAMING_SNAKE_CASE__       :  str						     =		Image.open(dataset[0]['''file''']							)
      SCREAMING_SNAKE_CASE__       :  Dict						     =		Image.open(dataset[1]['''file''']							)

      return image, map




def    a   (							)	-> str:




      '''simple docstring'''

      SCREAMING_SNAKE_CASE__       :  str						     =		load_dataset('''hf-internal-testing/fixtures_ade20k'''      ,    split='''test'''							)

      SCREAMING_SNAKE_CASE__       :  Any						     =		Image.open(ds[0]['''file''']							)
      SCREAMING_SNAKE_CASE__       :  Dict						     =		Image.open(ds[1]['''file''']							)
      SCREAMING_SNAKE_CASE__       :  List[str]						     =		Image.open(ds[2]['''file''']							)
      SCREAMING_SNAKE_CASE__       :  List[str]						     =		Image.open(ds[3]['''file''']							)

      return [imagea, imagea], [mapa, mapa]





@require_torch
@require_vision
class 							lowercase  (						_UpperCAmelCase    ,       unittest.TestCase			):
     lowerCamelCase			:			Tuple      = BeitImageProcessor if is_vision_available() else None







     def 							lowercase__       (						self :						List[Any]      ):
           SCREAMING_SNAKE_CASE__       :  Tuple						     =		BeitImageProcessingTester(self      )







     @property
     def 							lowercase__       (						self :						Optional[Any]      ):
           return self.image_processor_tester.prepare_image_processor_dict()







     def 							lowercase__       (						self :						Tuple      ):
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		self.image_processing_class(**self.image_processor_dict      )
           self.assertTrue(hasattr(_lowercase   ,   '''do_resize'''      )      )
           self.assertTrue(hasattr(_lowercase   ,   '''size'''      )      )
           self.assertTrue(hasattr(_lowercase   ,   '''do_center_crop'''      )      )
           self.assertTrue(hasattr(_lowercase   ,   '''center_crop'''      )      )
           self.assertTrue(hasattr(_lowercase   ,   '''do_normalize'''      )      )
           self.assertTrue(hasattr(_lowercase   ,   '''image_mean'''      )      )
           self.assertTrue(hasattr(_lowercase   ,   '''image_std'''      )      )







     def 							lowercase__       (						self :						Optional[Any]      ):
           SCREAMING_SNAKE_CASE__       :  List[Any]						     =		self.image_processing_class.from_dict(self.image_processor_dict      )
           self.assertEqual(image_processor.size   ,   {'''height''': 20, '''width''': 20}      )
           self.assertEqual(image_processor.crop_size   ,   {'''height''': 18, '''width''': 18}      )
           self.assertEqual(image_processor.do_reduce_labels   ,   _lowercase      )

           SCREAMING_SNAKE_CASE__       :  Tuple						     =		self.image_processing_class.from_dict(
               self.image_processor_dict   ,   size=42   ,   crop_size=84   ,   reduce_labels=_lowercase      )
           self.assertEqual(image_processor.size   ,   {'''height''': 42, '''width''': 42}      )
           self.assertEqual(image_processor.crop_size   ,   {'''height''': 84, '''width''': 84}      )
           self.assertEqual(image_processor.do_reduce_labels   ,   _lowercase      )







     def 							lowercase__       (						self :						str      ):
           pass







     def 							lowercase__       (						self :						int      ):
           # Initialize image_processing
           SCREAMING_SNAKE_CASE__       :  Any						     =		self.image_processing_class(**self.image_processor_dict      )
           # create random PIL images
           SCREAMING_SNAKE_CASE__       :  str						     =		prepare_image_inputs(self.image_processor_tester   ,   equal_resolution=_lowercase      )
           for image in image_inputs:
                 self.assertIsInstance(_lowercase   ,   Image.Image      )

           # Test not batched input
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		image_processing(image_inputs[0]   ,   return_tensors='''pt'''      ).pixel_values
           self.assertEqual(
               encoded_images.shape   ,   (
                   1,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size['''height'''],
                   self.image_processor_tester.crop_size['''width'''],
               )   ,   )

           # Test batched
           SCREAMING_SNAKE_CASE__       :  List[Any]						     =		image_processing(_lowercase   ,   return_tensors='''pt'''      ).pixel_values
           self.assertEqual(
               encoded_images.shape   ,   (
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size['''height'''],
                   self.image_processor_tester.crop_size['''width'''],
               )   ,   )







     def 							lowercase__       (						self :						Dict      ):
           # Initialize image_processing
           SCREAMING_SNAKE_CASE__       :  Dict						     =		self.image_processing_class(**self.image_processor_dict      )
           # create random numpy tensors
           SCREAMING_SNAKE_CASE__       :  Any						     =		prepare_image_inputs(self.image_processor_tester   ,   equal_resolution=_lowercase   ,   numpify=_lowercase      )
           for image in image_inputs:
                 self.assertIsInstance(_lowercase   ,   np.ndarray      )

           # Test not batched input
           SCREAMING_SNAKE_CASE__       :  Tuple						     =		image_processing(image_inputs[0]   ,   return_tensors='''pt'''      ).pixel_values
           self.assertEqual(
               encoded_images.shape   ,   (
                   1,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size['''height'''],
                   self.image_processor_tester.crop_size['''width'''],
               )   ,   )

           # Test batched
           SCREAMING_SNAKE_CASE__       :  int						     =		image_processing(_lowercase   ,   return_tensors='''pt'''      ).pixel_values
           self.assertEqual(
               encoded_images.shape   ,   (
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size['''height'''],
                   self.image_processor_tester.crop_size['''width'''],
               )   ,   )







     def 							lowercase__       (						self :						Optional[Any]      ):
           # Initialize image_processing
           SCREAMING_SNAKE_CASE__       :  List[Any]						     =		self.image_processing_class(**self.image_processor_dict      )
           # create random PyTorch tensors
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		prepare_image_inputs(self.image_processor_tester   ,   equal_resolution=_lowercase   ,   torchify=_lowercase      )
           for image in image_inputs:
                 self.assertIsInstance(_lowercase   ,   torch.Tensor      )

           # Test not batched input
           SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		image_processing(image_inputs[0]   ,   return_tensors='''pt'''      ).pixel_values
           self.assertEqual(
               encoded_images.shape   ,   (
                   1,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size['''height'''],
                   self.image_processor_tester.crop_size['''width'''],
               )   ,   )

           # Test batched
           SCREAMING_SNAKE_CASE__       :  Any						     =		image_processing(_lowercase   ,   return_tensors='''pt'''      ).pixel_values
           self.assertEqual(
               encoded_images.shape   ,   (
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size['''height'''],
                   self.image_processor_tester.crop_size['''width'''],
               )   ,   )







     def 							lowercase__       (						self :						Optional[int]      ):
           # Initialize image_processing
           SCREAMING_SNAKE_CASE__       :  Any						     =		self.image_processing_class(**self.image_processor_dict      )
           # create random PyTorch tensors
           SCREAMING_SNAKE_CASE__       :  Tuple						     =		prepare_image_inputs(self.image_processor_tester   ,   equal_resolution=_lowercase   ,   torchify=_lowercase      )
           SCREAMING_SNAKE_CASE__       :  Any						     =		[]
           for image in image_inputs:
                 self.assertIsInstance(_lowercase   ,   torch.Tensor      )
                 maps.append(torch.zeros(image.shape[-2:]      ).long()      )

           # Test not batched input
           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		image_processing(image_inputs[0]   ,   maps[0]   ,   return_tensors='''pt'''      )
           self.assertEqual(
               encoding['''pixel_values'''].shape   ,   (
                   1,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size['''height'''],
                   self.image_processor_tester.crop_size['''width'''],
               )   ,   )
           self.assertEqual(
               encoding['''labels'''].shape   ,   (
                   1,
                   self.image_processor_tester.crop_size['''height'''],
                   self.image_processor_tester.crop_size['''width'''],
               )   ,   )
           self.assertEqual(encoding['''labels'''].dtype   ,   torch.long      )
           self.assertTrue(encoding['''labels'''].min().item() >= 0      )
           self.assertTrue(encoding['''labels'''].max().item() <= 2_55      )

           # Test batched
           SCREAMING_SNAKE_CASE__       :  str						     =		image_processing(_lowercase   ,   _lowercase   ,   return_tensors='''pt'''      )
           self.assertEqual(
               encoding['''pixel_values'''].shape   ,   (
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size['''height'''],
                   self.image_processor_tester.crop_size['''width'''],
               )   ,   )
           self.assertEqual(
               encoding['''labels'''].shape   ,   (
                   self.image_processor_tester.batch_size,
                   self.image_processor_tester.crop_size['''height'''],
                   self.image_processor_tester.crop_size['''width'''],
               )   ,   )
           self.assertEqual(encoding['''labels'''].dtype   ,   torch.long      )
           self.assertTrue(encoding['''labels'''].min().item() >= 0      )
           self.assertTrue(encoding['''labels'''].max().item() <= 2_55      )

           # Test not batched input (PIL images)
           SCREAMING_SNAKE_CASE__     ,       SCREAMING_SNAKE_CASE__       :  Tuple						     =		prepare_semantic_single_inputs()

           SCREAMING_SNAKE_CASE__       :  int						     =		image_processing(_lowercase   ,   _lowercase   ,   return_tensors='''pt'''      )
           self.assertEqual(
               encoding['''pixel_values'''].shape   ,   (
                   1,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size['''height'''],
                   self.image_processor_tester.crop_size['''width'''],
               )   ,   )
           self.assertEqual(
               encoding['''labels'''].shape   ,   (
                   1,
                   self.image_processor_tester.crop_size['''height'''],
                   self.image_processor_tester.crop_size['''width'''],
               )   ,   )
           self.assertEqual(encoding['''labels'''].dtype   ,   torch.long      )
           self.assertTrue(encoding['''labels'''].min().item() >= 0      )
           self.assertTrue(encoding['''labels'''].max().item() <= 2_55      )

           # Test batched input (PIL images)
           SCREAMING_SNAKE_CASE__     ,       SCREAMING_SNAKE_CASE__       :  Dict						     =		prepare_semantic_batch_inputs()

           SCREAMING_SNAKE_CASE__       :  Any						     =		image_processing(_lowercase   ,   _lowercase   ,   return_tensors='''pt'''      )
           self.assertEqual(
               encoding['''pixel_values'''].shape   ,   (
                   2,
                   self.image_processor_tester.num_channels,
                   self.image_processor_tester.crop_size['''height'''],
                   self.image_processor_tester.crop_size['''width'''],
               )   ,   )
           self.assertEqual(
               encoding['''labels'''].shape   ,   (
                   2,
                   self.image_processor_tester.crop_size['''height'''],
                   self.image_processor_tester.crop_size['''width'''],
               )   ,   )
           self.assertEqual(encoding['''labels'''].dtype   ,   torch.long      )
           self.assertTrue(encoding['''labels'''].min().item() >= 0      )
           self.assertTrue(encoding['''labels'''].max().item() <= 2_55      )







     def 							lowercase__       (						self :						int      ):
           # Initialize image_processing
           SCREAMING_SNAKE_CASE__       :  str						     =		self.image_processing_class(**self.image_processor_dict      )

           # ADE20k has 150 classes, and the background is included, so labels should be between 0 and 150
           SCREAMING_SNAKE_CASE__     ,       SCREAMING_SNAKE_CASE__       :  List[str]						     =		prepare_semantic_single_inputs()
           SCREAMING_SNAKE_CASE__       :  List[str]						     =		image_processing(_lowercase   ,   _lowercase   ,   return_tensors='''pt'''      )
           self.assertTrue(encoding['''labels'''].min().item() >= 0      )
           self.assertTrue(encoding['''labels'''].max().item() <= 1_50      )

           SCREAMING_SNAKE_CASE__       :  List[Any]						     =		True
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		image_processing(_lowercase   ,   _lowercase   ,   return_tensors='''pt'''      )
           self.assertTrue(encoding['''labels'''].min().item() >= 0      )
           self.assertTrue(encoding['''labels'''].max().item() <= 2_55      )














import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class     lowerCamelCase_					(   lowerCamelCase      ,					unittest.TestCase			):
				a__       							=	MobileBertTokenizer
				a__       							=	MobileBertTokenizerFast
				a__       							=	True
				a__       							=	True
				a__       							=	filter_non_english
				a__       							=	'''google/mobilebert-uncased'''







				def       A						(    self  ):
								"""simple docstring"""
								super().setUp()

								__magic_name__		:Tuple   =   [
								    '''[UNK]''',
								    '''[CLS]''',
								    '''[SEP]''',
								    '''[PAD]''',
								    '''[MASK]''',
								    '''want''',
								    '''##want''',
								    '''##ed''',
								    '''wa''',
								    '''un''',
								    '''runn''',
								    '''##ing''',
								    ''',''',
								    '''low''',
								    '''lowest''',
								]
								__magic_name__		:Dict   =   os.path.join(self.tmpdirname							,       VOCAB_FILES_NAMES['''vocab_file''']  )
								with open(self.vocab_file							,       '''w'''							,       encoding='''utf-8'''  ) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )

								__magic_name__		:List[str]   =   [
								    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
								    for tokenizer_def in self.tokenizers_list
								]







				def       A						(    self							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]   =   '''UNwant\u00E9d,running'''
								__magic_name__		:int   =   '''unwanted, running'''
								return input_text, output_text







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[int]   =   self.tokenizer_class(self.vocab_file  )

								__magic_name__		:List[Any]   =   tokenizer.tokenize('''UNwant\u00E9d,running'''  )
								self.assertListEqual(__lowerCAmelCase							,       ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase  )							,       [9, 6, 7, 1_2, 1_0, 1_1]  )







				def       A						(    self  ):
								"""simple docstring"""
								if not self.test_rust_tokenizer:
												return

								__magic_name__		:int   =   self.get_tokenizer()
								__magic_name__		:Tuple   =   self.get_rust_tokenizer()

								__magic_name__		:List[str]   =   '''UNwant\u00E9d,running'''

								__magic_name__		:Optional[Any]   =   tokenizer.tokenize(__lowerCAmelCase  )
								__magic_name__		:List[Any]   =   rust_tokenizer.tokenize(__lowerCAmelCase  )
								self.assertListEqual(__lowerCAmelCase							,       __lowerCAmelCase  )

								__magic_name__		:int   =   tokenizer.encode(__lowerCAmelCase							,       add_special_tokens=__lowerCAmelCase  )
								__magic_name__		:str   =   rust_tokenizer.encode(__lowerCAmelCase							,       add_special_tokens=__lowerCAmelCase  )
								self.assertListEqual(__lowerCAmelCase							,       __lowerCAmelCase  )

								__magic_name__		:List[Any]   =   self.get_rust_tokenizer()
								__magic_name__		:Any   =   tokenizer.encode(__lowerCAmelCase  )
								__magic_name__		:Any   =   rust_tokenizer.encode(__lowerCAmelCase  )
								self.assertListEqual(__lowerCAmelCase							,       __lowerCAmelCase  )

								# With lower casing
								__magic_name__		:Any   =   self.get_tokenizer(do_lower_case=__lowerCAmelCase  )
								__magic_name__		:List[Any]   =   self.get_rust_tokenizer(do_lower_case=__lowerCAmelCase  )

								__magic_name__		:Dict   =   '''UNwant\u00E9d,running'''

								__magic_name__		:Tuple   =   tokenizer.tokenize(__lowerCAmelCase  )
								__magic_name__		:Union[str, Any]   =   rust_tokenizer.tokenize(__lowerCAmelCase  )
								self.assertListEqual(__lowerCAmelCase							,       __lowerCAmelCase  )

								__magic_name__		:Optional[Any]   =   tokenizer.encode(__lowerCAmelCase							,       add_special_tokens=__lowerCAmelCase  )
								__magic_name__		:Dict   =   rust_tokenizer.encode(__lowerCAmelCase							,       add_special_tokens=__lowerCAmelCase  )
								self.assertListEqual(__lowerCAmelCase							,       __lowerCAmelCase  )

								__magic_name__		:Tuple   =   self.get_rust_tokenizer()
								__magic_name__		:Dict   =   tokenizer.encode(__lowerCAmelCase  )
								__magic_name__		:List[Any]   =   rust_tokenizer.encode(__lowerCAmelCase  )
								self.assertListEqual(__lowerCAmelCase							,       __lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[int]   =   BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  )							,       ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[Any]   =   BasicTokenizer(do_lower_case=__lowerCAmelCase  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )							,       ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )							,       ['''hello''']  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]   =   BasicTokenizer(do_lower_case=__lowerCAmelCase							,       strip_accents=__lowerCAmelCase  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )							,       ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )							,       ['''h\u00E9llo''']  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Dict   =   BasicTokenizer(do_lower_case=__lowerCAmelCase							,       strip_accents=__lowerCAmelCase  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )							,       ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )							,       ['''hello''']  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[int]   =   BasicTokenizer(do_lower_case=__lowerCAmelCase  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )							,       ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )							,       ['''hello''']  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[str]   =   BasicTokenizer(do_lower_case=__lowerCAmelCase  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )							,       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:int   =   BasicTokenizer(do_lower_case=__lowerCAmelCase							,       strip_accents=__lowerCAmelCase  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )							,       ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[int]   =   BasicTokenizer(do_lower_case=__lowerCAmelCase							,       strip_accents=__lowerCAmelCase  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )							,       ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[Any]   =   BasicTokenizer(do_lower_case=__lowerCAmelCase							,       never_split=['''[UNK]''']  )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  )							,       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:int   =   ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

								__magic_name__		:Union[str, Any]   =   {}
								for i, token in enumerate(__lowerCAmelCase  ):
												__magic_name__		:Tuple   =   i
								__magic_name__		:List[Any]   =   WordpieceTokenizer(vocab=__lowerCAmelCase							,       unk_token='''[UNK]'''  )

								self.assertListEqual(tokenizer.tokenize(''''''  )							,       []  )

								self.assertListEqual(tokenizer.tokenize('''unwanted running'''  )							,       ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )

								self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  )							,       ['''[UNK]''', '''runn''', '''##ing''']  )







				def       A						(    self  ):
								"""simple docstring"""
								self.assertTrue(_is_whitespace(''' '''  )  )
								self.assertTrue(_is_whitespace('''\t'''  )  )
								self.assertTrue(_is_whitespace('''\r'''  )  )
								self.assertTrue(_is_whitespace('''\n'''  )  )
								self.assertTrue(_is_whitespace('''\u00A0'''  )  )

								self.assertFalse(_is_whitespace('''A'''  )  )
								self.assertFalse(_is_whitespace('''-'''  )  )







				def       A						(    self  ):
								"""simple docstring"""
								self.assertTrue(_is_control('''\u0005'''  )  )

								self.assertFalse(_is_control('''A'''  )  )
								self.assertFalse(_is_control(''' '''  )  )
								self.assertFalse(_is_control('''\t'''  )  )
								self.assertFalse(_is_control('''\r'''  )  )







				def       A						(    self  ):
								"""simple docstring"""
								self.assertTrue(_is_punctuation('''-'''  )  )
								self.assertTrue(_is_punctuation('''$'''  )  )
								self.assertTrue(_is_punctuation('''`'''  )  )
								self.assertTrue(_is_punctuation('''.'''  )  )

								self.assertFalse(_is_punctuation('''A'''  )  )
								self.assertFalse(_is_punctuation(''' '''  )  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Any   =   self.get_tokenizer()
								__magic_name__		:Any   =   self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(__lowerCAmelCase  ) for t in ['''Test''', '''\xad''', '''test''']]							,       [['''[UNK]'''], [], ['''[UNK]''']]  )

								self.assertListEqual(
								    [rust_tokenizer.tokenize(__lowerCAmelCase  ) for t in ['''Test''', '''\xad''', '''test''']]							,       [['''[UNK]'''], [], ['''[UNK]''']]  )







				@slow
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[int]   =   self.tokenizer_class.from_pretrained('''google/mobilebert-uncased'''  )

								__magic_name__		:Optional[int]   =   tokenizer.encode('''sequence builders'''							,       add_special_tokens=__lowerCAmelCase  )
								__magic_name__		:List[Any]   =   tokenizer.encode('''multi-sequence build'''							,       add_special_tokens=__lowerCAmelCase  )

								__magic_name__		:Union[str, Any]   =   tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase  )
								__magic_name__		:List[Any]   =   tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase							,       __lowerCAmelCase  )

								assert encoded_sentence == [1_0_1] + text + [1_0_2]
								assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]







				def       A						(    self  ):
								"""simple docstring"""
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																__magic_name__		:Optional[Any]   =   self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase							,       **__lowerCAmelCase  )

																__magic_name__		:Optional[int]   =   F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																__magic_name__		:Optional[Any]   =   tokenizer_r.encode_plus(
																    __lowerCAmelCase							,       return_attention_mask=__lowerCAmelCase							,       return_token_type_ids=__lowerCAmelCase							,       return_offsets_mapping=__lowerCAmelCase							,       add_special_tokens=__lowerCAmelCase							,       )

																__magic_name__		:Any   =   tokenizer_r.do_lower_case if hasattr(__lowerCAmelCase							,       '''do_lower_case'''  ) else False
																__magic_name__		:Optional[int]   =   (
																    [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), '''A'''),
																        ((1, 2), ''','''),
																        ((3, 5), '''na'''),
																        ((5, 6), '''##ï'''),
																        ((6, 8), '''##ve'''),
																        ((9, 1_5), tokenizer_r.mask_token),
																        ((1_6, 2_1), '''Allen'''),
																        ((2_1, 2_3), '''##NL'''),
																        ((2_3, 2_4), '''##P'''),
																        ((2_5, 3_3), '''sentence'''),
																        ((3_3, 3_4), '''.'''),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																    if not do_lower_case
																    else [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), '''a'''),
																        ((1, 2), ''','''),
																        ((3, 8), '''naive'''),
																        ((9, 1_5), tokenizer_r.mask_token),
																        ((1_6, 2_1), '''allen'''),
																        ((2_1, 2_3), '''##nl'''),
																        ((2_3, 2_4), '''##p'''),
																        ((2_5, 3_3), '''sentence'''),
																        ((3_3, 3_4), '''.'''),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																)

																self.assertEqual(
																    [e[1] for e in expected_results]							,       tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']  )  )
																self.assertEqual([e[0] for e in expected_results]							,       tokens['''offset_mapping''']  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Dict   =   ['''的''', '''人''', '''有''']
								__magic_name__		:Any   =   ''''''.join(__lowerCAmelCase  )
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
																__magic_name__		:Optional[Any]   =   True
																__magic_name__		:Optional[int]   =   self.tokenizer_class.from_pretrained(__lowerCAmelCase							,       **__lowerCAmelCase  )
																__magic_name__		:Tuple   =   self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase							,       **__lowerCAmelCase  )

																__magic_name__		:Dict   =   tokenizer_p.encode(__lowerCAmelCase							,       add_special_tokens=__lowerCAmelCase  )
																__magic_name__		:List[str]   =   tokenizer_r.encode(__lowerCAmelCase							,       add_special_tokens=__lowerCAmelCase  )

																__magic_name__		:Dict   =   tokenizer_r.convert_ids_to_tokens(__lowerCAmelCase  )
																__magic_name__		:Union[str, Any]   =   tokenizer_p.convert_ids_to_tokens(__lowerCAmelCase  )

																# it is expected that each Chinese character is not preceded by "##"
																self.assertListEqual(__lowerCAmelCase							,       __lowerCAmelCase  )
																self.assertListEqual(__lowerCAmelCase							,       __lowerCAmelCase  )

																__magic_name__		:List[str]   =   False
																__magic_name__		:Tuple   =   self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase							,       **__lowerCAmelCase  )
																__magic_name__		:List[str]   =   self.tokenizer_class.from_pretrained(__lowerCAmelCase							,       **__lowerCAmelCase  )

																__magic_name__		:Optional[Any]   =   tokenizer_r.encode(__lowerCAmelCase							,       add_special_tokens=__lowerCAmelCase  )
																__magic_name__		:Union[str, Any]   =   tokenizer_p.encode(__lowerCAmelCase							,       add_special_tokens=__lowerCAmelCase  )

																__magic_name__		:List[str]   =   tokenizer_r.convert_ids_to_tokens(__lowerCAmelCase  )
																__magic_name__		:Optional[int]   =   tokenizer_p.convert_ids_to_tokens(__lowerCAmelCase  )

																# it is expected that only the first Chinese character is not preceded by "##".
																__magic_name__		:Dict   =   [
																    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__lowerCAmelCase  )
																]
																self.assertListEqual(__lowerCAmelCase							,       __lowerCAmelCase  )
																self.assertListEqual(__lowerCAmelCase							,       __lowerCAmelCase  )






