






import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class        lowerCAmelCase_				(      lowercase						,      unittest.TestCase	):







				"""simple docstring"""




				_snake_case				:   Tuple        =			BertTokenizer
				_snake_case				:   str        =			BertTokenizerFast
				_snake_case				:   Optional[Any]        =			True
				_snake_case				:   Optional[int]        =			True
				_snake_case				:   Dict        =			filter_non_english







				def 				__a   ( self     :List[str]     ):
					super().setUp()

					UpperCamelCase__ :str						=      [
					    """[UNK]""",
					    """[CLS]""",
					    """[SEP]""",
					    """[PAD]""",
					    """[MASK]""",
					    """want""",
					    """##want""",
					    """##ed""",
					    """wa""",
					    """un""",
					    """runn""",
					    """##ing""",
					    """,""",
					    """low""",
					    """lowest""",
					]
					UpperCamelCase__ :List[str]						=      os.path.join(self.tmpdirname  ,     VOCAB_FILES_NAMES["""vocab_file"""]     )
					with open(self.vocab_file  ,     """w"""  ,     encoding="""utf-8"""     ) as vocab_writer:
						vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )







				def 				__a   ( self     :Tuple  ,     lowerCamelCase__     :Dict     ):
					UpperCamelCase__ :Tuple						=      """UNwant\u00E9d,running"""
					UpperCamelCase__ :Tuple						=      """unwanted, running"""
					return input_text, output_text







				def 				__a   ( self     :int     ):
					UpperCamelCase__ :Optional[Any]						=      self.tokenizer_class(self.vocab_file     )

					UpperCamelCase__ :Union[str, Any]						=      tokenizer.tokenize("""UNwant\u00E9d,running"""     )
					self.assertListEqual(lowerCamelCase__  ,     ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]     )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__     )  ,     [9, 6, 7, 12, 10, 11]     )







				def 				__a   ( self     :Dict     ):
					if not self.test_rust_tokenizer:
						return

					UpperCamelCase__ :Tuple						=      self.get_tokenizer()
					UpperCamelCase__ :Dict						=      self.get_rust_tokenizer()

					UpperCamelCase__ :str						=      """UNwant\u00E9d,running"""

					UpperCamelCase__ :Optional[int]						=      tokenizer.tokenize(lowerCamelCase__     )
					UpperCamelCase__ :List[str]						=      rust_tokenizer.tokenize(lowerCamelCase__     )
					self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )

					UpperCamelCase__ :Union[str, Any]						=      tokenizer.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )
					UpperCamelCase__ :int						=      rust_tokenizer.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )
					self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )

					UpperCamelCase__ :List[Any]						=      self.get_rust_tokenizer()
					UpperCamelCase__ :str						=      tokenizer.encode(lowerCamelCase__     )
					UpperCamelCase__ :Any						=      rust_tokenizer.encode(lowerCamelCase__     )
					self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )

					# With lower casing
					UpperCamelCase__ :Dict						=      self.get_tokenizer(do_lower_case=lowerCamelCase__     )
					UpperCamelCase__ :List[Any]						=      self.get_rust_tokenizer(do_lower_case=lowerCamelCase__     )

					UpperCamelCase__ :List[str]						=      """UNwant\u00E9d,running"""

					UpperCamelCase__ :List[Any]						=      tokenizer.tokenize(lowerCamelCase__     )
					UpperCamelCase__ :Union[str, Any]						=      rust_tokenizer.tokenize(lowerCamelCase__     )
					self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )

					UpperCamelCase__ :Dict						=      tokenizer.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )
					UpperCamelCase__ :Union[str, Any]						=      rust_tokenizer.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )
					self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )

					UpperCamelCase__ :Union[str, Any]						=      self.get_rust_tokenizer()
					UpperCamelCase__ :Optional[Any]						=      tokenizer.encode(lowerCamelCase__     )
					UpperCamelCase__ :Any						=      rust_tokenizer.encode(lowerCamelCase__     )
					self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )







				def 				__a   ( self     :str     ):
					UpperCamelCase__ :str						=      BasicTokenizer()

					self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""     )  ,     ["""ah""", """\u535A""", """\u63A8""", """zz"""]     )







				def 				__a   ( self     :Union[str, Any]     ):
					UpperCamelCase__ :str						=      BasicTokenizer(do_lower_case=lowerCamelCase__     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )  ,     ["""hello""", """!""", """how""", """are""", """you""", """?"""]     )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )  ,     ["""hello"""]     )







				def 				__a   ( self     :str     ):
					UpperCamelCase__ :Optional[int]						=      BasicTokenizer(do_lower_case=lowerCamelCase__  ,     strip_accents=lowerCamelCase__     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )  ,     ["""hällo""", """!""", """how""", """are""", """you""", """?"""]     )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )  ,     ["""h\u00E9llo"""]     )







				def 				__a   ( self     :Optional[Any]     ):
					UpperCamelCase__ :Any						=      BasicTokenizer(do_lower_case=lowerCamelCase__  ,     strip_accents=lowerCamelCase__     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )  ,     ["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )  ,     ["""hello"""]     )







				def 				__a   ( self     :int     ):
					UpperCamelCase__ :int						=      BasicTokenizer(do_lower_case=lowerCamelCase__     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )  ,     ["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
					self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )  ,     ["""hello"""]     )







				def 				__a   ( self     :List[str]     ):
					UpperCamelCase__ :Optional[Any]						=      BasicTokenizer(do_lower_case=lowerCamelCase__     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )  ,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )







				def 				__a   ( self     :List[Any]     ):
					UpperCamelCase__ :List[str]						=      BasicTokenizer(do_lower_case=lowerCamelCase__  ,     strip_accents=lowerCamelCase__     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )  ,     ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )







				def 				__a   ( self     :List[str]     ):
					UpperCamelCase__ :List[Any]						=      BasicTokenizer(do_lower_case=lowerCamelCase__  ,     strip_accents=lowerCamelCase__     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )  ,     ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )







				def 				__a   ( self     :str     ):
					UpperCamelCase__ :str						=      BasicTokenizer(do_lower_case=lowerCamelCase__  ,     never_split=["""[UNK]"""]     )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""     )  ,     ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]     )







				def 				__a   ( self     :str     ):
					UpperCamelCase__ :List[str]						=      BasicTokenizer()
					UpperCamelCase__ :List[str]						=      """a\n'll !!to?'d of, can't."""
					UpperCamelCase__ :Dict						=      ["""a""", """'""", """ll""", """!""", """!""", """to""", """?""", """'""", """d""", """of""", """,""", """can""", """'""", """t""", """."""]
					self.assertListEqual(tokenizer.tokenize(lowerCamelCase__     )  ,     lowerCamelCase__     )







				def 				__a   ( self     :Optional[Any]     ):
					UpperCamelCase__ :str						=      ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

					UpperCamelCase__ :Optional[Any]						=      {}
					for i, token in enumerate(lowerCamelCase__     ):
						UpperCamelCase__ :List[Any]						=      i
					UpperCamelCase__ :Optional[int]						=      WordpieceTokenizer(vocab=lowerCamelCase__  ,     unk_token="""[UNK]"""     )

					self.assertListEqual(tokenizer.tokenize(""""""     )  ,     []     )

					self.assertListEqual(tokenizer.tokenize("""unwanted running"""     )  ,     ["""un""", """##want""", """##ed""", """runn""", """##ing"""]     )

					self.assertListEqual(tokenizer.tokenize("""unwantedX running"""     )  ,     ["""[UNK]""", """runn""", """##ing"""]     )







				def 				__a   ( self     :Optional[int]     ):
					self.assertTrue(_is_whitespace(""" """     )     )
					self.assertTrue(_is_whitespace("""\t"""     )     )
					self.assertTrue(_is_whitespace("""\r"""     )     )
					self.assertTrue(_is_whitespace("""\n"""     )     )
					self.assertTrue(_is_whitespace("""\u00A0"""     )     )

					self.assertFalse(_is_whitespace("""A"""     )     )
					self.assertFalse(_is_whitespace("""-"""     )     )







				def 				__a   ( self     :Optional[int]     ):
					self.assertTrue(_is_control("""\u0005"""     )     )

					self.assertFalse(_is_control("""A"""     )     )
					self.assertFalse(_is_control(""" """     )     )
					self.assertFalse(_is_control("""\t"""     )     )
					self.assertFalse(_is_control("""\r"""     )     )







				def 				__a   ( self     :List[Any]     ):
					self.assertTrue(_is_punctuation("""-"""     )     )
					self.assertTrue(_is_punctuation("""$"""     )     )
					self.assertTrue(_is_punctuation("""`"""     )     )
					self.assertTrue(_is_punctuation("""."""     )     )

					self.assertFalse(_is_punctuation("""A"""     )     )
					self.assertFalse(_is_punctuation(""" """     )     )







				def 				__a   ( self     :Tuple     ):
					UpperCamelCase__ :List[str]						=      self.get_tokenizer()
					UpperCamelCase__ :Dict						=      self.get_rust_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(lowerCamelCase__     ) for t in ["""Test""", """\xad""", """test"""]]  ,     [["""[UNK]"""], [], ["""[UNK]"""]]     )

					self.assertListEqual(
					    [rust_tokenizer.tokenize(lowerCamelCase__     ) for t in ["""Test""", """\xad""", """test"""]]  ,     [["""[UNK]"""], [], ["""[UNK]"""]]     )







				@slow
				def 				__a   ( self     :List[str]     ):
					UpperCamelCase__ :Optional[int]						=      self.tokenizer_class.from_pretrained("""bert-base-uncased"""     )

					UpperCamelCase__ :str						=      tokenizer.encode("""sequence builders"""  ,     add_special_tokens=lowerCamelCase__     )
					UpperCamelCase__ :int						=      tokenizer.encode("""multi-sequence build"""  ,     add_special_tokens=lowerCamelCase__     )

					UpperCamelCase__ :List[str]						=      tokenizer.build_inputs_with_special_tokens(lowerCamelCase__     )
					UpperCamelCase__ :Dict						=      tokenizer.build_inputs_with_special_tokens(lowerCamelCase__  ,     lowerCamelCase__     )

					assert encoded_sentence == [1_01] + text + [1_02]
					assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]







				def 				__a   ( self     :int     ):
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
							UpperCamelCase__ :Any						=      self.rust_tokenizer_class.from_pretrained(lowerCamelCase__  ,     **lowerCamelCase__     )

							UpperCamelCase__ :Optional[int]						=      f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
							UpperCamelCase__ :Any						=      tokenizer_r.encode_plus(
							    lowerCamelCase__  ,     return_attention_mask=lowerCamelCase__  ,     return_token_type_ids=lowerCamelCase__  ,     return_offsets_mapping=lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__  ,     )

							UpperCamelCase__ :Optional[Any]						=      tokenizer_r.do_lower_case if hasattr(lowerCamelCase__  ,     """do_lower_case"""     ) else False
							UpperCamelCase__ :Dict						=      (
							    [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), """A"""),
							        ((1, 2), ""","""),
							        ((3, 5), """na"""),
							        ((5, 6), """##ï"""),
							        ((6, 8), """##ve"""),
							        ((9, 15), tokenizer_r.mask_token),
							        ((16, 21), """Allen"""),
							        ((21, 23), """##NL"""),
							        ((23, 24), """##P"""),
							        ((25, 33), """sentence"""),
							        ((33, 34), """."""),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							    if not do_lower_case
							    else [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), """a"""),
							        ((1, 2), ""","""),
							        ((3, 8), """naive"""),
							        ((9, 15), tokenizer_r.mask_token),
							        ((16, 21), """allen"""),
							        ((21, 23), """##nl"""),
							        ((23, 24), """##p"""),
							        ((25, 33), """sentence"""),
							        ((33, 34), """."""),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							)

							self.assertEqual(
							    [e[1] for e in expected_results]  ,     tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]     )     )
							self.assertEqual([e[0] for e in expected_results]  ,     tokens["""offset_mapping"""]     )







				def 				__a   ( self     :Dict     ):
					UpperCamelCase__ :Any						=      ["""的""", """人""", """有"""]
					UpperCamelCase__ :Optional[Any]						=      """""".join(lowerCamelCase__     )
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
							UpperCamelCase__ :str						=      True
							UpperCamelCase__ :int						=      self.tokenizer_class.from_pretrained(lowerCamelCase__  ,     **lowerCamelCase__     )
							UpperCamelCase__ :List[str]						=      self.rust_tokenizer_class.from_pretrained(lowerCamelCase__  ,     **lowerCamelCase__     )

							UpperCamelCase__ :Tuple						=      tokenizer_p.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )
							UpperCamelCase__ :Optional[int]						=      tokenizer_r.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )

							UpperCamelCase__ :List[str]						=      tokenizer_r.convert_ids_to_tokens(lowerCamelCase__     )
							UpperCamelCase__ :List[Any]						=      tokenizer_p.convert_ids_to_tokens(lowerCamelCase__     )

							# it is expected that each Chinese character is not preceded by "##"
							self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )
							self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )

							UpperCamelCase__ :List[Any]						=      False
							UpperCamelCase__ :str						=      self.rust_tokenizer_class.from_pretrained(lowerCamelCase__  ,     **lowerCamelCase__     )
							UpperCamelCase__ :int						=      self.tokenizer_class.from_pretrained(lowerCamelCase__  ,     **lowerCamelCase__     )

							UpperCamelCase__ :Optional[int]						=      tokenizer_r.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )
							UpperCamelCase__ :str						=      tokenizer_p.encode(lowerCamelCase__  ,     add_special_tokens=lowerCamelCase__     )

							UpperCamelCase__ :Union[str, Any]						=      tokenizer_r.convert_ids_to_tokens(lowerCamelCase__     )
							UpperCamelCase__ :Dict						=      tokenizer_p.convert_ids_to_tokens(lowerCamelCase__     )

							# it is expected that only the first Chinese character is not preceded by "##".
							UpperCamelCase__ :Any						=      [
							    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCamelCase__     )
							]
							self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )
							self.assertListEqual(lowerCamelCase__  ,     lowerCamelCase__     )







'''simple docstring'''



from typing import List, Optional, Union

import numpy as np
import PIL.Image

from ...image_processing_utils import BaseImageProcessor, BatchFeature
from ...image_transforms import rescale, resize, to_channel_dimension_format
from ...image_utils import (
    ChannelDimension,
    PILImageResampling,
    get_image_size,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


__UpperCamelCase   :							Dict         =   logging.get_logger(__name__)
class 				__SCREAMING_SNAKE_CASE							(				_lowerCAmelCase  ):
      __a	       =["pixel_values"]




      def __init__(	self			, lowerCamelCase = True			, lowerCamelCase = 32			, lowerCamelCase=PILImageResampling.BILINEAR			, lowerCamelCase = True			, **lowerCamelCase			, )				->None:



         '''simple docstring'''
         __a					  =     do_resize
         __a					  =     do_rescale
         __a					  =     size_divisor
         __a					  =     resample
         super().__init__(**lowerCamelCase   )




      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase = None			, **lowerCamelCase   )				->np.ndarray:



         '''simple docstring'''
         __a  ,      __a					  =     get_image_size(lowerCamelCase   )
         # Rounds the height and width down to the closest multiple of size_divisor
         __a					  =     height // size_divisor * size_divisor
         __a					  =     width // size_divisor * size_divisor
         __a					  =     resize(lowerCamelCase			, (new_h, new_w)			, resample=lowerCamelCase			, data_format=lowerCamelCase			, **lowerCamelCase   )
         return image




      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase			, lowerCamelCase = None			, **lowerCamelCase   )				->np.ndarray:



         '''simple docstring'''
         return rescale(image=lowerCamelCase			, scale=lowerCamelCase			, data_format=lowerCamelCase			, **lowerCamelCase   )






      def 			__UpperCamelCase				(	self			, lowerCamelCase			, lowerCamelCase = None			, lowerCamelCase = None			, lowerCamelCase=None			, lowerCamelCase = None			, lowerCamelCase = None			, lowerCamelCase = ChannelDimension.FIRST			, **lowerCamelCase			, )				->BatchFeature:



         '''simple docstring'''
         __a					  =     do_resize if do_resize is not None else self.do_resize
         __a					  =     do_rescale if do_rescale is not None else self.do_rescale
         __a					  =     size_divisor if size_divisor is not None else self.size_divisor
         __a					  =     resample if resample is not None else self.resample

         if do_resize and size_divisor is None:
            raise ValueError('size_divisor is required for resizing'   )

         __a					  =     make_list_of_images(lowerCamelCase   )

         if not valid_images(lowerCamelCase   ):
            raise ValueError('Invalid image(s)'   )

         # All transformations expect numpy arrays.
         __a					  =     [to_numpy_array(lowerCamelCase   ) for img in images]

         if do_resize:
            __a					  =     [self.resize(lowerCamelCase			, size_divisor=lowerCamelCase			, resample=lowerCamelCase   ) for image in images]

         if do_rescale:
            __a					  =     [self.rescale(lowerCamelCase			, scale=1 / 255   ) for image in images]

         __a					  =     [to_channel_dimension_format(lowerCamelCase			, lowerCamelCase   ) for image in images]

         __a					  =     {'pixel_values': images}
         return BatchFeature(data=lowerCamelCase			, tensor_type=lowerCamelCase   )