import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class    A_ (       a_			,       unittest.TestCase ):
		_SCREAMING_SNAKE_CASE    				=			BertTokenizer
		_SCREAMING_SNAKE_CASE    				=			BertTokenizerFast
		_SCREAMING_SNAKE_CASE    				=			True
		_SCREAMING_SNAKE_CASE    				=			True
		_SCREAMING_SNAKE_CASE    				=			filter_non_english

		def 		_UpperCAmelCase   (		self :      str       ):
			super().setUp()

			__a       =  [
			    "[UNK]",
			    "[CLS]",
			    "[SEP]",
			    "[PAD]",
			    "[MASK]",
			    "want",
			    "##want",
			    "##ed",
			    "wa",
			    "un",
			    "runn",
			    "##ing",
			    ",",
			    "low",
			    "lowest",
			]
			__a       =  os.path.join(self.tmpdirname     ,							VOCAB_FILES_NAMES["vocab_file"]       )
			with open(self.vocab_file     ,							"w"     ,							encoding="utf-8"       ) as vocab_writer:
				vocab_writer.write("".join([x + "\n" for x in vocab_tokens]       )       )

		def 		_UpperCAmelCase   (		self :      Any     ,							__SCREAMING_SNAKE_CASE :      List[str]       ):
			__a       =  "UNwant\u00E9d,running"
			__a       =  "unwanted, running"
			return input_text, output_text

		def 		_UpperCAmelCase   (		self :      Tuple       ):
			__a       =  self.tokenizer_class(self.vocab_file       )

			__a       =  tokenizer.tokenize("UNwant\u00E9d,running"       )
			self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							["un", "##want", "##ed", ",", "runn", "##ing"]       )
			self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE       )     ,							[9, 6, 7, 12, 10, 11]       )

		def 		_UpperCAmelCase   (		self :      List[str]       ):
			if not self.test_rust_tokenizer:
				return

			__a       =  self.get_tokenizer()
			__a       =  self.get_rust_tokenizer()

			__a       =  "UNwant\u00E9d,running"

			__a       =  tokenizer.tokenize(__SCREAMING_SNAKE_CASE       )
			__a       =  rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE       )
			self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

			__a       =  tokenizer.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )
			__a       =  rust_tokenizer.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )
			self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

			__a       =  self.get_rust_tokenizer()
			__a       =  tokenizer.encode(__SCREAMING_SNAKE_CASE       )
			__a       =  rust_tokenizer.encode(__SCREAMING_SNAKE_CASE       )
			self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

			# With lower casing
			__a       =  self.get_tokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       )
			__a       =  self.get_rust_tokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       )

			__a       =  "UNwant\u00E9d,running"

			__a       =  tokenizer.tokenize(__SCREAMING_SNAKE_CASE       )
			__a       =  rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE       )
			self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

			__a       =  tokenizer.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )
			__a       =  rust_tokenizer.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )
			self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

			__a       =  self.get_rust_tokenizer()
			__a       =  tokenizer.encode(__SCREAMING_SNAKE_CASE       )
			__a       =  rust_tokenizer.encode(__SCREAMING_SNAKE_CASE       )
			self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

		def 		_UpperCAmelCase   (		self :      int       ):
			__a       =  BasicTokenizer()

			self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"       )     ,							["ah", "\u535A", "\u63A8", "zz"]       )

		def 		_UpperCAmelCase   (		self :      List[str]       ):
			__a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       )

			self.assertListEqual(
			    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )     ,							["hello", "!", "how", "are", "you", "?"]       )
			self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )     ,							["hello"]       )

		def 		_UpperCAmelCase   (		self :      List[str]       ):
			__a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,							strip_accents=__SCREAMING_SNAKE_CASE       )

			self.assertListEqual(
			    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )     ,							["hällo", "!", "how", "are", "you", "?"]       )
			self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )     ,							["h\u00E9llo"]       )

		def 		_UpperCAmelCase   (		self :      Optional[int]       ):
			__a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,							strip_accents=__SCREAMING_SNAKE_CASE       )

			self.assertListEqual(
			    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )     ,							["hallo", "!", "how", "are", "you", "?"]       )
			self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )     ,							["hello"]       )

		def 		_UpperCAmelCase   (		self :      List[Any]       ):
			__a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       )

			self.assertListEqual(
			    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )     ,							["hallo", "!", "how", "are", "you", "?"]       )
			self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )     ,							["hello"]       )

		def 		_UpperCAmelCase   (		self :      List[Any]       ):
			__a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       )

			self.assertListEqual(
			    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )     ,							["HeLLo", "!", "how", "Are", "yoU", "?"]       )

		def 		_UpperCAmelCase   (		self :      Optional[int]       ):
			__a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,							strip_accents=__SCREAMING_SNAKE_CASE       )

			self.assertListEqual(
			    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )     ,							["HäLLo", "!", "how", "Are", "yoU", "?"]       )

		def 		_UpperCAmelCase   (		self :      List[Any]       ):
			__a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,							strip_accents=__SCREAMING_SNAKE_CASE       )

			self.assertListEqual(
			    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )     ,							["HaLLo", "!", "how", "Are", "yoU", "?"]       )

		def 		_UpperCAmelCase   (		self :      Optional[int]       ):
			__a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,							never_split=["[UNK]"]       )

			self.assertListEqual(
			    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"       )     ,							["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]       )

		def 		_UpperCAmelCase   (		self :      Any       ):
			__a       =  BasicTokenizer()
			__a       =  "a\n'll !!to?'d of, can't."
			__a       =  ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
			self.assertListEqual(tokenizer.tokenize(__SCREAMING_SNAKE_CASE       )     ,							__SCREAMING_SNAKE_CASE       )

		def 		_UpperCAmelCase   (		self :      Dict       ):
			__a       =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

			__a       =  {}
			for i, token in enumerate(__SCREAMING_SNAKE_CASE       ):
				__a       =  i
			__a       =  WordpieceTokenizer(vocab=__SCREAMING_SNAKE_CASE     ,							unk_token="[UNK]"       )

			self.assertListEqual(tokenizer.tokenize(""       )     ,							[]       )

			self.assertListEqual(tokenizer.tokenize("unwanted running"       )     ,							["un", "##want", "##ed", "runn", "##ing"]       )

			self.assertListEqual(tokenizer.tokenize("unwantedX running"       )     ,							["[UNK]", "runn", "##ing"]       )

		def 		_UpperCAmelCase   (		self :      Any       ):
			self.assertTrue(_is_whitespace(" "       )       )
			self.assertTrue(_is_whitespace("\t"       )       )
			self.assertTrue(_is_whitespace("\r"       )       )
			self.assertTrue(_is_whitespace("\n"       )       )
			self.assertTrue(_is_whitespace("\u00A0"       )       )

			self.assertFalse(_is_whitespace("A"       )       )
			self.assertFalse(_is_whitespace("-"       )       )

		def 		_UpperCAmelCase   (		self :      Tuple       ):
			self.assertTrue(_is_control("\u0005"       )       )

			self.assertFalse(_is_control("A"       )       )
			self.assertFalse(_is_control(" "       )       )
			self.assertFalse(_is_control("\t"       )       )
			self.assertFalse(_is_control("\r"       )       )

		def 		_UpperCAmelCase   (		self :      str       ):
			self.assertTrue(_is_punctuation("-"       )       )
			self.assertTrue(_is_punctuation("$"       )       )
			self.assertTrue(_is_punctuation("`"       )       )
			self.assertTrue(_is_punctuation("."       )       )

			self.assertFalse(_is_punctuation("A"       )       )
			self.assertFalse(_is_punctuation(" "       )       )

		def 		_UpperCAmelCase   (		self :      int       ):
			__a       =  self.get_tokenizer()
			__a       =  self.get_rust_tokenizer()

			# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
			self.assertListEqual([tokenizer.tokenize(__SCREAMING_SNAKE_CASE       ) for t in ["Test", "\xad", "test"]]     ,							[["[UNK]"], [], ["[UNK]"]]       )

			self.assertListEqual(
			    [rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE       ) for t in ["Test", "\xad", "test"]]     ,							[["[UNK]"], [], ["[UNK]"]]       )

		@slow
		def 		_UpperCAmelCase   (		self :      str       ):
			__a       =  self.tokenizer_class.from_pretrained("bert-base-uncased"       )

			__a       =  tokenizer.encode("sequence builders"     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )
			__a       =  tokenizer.encode("multi-sequence build"     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )

			__a       =  tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE       )
			__a       =  tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

			assert encoded_sentence == [1_01] + text + [1_02]
			assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]

		def 		_UpperCAmelCase   (		self :      str       ):
			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
				with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
					__a       =  self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,							**__SCREAMING_SNAKE_CASE       )

					__a       =  f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
					__a       =  tokenizer_r.encode_plus(
					    __SCREAMING_SNAKE_CASE     ,							return_attention_mask=__SCREAMING_SNAKE_CASE     ,							return_token_type_ids=__SCREAMING_SNAKE_CASE     ,							return_offsets_mapping=__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE     ,							)

					__a       =  tokenizer_r.do_lower_case if hasattr(__SCREAMING_SNAKE_CASE     ,							"do_lower_case"       ) else False
					__a       =  (
					    [
					        ((0, 0), tokenizer_r.cls_token),
					        ((0, 1), "A"),
					        ((1, 2), ","),
					        ((3, 5), "na"),
					        ((5, 6), "##ï"),
					        ((6, 8), "##ve"),
					        ((9, 15), tokenizer_r.mask_token),
					        ((16, 21), "Allen"),
					        ((21, 23), "##NL"),
					        ((23, 24), "##P"),
					        ((25, 33), "sentence"),
					        ((33, 34), "."),
					        ((0, 0), tokenizer_r.sep_token),
					    ]
					    if not do_lower_case
					    else [
					        ((0, 0), tokenizer_r.cls_token),
					        ((0, 1), "a"),
					        ((1, 2), ","),
					        ((3, 8), "naive"),
					        ((9, 15), tokenizer_r.mask_token),
					        ((16, 21), "allen"),
					        ((21, 23), "##nl"),
					        ((23, 24), "##p"),
					        ((25, 33), "sentence"),
					        ((33, 34), "."),
					        ((0, 0), tokenizer_r.sep_token),
					    ]
					)

					self.assertEqual(
					    [e[1] for e in expected_results]     ,							tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]       )       )
					self.assertEqual([e[0] for e in expected_results]     ,							tokens["offset_mapping"]       )

		def 		_UpperCAmelCase   (		self :      Union[str, Any]       ):
			__a       =  ["的", "人", "有"]
			__a       =  "".join(__SCREAMING_SNAKE_CASE       )
			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
				with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
					__a       =  True
					__a       =  self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,							**__SCREAMING_SNAKE_CASE       )
					__a       =  self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,							**__SCREAMING_SNAKE_CASE       )

					__a       =  tokenizer_p.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )
					__a       =  tokenizer_r.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )

					__a       =  tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE       )
					__a       =  tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE       )

					# it is expected that each Chinese character is not preceded by "##"
					self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )
					self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

					__a       =  False
					__a       =  self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,							**__SCREAMING_SNAKE_CASE       )
					__a       =  self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,							**__SCREAMING_SNAKE_CASE       )

					__a       =  tokenizer_r.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )
					__a       =  tokenizer_p.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )

					__a       =  tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE       )
					__a       =  tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE       )

					# it is expected that only the first Chinese character is not preceded by "##".
					__a       =  [
					    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(__SCREAMING_SNAKE_CASE       )
					]
					self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )
					self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )



from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
  if not (is_transformers_available() and is_torch_available() and is_transformers_version(""">=""", """4.25.0""")):
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  from ...utils.dummy_torch_and_transformers_objects import UnCLIPImageVariationPipeline, UnCLIPPipeline
else:
  from .pipeline_unclip import UnCLIPPipeline
  from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline
  from .text_proj import UnCLIPTextProjModel


