import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class    A_ (       a_			,       unittest.TestCase ):
  _SCREAMING_SNAKE_CASE    				=			MobileBertTokenizer
  _SCREAMING_SNAKE_CASE    				=			MobileBertTokenizerFast
  _SCREAMING_SNAKE_CASE    				=			True
  _SCREAMING_SNAKE_CASE    				=			True
  _SCREAMING_SNAKE_CASE    				=			filter_non_english
  _SCREAMING_SNAKE_CASE    				=			"""google/mobilebert-uncased"""

  def 		_UpperCAmelCase   (		self :      Optional[Any]       ):
   super().setUp()

   __a       =  [
       "[UNK]",
       "[CLS]",
       "[SEP]",
       "[PAD]",
       "[MASK]",
       "want",
       "##want",
       "##ed",
       "wa",
       "un",
       "runn",
       "##ing",
       ",",
       "low",
       "lowest",
   ]
   __a       =  os.path.join(self.tmpdirname     ,							VOCAB_FILES_NAMES["vocab_file"]       )
   with open(self.vocab_file     ,							"w"     ,							encoding="utf-8"       ) as vocab_writer:
    vocab_writer.write("".join([x + "\n" for x in vocab_tokens]       )       )

   __a       =  [
       (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
       for tokenizer_def in self.tokenizers_list
   ]

  def 		_UpperCAmelCase   (		self :      List[str]     ,							__SCREAMING_SNAKE_CASE :      List[Any]       ):
   __a       =  "UNwant\u00E9d,running"
   __a       =  "unwanted, running"
   return input_text, output_text

  def 		_UpperCAmelCase   (		self :      Dict       ):
   __a       =  self.tokenizer_class(self.vocab_file       )

   __a       =  tokenizer.tokenize("UNwant\u00E9d,running"       )
   self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							["un", "##want", "##ed", ",", "runn", "##ing"]       )
   self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE       )     ,							[9, 6, 7, 12, 10, 11]       )

  def 		_UpperCAmelCase   (		self :      Optional[int]       ):
   if not self.test_rust_tokenizer:
    return

   __a       =  self.get_tokenizer()
   __a       =  self.get_rust_tokenizer()

   __a       =  "UNwant\u00E9d,running"

   __a       =  tokenizer.tokenize(__SCREAMING_SNAKE_CASE       )
   __a       =  rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE       )
   self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

   __a       =  tokenizer.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )
   __a       =  rust_tokenizer.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )
   self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

   __a       =  self.get_rust_tokenizer()
   __a       =  tokenizer.encode(__SCREAMING_SNAKE_CASE       )
   __a       =  rust_tokenizer.encode(__SCREAMING_SNAKE_CASE       )
   self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

   # With lower casing
   __a       =  self.get_tokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       )
   __a       =  self.get_rust_tokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       )

   __a       =  "UNwant\u00E9d,running"

   __a       =  tokenizer.tokenize(__SCREAMING_SNAKE_CASE       )
   __a       =  rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE       )
   self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

   __a       =  tokenizer.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )
   __a       =  rust_tokenizer.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )
   self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

   __a       =  self.get_rust_tokenizer()
   __a       =  tokenizer.encode(__SCREAMING_SNAKE_CASE       )
   __a       =  rust_tokenizer.encode(__SCREAMING_SNAKE_CASE       )
   self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

  def 		_UpperCAmelCase   (		self :      Union[str, Any]       ):
   __a       =  BasicTokenizer()

   self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"       )     ,							["ah", "\u535A", "\u63A8", "zz"]       )

  def 		_UpperCAmelCase   (		self :      List[str]       ):
   __a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       )

   self.assertListEqual(
       tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )     ,							["hello", "!", "how", "are", "you", "?"]       )
   self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )     ,							["hello"]       )

  def 		_UpperCAmelCase   (		self :      Optional[int]       ):
   __a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,							strip_accents=__SCREAMING_SNAKE_CASE       )

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )     ,							["hällo", "!", "how", "are", "you", "?"]       )
   self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )     ,							["h\u00E9llo"]       )

  def 		_UpperCAmelCase   (		self :      Optional[int]       ):
   __a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,							strip_accents=__SCREAMING_SNAKE_CASE       )

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )     ,							["hallo", "!", "how", "are", "you", "?"]       )
   self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )     ,							["hello"]       )

  def 		_UpperCAmelCase   (		self :      Any       ):
   __a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       )

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )     ,							["hallo", "!", "how", "are", "you", "?"]       )
   self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )     ,							["hello"]       )

  def 		_UpperCAmelCase   (		self :      str       ):
   __a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE       )

   self.assertListEqual(
       tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )     ,							["HeLLo", "!", "how", "Are", "yoU", "?"]       )

  def 		_UpperCAmelCase   (		self :      int       ):
   __a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,							strip_accents=__SCREAMING_SNAKE_CASE       )

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )     ,							["HäLLo", "!", "how", "Are", "yoU", "?"]       )

  def 		_UpperCAmelCase   (		self :      Union[str, Any]       ):
   __a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,							strip_accents=__SCREAMING_SNAKE_CASE       )

   self.assertListEqual(
       tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )     ,							["HaLLo", "!", "how", "Are", "yoU", "?"]       )

  def 		_UpperCAmelCase   (		self :      int       ):
   __a       =  BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE     ,							never_split=["[UNK]"]       )

   self.assertListEqual(
       tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"       )     ,							["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]       )

  def 		_UpperCAmelCase   (		self :      str       ):
   __a       =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

   __a       =  {}
   for i, token in enumerate(__SCREAMING_SNAKE_CASE       ):
    __a       =  i
   __a       =  WordpieceTokenizer(vocab=__SCREAMING_SNAKE_CASE     ,							unk_token="[UNK]"       )

   self.assertListEqual(tokenizer.tokenize(""       )     ,							[]       )

   self.assertListEqual(tokenizer.tokenize("unwanted running"       )     ,							["un", "##want", "##ed", "runn", "##ing"]       )

   self.assertListEqual(tokenizer.tokenize("unwantedX running"       )     ,							["[UNK]", "runn", "##ing"]       )

  def 		_UpperCAmelCase   (		self :      Union[str, Any]       ):
   self.assertTrue(_is_whitespace(" "       )       )
   self.assertTrue(_is_whitespace("\t"       )       )
   self.assertTrue(_is_whitespace("\r"       )       )
   self.assertTrue(_is_whitespace("\n"       )       )
   self.assertTrue(_is_whitespace("\u00A0"       )       )

   self.assertFalse(_is_whitespace("A"       )       )
   self.assertFalse(_is_whitespace("-"       )       )

  def 		_UpperCAmelCase   (		self :      Optional[int]       ):
   self.assertTrue(_is_control("\u0005"       )       )

   self.assertFalse(_is_control("A"       )       )
   self.assertFalse(_is_control(" "       )       )
   self.assertFalse(_is_control("\t"       )       )
   self.assertFalse(_is_control("\r"       )       )

  def 		_UpperCAmelCase   (		self :      Optional[int]       ):
   self.assertTrue(_is_punctuation("-"       )       )
   self.assertTrue(_is_punctuation("$"       )       )
   self.assertTrue(_is_punctuation("`"       )       )
   self.assertTrue(_is_punctuation("."       )       )

   self.assertFalse(_is_punctuation("A"       )       )
   self.assertFalse(_is_punctuation(" "       )       )

  def 		_UpperCAmelCase   (		self :      Dict       ):
   __a       =  self.get_tokenizer()
   __a       =  self.get_rust_tokenizer()

   # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
   self.assertListEqual([tokenizer.tokenize(__SCREAMING_SNAKE_CASE       ) for t in ["Test", "\xad", "test"]]     ,							[["[UNK]"], [], ["[UNK]"]]       )

   self.assertListEqual(
       [rust_tokenizer.tokenize(__SCREAMING_SNAKE_CASE       ) for t in ["Test", "\xad", "test"]]     ,							[["[UNK]"], [], ["[UNK]"]]       )

  @slow
  def 		_UpperCAmelCase   (		self :      Optional[Any]       ):
   __a       =  self.tokenizer_class.from_pretrained("google/mobilebert-uncased"       )

   __a       =  tokenizer.encode("sequence builders"     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )
   __a       =  tokenizer.encode("multi-sequence build"     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )

   __a       =  tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE       )
   __a       =  tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

   assert encoded_sentence == [1_01] + text + [1_02]
   assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]

  def 		_UpperCAmelCase   (		self :      List[Any]       ):
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
    with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
     __a       =  self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,							**__SCREAMING_SNAKE_CASE       )

     __a       =  f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
     __a       =  tokenizer_r.encode_plus(
         __SCREAMING_SNAKE_CASE     ,							return_attention_mask=__SCREAMING_SNAKE_CASE     ,							return_token_type_ids=__SCREAMING_SNAKE_CASE     ,							return_offsets_mapping=__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE     ,							)

     __a       =  tokenizer_r.do_lower_case if hasattr(__SCREAMING_SNAKE_CASE     ,							"do_lower_case"       ) else False
     __a       =  (
         [
             ((0, 0), tokenizer_r.cls_token),
             ((0, 1), "A"),
             ((1, 2), ","),
             ((3, 5), "na"),
             ((5, 6), "##ï"),
             ((6, 8), "##ve"),
             ((9, 15), tokenizer_r.mask_token),
             ((16, 21), "Allen"),
             ((21, 23), "##NL"),
             ((23, 24), "##P"),
             ((25, 33), "sentence"),
             ((33, 34), "."),
             ((0, 0), tokenizer_r.sep_token),
         ]
         if not do_lower_case
         else [
             ((0, 0), tokenizer_r.cls_token),
             ((0, 1), "a"),
             ((1, 2), ","),
             ((3, 8), "naive"),
             ((9, 15), tokenizer_r.mask_token),
             ((16, 21), "allen"),
             ((21, 23), "##nl"),
             ((23, 24), "##p"),
             ((25, 33), "sentence"),
             ((33, 34), "."),
             ((0, 0), tokenizer_r.sep_token),
         ]
     )

     self.assertEqual(
         [e[1] for e in expected_results]     ,							tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]       )       )
     self.assertEqual([e[0] for e in expected_results]     ,							tokens["offset_mapping"]       )

  def 		_UpperCAmelCase   (		self :      List[Any]       ):
   __a       =  ["的", "人", "有"]
   __a       =  "".join(__SCREAMING_SNAKE_CASE       )
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
    with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
     __a       =  True
     __a       =  self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,							**__SCREAMING_SNAKE_CASE       )
     __a       =  self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,							**__SCREAMING_SNAKE_CASE       )

     __a       =  tokenizer_p.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )
     __a       =  tokenizer_r.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )

     __a       =  tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE       )
     __a       =  tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE       )

     # it is expected that each Chinese character is not preceded by "##"
     self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )
     self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )

     __a       =  False
     __a       =  self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,							**__SCREAMING_SNAKE_CASE       )
     __a       =  self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE     ,							**__SCREAMING_SNAKE_CASE       )

     __a       =  tokenizer_r.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )
     __a       =  tokenizer_p.encode(__SCREAMING_SNAKE_CASE     ,							add_special_tokens=__SCREAMING_SNAKE_CASE       )

     __a       =  tokenizer_r.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE       )
     __a       =  tokenizer_p.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE       )

     # it is expected that only the first Chinese character is not preceded by "##".
     __a       =  [
         f"""##{token}""" if idx != 0 else token for idx, token in enumerate(__SCREAMING_SNAKE_CASE       )
     ]
     self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )
     self.assertListEqual(__SCREAMING_SNAKE_CASE     ,							__SCREAMING_SNAKE_CASE       )



from ...utils import (
    OptionalDependencyNotAvailable,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)


try:
  if not (is_transformers_available() and is_torch_available() and is_transformers_version(""">=""", """4.25.0""")):
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  from ...utils.dummy_torch_and_transformers_objects import UnCLIPImageVariationPipeline, UnCLIPPipeline
else:
  from .pipeline_unclip import UnCLIPPipeline
  from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline
  from .text_proj import UnCLIPTextProjModel


