import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 			UpperCAmelCase__    (		snake_case		,	unittest.TestCase					):



  """simple docstring"""






  lowerCAmelCase__		:	Optional[int]    		= BertTokenizer
  lowerCAmelCase__		:	Dict    		= BertTokenizerFast
  lowerCAmelCase__		:	str    		= True
  lowerCAmelCase__		:	Optional[Any]    		= True
  lowerCAmelCase__		:	List[str]    		= filter_non_english






  def 	_UpperCAmelCase				(				self:			List[Any]   ) -> Tuple:




    '''simple docstring'''




    super().setUp()

    __UpperCAmelCase						    =       [
        "[UNK]",
        "[CLS]",
        "[SEP]",
        "[PAD]",
        "[MASK]",
        "want",
        "##want",
        "##ed",
        "wa",
        "un",
        "runn",
        "##ing",
        ",",
        "low",
        "lowest",
    ]
    __UpperCAmelCase						    =       os.path.join(self.tmpdirname				,				VOCAB_FILES_NAMES["vocab_file"]   )
    with open(self.vocab_file				,				"w"				,				encoding="utf-8"   ) as vocab_writer:
      vocab_writer.write("".join([x + "\n" for x in vocab_tokens]   )   )






  def 	_UpperCAmelCase				(				self:			int				,				__lowerCAmelCase:			Tuple   ) -> int:




    '''simple docstring'''




    __UpperCAmelCase						    =       "UNwant\u00E9d,running"
    __UpperCAmelCase						    =       "unwanted, running"
    return input_text, output_text






  def 	_UpperCAmelCase				(				self:			Optional[Any]   ) -> Union[str, Any]:




    '''simple docstring'''




    __UpperCAmelCase						    =       self.tokenizer_class(self.vocab_file   )

    __UpperCAmelCase						    =       tokenizer.tokenize("UNwant\u00E9d,running"   )
    self.assertListEqual(__lowerCAmelCase				,				["un", "##want", "##ed", ",", "runn", "##ing"]   )
    self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase   )				,				[9, 6, 7, 12, 10, 11]   )






  def 	_UpperCAmelCase				(				self:			Any   ) -> Any:




    '''simple docstring'''




    if not self.test_rust_tokenizer:
      return

    __UpperCAmelCase						    =       self.get_tokenizer()
    __UpperCAmelCase						    =       self.get_rust_tokenizer()

    __UpperCAmelCase						    =       "UNwant\u00E9d,running"

    __UpperCAmelCase						    =       tokenizer.tokenize(__lowerCAmelCase   )
    __UpperCAmelCase						    =       rust_tokenizer.tokenize(__lowerCAmelCase   )
    self.assertListEqual(__lowerCAmelCase				,				__lowerCAmelCase   )

    __UpperCAmelCase						    =       tokenizer.encode(__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )
    __UpperCAmelCase						    =       rust_tokenizer.encode(__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )
    self.assertListEqual(__lowerCAmelCase				,				__lowerCAmelCase   )

    __UpperCAmelCase						    =       self.get_rust_tokenizer()
    __UpperCAmelCase						    =       tokenizer.encode(__lowerCAmelCase   )
    __UpperCAmelCase						    =       rust_tokenizer.encode(__lowerCAmelCase   )
    self.assertListEqual(__lowerCAmelCase				,				__lowerCAmelCase   )

    # With lower casing
    __UpperCAmelCase						    =       self.get_tokenizer(do_lower_case=__lowerCAmelCase   )
    __UpperCAmelCase						    =       self.get_rust_tokenizer(do_lower_case=__lowerCAmelCase   )

    __UpperCAmelCase						    =       "UNwant\u00E9d,running"

    __UpperCAmelCase						    =       tokenizer.tokenize(__lowerCAmelCase   )
    __UpperCAmelCase						    =       rust_tokenizer.tokenize(__lowerCAmelCase   )
    self.assertListEqual(__lowerCAmelCase				,				__lowerCAmelCase   )

    __UpperCAmelCase						    =       tokenizer.encode(__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )
    __UpperCAmelCase						    =       rust_tokenizer.encode(__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )
    self.assertListEqual(__lowerCAmelCase				,				__lowerCAmelCase   )

    __UpperCAmelCase						    =       self.get_rust_tokenizer()
    __UpperCAmelCase						    =       tokenizer.encode(__lowerCAmelCase   )
    __UpperCAmelCase						    =       rust_tokenizer.encode(__lowerCAmelCase   )
    self.assertListEqual(__lowerCAmelCase				,				__lowerCAmelCase   )






  def 	_UpperCAmelCase				(				self:			List[str]   ) -> Optional[Any]:




    '''simple docstring'''




    __UpperCAmelCase						    =       BasicTokenizer()

    self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"   )				,				["ah", "\u535A", "\u63A8", "zz"]   )






  def 	_UpperCAmelCase				(				self:			List[Any]   ) -> Any:




    '''simple docstring'''




    __UpperCAmelCase						    =       BasicTokenizer(do_lower_case=__lowerCAmelCase   )

    self.assertListEqual(
        tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "   )				,				["hello", "!", "how", "are", "you", "?"]   )
    self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   )				,				["hello"]   )






  def 	_UpperCAmelCase				(				self:			Tuple   ) -> Tuple:




    '''simple docstring'''




    __UpperCAmelCase						    =       BasicTokenizer(do_lower_case=__lowerCAmelCase				,				strip_accents=__lowerCAmelCase   )

    self.assertListEqual(
        tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   )				,				["hällo", "!", "how", "are", "you", "?"]   )
    self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   )				,				["h\u00E9llo"]   )






  def 	_UpperCAmelCase				(				self:			Tuple   ) -> Optional[Any]:




    '''simple docstring'''




    __UpperCAmelCase						    =       BasicTokenizer(do_lower_case=__lowerCAmelCase				,				strip_accents=__lowerCAmelCase   )

    self.assertListEqual(
        tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   )				,				["hallo", "!", "how", "are", "you", "?"]   )
    self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   )				,				["hello"]   )






  def 	_UpperCAmelCase				(				self:			Union[str, Any]   ) -> Tuple:




    '''simple docstring'''




    __UpperCAmelCase						    =       BasicTokenizer(do_lower_case=__lowerCAmelCase   )

    self.assertListEqual(
        tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   )				,				["hallo", "!", "how", "are", "you", "?"]   )
    self.assertListEqual(tokenizer.tokenize("H\u00E9llo"   )				,				["hello"]   )






  def 	_UpperCAmelCase				(				self:			str   ) -> Any:




    '''simple docstring'''




    __UpperCAmelCase						    =       BasicTokenizer(do_lower_case=__lowerCAmelCase   )

    self.assertListEqual(
        tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "   )				,				["HeLLo", "!", "how", "Are", "yoU", "?"]   )






  def 	_UpperCAmelCase				(				self:			Tuple   ) -> Any:




    '''simple docstring'''




    __UpperCAmelCase						    =       BasicTokenizer(do_lower_case=__lowerCAmelCase				,				strip_accents=__lowerCAmelCase   )

    self.assertListEqual(
        tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   )				,				["HäLLo", "!", "how", "Are", "yoU", "?"]   )






  def 	_UpperCAmelCase				(				self:			Optional[int]   ) -> Optional[Any]:




    '''simple docstring'''




    __UpperCAmelCase						    =       BasicTokenizer(do_lower_case=__lowerCAmelCase				,				strip_accents=__lowerCAmelCase   )

    self.assertListEqual(
        tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "   )				,				["HaLLo", "!", "how", "Are", "yoU", "?"]   )






  def 	_UpperCAmelCase				(				self:			Optional[Any]   ) -> Optional[int]:




    '''simple docstring'''




    __UpperCAmelCase						    =       BasicTokenizer(do_lower_case=__lowerCAmelCase				,				never_split=["[UNK]"]   )

    self.assertListEqual(
        tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"   )				,				["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]   )






  def 	_UpperCAmelCase				(				self:			Optional[int]   ) -> List[Any]:




    '''simple docstring'''




    __UpperCAmelCase						    =       BasicTokenizer()
    __UpperCAmelCase						    =       "a\n'll !!to?'d of, can't."
    __UpperCAmelCase						    =       ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
    self.assertListEqual(tokenizer.tokenize(__lowerCAmelCase   )				,				__lowerCAmelCase   )






  def 	_UpperCAmelCase				(				self:			List[Any]   ) -> Dict:




    '''simple docstring'''




    __UpperCAmelCase						    =       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

    __UpperCAmelCase						    =       {}
    for i, token in enumerate(__lowerCAmelCase   ):
      __UpperCAmelCase						    =       i
    __UpperCAmelCase						    =       WordpieceTokenizer(vocab=__lowerCAmelCase				,				unk_token="[UNK]"   )

    self.assertListEqual(tokenizer.tokenize(""   )				,				[]   )

    self.assertListEqual(tokenizer.tokenize("unwanted running"   )				,				["un", "##want", "##ed", "runn", "##ing"]   )

    self.assertListEqual(tokenizer.tokenize("unwantedX running"   )				,				["[UNK]", "runn", "##ing"]   )






  def 	_UpperCAmelCase				(				self:			Union[str, Any]   ) -> str:




    '''simple docstring'''




    self.assertTrue(_is_whitespace(" "   )   )
    self.assertTrue(_is_whitespace("\t"   )   )
    self.assertTrue(_is_whitespace("\r"   )   )
    self.assertTrue(_is_whitespace("\n"   )   )
    self.assertTrue(_is_whitespace("\u00A0"   )   )

    self.assertFalse(_is_whitespace("A"   )   )
    self.assertFalse(_is_whitespace("-"   )   )






  def 	_UpperCAmelCase				(				self:			Optional[Any]   ) -> Dict:




    '''simple docstring'''




    self.assertTrue(_is_control("\u0005"   )   )

    self.assertFalse(_is_control("A"   )   )
    self.assertFalse(_is_control(" "   )   )
    self.assertFalse(_is_control("\t"   )   )
    self.assertFalse(_is_control("\r"   )   )






  def 	_UpperCAmelCase				(				self:			str   ) -> int:




    '''simple docstring'''




    self.assertTrue(_is_punctuation("-"   )   )
    self.assertTrue(_is_punctuation("$"   )   )
    self.assertTrue(_is_punctuation("`"   )   )
    self.assertTrue(_is_punctuation("."   )   )

    self.assertFalse(_is_punctuation("A"   )   )
    self.assertFalse(_is_punctuation(" "   )   )






  def 	_UpperCAmelCase				(				self:			Dict   ) -> Tuple:




    '''simple docstring'''




    __UpperCAmelCase						    =       self.get_tokenizer()
    __UpperCAmelCase						    =       self.get_rust_tokenizer()

    # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
    self.assertListEqual([tokenizer.tokenize(__lowerCAmelCase   ) for t in ["Test", "\xad", "test"]]				,				[["[UNK]"], [], ["[UNK]"]]   )

    self.assertListEqual(
        [rust_tokenizer.tokenize(__lowerCAmelCase   ) for t in ["Test", "\xad", "test"]]				,				[["[UNK]"], [], ["[UNK]"]]   )






  @slow
  def 	_UpperCAmelCase				(				self:			Dict   ) -> Optional[Any]:




    '''simple docstring'''




    __UpperCAmelCase						    =       self.tokenizer_class.from_pretrained("bert-base-uncased"   )

    __UpperCAmelCase						    =       tokenizer.encode("sequence builders"				,				add_special_tokens=__lowerCAmelCase   )
    __UpperCAmelCase						    =       tokenizer.encode("multi-sequence build"				,				add_special_tokens=__lowerCAmelCase   )

    __UpperCAmelCase						    =       tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase   )
    __UpperCAmelCase						    =       tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase				,				__lowerCAmelCase   )

    assert encoded_sentence == [101] + text + [102]
    assert encoded_pair == [101] + text + [102] + text_a + [102]






  def 	_UpperCAmelCase				(				self:			List[str]   ) -> List[str]:




    '''simple docstring'''




    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
        __UpperCAmelCase						    =       self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase				,				**__lowerCAmelCase   )

        __UpperCAmelCase						    =       F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
        __UpperCAmelCase						    =       tokenizer_r.encode_plus(
            __lowerCAmelCase				,				return_attention_mask=__lowerCAmelCase				,				return_token_type_ids=__lowerCAmelCase				,				return_offsets_mapping=__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase				,				)

        __UpperCAmelCase						    =       tokenizer_r.do_lower_case if hasattr(__lowerCAmelCase				,				"do_lower_case"   ) else False
        __UpperCAmelCase						    =       (
            [
                ((0, 0), tokenizer_r.cls_token),
                ((0, 1), "A"),
                ((1, 2), ","),
                ((3, 5), "na"),
                ((5, 6), "##ï"),
                ((6, 8), "##ve"),
                ((9, 15), tokenizer_r.mask_token),
                ((16, 21), "Allen"),
                ((21, 23), "##NL"),
                ((23, 24), "##P"),
                ((25, 33), "sentence"),
                ((33, 34), "."),
                ((0, 0), tokenizer_r.sep_token),
            ]
            if not do_lower_case
            else [
                ((0, 0), tokenizer_r.cls_token),
                ((0, 1), "a"),
                ((1, 2), ","),
                ((3, 8), "naive"),
                ((9, 15), tokenizer_r.mask_token),
                ((16, 21), "allen"),
                ((21, 23), "##nl"),
                ((23, 24), "##p"),
                ((25, 33), "sentence"),
                ((33, 34), "."),
                ((0, 0), tokenizer_r.sep_token),
            ]
        )

        self.assertEqual(
            [e[1] for e in expected_results]				,				tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]   )   )
        self.assertEqual([e[0] for e in expected_results]				,				tokens["offset_mapping"]   )






  def 	_UpperCAmelCase				(				self:			List[str]   ) -> List[Any]:




    '''simple docstring'''




    __UpperCAmelCase						    =       ["的", "人", "有"]
    __UpperCAmelCase						    =       "".join(__lowerCAmelCase   )
    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
        __UpperCAmelCase						    =       True
        __UpperCAmelCase						    =       self.tokenizer_class.from_pretrained(__lowerCAmelCase				,				**__lowerCAmelCase   )
        __UpperCAmelCase						    =       self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase				,				**__lowerCAmelCase   )

        __UpperCAmelCase						    =       tokenizer_p.encode(__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )
        __UpperCAmelCase						    =       tokenizer_r.encode(__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )

        __UpperCAmelCase						    =       tokenizer_r.convert_ids_to_tokens(__lowerCAmelCase   )
        __UpperCAmelCase						    =       tokenizer_p.convert_ids_to_tokens(__lowerCAmelCase   )

        # it is expected that each Chinese character is not preceded by "##"
        self.assertListEqual(__lowerCAmelCase				,				__lowerCAmelCase   )
        self.assertListEqual(__lowerCAmelCase				,				__lowerCAmelCase   )

        __UpperCAmelCase						    =       False
        __UpperCAmelCase						    =       self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase				,				**__lowerCAmelCase   )
        __UpperCAmelCase						    =       self.tokenizer_class.from_pretrained(__lowerCAmelCase				,				**__lowerCAmelCase   )

        __UpperCAmelCase						    =       tokenizer_r.encode(__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )
        __UpperCAmelCase						    =       tokenizer_p.encode(__lowerCAmelCase				,				add_special_tokens=__lowerCAmelCase   )

        __UpperCAmelCase						    =       tokenizer_r.convert_ids_to_tokens(__lowerCAmelCase   )
        __UpperCAmelCase						    =       tokenizer_p.convert_ids_to_tokens(__lowerCAmelCase   )

        # it is expected that only the first Chinese character is not preceded by "##".
        __UpperCAmelCase						    =       [
            F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__lowerCAmelCase   )
        ]
        self.assertListEqual(__lowerCAmelCase				,				__lowerCAmelCase   )
        self.assertListEqual(__lowerCAmelCase				,				__lowerCAmelCase   )




def       __lowerCAmelCase      (						A_   : str					)    ->  Any:
  if not head:
    return True
  # split the list to two parts
  __UpperCAmelCase    ,		__UpperCAmelCase						    =       head.next, head
  while fast and fast.next:
    __UpperCAmelCase						    =       fast.next.next
    __UpperCAmelCase						    =       slow.next
  __UpperCAmelCase						    =       slow.next
  __UpperCAmelCase						    =       None  # Don't forget here! But forget still works!
  # reverse the second part
  __UpperCAmelCase						    =       None
  while second:
    __UpperCAmelCase						    =       second.next
    __UpperCAmelCase						    =       node
    __UpperCAmelCase						    =       second
    __UpperCAmelCase						    =       nxt
  # compare two parts
  # second part has the same or one less node
  while node:
    if node.val != head.val:
      return False
    __UpperCAmelCase						    =       node.next
    __UpperCAmelCase						    =       head.next
  return True




def       __lowerCAmelCase      (						A_   : Optional[Any]					)    ->  Optional[Any]:
  if not head or not head.next:
    return True

  # 1. Get the midpoint (slow)
  __UpperCAmelCase						    =       __UpperCAmelCase						    =       __UpperCAmelCase						    =       head
  while fast and fast.next:
    __UpperCAmelCase    ,		__UpperCAmelCase						    =       fast.next.next, slow.next

  # 2. Push the second half into the stack
  __UpperCAmelCase						    =       [slow.val]
  while slow.next:
    __UpperCAmelCase						    =       slow.next
    stack.append(slow.val					)

  # 3. Comparison
  while stack:
    if stack.pop() != cur.val:
      return False
    __UpperCAmelCase						    =       cur.next

  return True





def       __lowerCAmelCase      (						A_   : int					)    ->  Optional[int]:
  if not head or not head.next:
    return True
  __UpperCAmelCase						    =       {}
  __UpperCAmelCase						    =       0
  while head:
    if head.val in d:
      d[head.val].append(A_					)
    else:
      __UpperCAmelCase						    =       [pos]
    __UpperCAmelCase						    =       head.next
    pos += 1
  __UpperCAmelCase						    =       pos - 1
  __UpperCAmelCase						    =       0
  for v in d.values():
    if len(A_					) % 2 != 0:
      middle += 1
    else:
      __UpperCAmelCase						    =       0
      for i in range(0						,	len(A_					)					):
        if v[i] + v[len(A_					) - 1 - step] != checksum:
          return False
        step += 1
    if middle > 1:
      return False
  return True



