

import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class   UpperCamelCase			(  SCREAMING_SNAKE_CASE						,      unittest.TestCase ):
    __UpperCamelCase    =MobileBertTokenizer
    __UpperCamelCase    =MobileBertTokenizerFast
    __UpperCamelCase    =True
    __UpperCamelCase    =True
    __UpperCamelCase    =filter_non_english
    __UpperCamelCase    ="google/mobilebert-uncased"

    def UpperCamelCase      (      self			:   List[Any]     ):

     """simple docstring"""






     super().setUp()

     SCREAMING_SNAKE_CASE		    =      [
         '[UNK]',
         '[CLS]',
         '[SEP]',
         '[PAD]',
         '[MASK]',
         'want',
         '##want',
         '##ed',
         'wa',
         'un',
         'runn',
         '##ing',
         ',',
         'low',
         'lowest',
     ]
     SCREAMING_SNAKE_CASE		    =      os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES['vocab_file']     )
     with open(self.vocab_file				,     'w'				,     encoding='utf-8'     ) as vocab_writer:
      vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )

     SCREAMING_SNAKE_CASE		    =      [
         (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
         for tokenizer_def in self.tokenizers_list
     ]

    def UpperCamelCase      (      self			:   Tuple				,     snake_case__			:   Union[str, Any]     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      'UNwant\u00E9d,running'
     SCREAMING_SNAKE_CASE		    =      'unwanted, running'
     return input_text, output_text

    def UpperCamelCase      (      self			:   Any     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      self.tokenizer_class(self.vocab_file     )

     SCREAMING_SNAKE_CASE		    =      tokenizer.tokenize('UNwant\u00E9d,running'     )
     self.assertListEqual(snake_case__				,     ['un', '##want', '##ed', ',', 'runn', '##ing']     )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__     )				,     [9, 6, 7, 1_2, 1_0, 1_1]     )

    def UpperCamelCase      (      self			:   List[Any]     ):

     """simple docstring"""






     if not self.test_rust_tokenizer:
      return

     SCREAMING_SNAKE_CASE		    =      self.get_tokenizer()
     SCREAMING_SNAKE_CASE		    =      self.get_rust_tokenizer()

     SCREAMING_SNAKE_CASE		    =      'UNwant\u00E9d,running'

     SCREAMING_SNAKE_CASE		    =      tokenizer.tokenize(snake_case__     )
     SCREAMING_SNAKE_CASE		    =      rust_tokenizer.tokenize(snake_case__     )
     self.assertListEqual(snake_case__				,     snake_case__     )

     SCREAMING_SNAKE_CASE		    =      tokenizer.encode(snake_case__				,     add_special_tokens=snake_case__     )
     SCREAMING_SNAKE_CASE		    =      rust_tokenizer.encode(snake_case__				,     add_special_tokens=snake_case__     )
     self.assertListEqual(snake_case__				,     snake_case__     )

     SCREAMING_SNAKE_CASE		    =      self.get_rust_tokenizer()
     SCREAMING_SNAKE_CASE		    =      tokenizer.encode(snake_case__     )
     SCREAMING_SNAKE_CASE		    =      rust_tokenizer.encode(snake_case__     )
     self.assertListEqual(snake_case__				,     snake_case__     )

     # With lower casing
     SCREAMING_SNAKE_CASE		    =      self.get_tokenizer(do_lower_case=snake_case__     )
     SCREAMING_SNAKE_CASE		    =      self.get_rust_tokenizer(do_lower_case=snake_case__     )

     SCREAMING_SNAKE_CASE		    =      'UNwant\u00E9d,running'

     SCREAMING_SNAKE_CASE		    =      tokenizer.tokenize(snake_case__     )
     SCREAMING_SNAKE_CASE		    =      rust_tokenizer.tokenize(snake_case__     )
     self.assertListEqual(snake_case__				,     snake_case__     )

     SCREAMING_SNAKE_CASE		    =      tokenizer.encode(snake_case__				,     add_special_tokens=snake_case__     )
     SCREAMING_SNAKE_CASE		    =      rust_tokenizer.encode(snake_case__				,     add_special_tokens=snake_case__     )
     self.assertListEqual(snake_case__				,     snake_case__     )

     SCREAMING_SNAKE_CASE		    =      self.get_rust_tokenizer()
     SCREAMING_SNAKE_CASE		    =      tokenizer.encode(snake_case__     )
     SCREAMING_SNAKE_CASE		    =      rust_tokenizer.encode(snake_case__     )
     self.assertListEqual(snake_case__				,     snake_case__     )

    def UpperCamelCase      (      self			:   int     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'     )				,     ['ah', '\u535A', '\u63A8', 'zz']     )

    def UpperCamelCase      (      self			:   List[Any]     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      BasicTokenizer(do_lower_case=snake_case__     )

     self.assertListEqual(
         tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '     )				,     ['hello', '!', 'how', 'are', 'you', '?']     )
     self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )				,     ['hello']     )

    def UpperCamelCase      (      self			:   Tuple     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      BasicTokenizer(do_lower_case=snake_case__				,     strip_accents=snake_case__     )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )				,     ['hällo', '!', 'how', 'are', 'you', '?']     )
     self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )				,     ['h\u00E9llo']     )

    def UpperCamelCase      (      self			:   int     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      BasicTokenizer(do_lower_case=snake_case__				,     strip_accents=snake_case__     )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )				,     ['hallo', '!', 'how', 'are', 'you', '?']     )
     self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )				,     ['hello']     )

    def UpperCamelCase      (      self			:   Optional[int]     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      BasicTokenizer(do_lower_case=snake_case__     )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )				,     ['hallo', '!', 'how', 'are', 'you', '?']     )
     self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )				,     ['hello']     )

    def UpperCamelCase      (      self			:   Tuple     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      BasicTokenizer(do_lower_case=snake_case__     )

     self.assertListEqual(
         tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '     )				,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?']     )

    def UpperCamelCase      (      self			:   List[Any]     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      BasicTokenizer(do_lower_case=snake_case__				,     strip_accents=snake_case__     )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )				,     ['HäLLo', '!', 'how', 'Are', 'yoU', '?']     )

    def UpperCamelCase      (      self			:   List[str]     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      BasicTokenizer(do_lower_case=snake_case__				,     strip_accents=snake_case__     )

     self.assertListEqual(
         tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )				,     ['HaLLo', '!', 'how', 'Are', 'yoU', '?']     )

    def UpperCamelCase      (      self			:   Optional[int]     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      BasicTokenizer(do_lower_case=snake_case__				,     never_split=['[UNK]']     )

     self.assertListEqual(
         tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'     )				,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']     )

    def UpperCamelCase      (      self			:   Tuple     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

     SCREAMING_SNAKE_CASE		    =      {}
     for i, token in enumerate(snake_case__     ):
      SCREAMING_SNAKE_CASE		    =      i
     SCREAMING_SNAKE_CASE		    =      WordpieceTokenizer(vocab=snake_case__				,     unk_token='[UNK]'     )

     self.assertListEqual(tokenizer.tokenize(''     )				,     []     )

     self.assertListEqual(tokenizer.tokenize('unwanted running'     )				,     ['un', '##want', '##ed', 'runn', '##ing']     )

     self.assertListEqual(tokenizer.tokenize('unwantedX running'     )				,     ['[UNK]', 'runn', '##ing']     )

    def UpperCamelCase      (      self			:   Any     ):

     """simple docstring"""






     self.assertTrue(_is_whitespace(' '     )     )
     self.assertTrue(_is_whitespace('\t'     )     )
     self.assertTrue(_is_whitespace('\r'     )     )
     self.assertTrue(_is_whitespace('\n'     )     )
     self.assertTrue(_is_whitespace('\u00A0'     )     )

     self.assertFalse(_is_whitespace('A'     )     )
     self.assertFalse(_is_whitespace('-'     )     )

    def UpperCamelCase      (      self			:   Tuple     ):

     """simple docstring"""






     self.assertTrue(_is_control('\u0005'     )     )

     self.assertFalse(_is_control('A'     )     )
     self.assertFalse(_is_control(' '     )     )
     self.assertFalse(_is_control('\t'     )     )
     self.assertFalse(_is_control('\r'     )     )

    def UpperCamelCase      (      self			:   str     ):

     """simple docstring"""






     self.assertTrue(_is_punctuation('-'     )     )
     self.assertTrue(_is_punctuation('$'     )     )
     self.assertTrue(_is_punctuation('`'     )     )
     self.assertTrue(_is_punctuation('.'     )     )

     self.assertFalse(_is_punctuation('A'     )     )
     self.assertFalse(_is_punctuation(' '     )     )

    def UpperCamelCase      (      self			:   Optional[int]     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      self.get_tokenizer()
     SCREAMING_SNAKE_CASE		    =      self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(snake_case__     ) for t in ['Test', '\xad', 'test']]				,     [['[UNK]'], [], ['[UNK]']]     )

     self.assertListEqual(
         [rust_tokenizer.tokenize(snake_case__     ) for t in ['Test', '\xad', 'test']]				,     [['[UNK]'], [], ['[UNK]']]     )

    @slow
    def UpperCamelCase      (      self			:   Dict     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      self.tokenizer_class.from_pretrained('google/mobilebert-uncased'     )

     SCREAMING_SNAKE_CASE		    =      tokenizer.encode('sequence builders'				,     add_special_tokens=snake_case__     )
     SCREAMING_SNAKE_CASE		    =      tokenizer.encode('multi-sequence build'				,     add_special_tokens=snake_case__     )

     SCREAMING_SNAKE_CASE		    =      tokenizer.build_inputs_with_special_tokens(snake_case__     )
     SCREAMING_SNAKE_CASE		    =      tokenizer.build_inputs_with_special_tokens(snake_case__				,     snake_case__     )

     assert encoded_sentence == [1_0_1] + text + [1_0_2]
     assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]

    def UpperCamelCase      (      self			:   str     ):

     """simple docstring"""






     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
       SCREAMING_SNAKE_CASE		    =      self.rust_tokenizer_class.from_pretrained(snake_case__				,     **snake_case__     )

       SCREAMING_SNAKE_CASE		    =      F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
       SCREAMING_SNAKE_CASE		    =      tokenizer_r.encode_plus(
           snake_case__				,     return_attention_mask=snake_case__				,     return_token_type_ids=snake_case__				,     return_offsets_mapping=snake_case__				,     add_special_tokens=snake_case__				,     )

       SCREAMING_SNAKE_CASE		    =      tokenizer_r.do_lower_case if hasattr(snake_case__				,     'do_lower_case'     ) else False
       SCREAMING_SNAKE_CASE		    =      (
           [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), 'A'),
               ((1, 2), ','),
               ((3, 5), 'na'),
               ((5, 6), '##ï'),
               ((6, 8), '##ve'),
               ((9, 1_5), tokenizer_r.mask_token),
               ((1_6, 2_1), 'Allen'),
               ((2_1, 2_3), '##NL'),
               ((2_3, 2_4), '##P'),
               ((2_5, 3_3), 'sentence'),
               ((3_3, 3_4), '.'),
               ((0, 0), tokenizer_r.sep_token),
           ]
           if not do_lower_case
           else [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), 'a'),
               ((1, 2), ','),
               ((3, 8), 'naive'),
               ((9, 1_5), tokenizer_r.mask_token),
               ((1_6, 2_1), 'allen'),
               ((2_1, 2_3), '##nl'),
               ((2_3, 2_4), '##p'),
               ((2_5, 3_3), 'sentence'),
               ((3_3, 3_4), '.'),
               ((0, 0), tokenizer_r.sep_token),
           ]
       )

       self.assertEqual(
           [e[1] for e in expected_results]				,     tokenizer_r.convert_ids_to_tokens(tokens['input_ids']     )     )
       self.assertEqual([e[0] for e in expected_results]				,     tokens['offset_mapping']     )




    def UpperCamelCase      (      self			:   List[str]     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      ['的', '人', '有']
     SCREAMING_SNAKE_CASE		    =      ''.join(snake_case__     )
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
       SCREAMING_SNAKE_CASE		    =      True
       SCREAMING_SNAKE_CASE		    =      self.tokenizer_class.from_pretrained(snake_case__				,     **snake_case__     )
       SCREAMING_SNAKE_CASE		    =      self.rust_tokenizer_class.from_pretrained(snake_case__				,     **snake_case__     )

       SCREAMING_SNAKE_CASE		    =      tokenizer_p.encode(snake_case__				,     add_special_tokens=snake_case__     )
       SCREAMING_SNAKE_CASE		    =      tokenizer_r.encode(snake_case__				,     add_special_tokens=snake_case__     )

       SCREAMING_SNAKE_CASE		    =      tokenizer_r.convert_ids_to_tokens(snake_case__     )
       SCREAMING_SNAKE_CASE		    =      tokenizer_p.convert_ids_to_tokens(snake_case__     )

       # it is expected that each Chinese character is not preceded by "##"
       self.assertListEqual(snake_case__				,     snake_case__     )
       self.assertListEqual(snake_case__				,     snake_case__     )

       SCREAMING_SNAKE_CASE		    =      False
       SCREAMING_SNAKE_CASE		    =      self.rust_tokenizer_class.from_pretrained(snake_case__				,     **snake_case__     )
       SCREAMING_SNAKE_CASE		    =      self.tokenizer_class.from_pretrained(snake_case__				,     **snake_case__     )

       SCREAMING_SNAKE_CASE		    =      tokenizer_r.encode(snake_case__				,     add_special_tokens=snake_case__     )
       SCREAMING_SNAKE_CASE		    =      tokenizer_p.encode(snake_case__				,     add_special_tokens=snake_case__     )

       SCREAMING_SNAKE_CASE		    =      tokenizer_r.convert_ids_to_tokens(snake_case__     )
       SCREAMING_SNAKE_CASE		    =      tokenizer_p.convert_ids_to_tokens(snake_case__     )

       # it is expected that only the first Chinese character is not preceded by "##".
       SCREAMING_SNAKE_CASE		    =      [
           F"""##{token}""" if idx != 0 else token for idx, token in enumerate(snake_case__     )
       ]
       self.assertListEqual(snake_case__				,     snake_case__     )
       self.assertListEqual(snake_case__				,     snake_case__     )




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_rembert import RemBertTokenizer
else:
      a_     :    int     		=   None

a_     :    List[Any]     		=   logging.get_logger(__name__)
a_     :    List[Any]     		=   {"vocab_file": "sentencepiece.model", "tokenizer_file": "tokenizer.json"}

a_     :    Optional[int]     		=   {
    "vocab_file": {
        "google/rembert": "https://huggingface.co/google/rembert/resolve/main/sentencepiece.model",
    },
    "tokenizer_file": {
        "google/rembert": "https://huggingface.co/google/rembert/resolve/main/tokenizer.json",
    },
}

a_     :    Union[str, Any]     		=   {
    "google/rembert": 256,
}

a_     :    List[Any]     		=   "▁"




class   UpperCamelCase			(  SCREAMING_SNAKE_CASE ):
    __UpperCamelCase    =VOCAB_FILES_NAMES
    __UpperCamelCase    =PRETRAINED_VOCAB_FILES_MAP
    __UpperCamelCase    =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __UpperCamelCase    =RemBertTokenizer

    def __init__(      self			:   int				,     snake_case__			:   Any=None				,     snake_case__			:   Optional[Any]=None				,     snake_case__			:   Optional[int]=True				,     snake_case__			:   Any=True				,     snake_case__			:   Any=False				,     snake_case__			:   Dict="[CLS]"				,     snake_case__			:   List[str]="[SEP]"				,     snake_case__			:   List[str]="<unk>"				,     snake_case__			:   Union[str, Any]="[SEP]"				,     snake_case__			:   List[Any]="<pad>"				,     snake_case__			:   Tuple="[CLS]"				,     snake_case__			:   str="[MASK]"				,     **snake_case__			:   Union[str, Any]				,     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      AddedToken(snake_case__				,     lstrip=snake_case__				,     rstrip=snake_case__     ) if isinstance(snake_case__				,     snake_case__     ) else mask_token

     super().__init__(
         snake_case__				,     tokenizer_file=snake_case__				,     do_lower_case=snake_case__				,     remove_space=snake_case__				,     keep_accents=snake_case__				,     bos_token=snake_case__				,     eos_token=snake_case__				,     unk_token=snake_case__				,     sep_token=snake_case__				,     pad_token=snake_case__				,     cls_token=snake_case__				,     mask_token=snake_case__				,     **snake_case__				,     )

     SCREAMING_SNAKE_CASE		    =      do_lower_case
     SCREAMING_SNAKE_CASE		    =      remove_space
     SCREAMING_SNAKE_CASE		    =      keep_accents
     SCREAMING_SNAKE_CASE		    =      vocab_file
     SCREAMING_SNAKE_CASE		    =      False if not self.vocab_file else True

    def UpperCamelCase      (      self			:   Tuple				,     snake_case__			:   List[int]				,     snake_case__			:   Optional[List[int]] = None     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      [self.sep_token_id]
     SCREAMING_SNAKE_CASE		    =      [self.cls_token_id]
     if token_ids_a is None:
      return cls + token_ids_a + sep
     return cls + token_ids_a + sep + token_ids_a + sep

    def UpperCamelCase      (      self			:   Tuple				,     snake_case__			:   List[int]				,     snake_case__			:   Optional[List[int]] = None				,     snake_case__			:   bool = False     ):

     """simple docstring"""






     if already_has_special_tokens:
      if token_ids_a is not None:
       raise ValueError(
           'You should not supply a second sequence if the provided sequence of '
           'ids is already formatted with special tokens for the model.'     )
      return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

     if token_ids_a is not None:
      return [1] + ([0] * len(snake_case__     )) + [1] + ([0] * len(snake_case__     )) + [1]
     return [1] + ([0] * len(snake_case__     )) + [1]

    def UpperCamelCase      (      self			:   int				,     snake_case__			:   List[int]				,     snake_case__			:   Optional[List[int]] = None     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      [self.sep_token_id]
     SCREAMING_SNAKE_CASE		    =      [self.cls_token_id]

     if token_ids_a is None:
      return len(cls + token_ids_a + sep     ) * [0]
     return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]




    def UpperCamelCase      (      self			:   Dict				,     snake_case__			:   str				,     snake_case__			:   Optional[str] = None     ):

     """simple docstring"""






     if not os.path.isdir(snake_case__     ):
      logger.error('Vocabulary path ({}) should be a directory'.format(snake_case__     )     )
      return
     SCREAMING_SNAKE_CASE		    =      os.path.join(
         snake_case__				,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )

     if os.path.abspath(self.vocab_file     ) != os.path.abspath(snake_case__     ):
      copyfile(self.vocab_file				,     snake_case__     )

     return (out_vocab_file,)

