




import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 			_UpperCamelCase      (				lowerCAmelCase	,       unittest.TestCase		):
      UpperCAmelCase_        =		MobileBertTokenizer
      UpperCAmelCase_        =		MobileBertTokenizerFast
      UpperCAmelCase_        =		True
      UpperCAmelCase_        =		True
      UpperCAmelCase_        =		filter_non_english
      UpperCAmelCase_        =		"""google/mobilebert-uncased"""
      def 	UpperCAmelCase_						(			self   :int      )     ->			List[str]:
       super().setUp()

       UpperCAmelCase__							 =						[
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "[PAD]",
           "[MASK]",
           "want",
           "##want",
           "##ed",
           "wa",
           "un",
           "runn",
           "##ing",
           ",",
           "low",
           "lowest",
       ]
       UpperCAmelCase__							 =						os.path.join(self.tmpdirname ,      VOCAB_FILES_NAMES["vocab_file"]      )
       with open(self.vocab_file ,      "w" ,      encoding="utf-8"      ) as vocab_writer:
        vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )

       UpperCAmelCase__							 =						[
           (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
           for tokenizer_def in self.tokenizers_list
       ]
      def 	UpperCAmelCase_						(			self   :Dict ,      lowerCamelCase   :int      )     ->			Optional[int]:
       UpperCAmelCase__							 =						"UNwant\u00E9d,running"
       UpperCAmelCase__							 =						"unwanted, running"
       return input_text, output_text
      def 	UpperCAmelCase_						(			self   :List[str]      )     ->			str:
       UpperCAmelCase__							 =						self.tokenizer_class(self.vocab_file      )

       UpperCAmelCase__							 =						tokenizer.tokenize("UNwant\u00E9d,running"      )
       self.assertListEqual(lowerCamelCase ,      ["un", "##want", "##ed", ",", "runn", "##ing"]      )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase      ) ,      [9, 6, 7, 12, 10, 11]      )
      def 	UpperCAmelCase_						(			self   :Union[str, Any]      )     ->			List[Any]:
       if not self.test_rust_tokenizer:
        return

       UpperCAmelCase__							 =						self.get_tokenizer()
       UpperCAmelCase__							 =						self.get_rust_tokenizer()

       UpperCAmelCase__							 =						"UNwant\u00E9d,running"

       UpperCAmelCase__							 =						tokenizer.tokenize(lowerCamelCase      )
       UpperCAmelCase__							 =						rust_tokenizer.tokenize(lowerCamelCase      )
       self.assertListEqual(lowerCamelCase ,      lowerCamelCase      )

       UpperCAmelCase__							 =						tokenizer.encode(lowerCamelCase ,      add_special_tokens=lowerCamelCase      )
       UpperCAmelCase__							 =						rust_tokenizer.encode(lowerCamelCase ,      add_special_tokens=lowerCamelCase      )
       self.assertListEqual(lowerCamelCase ,      lowerCamelCase      )

       UpperCAmelCase__							 =						self.get_rust_tokenizer()
       UpperCAmelCase__							 =						tokenizer.encode(lowerCamelCase      )
       UpperCAmelCase__							 =						rust_tokenizer.encode(lowerCamelCase      )
       self.assertListEqual(lowerCamelCase ,      lowerCamelCase      )

       # With lower casing
       UpperCAmelCase__							 =						self.get_tokenizer(do_lower_case=lowerCamelCase      )
       UpperCAmelCase__							 =						self.get_rust_tokenizer(do_lower_case=lowerCamelCase      )

       UpperCAmelCase__							 =						"UNwant\u00E9d,running"

       UpperCAmelCase__							 =						tokenizer.tokenize(lowerCamelCase      )
       UpperCAmelCase__							 =						rust_tokenizer.tokenize(lowerCamelCase      )
       self.assertListEqual(lowerCamelCase ,      lowerCamelCase      )

       UpperCAmelCase__							 =						tokenizer.encode(lowerCamelCase ,      add_special_tokens=lowerCamelCase      )
       UpperCAmelCase__							 =						rust_tokenizer.encode(lowerCamelCase ,      add_special_tokens=lowerCamelCase      )
       self.assertListEqual(lowerCamelCase ,      lowerCamelCase      )

       UpperCAmelCase__							 =						self.get_rust_tokenizer()
       UpperCAmelCase__							 =						tokenizer.encode(lowerCamelCase      )
       UpperCAmelCase__							 =						rust_tokenizer.encode(lowerCamelCase      )
       self.assertListEqual(lowerCamelCase ,      lowerCamelCase      )
      def 	UpperCAmelCase_						(			self   :Optional[int]      )     ->			List[Any]:
       UpperCAmelCase__							 =						BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"      ) ,      ["ah", "\u535A", "\u63A8", "zz"]      )
      def 	UpperCAmelCase_						(			self   :Optional[Any]      )     ->			str:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      ) ,      ["hello", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      ) ,      ["hello"]      )
      def 	UpperCAmelCase_						(			self   :Optional[Any]      )     ->			Dict:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase ,      strip_accents=lowerCamelCase      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      ) ,      ["hällo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      ) ,      ["h\u00E9llo"]      )
      def 	UpperCAmelCase_						(			self   :str      )     ->			Optional[Any]:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase ,      strip_accents=lowerCamelCase      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      ) ,      ["hallo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      ) ,      ["hello"]      )
      def 	UpperCAmelCase_						(			self   :Union[str, Any]      )     ->			int:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      ) ,      ["hallo", "!", "how", "are", "you", "?"]      )
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"      ) ,      ["hello"]      )
      def 	UpperCAmelCase_						(			self   :Tuple      )     ->			Optional[int]:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "      ) ,      ["HeLLo", "!", "how", "Are", "yoU", "?"]      )
      def 	UpperCAmelCase_						(			self   :List[str]      )     ->			Union[str, Any]:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase ,      strip_accents=lowerCamelCase      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      ) ,      ["HäLLo", "!", "how", "Are", "yoU", "?"]      )
      def 	UpperCAmelCase_						(			self   :Tuple      )     ->			Optional[Any]:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase ,      strip_accents=lowerCamelCase      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "      ) ,      ["HaLLo", "!", "how", "Are", "yoU", "?"]      )
      def 	UpperCAmelCase_						(			self   :Any      )     ->			Optional[int]:
       UpperCAmelCase__							 =						BasicTokenizer(do_lower_case=lowerCamelCase ,      never_split=["[UNK]"]      )

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"      ) ,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]      )
      def 	UpperCAmelCase_						(			self   :Dict      )     ->			Union[str, Any]:
       UpperCAmelCase__							 =						["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

       UpperCAmelCase__							 =						{}
       for i, token in enumerate(lowerCamelCase      ):
        UpperCAmelCase__							 =						i
       UpperCAmelCase__							 =						WordpieceTokenizer(vocab=lowerCamelCase ,      unk_token="[UNK]"      )

       self.assertListEqual(tokenizer.tokenize(""      ) ,      []      )

       self.assertListEqual(tokenizer.tokenize("unwanted running"      ) ,      ["un", "##want", "##ed", "runn", "##ing"]      )

       self.assertListEqual(tokenizer.tokenize("unwantedX running"      ) ,      ["[UNK]", "runn", "##ing"]      )
      def 	UpperCAmelCase_						(			self   :str      )     ->			Union[str, Any]:
       self.assertTrue(_is_whitespace(" "      )      )
       self.assertTrue(_is_whitespace("\t"      )      )
       self.assertTrue(_is_whitespace("\r"      )      )
       self.assertTrue(_is_whitespace("\n"      )      )
       self.assertTrue(_is_whitespace("\u00A0"      )      )

       self.assertFalse(_is_whitespace("A"      )      )
       self.assertFalse(_is_whitespace("-"      )      )
      def 	UpperCAmelCase_						(			self   :List[Any]      )     ->			Dict:
       self.assertTrue(_is_control("\u0005"      )      )

       self.assertFalse(_is_control("A"      )      )
       self.assertFalse(_is_control(" "      )      )
       self.assertFalse(_is_control("\t"      )      )
       self.assertFalse(_is_control("\r"      )      )
      def 	UpperCAmelCase_						(			self   :List[str]      )     ->			Optional[int]:
       self.assertTrue(_is_punctuation("-"      )      )
       self.assertTrue(_is_punctuation("$"      )      )
       self.assertTrue(_is_punctuation("`"      )      )
       self.assertTrue(_is_punctuation("."      )      )

       self.assertFalse(_is_punctuation("A"      )      )
       self.assertFalse(_is_punctuation(" "      )      )
      def 	UpperCAmelCase_						(			self   :Optional[Any]      )     ->			Tuple:
       UpperCAmelCase__							 =						self.get_tokenizer()
       UpperCAmelCase__							 =						self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(lowerCamelCase      ) for t in ["Test", "\xad", "test"]] ,      [["[UNK]"], [], ["[UNK]"]]      )

       self.assertListEqual(
           [rust_tokenizer.tokenize(lowerCamelCase      ) for t in ["Test", "\xad", "test"]] ,      [["[UNK]"], [], ["[UNK]"]]      )
      @slow
      def 	UpperCAmelCase_						(			self   :Union[str, Any]      )     ->			str:
       UpperCAmelCase__							 =						self.tokenizer_class.from_pretrained("google/mobilebert-uncased"      )

       UpperCAmelCase__							 =						tokenizer.encode("sequence builders" ,      add_special_tokens=lowerCamelCase      )
       UpperCAmelCase__							 =						tokenizer.encode("multi-sequence build" ,      add_special_tokens=lowerCamelCase      )

       UpperCAmelCase__							 =						tokenizer.build_inputs_with_special_tokens(lowerCamelCase      )
       UpperCAmelCase__							 =						tokenizer.build_inputs_with_special_tokens(lowerCamelCase ,      lowerCamelCase      )

       assert encoded_sentence == [101] + text + [102]
       assert encoded_pair == [101] + text + [102] + text_a + [102]
      def 	UpperCAmelCase_						(			self   :Any      )     ->			Optional[int]:
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
         UpperCAmelCase__							 =						self.rust_tokenizer_class.from_pretrained(lowerCamelCase ,      **lowerCamelCase      )

         UpperCAmelCase__							 =						f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
         UpperCAmelCase__							 =						tokenizer_r.encode_plus(
             lowerCamelCase ,      return_attention_mask=lowerCamelCase ,      return_token_type_ids=lowerCamelCase ,      return_offsets_mapping=lowerCamelCase ,      add_special_tokens=lowerCamelCase ,      )

         UpperCAmelCase__							 =						tokenizer_r.do_lower_case if hasattr(lowerCamelCase ,      "do_lower_case"      ) else False
         UpperCAmelCase__							 =						(
             [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), "A"),
                 ((1, 2), ","),
                 ((3, 5), "na"),
                 ((5, 6), "##ï"),
                 ((6, 8), "##ve"),
                 ((9, 15), tokenizer_r.mask_token),
                 ((16, 21), "Allen"),
                 ((21, 23), "##NL"),
                 ((23, 24), "##P"),
                 ((25, 33), "sentence"),
                 ((33, 34), "."),
                 ((0, 0), tokenizer_r.sep_token),
             ]
             if not do_lower_case
             else [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), "a"),
                 ((1, 2), ","),
                 ((3, 8), "naive"),
                 ((9, 15), tokenizer_r.mask_token),
                 ((16, 21), "allen"),
                 ((21, 23), "##nl"),
                 ((23, 24), "##p"),
                 ((25, 33), "sentence"),
                 ((33, 34), "."),
                 ((0, 0), tokenizer_r.sep_token),
             ]
         )

         self.assertEqual(
             [e[1] for e in expected_results] ,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]      )      )
         self.assertEqual([e[0] for e in expected_results] ,      tokens["offset_mapping"]      )







      def 	UpperCAmelCase_						(			self   :Optional[Any]      )     ->			Any:
       UpperCAmelCase__							 =						["的", "人", "有"]
       UpperCAmelCase__							 =						"".join(lowerCamelCase      )
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
         UpperCAmelCase__							 =						True
         UpperCAmelCase__							 =						self.tokenizer_class.from_pretrained(lowerCamelCase ,      **lowerCamelCase      )
         UpperCAmelCase__							 =						self.rust_tokenizer_class.from_pretrained(lowerCamelCase ,      **lowerCamelCase      )

         UpperCAmelCase__							 =						tokenizer_p.encode(lowerCamelCase ,      add_special_tokens=lowerCamelCase      )
         UpperCAmelCase__							 =						tokenizer_r.encode(lowerCamelCase ,      add_special_tokens=lowerCamelCase      )

         UpperCAmelCase__							 =						tokenizer_r.convert_ids_to_tokens(lowerCamelCase      )
         UpperCAmelCase__							 =						tokenizer_p.convert_ids_to_tokens(lowerCamelCase      )

         # it is expected that each Chinese character is not preceded by "##"
         self.assertListEqual(lowerCamelCase ,      lowerCamelCase      )
         self.assertListEqual(lowerCamelCase ,      lowerCamelCase      )

         UpperCAmelCase__							 =						False
         UpperCAmelCase__							 =						self.rust_tokenizer_class.from_pretrained(lowerCamelCase ,      **lowerCamelCase      )
         UpperCAmelCase__							 =						self.tokenizer_class.from_pretrained(lowerCamelCase ,      **lowerCamelCase      )

         UpperCAmelCase__							 =						tokenizer_r.encode(lowerCamelCase ,      add_special_tokens=lowerCamelCase      )
         UpperCAmelCase__							 =						tokenizer_p.encode(lowerCamelCase ,      add_special_tokens=lowerCamelCase      )

         UpperCAmelCase__							 =						tokenizer_r.convert_ids_to_tokens(lowerCamelCase      )
         UpperCAmelCase__							 =						tokenizer_p.convert_ids_to_tokens(lowerCamelCase      )

         # it is expected that only the first Chinese character is not preceded by "##".
         UpperCAmelCase__							 =						[
             f'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCamelCase      )
         ]
         self.assertListEqual(lowerCamelCase ,      lowerCamelCase      )
         self.assertListEqual(lowerCamelCase ,      lowerCamelCase      )











from typing import List, Optional, Union

import numpy as np

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


_lowerCAmelCase						:   Any		=   logging.get_logger(__name__)
class 			_UpperCamelCase      (				lowerCAmelCase		):
      UpperCAmelCase_        =		["""input_values""", """padding_mask"""]
      def __init__(			self   :Dict ,      lowerCamelCase   :int = 1 ,      lowerCamelCase   :int = 2_4000 ,      lowerCamelCase   :float = 0.0 ,      lowerCamelCase   :float = None ,      lowerCamelCase   :float = None ,      **lowerCamelCase   :Optional[Any] ,      )     ->			str:
       super().__init__(feature_size=lowerCamelCase ,      sampling_rate=lowerCamelCase ,      padding_value=lowerCamelCase ,      **lowerCamelCase      )
       UpperCAmelCase__							 =						chunk_length_s
       UpperCAmelCase__							 =						overlap
      @property
      def 	UpperCAmelCase_						(			self   :Optional[Any]      )     ->			Optional[int]:
       if self.chunk_length_s is None:
        return None
       else:
        return int(self.chunk_length_s * self.sampling_rate      )
      @property
      def 	UpperCAmelCase_						(			self   :str      )     ->			Optional[int]:
       if self.chunk_length_s is None or self.overlap is None:
        return None
       else:
        return max(1 ,      int((1.0 - self.overlap) * self.chunk_length      )      )







      def __call__(			self   :List[Any] ,      lowerCamelCase   :Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]] ,      lowerCamelCase   :Optional[Union[bool, str, PaddingStrategy]] = None ,      lowerCamelCase   :Optional[bool] = False ,      lowerCamelCase   :Optional[int] = None ,      lowerCamelCase   :Optional[Union[str, TensorType]] = None ,      lowerCamelCase   :Optional[int] = None ,      )     ->			BatchFeature:
       if sampling_rate is not None:
        if sampling_rate != self.sampling_rate:
         raise ValueError(
             f'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
             f''' {self.sampling_rate}. Please make sure that the provided audio input was sampled with'''
             f''' {self.sampling_rate} and not {sampling_rate}.'''      )
       else:
        logger.warning(
            "It is strongly recommended to pass the `sampling_rate` argument to this function. "
            "Failing to do so can result in silent errors that might be hard to debug."      )

       if padding and truncation:
        raise ValueError("Both padding and truncation were set. Make sure you only set one."      )
       elif padding is None:
        # by default let's pad the inputs
        UpperCAmelCase__							 =						True

       UpperCAmelCase__							 =						bool(
           isinstance(lowerCamelCase ,      (list, tuple)      ) and (isinstance(raw_audio[0] ,      (np.ndarray, tuple, list)      ))      )

       if is_batched:
        UpperCAmelCase__							 =						[np.asarray(lowerCamelCase ,      dtype=np.floataa      ).T for audio in raw_audio]
       elif not is_batched and not isinstance(lowerCamelCase ,      np.ndarray      ):
        UpperCAmelCase__							 =						np.asarray(lowerCamelCase ,      dtype=np.floataa      )
       elif isinstance(lowerCamelCase ,      np.ndarray      ) and raw_audio.dtype is np.dtype(np.floataa      ):
        UpperCAmelCase__							 =						raw_audio.astype(np.floataa      )

       # always return batch
       if not is_batched:
        UpperCAmelCase__							 =						[np.asarray(lowerCamelCase      ).T]

       # verify inputs are valid
       for idx, example in enumerate(lowerCamelCase      ):
        if example.ndim > 2:
         raise ValueError(f'''Expected input shape (channels, length) but got shape {example.shape}'''      )
        if self.feature_size == 1 and example.ndim != 1:
         raise ValueError(f'''Expected mono audio but example has {example.shape[-1]} channels'''      )
        if self.feature_size == 2 and example.shape[-1] != 2:
         raise ValueError(f'''Expected stereo audio but example has {example.shape[-1]} channels'''      )

       UpperCAmelCase__							 =						None
       UpperCAmelCase__							 =						BatchFeature({"input_values": raw_audio}      )
       if self.chunk_stride is not None and self.chunk_length is not None and max_length is None:
        if truncation:
         UpperCAmelCase__							 =						min(array.shape[0] for array in raw_audio      )
         UpperCAmelCase__							 =						int(np.floor(max_length / self.chunk_stride      )      )
         UpperCAmelCase__							 =						(nb_step - 1) * self.chunk_stride + self.chunk_length
        elif padding:
         UpperCAmelCase__							 =						max(array.shape[0] for array in raw_audio      )
         UpperCAmelCase__							 =						int(np.ceil(max_length / self.chunk_stride      )      )
         UpperCAmelCase__							 =						(nb_step - 1) * self.chunk_stride + self.chunk_length
         UpperCAmelCase__							 =						"max_length"
        else:
         UpperCAmelCase__							 =						input_values

        # normal padding on batch
       if padded_inputs is None:
        UpperCAmelCase__							 =						self.pad(
            lowerCamelCase ,      max_length=lowerCamelCase ,      truncation=lowerCamelCase ,      padding=lowerCamelCase ,      return_attention_mask=lowerCamelCase ,      )
        if padding:
         UpperCAmelCase__							 =						padded_inputs.pop("attention_mask"      )

       UpperCAmelCase__							 =						[]
       for example in padded_inputs.pop("input_values"      ):
        if self.feature_size == 1:
         UpperCAmelCase__							 =						example[..., None]
        input_values.append(example.T      )

       UpperCAmelCase__							 =						input_values
       if return_tensors is not None:
        UpperCAmelCase__							 =						padded_inputs.convert_to_tensors(lowerCamelCase      )

       return padded_inputs





