


"""simple docstring"""



import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class _lowerCAmelCase							( snake_case_       ,				unittest.TestCase      ):
   __UpperCAmelCase					:			Optional[int]				    =    MobileBertTokenizer
   __UpperCAmelCase					:			Dict				    =    MobileBertTokenizerFast
   __UpperCAmelCase					:			Optional[int]				    =    True
   __UpperCAmelCase					:			Union[str, Any]				    =    True
   __UpperCAmelCase					:			List[str]				    =    filter_non_english
   __UpperCAmelCase					:			Any				    =    '''google/mobilebert-uncased'''







   def 					lowerCamelCase						(      self		)      ->						Union[str, Any]:

       '''simple docstring'''




       super().setUp()

       snake_case       : int    =       [
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "[PAD]",
           "[MASK]",
           "want",
           "##want",
           "##ed",
           "wa",
           "un",
           "runn",
           "##ing",
           ",",
           "low",
           "lowest",
       ]
       snake_case       : Dict    =       os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES["vocab_file"]		)
       with open(self.vocab_file		,      "w"		,      encoding="utf-8"		) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)

       snake_case       : Optional[int]    =       [
           (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
           for tokenizer_def in self.tokenizers_list
       ]







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						List[str]:

       '''simple docstring'''




       snake_case       : List[str]    =       "UNwant\u00E9d,running"
       snake_case       : str    =       "unwanted, running"
       return input_text, output_text







   def 					lowerCamelCase						(      self		)      ->						Union[str, Any]:

       '''simple docstring'''




       snake_case       : Dict    =       self.tokenizer_class(self.vocab_file		)

       snake_case       : Any    =       tokenizer.tokenize("UNwant\u00E9d,running"		)
       self.assertListEqual(UpperCamelCase__		,      ["un", "##want", "##ed", ",", "runn", "##ing"]		)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__		)		,      [9, 6, 7, 12, 10, 11]		)







   def 					lowerCamelCase						(      self		)      ->						int:

       '''simple docstring'''




       if not self.test_rust_tokenizer:
           return

       snake_case       : List[Any]    =       self.get_tokenizer()
       snake_case       : List[Any]    =       self.get_rust_tokenizer()

       snake_case       : str    =       "UNwant\u00E9d,running"

       snake_case       : List[Any]    =       tokenizer.tokenize(UpperCamelCase__		)
       snake_case       : List[Any]    =       rust_tokenizer.tokenize(UpperCamelCase__		)
       self.assertListEqual(UpperCamelCase__		,      UpperCamelCase__		)

       snake_case       : Any    =       tokenizer.encode(UpperCamelCase__		,      add_special_tokens=UpperCamelCase__		)
       snake_case       : Any    =       rust_tokenizer.encode(UpperCamelCase__		,      add_special_tokens=UpperCamelCase__		)
       self.assertListEqual(UpperCamelCase__		,      UpperCamelCase__		)

       snake_case       : str    =       self.get_rust_tokenizer()
       snake_case       : Dict    =       tokenizer.encode(UpperCamelCase__		)
       snake_case       : Dict    =       rust_tokenizer.encode(UpperCamelCase__		)
       self.assertListEqual(UpperCamelCase__		,      UpperCamelCase__		)

       # With lower casing
       snake_case       : Dict    =       self.get_tokenizer(do_lower_case=UpperCamelCase__		)
       snake_case       : Union[str, Any]    =       self.get_rust_tokenizer(do_lower_case=UpperCamelCase__		)

       snake_case       : Union[str, Any]    =       "UNwant\u00E9d,running"

       snake_case       : Optional[int]    =       tokenizer.tokenize(UpperCamelCase__		)
       snake_case       : Dict    =       rust_tokenizer.tokenize(UpperCamelCase__		)
       self.assertListEqual(UpperCamelCase__		,      UpperCamelCase__		)

       snake_case       : Optional[int]    =       tokenizer.encode(UpperCamelCase__		,      add_special_tokens=UpperCamelCase__		)
       snake_case       : Optional[int]    =       rust_tokenizer.encode(UpperCamelCase__		,      add_special_tokens=UpperCamelCase__		)
       self.assertListEqual(UpperCamelCase__		,      UpperCamelCase__		)

       snake_case       : int    =       self.get_rust_tokenizer()
       snake_case       : int    =       tokenizer.encode(UpperCamelCase__		)
       snake_case       : Any    =       rust_tokenizer.encode(UpperCamelCase__		)
       self.assertListEqual(UpperCamelCase__		,      UpperCamelCase__		)







   def 					lowerCamelCase						(      self		)      ->						Tuple:

       '''simple docstring'''




       snake_case       : List[str]    =       BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"		)		,      ["ah", "\u535A", "\u63A8", "zz"]		)







   def 					lowerCamelCase						(      self		)      ->						Union[str, Any]:

       '''simple docstring'''




       snake_case       : Any    =       BasicTokenizer(do_lower_case=UpperCamelCase__		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)		,      ["hello", "!", "how", "are", "you", "?"]		)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)		,      ["hello"]		)







   def 					lowerCamelCase						(      self		)      ->						int:

       '''simple docstring'''




       snake_case       : Dict    =       BasicTokenizer(do_lower_case=UpperCamelCase__		,      strip_accents=UpperCamelCase__		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)		,      ["hällo", "!", "how", "are", "you", "?"]		)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)		,      ["h\u00E9llo"]		)







   def 					lowerCamelCase						(      self		)      ->						Union[str, Any]:

       '''simple docstring'''




       snake_case       : Optional[int]    =       BasicTokenizer(do_lower_case=UpperCamelCase__		,      strip_accents=UpperCamelCase__		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)		,      ["hallo", "!", "how", "are", "you", "?"]		)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)		,      ["hello"]		)







   def 					lowerCamelCase						(      self		)      ->						Tuple:

       '''simple docstring'''




       snake_case       : Tuple    =       BasicTokenizer(do_lower_case=UpperCamelCase__		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)		,      ["hallo", "!", "how", "are", "you", "?"]		)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)		,      ["hello"]		)







   def 					lowerCamelCase						(      self		)      ->						List[Any]:

       '''simple docstring'''




       snake_case       : List[str]    =       BasicTokenizer(do_lower_case=UpperCamelCase__		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)		,      ["HeLLo", "!", "how", "Are", "yoU", "?"]		)







   def 					lowerCamelCase						(      self		)      ->						List[Any]:

       '''simple docstring'''




       snake_case       : Optional[int]    =       BasicTokenizer(do_lower_case=UpperCamelCase__		,      strip_accents=UpperCamelCase__		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)		,      ["HäLLo", "!", "how", "Are", "yoU", "?"]		)







   def 					lowerCamelCase						(      self		)      ->						Tuple:

       '''simple docstring'''




       snake_case       : List[Any]    =       BasicTokenizer(do_lower_case=UpperCamelCase__		,      strip_accents=UpperCamelCase__		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)		,      ["HaLLo", "!", "how", "Are", "yoU", "?"]		)







   def 					lowerCamelCase						(      self		)      ->						Any:

       '''simple docstring'''




       snake_case       : List[str]    =       BasicTokenizer(do_lower_case=UpperCamelCase__		,      never_split=["[UNK]"]		)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"		)		,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]		)







   def 					lowerCamelCase						(      self		)      ->						Any:

       '''simple docstring'''




       snake_case       : Any    =       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

       snake_case       : Any    =       {}
       for i, token in enumerate(UpperCamelCase__		):
           snake_case       : Optional[int]    =       i
       snake_case       : List[str]    =       WordpieceTokenizer(vocab=UpperCamelCase__		,      unk_token="[UNK]"		)

       self.assertListEqual(tokenizer.tokenize(""		)		,      []		)

       self.assertListEqual(tokenizer.tokenize("unwanted running"		)		,      ["un", "##want", "##ed", "runn", "##ing"]		)

       self.assertListEqual(tokenizer.tokenize("unwantedX running"		)		,      ["[UNK]", "runn", "##ing"]		)







   def 					lowerCamelCase						(      self		)      ->						Tuple:

       '''simple docstring'''




       self.assertTrue(_is_whitespace(" "		)		)
       self.assertTrue(_is_whitespace("\t"		)		)
       self.assertTrue(_is_whitespace("\r"		)		)
       self.assertTrue(_is_whitespace("\n"		)		)
       self.assertTrue(_is_whitespace("\u00A0"		)		)

       self.assertFalse(_is_whitespace("A"		)		)
       self.assertFalse(_is_whitespace("-"		)		)







   def 					lowerCamelCase						(      self		)      ->						Tuple:

       '''simple docstring'''




       self.assertTrue(_is_control("\u0005"		)		)

       self.assertFalse(_is_control("A"		)		)
       self.assertFalse(_is_control(" "		)		)
       self.assertFalse(_is_control("\t"		)		)
       self.assertFalse(_is_control("\r"		)		)







   def 					lowerCamelCase						(      self		)      ->						List[Any]:

       '''simple docstring'''




       self.assertTrue(_is_punctuation("-"		)		)
       self.assertTrue(_is_punctuation("$"		)		)
       self.assertTrue(_is_punctuation("`"		)		)
       self.assertTrue(_is_punctuation("."		)		)

       self.assertFalse(_is_punctuation("A"		)		)
       self.assertFalse(_is_punctuation(" "		)		)







   def 					lowerCamelCase						(      self		)      ->						List[str]:

       '''simple docstring'''




       snake_case       : Any    =       self.get_tokenizer()
       snake_case       : int    =       self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(UpperCamelCase__		) for t in ["Test", "\xad", "test"]]		,      [["[UNK]"], [], ["[UNK]"]]		)

       self.assertListEqual(
           [rust_tokenizer.tokenize(UpperCamelCase__		) for t in ["Test", "\xad", "test"]]		,      [["[UNK]"], [], ["[UNK]"]]		)







   @slow
   def 					lowerCamelCase						(      self		)      ->						Optional[int]:

       '''simple docstring'''




       snake_case       : Optional[Any]    =       self.tokenizer_class.from_pretrained("google/mobilebert-uncased"		)

       snake_case       : Optional[int]    =       tokenizer.encode("sequence builders"		,      add_special_tokens=UpperCamelCase__		)
       snake_case       : List[str]    =       tokenizer.encode("multi-sequence build"		,      add_special_tokens=UpperCamelCase__		)

       snake_case       : int    =       tokenizer.build_inputs_with_special_tokens(UpperCamelCase__		)
       snake_case       : Optional[int]    =       tokenizer.build_inputs_with_special_tokens(UpperCamelCase__		,      UpperCamelCase__		)

       assert encoded_sentence == [101] + text + [102]
       assert encoded_pair == [101] + text + [102] + text_a + [102]







   def 					lowerCamelCase						(      self		)      ->						Tuple:

       '''simple docstring'''




       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'		):
               snake_case       : List[str]    =       self.rust_tokenizer_class.from_pretrained(UpperCamelCase__		,      **UpperCamelCase__		)

               snake_case       : Optional[int]    =       F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
               snake_case       : Union[str, Any]    =       tokenizer_r.encode_plus(
                   UpperCamelCase__		,      return_attention_mask=UpperCamelCase__		,      return_token_type_ids=UpperCamelCase__		,      return_offsets_mapping=UpperCamelCase__		,      add_special_tokens=UpperCamelCase__		,      )

               snake_case       : Optional[Any]    =       tokenizer_r.do_lower_case if hasattr(UpperCamelCase__		,      "do_lower_case"		) else False
               snake_case       : Optional[Any]    =       (
                   [
                       ((0, 0), tokenizer_r.cls_token),
                       ((0, 1), "A"),
                       ((1, 2), ","),
                       ((3, 5), "na"),
                       ((5, 6), "##ï"),
                       ((6, 8), "##ve"),
                       ((9, 15), tokenizer_r.mask_token),
                       ((16, 21), "Allen"),
                       ((21, 23), "##NL"),
                       ((23, 24), "##P"),
                       ((25, 33), "sentence"),
                       ((33, 34), "."),
                       ((0, 0), tokenizer_r.sep_token),
                   ]
                   if not do_lower_case
                   else [
                       ((0, 0), tokenizer_r.cls_token),
                       ((0, 1), "a"),
                       ((1, 2), ","),
                       ((3, 8), "naive"),
                       ((9, 15), tokenizer_r.mask_token),
                       ((16, 21), "allen"),
                       ((21, 23), "##nl"),
                       ((23, 24), "##p"),
                       ((25, 33), "sentence"),
                       ((33, 34), "."),
                       ((0, 0), tokenizer_r.sep_token),
                   ]
               )

               self.assertEqual(
                   [e[1] for e in expected_results]		,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]		)		)
               self.assertEqual([e[0] for e in expected_results]		,      tokens["offset_mapping"]		)







   def 					lowerCamelCase						(      self		)      ->						str:

       '''simple docstring'''




       snake_case       : List[str]    =       ["的", "人", "有"]
       snake_case       : Optional[Any]    =       "".join(UpperCamelCase__		)
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'		):
               snake_case       : Union[str, Any]    =       True
               snake_case       : Optional[Any]    =       self.tokenizer_class.from_pretrained(UpperCamelCase__		,      **UpperCamelCase__		)
               snake_case       : List[Any]    =       self.rust_tokenizer_class.from_pretrained(UpperCamelCase__		,      **UpperCamelCase__		)

               snake_case       : Optional[Any]    =       tokenizer_p.encode(UpperCamelCase__		,      add_special_tokens=UpperCamelCase__		)
               snake_case       : Union[str, Any]    =       tokenizer_r.encode(UpperCamelCase__		,      add_special_tokens=UpperCamelCase__		)

               snake_case       : Optional[Any]    =       tokenizer_r.convert_ids_to_tokens(UpperCamelCase__		)
               snake_case       : Dict    =       tokenizer_p.convert_ids_to_tokens(UpperCamelCase__		)

               # it is expected that each Chinese character is not preceded by "##"
               self.assertListEqual(UpperCamelCase__		,      UpperCamelCase__		)
               self.assertListEqual(UpperCamelCase__		,      UpperCamelCase__		)

               snake_case       : List[str]    =       False
               snake_case       : str    =       self.rust_tokenizer_class.from_pretrained(UpperCamelCase__		,      **UpperCamelCase__		)
               snake_case       : Dict    =       self.tokenizer_class.from_pretrained(UpperCamelCase__		,      **UpperCamelCase__		)

               snake_case       : Optional[Any]    =       tokenizer_r.encode(UpperCamelCase__		,      add_special_tokens=UpperCamelCase__		)
               snake_case       : Union[str, Any]    =       tokenizer_p.encode(UpperCamelCase__		,      add_special_tokens=UpperCamelCase__		)

               snake_case       : Optional[Any]    =       tokenizer_r.convert_ids_to_tokens(UpperCamelCase__		)
               snake_case       : str    =       tokenizer_p.convert_ids_to_tokens(UpperCamelCase__		)

               # it is expected that only the first Chinese character is not preceded by "##".
               snake_case       : int    =       [
                   F'##{token}' if idx != 0 else token for idx, token in enumerate(UpperCamelCase__		)
               ]
               self.assertListEqual(UpperCamelCase__		,      UpperCamelCase__		)
               self.assertListEqual(UpperCamelCase__		,      UpperCamelCase__		)









"""simple docstring"""



import argparse

import torch

from ...utils import logging
from . import AlbertConfig, AlbertForPreTraining, load_tf_weights_in_albert


logging.set_verbosity_info()







def     __lowerCAmelCase		(		lowercase    :    List[Any]		,				lowercase    :    Dict		,				lowercase    :    str					)      ->		Optional[int]:


    """simple docstring"""





    snake_case       : Tuple    =       AlbertConfig.from_json_file(lowercase					)
    print(F'Building PyTorch model from configuration: {config}'					)
    snake_case       : int    =       AlbertForPreTraining(lowercase					)

    # Load weights from tf checkpoint
    load_tf_weights_in_albert(lowercase		,				lowercase		,				lowercase					)

    # Save pytorch-model
    print(F'Save PyTorch model to {pytorch_dump_path}'					)
    torch.save(model.state_dict()		,				lowercase					)


if __name__ == "__main__":
  __snake_case       =       argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      """--tf_checkpoint_path""", default=None, type=str, required=True, help="""Path to the TensorFlow checkpoint path."""
  )
  parser.add_argument(
      """--albert_config_file""",
      default=None,
      type=str,
      required=True,
      help=(
          """The config json file corresponding to the pre-trained ALBERT model. \n"""
          """This specifies the model architecture."""
      ),
  )
  parser.add_argument(
      """--pytorch_dump_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
  )
  __snake_case       =       parser.parse_args()
  convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.albert_config_file, args.pytorch_dump_path)





