
'''simple docstring'''







import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase   =       get_tests_dir("""fixtures/spiece.model""")







@require_sentencepiece
@require_tokenizers
class UpperCAmelCase__      (					lowercase__       ,							unittest.TestCase      ):





  """simple docstring"""

  __UpperCAmelCase							:							int    					=   AlbertTokenizer
  __UpperCAmelCase							:							Optional[Any]    					=   AlbertTokenizerFast
  __UpperCAmelCase							:							List[str]    					=   True
  __UpperCAmelCase							:							List[Any]    					=   True
  __UpperCAmelCase							:							Union[str, Any]    					=   True


  def     __lowercase						(						self :	List[str]			):


     '''simple docstring'''






     super().setUp()

     # We have a SentencePiece fixture for testing
     _a							:      Any     =  AlbertTokenizer(__a			)
     tokenizer.save_pretrained(self.tmpdirname			)


  def     __lowercase						(						self :	Tuple				,_a :	Dict			):


     '''simple docstring'''






     _a							:      List[str]     =  'this is a test'
     _a							:      Dict     =  'this is a test'
     return input_text, output_text


  def     __lowercase						(						self :	str			):


     '''simple docstring'''






     _a							:      Any     =  '<pad>'
     _a							:      int     =  0

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a			)				,__a			)
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a			)				,__a			)


  def     __lowercase						(						self :	Any			):


     '''simple docstring'''






     _a							:      Any     =  list(self.get_tokenizer().get_vocab().keys()			)

     self.assertEqual(vocab_keys[0]				,'<pad>'			)
     self.assertEqual(vocab_keys[1]				,'<unk>'			)
     self.assertEqual(vocab_keys[-1]				,'▁eloquent'			)
     self.assertEqual(len(__a			)				,3_0000			)


  def     __lowercase						(						self :	int			):


     '''simple docstring'''






     self.assertEqual(self.get_tokenizer().vocab_size				,3_0000			)


  def     __lowercase						(						self :	Optional[Any]			):


     '''simple docstring'''






     if not self.test_rust_tokenizer:
        return

     _a							:      Optional[Any]     =  self.get_tokenizer()
     _a							:      Dict     =  self.get_rust_tokenizer()

     _a							:      Any     =  'I was born in 92000, and this is falsé.'

     _a							:      Union[str, Any]     =  tokenizer.tokenize(__a			)
     _a							:      Any     =  rust_tokenizer.tokenize(__a			)
     self.assertListEqual(__a				,__a			)

     _a							:      List[Any]     =  tokenizer.encode(__a				,add_special_tokens=__a			)
     _a							:      Tuple     =  rust_tokenizer.encode(__a				,add_special_tokens=__a			)
     self.assertListEqual(__a				,__a			)

     _a							:      Dict     =  self.get_rust_tokenizer()
     _a							:      Optional[int]     =  tokenizer.encode(__a			)
     _a							:      Tuple     =  rust_tokenizer.encode(__a			)
     self.assertListEqual(__a				,__a			)


  def     __lowercase						(						self :	Union[str, Any]			):


     '''simple docstring'''






     _a							:      int     =  AlbertTokenizer(__a				,keep_accents=__a			)

     _a							:      Union[str, Any]     =  tokenizer.tokenize('This is a test'			)
     self.assertListEqual(__a				,['▁this', '▁is', '▁a', '▁test']			)

     self.assertListEqual(tokenizer.convert_tokens_to_ids(__a			)				,[48, 25, 21, 1289]			)

     _a							:      Tuple     =  tokenizer.tokenize('I was born in 92000, and this is falsé.'			)
     self.assertListEqual(
         __a				,['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.']			)
     _a							:      List[str]     =  tokenizer.convert_tokens_to_ids(__a			)
     self.assertListEqual(__a				,[31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]			)

     _a							:      int     =  tokenizer.convert_ids_to_tokens(__a			)
     self.assertListEqual(
         __a				,['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.']				,)


  def     __lowercase						(						self :	str			):


     '''simple docstring'''






     _a							:      str     =  AlbertTokenizer(__a			)

     _a							:      Tuple     =  tokenizer.encode('sequence builders'			)
     _a							:      List[Any]     =  tokenizer.encode('multi-sequence build'			)

     _a							:      Union[str, Any]     =  tokenizer.build_inputs_with_special_tokens(__a			)
     _a							:      Tuple     =  tokenizer.build_inputs_with_special_tokens(__a				,__a			)

     assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
     assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
         tokenizer.sep_token_id
     ]


  @slow
  def     __lowercase						(						self :	Tuple			):


     '''simple docstring'''






     _a							:      Any     =  {'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'input_ids': [[2, 2_1970, 13, 5, 6092, 167, 28, 7103, 2153, 673, 8, 7028, 1_2051, 18, 17, 7103, 2153, 673, 8, 3515, 1_8684, 8, 4461, 6, 1927, 297, 8, 1_2060, 2607, 18, 13, 5, 4461, 15, 1_0538, 38, 8, 135, 15, 822, 58, 15, 993, 1_0363, 15, 1460, 8005, 4461, 15, 993, 255, 2328, 9, 9, 9, 6, 26, 1112, 816, 3260, 13, 5, 103, 2377, 6, 17, 1112, 816, 2782, 13, 5, 103, 1_0641, 6, 29, 84, 2512, 2430, 782, 1_8684, 2761, 19, 808, 2430, 2556, 17, 855, 1480, 9477, 4091, 128, 1_1712, 15, 7103, 2153, 673, 17, 2_4883, 9990, 9, 3], [2, 1_1502, 25, 1006, 20, 782, 8, 1_1809, 855, 1732, 1_9393, 1_8667, 37, 367, 2_1018, 69, 1854, 34, 1_1860, 1_9124, 27, 156, 225, 17, 193, 4141, 19, 65, 9124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2231, 886, 2385, 1_7659, 84, 14, 1_6792, 1952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=__a				,model_name='albert-base-v2'				,revision='6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'				,)




'''simple docstring'''


import argparse

import torch

from transformers import BertConfig, BertForPreTraining, load_tf_weights_in_bert
from transformers.utils import logging


logging.set_verbosity_info()





def    lowerCAmelCase       (		UpperCamelCase__					: Optional[int]     ,       UpperCamelCase__					: List[str]     ,       UpperCamelCase__					: Optional[Any]							):


	"""simple docstring"""
	# Initialise PyTorch model
	__UpperCAmelCase      					=      BertConfig.from_json_file(UpperCamelCase__							)
	print(f"""Building PyTorch model from configuration: {config}"""							)
	__UpperCAmelCase      					=      BertForPreTraining(UpperCamelCase__							)

	# Load weights from tf checkpoint
	load_tf_weights_in_bert(UpperCamelCase__     ,       UpperCamelCase__     ,       UpperCamelCase__							)

	# Save pytorch-model
	print(f"""Save PyTorch model to {pytorch_dump_path}"""							)
	torch.save(model.state_dict()     ,       UpperCamelCase__							)


if __name__ == "__main__":
					__lowerCAmelCase						:							Tuple    							=						argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    "--tf_checkpoint_path", default=None, type=str, required=True, help="Path to the TensorFlow checkpoint path."
					)
					parser.add_argument(
					    "--bert_config_file",
					    default=None,
					    type=str,
					    required=True,
					    help=(
					        "The config json file corresponding to the pre-trained BERT model. \n"
					        "This specifies the model architecture."
					    ),
					)
					parser.add_argument(
					    "--pytorch_dump_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
					)
					__lowerCAmelCase						:							Optional[Any]    							=						parser.parse_args()
					convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)
