




"""simple docstring"""







import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


a      =							get_tests_dir('fixtures/test_sentencepiece.model')






@require_sentencepiece
@require_tokenizers
class       SCREAMING_SNAKE_CASE__					(					__lowercase						,				unittest.TestCase					):
						_a    						=						ReformerTokenizer
						_a    						=						ReformerTokenizerFast
						_a    						=						True
						_a    						=						False
						_a    						=						True


						def   __lowercase (							self				:			str					):
										super().setUp()

										lowerCAmelCase  =				ReformerTokenizer(lowerCAmelCase						,						keep_accents=lowerCAmelCase					)
										tokenizer.save_pretrained(self.tmpdirname					)


						def   __lowercase (							self				:			List[str]					):
										lowerCAmelCase  =				"<s>"
										lowerCAmelCase  =				1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase					)						,						lowerCAmelCase					)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase					)						,						lowerCAmelCase					)


						def   __lowercase (							self				:			int					):
										lowerCAmelCase  =				list(self.get_tokenizer().get_vocab().keys()					)

										self.assertEqual(vocab_keys[0]						,						"""<unk>"""					)
										self.assertEqual(vocab_keys[1]						,						"""<s>"""					)
										self.assertEqual(vocab_keys[-1]						,						"""j"""					)
										self.assertEqual(len(lowerCAmelCase					)						,						1000					)


						def   __lowercase (							self				:			Union[str, Any]					):
										self.assertEqual(self.get_tokenizer().vocab_size						,						1000					)


						def   __lowercase (							self				:			Tuple					):
										if not self.test_rust_tokenizer:
														return

										lowerCAmelCase  =				self.get_tokenizer()
										lowerCAmelCase  =				self.get_rust_tokenizer()

										lowerCAmelCase  =				"I was born in 92000, and this is falsé."

										lowerCAmelCase  =				tokenizer.tokenize(lowerCAmelCase					)
										lowerCAmelCase  =				rust_tokenizer.tokenize(lowerCAmelCase					)
										self.assertListEqual(lowerCAmelCase						,						lowerCAmelCase					)

										lowerCAmelCase  =				tokenizer.encode(lowerCAmelCase						,						add_special_tokens=lowerCAmelCase					)
										lowerCAmelCase  =				rust_tokenizer.encode(lowerCAmelCase						,						add_special_tokens=lowerCAmelCase					)
										self.assertListEqual(lowerCAmelCase						,						lowerCAmelCase					)

										lowerCAmelCase  =				self.get_rust_tokenizer()
										lowerCAmelCase  =				tokenizer.encode(lowerCAmelCase					)
										lowerCAmelCase  =				rust_tokenizer.encode(lowerCAmelCase					)
										self.assertListEqual(lowerCAmelCase						,						lowerCAmelCase					)


						def   __lowercase (							self				:			List[str]						,						lowerCAmelCase				:			List[str]=15					):
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																		lowerCAmelCase  =				self.rust_tokenizer_class.from_pretrained(lowerCAmelCase						,						**lowerCAmelCase					)

																		# Simple input
																		lowerCAmelCase  =				"This is a simple input"
																		lowerCAmelCase  =				["This is a simple input 1", "This is a simple input 2"]
																		lowerCAmelCase  =				("This is a simple input", "This is a pair")
																		lowerCAmelCase  =				[
																		    ("This is a simple input 1", "This is a simple input 2"),
																		    ("This is a simple pair 1", "This is a simple pair 2"),
																		]

																		# Simple input tests
																		self.assertRaises(lowerCAmelCase						,						tokenizer_r.encode						,						lowerCAmelCase						,						max_length=lowerCAmelCase						,						padding="""max_length"""					)

																		# Simple input
																		self.assertRaises(lowerCAmelCase						,						tokenizer_r.encode_plus						,						lowerCAmelCase						,						max_length=lowerCAmelCase						,						padding="""max_length"""					)

																		# Simple input
																		self.assertRaises(
																		    lowerCAmelCase						,						tokenizer_r.batch_encode_plus						,						lowerCAmelCase						,						max_length=lowerCAmelCase						,						padding="""max_length"""						,						)

																		# Pair input
																		self.assertRaises(lowerCAmelCase						,						tokenizer_r.encode						,						lowerCAmelCase						,						max_length=lowerCAmelCase						,						padding="""max_length"""					)

																		# Pair input
																		self.assertRaises(lowerCAmelCase						,						tokenizer_r.encode_plus						,						lowerCAmelCase						,						max_length=lowerCAmelCase						,						padding="""max_length"""					)

																		# Pair input
																		self.assertRaises(
																		    lowerCAmelCase						,						tokenizer_r.batch_encode_plus						,						lowerCAmelCase						,						max_length=lowerCAmelCase						,						padding="""max_length"""						,						)


						def   __lowercase (							self				:			List[Any]					):
										pass


						def   __lowercase (							self				:			Optional[Any]					):
										lowerCAmelCase  =				ReformerTokenizer(lowerCAmelCase						,						keep_accents=lowerCAmelCase					)

										lowerCAmelCase  =				tokenizer.tokenize("""This is a test"""					)
										self.assertListEqual(lowerCAmelCase						,						["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]					)

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(lowerCAmelCase					)						,						[285, 46, 10, 170, 382]						,						)

										lowerCAmelCase  =				tokenizer.tokenize("""I was born in 92000, and this is falsé."""					)
										self.assertListEqual(
										    lowerCAmelCase						,						[
										        SPIECE_UNDERLINE + """I""",
										        SPIECE_UNDERLINE + """was""",
										        SPIECE_UNDERLINE + """b""",
										        """or""",
										        """n""",
										        SPIECE_UNDERLINE + """in""",
										        SPIECE_UNDERLINE + """""",
										        """9""",
										        """2""",
										        """0""",
										        """0""",
										        """0""",
										        """,""",
										        SPIECE_UNDERLINE + """and""",
										        SPIECE_UNDERLINE + """this""",
										        SPIECE_UNDERLINE + """is""",
										        SPIECE_UNDERLINE + """f""",
										        """al""",
										        """s""",
										        """é""",
										        """.""",
										    ]						,						)
										lowerCAmelCase  =				tokenizer.convert_tokens_to_ids(lowerCAmelCase					)
										self.assertListEqual(
										    lowerCAmelCase						,						[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]						,						)

										lowerCAmelCase  =				tokenizer.convert_ids_to_tokens(lowerCAmelCase					)
										self.assertListEqual(
										    lowerCAmelCase						,						[
										        SPIECE_UNDERLINE + """I""",
										        SPIECE_UNDERLINE + """was""",
										        SPIECE_UNDERLINE + """b""",
										        """or""",
										        """n""",
										        SPIECE_UNDERLINE + """in""",
										        SPIECE_UNDERLINE + """""",
										        """<unk>""",
										        """2""",
										        """0""",
										        """0""",
										        """0""",
										        """,""",
										        SPIECE_UNDERLINE + """and""",
										        SPIECE_UNDERLINE + """this""",
										        SPIECE_UNDERLINE + """is""",
										        SPIECE_UNDERLINE + """f""",
										        """al""",
										        """s""",
										        """<unk>""",
										        """.""",
										    ]						,						)


						@cached_property
						def   __lowercase (							self				:			Dict					):
										return ReformerTokenizer.from_pretrained("""google/reformer-crime-and-punishment"""					)


						@slow
						def   __lowercase (							self				:			Union[str, Any]					):
										lowerCAmelCase  =				"Hello World!"
										lowerCAmelCase  =				[126, 32, 262, 152, 38, 72, 287]

										self.assertListEqual(lowerCAmelCase						,						self.big_tokenizer.encode(lowerCAmelCase					)					)


						@slow
						def   __lowercase (							self				:			Dict					):
										lowerCAmelCase  =				(
										    "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
										    " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
										)
										lowerCAmelCase  =				[
										    108,
										    265,
										    24,
										    111,
										    4,
										    258,
										    156,
										    35,
										    28,
										    275,
										    3,
										    259,
										    297,
										    260,
										    84,
										    4,
										    35,
										    110,
										    44,
										    8,
										    259,
										    91,
										    268,
										    21,
										    11,
										    209,
										    274,
										    109,
										    266,
										    277,
										    117,
										    86,
										    93,
										    315,
										    258,
										    278,
										    258,
										    277,
										    258,
										    0,
										    258,
										    288,
										    258,
										    319,
										    258,
										    0,
										    258,
										    0,
										    258,
										    0,
										    258,
										    0,
										    258,
										    287,
										    258,
										    315,
										    258,
										    289,
										    258,
										    278,
										    99,
										    269,
										    266,
										    262,
										    8,
										    259,
										    241,
										    4,
										    217,
										    230,
										    268,
										    266,
										    55,
										    168,
										    106,
										    75,
										    193,
										    266,
										    223,
										    27,
										    49,
										    26,
										    282,
										    25,
										    264,
										    299,
										    19,
										    26,
										    0,
										    258,
										    277,
										    117,
										    86,
										    93,
										    176,
										    183,
										    270,
										    11,
										    262,
										    42,
										    61,
										    265,
										]

										self.assertListEqual(lowerCAmelCase						,						self.big_tokenizer.encode(lowerCAmelCase					)					)


						@require_torch
						@slow
						def   __lowercase (							self				:			Any					):
										import torch

										from transformers import ReformerConfig, ReformerModel

										# Build sequence
										lowerCAmelCase  =				list(self.big_tokenizer.get_vocab().keys()					)[:10]
										lowerCAmelCase  =				" ".join(lowerCAmelCase					)
										lowerCAmelCase  =				self.big_tokenizer.encode_plus(lowerCAmelCase						,						return_tensors="""pt"""					)
										lowerCAmelCase  =				self.big_tokenizer.batch_encode_plus([sequence, sequence]						,						return_tensors="""pt"""					)

										lowerCAmelCase  =				ReformerConfig()
										# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
										lowerCAmelCase  =				encoded_sequence["input_ids"].shape
										lowerCAmelCase  =				ReformerModel(lowerCAmelCase					)

										# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
										assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

										with torch.no_grad():
														model(**lowerCAmelCase					)
														model(**lowerCAmelCase					)




						@slow
						def   __lowercase (							self				:			int					):
										# fmt: off
										lowerCAmelCase  =				{"input_ids": [[108, 265, 24, 111, 4, 258, 156, 7, 51, 279, 58, 7, 76, 25, 69, 278], [140, 243, 264, 134, 17, 267, 77, 263, 22, 262, 297, 258, 304, 177, 279, 266, 14, 89, 13, 35, 261, 299, 272, 137, 275, 278]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
										# fmt: on

										# This tokenizer does not know some characters like ")".
										# That is the reason why we use very simple texts here.
										# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
										lowerCAmelCase  =				[
										    "This is a very simple sentence.",
										    "The quick brown fox jumps over the lazy dog.",
										]

										self.tokenizer_integration_test_util(
										    expected_encoding=lowerCAmelCase						,						model_name="""google/reformer-crime-and-punishment"""						,						revision="""0e6c3decb8211d49bf881013425dc8b0448b3f5a"""						,						padding=lowerCAmelCase						,						sequences=lowerCAmelCase						,						)











"""simple docstring"""







import argparse
import os
import re

import tensorflow as tf
import torch

from transformers import BertConfig, BertModel
from transformers.utils import logging


logging.set_verbosity_info()
a      =							logging.get_logger(__name__)





def 							lowercase					(snake_case__   : str      ,       snake_case__   : Tuple      ,       snake_case__   : Dict )     ->  int:


				'''simple docstring'''






				lowerCAmelCase  =				os.path.abspath(snake_case__ )
				logger.info(f'''Converting TensorFlow checkpoint from {tf_path}''' )
				# Load weights from TF model
				lowerCAmelCase  =				tf.train.list_variables(snake_case__ )
				lowerCAmelCase  =				[]
				lowerCAmelCase  =				[]
				lowerCAmelCase  =				[]
				for full_name, shape in init_vars:
								# logger.info(f"Loading TF weight {name} with shape {shape}")
								lowerCAmelCase  =				full_name.split("""/""" )
								if full_name == "_CHECKPOINTABLE_OBJECT_GRAPH" or name[0] in ["global_step", "save_counter"]:
												logger.info(f'''Skipping non-model layer {full_name}''' )
												continue
								if "optimizer" in full_name:
												logger.info(f'''Skipping optimization layer {full_name}''' )
												continue
								if name[0] == "model":
												# ignore initial 'model'
												lowerCAmelCase  =				name[1:]
								# figure out how many levels deep the name is
								lowerCAmelCase  =				0
								for _name in name:
												if _name.startswith("""layer_with_weights""" ):
																depth += 1
												else:
																break
								layer_depth.append(snake_case__ )
								# read data
								lowerCAmelCase  =				tf.train.load_variable(snake_case__      ,       snake_case__ )
								names.append("""/""".join(snake_case__ ) )
								arrays.append(snake_case__ )
				logger.info(f'''Read a total of {len(snake_case__ ):,} layers''' )

				# Sanity check
				if len(set(snake_case__ ) ) != 1:
								raise ValueError(f'''Found layer names with different depths (layer depth {list(set(snake_case__ ) )})''' )
				lowerCAmelCase  =				list(set(snake_case__ ) )[0]
				if layer_depth != 1:
								raise ValueError(
								    """The model contains more than just the embedding/encoder layers. This script does not handle MLM/NSP"""
								    """ heads.""" )

				# convert layers
				logger.info("""Converting weights...""" )
				for full_name, array in zip(snake_case__      ,       snake_case__ ):
								lowerCAmelCase  =				full_name.split("""/""" )
								lowerCAmelCase  =				model
								lowerCAmelCase  =				[]
								for i, m_name in enumerate(snake_case__ ):
												if m_name == ".ATTRIBUTES":
																# variable names end with .ATTRIBUTES/VARIABLE_VALUE
																break
												if m_name.startswith("""layer_with_weights""" ):
																lowerCAmelCase  =				int(m_name.split("""-""" )[-1] )
																if layer_num <= 2:
																				# embedding layers
																				# layer_num 0: word_embeddings
																				# layer_num 1: position_embeddings
																				# layer_num 2: token_type_embeddings
																				continue
																elif layer_num == 3:
																				# embedding LayerNorm
																				trace.extend(["""embeddings""", """LayerNorm"""] )
																				lowerCAmelCase  =				getattr(snake_case__      ,       """embeddings""" )
																				lowerCAmelCase  =				getattr(snake_case__      ,       """LayerNorm""" )
																elif layer_num > 3 and layer_num < config.num_hidden_layers + 4:
																				# encoder layers
																				trace.extend(["""encoder""", """layer""", str(layer_num - 4 )] )
																				lowerCAmelCase  =				getattr(snake_case__      ,       """encoder""" )
																				lowerCAmelCase  =				getattr(snake_case__      ,       """layer""" )
																				lowerCAmelCase  =				pointer[layer_num - 4]
																elif layer_num == config.num_hidden_layers + 4:
																				# pooler layer
																				trace.extend(["""pooler""", """dense"""] )
																				lowerCAmelCase  =				getattr(snake_case__      ,       """pooler""" )
																				lowerCAmelCase  =				getattr(snake_case__      ,       """dense""" )
												elif m_name == "embeddings":
																trace.append("""embeddings""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """embeddings""" )
																if layer_num == 0:
																				trace.append("""word_embeddings""" )
																				lowerCAmelCase  =				getattr(snake_case__      ,       """word_embeddings""" )
																elif layer_num == 1:
																				trace.append("""position_embeddings""" )
																				lowerCAmelCase  =				getattr(snake_case__      ,       """position_embeddings""" )
																elif layer_num == 2:
																				trace.append("""token_type_embeddings""" )
																				lowerCAmelCase  =				getattr(snake_case__      ,       """token_type_embeddings""" )
																else:
																				raise ValueError(f'''Unknown embedding layer with name {full_name}''' )
																trace.append("""weight""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """weight""" )
												elif m_name == "_attention_layer":
																# self-attention layer
																trace.extend(["""attention""", """self"""] )
																lowerCAmelCase  =				getattr(snake_case__      ,       """attention""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """self""" )
												elif m_name == "_attention_layer_norm":
																# output attention norm
																trace.extend(["""attention""", """output""", """LayerNorm"""] )
																lowerCAmelCase  =				getattr(snake_case__      ,       """attention""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """output""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """LayerNorm""" )
												elif m_name == "_attention_output_dense":
																# output attention dense
																trace.extend(["""attention""", """output""", """dense"""] )
																lowerCAmelCase  =				getattr(snake_case__      ,       """attention""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """output""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """dense""" )
												elif m_name == "_output_dense":
																# output dense
																trace.extend(["""output""", """dense"""] )
																lowerCAmelCase  =				getattr(snake_case__      ,       """output""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """dense""" )
												elif m_name == "_output_layer_norm":
																# output dense
																trace.extend(["""output""", """LayerNorm"""] )
																lowerCAmelCase  =				getattr(snake_case__      ,       """output""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """LayerNorm""" )
												elif m_name == "_key_dense":
																# attention key
																trace.append("""key""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """key""" )
												elif m_name == "_query_dense":
																# attention query
																trace.append("""query""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """query""" )
												elif m_name == "_value_dense":
																# attention value
																trace.append("""value""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """value""" )
												elif m_name == "_intermediate_dense":
																# attention intermediate dense
																trace.extend(["""intermediate""", """dense"""] )
																lowerCAmelCase  =				getattr(snake_case__      ,       """intermediate""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """dense""" )
												elif m_name == "_output_layer_norm":
																# output layer norm
																trace.append("""output""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """output""" )
												# weights & biases
												elif m_name in ["bias", "beta"]:
																trace.append("""bias""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """bias""" )
												elif m_name in ["kernel", "gamma"]:
																trace.append("""weight""" )
																lowerCAmelCase  =				getattr(snake_case__      ,       """weight""" )
												else:
																logger.warning(f'''Ignored {m_name}''' )
        # for certain layers reshape is necessary
								lowerCAmelCase  =				""".""".join(snake_case__ )
								if re.match(R"""(\S+)\.attention\.self\.(key|value|query)\.(bias|weight)"""      ,       snake_case__ ) or re.match(
								    R"""(\S+)\.attention\.output\.dense\.weight"""      ,       snake_case__ ):
												lowerCAmelCase  =				array.reshape(pointer.data.shape )
								if "kernel" in full_name:
												lowerCAmelCase  =				array.transpose()
								if pointer.shape == array.shape:
												lowerCAmelCase  =				torch.from_numpy(snake_case__ )
								else:
												raise ValueError(
												    f'''Shape mismatch in layer {full_name}: Model expects shape {pointer.shape} but layer contains shape:'''
												    f''' {array.shape}''' )
								logger.info(f'''Successfully set variable {full_name} to PyTorch layer {trace}''' )
				return model





def 							lowercase					(snake_case__   : Tuple      ,       snake_case__   : Any      ,       snake_case__   : Any )     ->  str:


				'''simple docstring'''






				logger.info(f'''Loading model based on config from {config_path}...''' )
				lowerCAmelCase  =				BertConfig.from_json_file(snake_case__ )
				lowerCAmelCase  =				BertModel(snake_case__ )

				# Load weights from checkpoint
				logger.info(f'''Loading weights from checkpoint {tf_checkpoint_path}...''' )
				load_tfa_weights_in_bert(snake_case__      ,       snake_case__      ,       snake_case__ )

				# Save pytorch-model
				logger.info(f'''Saving PyTorch model to {pytorch_dump_path}...''' )
				torch.save(model.state_dict()      ,       snake_case__ )


if __name__ == "__main__":
		a      =							argparse.ArgumentParser()
		parser.add_argument(
		    '--tf_checkpoint_path', type=str, required=True, help='Path to the TensorFlow 2.x checkpoint path.'
		)
		parser.add_argument(
		    '--bert_config_file',
		    type=str,
		    required=True,
		    help='The config json file corresponding to the BERT model. This specifies the model architecture.',
		)
		parser.add_argument(
		    '--pytorch_dump_path',
		    type=str,
		    required=True,
		    help='Path to the output PyTorch model (must include filename).',
		)
		a      =							parser.parse_args()
		convert_tfa_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)





