
import os
import re
import shutil
import sys
import tempfile
import unittest

import black


_A     =  os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, "utils"))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
_A     =  "    def __init__(self, config):\n        super().__init__()\n        self.transform = BertPredictionHeadTransform(config)\n\n        # The output weights are the same as the input embeddings, but there is\n        # an output-only bias for each token.\n        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)\n\n        self.bias = nn.Parameter(torch.zeros(config.vocab_size))\n\n        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`\n        self.decoder.bias = self.bias\n\n    def forward(self, hidden_states):\n        hidden_states = self.transform(hidden_states)\n        hidden_states = self.decoder(hidden_states)\n        return hidden_states\n"





class 			_lowerCAmelCase      (			unittest.TestCase ):




				def   __a		(      self  )       ->   Any:
							lowerCAmelCase_      			=			tempfile.mkdtemp()
							os.makedirs(os.path.join(self.transformer_dir       ,       "models/bert/"  )  )
							lowerCAmelCase_      			=			self.transformer_dir
							shutil.copy(
							    os.path.join(_UpperCamelCase       ,       "src/transformers/models/bert/modeling_bert.py"  )       ,       os.path.join(self.transformer_dir       ,       "models/bert/modeling_bert.py"  )       ,       )




				def   __a		(      self  )       ->   List[Any]:
							lowerCAmelCase_      			=			"src/transformers"
							shutil.rmtree(self.transformer_dir  )




				def   __a		(      self       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase       ,       _UpperCamelCase=None  )       ->   Optional[int]:
							lowerCAmelCase_      			=			comment + f"""\nclass {class_name}(nn.Module):\n""" + class_code
							if overwrite_result is not None:
										lowerCAmelCase_      			=			comment + f"""\nclass {class_name}(nn.Module):\n""" + overwrite_result
							lowerCAmelCase_      			=			black.Mode(target_versions={black.TargetVersion.PYaa}       ,       line_length=119  )
							lowerCAmelCase_      			=			black.format_str(_UpperCamelCase       ,       mode=_UpperCamelCase  )
							lowerCAmelCase_      			=			os.path.join(self.transformer_dir       ,       "new_code.py"  )
							with open(_UpperCamelCase       ,       "w"       ,       newline="\n"  ) as f:
										f.write(_UpperCamelCase  )
							if overwrite_result is None:
										self.assertTrue(len(check_copies.is_copy_consistent(_UpperCamelCase  )  ) == 0  )
							else:
										check_copies.is_copy_consistent(f.name       ,       overwrite=_UpperCamelCase  )
										with open(_UpperCamelCase       ,       "r"  ) as f:
													self.assertTrue(f.read()       ,       _UpperCamelCase  )




				def   __a		(      self  )       ->   Any:
							lowerCAmelCase_      			=			check_copies.find_code_in_transformers("models.bert.modeling_bert.BertLMPredictionHead"  )
							self.assertEqual(_UpperCamelCase       ,       _UpperCamelCase  )




				def   __a		(      self  )       ->   Any:
							# Base copy consistency
							self.check_copy_consistency(
							    "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"       ,       "BertLMPredictionHead"       ,       REFERENCE_CODE + "\n"       ,       )

							# With no empty line at the end
							self.check_copy_consistency(
							    "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"       ,       "BertLMPredictionHead"       ,       _UpperCamelCase       ,       )

							# Copy consistency with rename
							self.check_copy_consistency(
							    "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"       ,       "TestModelLMPredictionHead"       ,       re.sub("Bert"       ,       "TestModel"       ,       _UpperCamelCase  )       ,       )

							# Copy consistency with a really long name
							lowerCAmelCase_      			=			"TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason"
							self.check_copy_consistency(
							    f"""# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}"""       ,       f"""{long_class_name}LMPredictionHead"""       ,       re.sub("Bert"       ,       _UpperCamelCase       ,       _UpperCamelCase  )       ,       )

							# Copy consistency with overwrite
							self.check_copy_consistency(
							    "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"       ,       "TestModelLMPredictionHead"       ,       _UpperCamelCase       ,       overwrite_result=re.sub("Bert"       ,       "TestModel"       ,       _UpperCamelCase  )       ,       )




				def   __a		(      self  )       ->   Any:
							lowerCAmelCase_      			=			check_copies.LOCALIZED_READMES["README_zh-hans.md"]

							lowerCAmelCase_      			=			(
							    "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"
							    " Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"
							    " Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"
							    " Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1."
							    " **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),"
							    " released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"
							    " lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same"
							    " method has been applied to compress GPT2 into"
							    " [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"
							    " [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"
							    " Multilingual BERT into"
							    " [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"
							    " version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**"
							    " (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders"
							    " as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang"
							    " Luong, Quoc V. Le, Christopher D. Manning."
							)
							lowerCAmelCase_      			=			(
							    "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"
							    " Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
							    " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
							    " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"
							)
							lowerCAmelCase_      			=			(
							    "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"
							    " Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
							    " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
							    " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1."
							    " **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文"
							    " [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"
							    " lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same"
							    " method has been applied to compress GPT2 into"
							    " [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"
							    " [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"
							    " Multilingual BERT into"
							    " [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"
							    " version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自"
							    " Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather"
							    " than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,"
							    " Christopher D. Manning 发布。\n"
							)

							lowerCAmelCase_			,						lowerCAmelCase_      			=			check_copies.convert_to_localized_md(
							    _UpperCamelCase       ,       _UpperCamelCase       ,       localized_readme["format_model_list"]  )

							self.assertFalse(_UpperCamelCase  )
							self.assertEqual(_UpperCamelCase       ,       _UpperCamelCase  )

							lowerCAmelCase_			,						lowerCAmelCase_      			=			check_copies.convert_to_localized_md(
							    _UpperCamelCase       ,       _UpperCamelCase       ,       localized_readme["format_model_list"]  )

							# Check whether the number of models is equal to README.md after conversion.
							self.assertTrue(_UpperCamelCase  )

							lowerCAmelCase_      			=			(
							    "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"
							    " Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"
							    " Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"
							    " Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut."
							)
							lowerCAmelCase_      			=			(
							    "1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and"
							    " the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
							    " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
							    " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"
							)
							lowerCAmelCase_      			=			(
							    "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"
							    " Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
							    " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
							    " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"
							)

							lowerCAmelCase_			,						lowerCAmelCase_      			=			check_copies.convert_to_localized_md(
							    _UpperCamelCase       ,       _UpperCamelCase       ,       localized_readme["format_model_list"]  )

							# Check if the model link is synchronized.
							self.assertEqual(_UpperCamelCase       ,       _UpperCamelCase  )





from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A     =  logging.get_logger(__name__)

_A     =  {
    "transfo-xl-wt103": "https://huggingface.co/transfo-xl-wt103/resolve/main/config.json",
}





class 			_lowerCAmelCase      (			__a ):
				_lowercase    	='''transfo-xl'''
				_lowercase    	=['''mems''']
				_lowercase    	={
				    '''n_token''': '''vocab_size''',
				    '''hidden_size''': '''d_model''',
				    '''num_attention_heads''': '''n_head''',
				    '''num_hidden_layers''': '''n_layer''',
				}




				def __init__(      self       ,       _UpperCamelCase=267_735       ,       _UpperCamelCase=[20_000, 40_000, 200_000]       ,       _UpperCamelCase=1_024       ,       _UpperCamelCase=1_024       ,       _UpperCamelCase=16       ,       _UpperCamelCase=64       ,       _UpperCamelCase=4_096       ,       _UpperCamelCase=4       ,       _UpperCamelCase=False       ,       _UpperCamelCase=18       ,       _UpperCamelCase=1_600       ,       _UpperCamelCase=1_000       ,       _UpperCamelCase=True       ,       _UpperCamelCase=True       ,       _UpperCamelCase=0       ,       _UpperCamelCase=-1       ,       _UpperCamelCase=True       ,       _UpperCamelCase=0.1       ,       _UpperCamelCase=0.0       ,       _UpperCamelCase=True       ,       _UpperCamelCase="normal"       ,       _UpperCamelCase=0.01       ,       _UpperCamelCase=0.01       ,       _UpperCamelCase=0.02       ,       _UpperCamelCase=1e-5       ,       _UpperCamelCase=0       ,       **_UpperCamelCase       ,       )       ->   Dict:
							lowerCAmelCase_      			=			vocab_size
							lowerCAmelCase_      			=			[]
							self.cutoffs.extend(_UpperCamelCase  )
							if proj_share_all_but_first:
										lowerCAmelCase_      			=			[False] + [True] * len(self.cutoffs  )
							else:
										lowerCAmelCase_      			=			[False] + [False] * len(self.cutoffs  )
							lowerCAmelCase_      			=			d_model
							lowerCAmelCase_      			=			d_embed
							lowerCAmelCase_      			=			d_head
							lowerCAmelCase_      			=			d_inner
							lowerCAmelCase_      			=			div_val
							lowerCAmelCase_      			=			pre_lnorm
							lowerCAmelCase_      			=			n_layer
							lowerCAmelCase_      			=			n_head
							lowerCAmelCase_      			=			mem_len
							lowerCAmelCase_      			=			same_length
							lowerCAmelCase_      			=			attn_type
							lowerCAmelCase_      			=			clamp_len
							lowerCAmelCase_      			=			sample_softmax
							lowerCAmelCase_      			=			adaptive
							lowerCAmelCase_      			=			dropout
							lowerCAmelCase_      			=			dropatt
							lowerCAmelCase_      			=			untie_r
							lowerCAmelCase_      			=			init
							lowerCAmelCase_      			=			init_range
							lowerCAmelCase_      			=			proj_init_std
							lowerCAmelCase_      			=			init_std
							lowerCAmelCase_      			=			layer_norm_epsilon
							super().__init__(eos_token_id=_UpperCamelCase       ,       **_UpperCamelCase  )




				@property
				def   __a		(      self  )       ->   List[Any]:
							# Message copied from Transformer-XL documentation
							logger.info(f"""The model {self.model_type} is one of the few models that has no sequence length limit."""  )
							return -1




				@max_position_embeddings.setter
				def   __a		(      self       ,       _UpperCamelCase  )       ->   str:
							# Message copied from Transformer-XL documentation
							raise NotImplementedError(
							    f"""The model {self.model_type} is one of the few models that has no sequence length limit."""  )



