





import os
import re
import shutil
import sys
import tempfile
import unittest

import black


__lowercase	: Optional[Any]       =os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, """utils"""))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
__lowercase	: Union[str, Any]       ="""    def __init__(self, config):
        super().__init__()
        self.transform = BertPredictionHeadTransform(config)

        # The output weights are the same as the input embeddings, but there is
        # an output-only bias for each token.
        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        self.bias = nn.Parameter(torch.zeros(config.vocab_size))

        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
        self.decoder.bias = self.bias

    def forward(self, hidden_states):
        hidden_states = self.transform(hidden_states)
        hidden_states = self.decoder(hidden_states)
        return hidden_states
"""

class   A							(						unittest.TestCase    ):



      def 						lowerCAmelCase__     (						self:    Optional[int]							)    ->     Optional[int]:
         '''simple docstring'''

         UpperCAmelCase_           =tempfile.mkdtemp()
         os.makedirs(os.path.join(self.transformer_dir		,	"models/bert/"							)							)
         UpperCAmelCase_           =self.transformer_dir
         shutil.copy(
             os.path.join(_lowerCAmelCase		,	"src/transformers/models/bert/modeling_bert.py"							)		,	os.path.join(self.transformer_dir		,	"models/bert/modeling_bert.py"							)		,	)



      def 						lowerCAmelCase__     (						self:    List[Any]							)    ->     List[str]:
         '''simple docstring'''

         UpperCAmelCase_           ="src/transformers"
         shutil.rmtree(self.transformer_dir							)



      def 						lowerCAmelCase__     (						self:    List[Any]		,	_lowerCAmelCase:    Optional[int]		,	_lowerCAmelCase:    Optional[int]		,	_lowerCAmelCase:    Union[str, Any]		,	_lowerCAmelCase:    Any=None							)    ->     List[str]:
         '''simple docstring'''

         UpperCAmelCase_           =comment + F'\nclass {class_name}(nn.Module):\n' + class_code
         if overwrite_result is not None:
            UpperCAmelCase_           =comment + F'\nclass {class_name}(nn.Module):\n' + overwrite_result
         UpperCAmelCase_           =black.Mode(target_versions={black.TargetVersion.PYaa}		,	line_length=119							)
         UpperCAmelCase_           =black.format_str(_lowerCAmelCase		,	mode=_lowerCAmelCase							)
         UpperCAmelCase_           =os.path.join(self.transformer_dir		,	"new_code.py"							)
         with open(_lowerCAmelCase		,	"w"		,	newline="\n"							) as f:
            f.write(_lowerCAmelCase							)
         if overwrite_result is None:
            self.assertTrue(len(check_copies.is_copy_consistent(_lowerCAmelCase							)							) == 0							)
         else:
            check_copies.is_copy_consistent(f.name		,	overwrite=_lowerCAmelCase							)
            with open(_lowerCAmelCase		,	"r"							) as f:
               self.assertTrue(f.read()		,	_lowerCAmelCase							)



      def 						lowerCAmelCase__     (						self:    List[str]							)    ->     str:
         '''simple docstring'''

         UpperCAmelCase_           =check_copies.find_code_in_transformers("models.bert.modeling_bert.BertLMPredictionHead"							)
         self.assertEqual(_lowerCAmelCase		,	_lowerCAmelCase							)



      def 						lowerCAmelCase__     (						self:    str							)    ->     List[str]:
         '''simple docstring'''

         self.check_copy_consistency(
             "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"		,	"BertLMPredictionHead"		,	REFERENCE_CODE + "\n"		,	)

         # With no empty line at the end
         self.check_copy_consistency(
             "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"		,	"BertLMPredictionHead"		,	_lowerCAmelCase		,	)

         # Copy consistency with rename
         self.check_copy_consistency(
             "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"		,	"TestModelLMPredictionHead"		,	re.sub("Bert"		,	"TestModel"		,	_lowerCAmelCase							)		,	)

         # Copy consistency with a really long name
         UpperCAmelCase_           ="TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason"
         self.check_copy_consistency(
             F'# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}'		,	F'{long_class_name}LMPredictionHead'		,	re.sub("Bert"		,	_lowerCAmelCase		,	_lowerCAmelCase							)		,	)

         # Copy consistency with overwrite
         self.check_copy_consistency(
             "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"		,	"TestModelLMPredictionHead"		,	_lowerCAmelCase		,	overwrite_result=re.sub("Bert"		,	"TestModel"		,	_lowerCAmelCase							)		,	)






      def 						lowerCAmelCase__     (						self:    Tuple							)    ->     List[Any]:
         '''simple docstring'''

         UpperCAmelCase_           =check_copies.LOCALIZED_READMES["README_zh-hans.md"]

         UpperCAmelCase_           =(
             "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"
             " Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"
             " Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"
             " Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1."
             " **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),"
             " released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"
             " lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same"
             " method has been applied to compress GPT2 into"
             " [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"
             " [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"
             " Multilingual BERT into"
             " [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"
             " version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**"
             " (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders"
             " as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang"
             " Luong, Quoc V. Le, Christopher D. Manning."
         )
         UpperCAmelCase_           =(
             "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"
             " Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
             " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
             " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"
         )
         UpperCAmelCase_           =(
             "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"
             " Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
             " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
             " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1."
             " **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文"
             " [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"
             " lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same"
             " method has been applied to compress GPT2 into"
             " [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"
             " [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"
             " Multilingual BERT into"
             " [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"
             " version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自"
             " Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather"
             " than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,"
             " Christopher D. Manning 发布。\n"
         )

         UpperCAmelCase_    ,							UpperCAmelCase_           =check_copies.convert_to_localized_md(
             _lowerCAmelCase		,	_lowerCAmelCase		,	localized_readme["format_model_list"]							)

         self.assertFalse(_lowerCAmelCase							)
         self.assertEqual(_lowerCAmelCase		,	_lowerCAmelCase							)

         UpperCAmelCase_    ,							UpperCAmelCase_           =check_copies.convert_to_localized_md(
             _lowerCAmelCase		,	_lowerCAmelCase		,	localized_readme["format_model_list"]							)

         # Check whether the number of models is equal to README.md after conversion.
         self.assertTrue(_lowerCAmelCase							)

         UpperCAmelCase_           =(
             "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"
             " Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"
             " Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"
             " Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut."
         )
         UpperCAmelCase_           =(
             "1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and"
             " the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
             " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
             " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"
         )
         UpperCAmelCase_           =(
             "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"
             " Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
             " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
             " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"
         )

         UpperCAmelCase_    ,							UpperCAmelCase_           =check_copies.convert_to_localized_md(
             _lowerCAmelCase		,	_lowerCAmelCase		,	localized_readme["format_model_list"]							)

         # Check if the model link is synchronized.
         self.assertEqual(_lowerCAmelCase		,	_lowerCAmelCase							)









import operator as op


lowerCamelCase		       =      """scaler.pt"""
lowerCamelCase		       =      """pytorch_model"""
lowerCamelCase		       =      """random_states"""
lowerCamelCase		       =      """optimizer"""
lowerCamelCase		       =      """scheduler"""
lowerCamelCase		       =      """pytorch_model.bin"""
lowerCamelCase		       =      """pytorch_model.bin.index.json"""
lowerCamelCase		       =      """model.safetensors"""
lowerCamelCase		       =      """model.safetensors.index.json"""
lowerCamelCase		       =      """1.10.2"""
lowerCamelCase		       =      """py38"""
lowerCamelCase		       =      """4.17.0"""
lowerCamelCase		       =      ["""ml.p3.16xlarge""", """ml.p3dn.24xlarge""", """ml.p4dn.24xlarge"""]
lowerCamelCase		       =      ["""FULL_SHARD""", """SHARD_GRAD_OP""", """NO_SHARD""", """HYBRID_SHARD""", """HYBRID_SHARD_ZERO2"""]
lowerCamelCase		       =      ["""TRANSFORMER_BASED_WRAP""", """SIZE_BASED_WRAP""", """NO_WRAP"""]
lowerCamelCase		       =      ["""BACKWARD_PRE""", """BACKWARD_POST""", """NO_PREFETCH"""]
lowerCamelCase		       =      ["""FULL_STATE_DICT""", """LOCAL_STATE_DICT""", """SHARDED_STATE_DICT"""]
lowerCamelCase		       =      """2.0.1"""
lowerCamelCase		       =      ["""pdsh""", """standard""", """openmpi""", """mvapich"""]
lowerCamelCase		       =      ["""default""", """reduce-overhead""", """max-autotune"""]

lowerCamelCase		       =      {""">""": op.gt, """>=""": op.ge, """==""": op.eq, """!=""": op.ne, """<=""": op.le, """<""": op.lt}

# These are the args for `torch.distributed.launch` for pytorch < 1.9
lowerCamelCase		       =      [
    """nnodes""",
    """nproc_per_node""",
    """rdzv_backend""",
    """rdzv_endpoint""",
    """rdzv_id""",
    """rdzv_conf""",
    """standalone""",
    """max_restarts""",
    """monitor_interval""",
    """start_method""",
    """role""",
    """module""",
    """m""",
    """no_python""",
    """run_path""",
    """log_dir""",
    """r""",
    """redirects""",
    """t""",
    """tee""",
    """node_rank""",
    """master_addr""",
    """master_port""",
]

lowerCamelCase		       =      ["""DEEPSPEED""", """MULTI_GPU""", """FSDP""", """MEGATRON_LM"""]
lowerCamelCase		       =      ["""DEEPSPEED""", """MULTI_XPU""", """FSDP"""]


