
"""simple docstring"""

import os
import re
import shutil
import sys
import tempfile
import unittest

import black


SCREAMING_SNAKE_CASE_    					=						os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, '''utils'''))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
SCREAMING_SNAKE_CASE_    					=						'''    def __init__(self, config):
        super().__init__()
        self.transform = BertPredictionHeadTransform(config)

        # The output weights are the same as the input embeddings, but there is
        # an output-only bias for each token.
        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        self.bias = nn.Parameter(torch.zeros(config.vocab_size))

        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
        self.decoder.bias = self.bias

    def forward(self, hidden_states):
        hidden_states = self.transform(hidden_states)
        hidden_states = self.decoder(hidden_states)
        return hidden_states
'''
class      _UpperCAmelCase							(		unittest.TestCase							):




      def     a_      (							self							)						->					Tuple:
          UpperCAmelCase									=					tempfile.mkdtemp()
          os.makedirs(os.path.join(self.transformer_dir   ,       'models/bert/'							)							)
          UpperCAmelCase									=					self.transformer_dir
          shutil.copy(
              os.path.join(lowercase_   ,       'src/transformers/models/bert/modeling_bert.py'							)   ,       os.path.join(self.transformer_dir   ,       'models/bert/modeling_bert.py'							)   ,       )




      def     a_      (							self							)						->					List[Any]:
          UpperCAmelCase									=					'src/transformers'
          shutil.rmtree(self.transformer_dir							)




      def     a_      (							self   ,       lowercase_   ,       lowercase_   ,       lowercase_   ,       lowercase_=None							)						->					Union[str, Any]:
          UpperCAmelCase									=					comment + F"\nclass {class_name}(nn.Module):\n" + class_code
          if overwrite_result is not None:
              UpperCAmelCase									=					comment + F"\nclass {class_name}(nn.Module):\n" + overwrite_result
          UpperCAmelCase									=					black.Mode(target_versions={black.TargetVersion.PYaa}   ,       line_length=1_1_9							)
          UpperCAmelCase									=					black.format_str(lowercase_   ,       mode=lowercase_							)
          UpperCAmelCase									=					os.path.join(self.transformer_dir   ,       'new_code.py'							)
          with open(lowercase_   ,       'w'   ,       newline='\n'							) as f:
              f.write(lowercase_							)
          if overwrite_result is None:
              self.assertTrue(len(check_copies.is_copy_consistent(lowercase_							)							) == 0							)
          else:
              check_copies.is_copy_consistent(f.name   ,       overwrite=lowercase_							)
              with open(lowercase_   ,       'r'							) as f:
                  self.assertTrue(f.read()   ,       lowercase_							)




      def     a_      (							self							)						->					Union[str, Any]:
          UpperCAmelCase									=					check_copies.find_code_in_transformers('models.bert.modeling_bert.BertLMPredictionHead'							)
          self.assertEqual(lowercase_   ,       lowercase_							)




      def     a_      (							self							)						->					List[str]:
          # Base copy consistency
          self.check_copy_consistency(
              '# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead'   ,       'BertLMPredictionHead'   ,       REFERENCE_CODE + '\n'   ,       )

          # With no empty line at the end
          self.check_copy_consistency(
              '# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead'   ,       'BertLMPredictionHead'   ,       lowercase_   ,       )

          # Copy consistency with rename
          self.check_copy_consistency(
              '# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel'   ,       'TestModelLMPredictionHead'   ,       re.sub('Bert'   ,       'TestModel'   ,       lowercase_							)   ,       )

          # Copy consistency with a really long name
          UpperCAmelCase									=					'TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason'
          self.check_copy_consistency(
              F"# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}"   ,       F"{long_class_name}LMPredictionHead"   ,       re.sub('Bert'   ,       lowercase_   ,       lowercase_							)   ,       )

          # Copy consistency with overwrite
          self.check_copy_consistency(
              '# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel'   ,       'TestModelLMPredictionHead'   ,       lowercase_   ,       overwrite_result=re.sub('Bert'   ,       'TestModel'   ,       lowercase_							)   ,       )




      def     a_      (							self							)						->					Union[str, Any]:
          UpperCAmelCase									=					check_copies.LOCALIZED_READMES['README_zh-hans.md']

          UpperCAmelCase									=					(
              '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the'
              ' Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for'
              ' Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong'
              ' Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1.'
              ' **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),'
              ' released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and'
              ' lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same'
              ' method has been applied to compress GPT2 into'
              ' [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into'
              ' [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),'
              ' Multilingual BERT into'
              ' [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German'
              ' version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**'
              ' (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders'
              ' as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang'
              ' Luong, Quoc V. Le, Christopher D. Manning.'
          )
          UpperCAmelCase									=					(
              '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'
              ' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
              ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
              ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'
          )
          UpperCAmelCase									=					(
              '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'
              ' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
              ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
              ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1.'
              ' **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文'
              ' [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and'
              ' lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same'
              ' method has been applied to compress GPT2 into'
              ' [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into'
              ' [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),'
              ' Multilingual BERT into'
              ' [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German'
              ' version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自'
              ' Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather'
              ' than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,'
              ' Christopher D. Manning 发布。\n'
          )

          UpperCAmelCase			,							UpperCAmelCase									=					check_copies.convert_to_localized_md(
              lowercase_   ,       lowercase_   ,       localized_readme['format_model_list']							)

          self.assertFalse(lowercase_							)
          self.assertEqual(lowercase_   ,       lowercase_							)

          UpperCAmelCase			,							UpperCAmelCase									=					check_copies.convert_to_localized_md(
              lowercase_   ,       lowercase_   ,       localized_readme['format_model_list']							)

          # Check whether the number of models is equal to README.md after conversion.
          self.assertTrue(lowercase_							)

          UpperCAmelCase									=					(
              '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the'
              ' Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for'
              ' Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong'
              ' Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.'
          )
          UpperCAmelCase									=					(
              '1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and'
              ' the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
              ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
              ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'
          )
          UpperCAmelCase									=					(
              '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'
              ' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
              ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
              ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'
          )

          UpperCAmelCase			,							UpperCAmelCase									=					check_copies.convert_to_localized_md(
              lowercase_   ,       lowercase_   ,       localized_readme['format_model_list']							)

          # Check if the model link is synchronized.
          self.assertEqual(lowercase_   ,       lowercase_							)





"""simple docstring"""

import unittest

import numpy as np

from transformers import RobertaPreLayerNormConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
   import jax.numpy as jnp

   from transformers.models.roberta_prelayernorm.modeling_flax_roberta_prelayernorm import (
       FlaxRobertaPreLayerNormForCausalLM,
       FlaxRobertaPreLayerNormForMaskedLM,
       FlaxRobertaPreLayerNormForMultipleChoice,
       FlaxRobertaPreLayerNormForQuestionAnswering,
       FlaxRobertaPreLayerNormForSequenceClassification,
       FlaxRobertaPreLayerNormForTokenClassification,
       FlaxRobertaPreLayerNormModel,
   )
class      _UpperCAmelCase							(		unittest.TestCase							):




      def __init__(							self   ,       lowercase_   ,       lowercase_=1_3   ,       lowercase_=7   ,       lowercase_=True   ,       lowercase_=True   ,       lowercase_=True   ,       lowercase_=True   ,       lowercase_=9_9   ,       lowercase_=3_2   ,       lowercase_=5   ,       lowercase_=4   ,       lowercase_=3_7   ,       lowercase_="gelu"   ,       lowercase_=0.1   ,       lowercase_=0.1   ,       lowercase_=5_1_2   ,       lowercase_=1_6   ,       lowercase_=2   ,       lowercase_=0.0_2   ,       lowercase_=4   ,       )						->					Dict:
          UpperCAmelCase									=					parent
          UpperCAmelCase									=					batch_size
          UpperCAmelCase									=					seq_length
          UpperCAmelCase									=					is_training
          UpperCAmelCase									=					use_attention_mask
          UpperCAmelCase									=					use_token_type_ids
          UpperCAmelCase									=					use_labels
          UpperCAmelCase									=					vocab_size
          UpperCAmelCase									=					hidden_size
          UpperCAmelCase									=					num_hidden_layers
          UpperCAmelCase									=					num_attention_heads
          UpperCAmelCase									=					intermediate_size
          UpperCAmelCase									=					hidden_act
          UpperCAmelCase									=					hidden_dropout_prob
          UpperCAmelCase									=					attention_probs_dropout_prob
          UpperCAmelCase									=					max_position_embeddings
          UpperCAmelCase									=					type_vocab_size
          UpperCAmelCase									=					type_sequence_label_size
          UpperCAmelCase									=					initializer_range
          UpperCAmelCase									=					num_choices




      def     a_      (							self							)						->					int:
          UpperCAmelCase									=					ids_tensor([self.batch_size, self.seq_length]   ,       self.vocab_size							)

          UpperCAmelCase									=					None
          if self.use_attention_mask:
              UpperCAmelCase									=					random_attention_mask([self.batch_size, self.seq_length]							)

          UpperCAmelCase									=					None
          if self.use_token_type_ids:
              UpperCAmelCase									=					ids_tensor([self.batch_size, self.seq_length]   ,       self.type_vocab_size							)

          UpperCAmelCase									=					RobertaPreLayerNormConfig(
              vocab_size=self.vocab_size   ,       hidden_size=self.hidden_size   ,       num_hidden_layers=self.num_hidden_layers   ,       num_attention_heads=self.num_attention_heads   ,       intermediate_size=self.intermediate_size   ,       hidden_act=self.hidden_act   ,       hidden_dropout_prob=self.hidden_dropout_prob   ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,       max_position_embeddings=self.max_position_embeddings   ,       type_vocab_size=self.type_vocab_size   ,       is_decoder=lowercase_   ,       initializer_range=self.initializer_range   ,       )

          return config, input_ids, token_type_ids, attention_mask




      def     a_      (							self							)						->					List[str]:
          UpperCAmelCase									=					self.prepare_config_and_inputs()
          UpperCAmelCase			,							UpperCAmelCase			,							UpperCAmelCase			,							UpperCAmelCase									=					config_and_inputs
          UpperCAmelCase									=					{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': attention_mask}
          return config, inputs_dict




      def     a_      (							self							)						->					List[Any]:
          UpperCAmelCase									=					self.prepare_config_and_inputs()
          UpperCAmelCase			,							UpperCAmelCase			,							UpperCAmelCase			,							UpperCAmelCase									=					config_and_inputs

          UpperCAmelCase									=					True
          UpperCAmelCase									=					floats_tensor([self.batch_size, self.seq_length, self.hidden_size]							)
          UpperCAmelCase									=					ids_tensor([self.batch_size, self.seq_length]   ,       vocab_size=2							)

          return (
              config,
              input_ids,
              token_type_ids,
              encoder_hidden_states,
              encoder_attention_mask,
          )

@require_flax
# Copied from tests.models.roberta.test_modelling_flax_roberta.FlaxRobertaPreLayerNormModelTest with ROBERTA->ROBERTA_PRELAYERNORM,Roberta->RobertaPreLayerNorm,roberta-base->andreasmadsen/efficient_mlm_m0.40
class      _UpperCAmelCase							(		SCREAMING_SNAKE_CASE_     ,			unittest.TestCase							):
      __SCREAMING_SNAKE_CASE   :							List[str]							      =	True

      __SCREAMING_SNAKE_CASE   :							int							      =	(
          (
              FlaxRobertaPreLayerNormModel,
              FlaxRobertaPreLayerNormForCausalLM,
              FlaxRobertaPreLayerNormForMaskedLM,
              FlaxRobertaPreLayerNormForSequenceClassification,
              FlaxRobertaPreLayerNormForTokenClassification,
              FlaxRobertaPreLayerNormForMultipleChoice,
              FlaxRobertaPreLayerNormForQuestionAnswering,
          )
          if is_flax_available()
          else ()
      )




      def     a_      (							self							)						->					Optional[int]:
          UpperCAmelCase									=					FlaxRobertaPreLayerNormModelTester(self							)




      @slow
      def     a_      (							self							)						->					Dict:
          for model_class_name in self.all_model_classes:
              UpperCAmelCase									=					model_class_name.from_pretrained('andreasmadsen/efficient_mlm_m0.40'   ,       from_pt=lowercase_							)
              UpperCAmelCase									=					model(np.ones((1, 1)							)							)
              self.assertIsNotNone(lowercase_							)

@require_flax
class      _UpperCAmelCase							(		unittest.TestCase							):




      @slow
      def     a_      (							self							)						->					Tuple:
          UpperCAmelCase									=					FlaxRobertaPreLayerNormForMaskedLM.from_pretrained('andreasmadsen/efficient_mlm_m0.40'   ,       from_pt=lowercase_							)

          UpperCAmelCase									=					np.array([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]   ,       dtype=jnp.intaa							)
          UpperCAmelCase									=					model(lowercase_							)[0]
          UpperCAmelCase									=					[1, 1_1, 5_0_2_6_5]
          self.assertEqual(list(output.shape							)   ,       lowercase_							)
          # compare the actual values for a slice.
          UpperCAmelCase									=					np.array(
              [[[4_0.4_8_8_0, 1_8.0_1_9_9, -5.2_3_6_7], [-1.8_8_7_7, -4.0_8_8_5, 1_0.7_0_8_5], [-2.2_6_1_3, -5.6_1_1_0, 7.2_6_6_5]]]   ,       dtype=np.floataa							)
          self.assertTrue(np.allclose(output[:, :3, :3]   ,       lowercase_   ,       atol=1E-4							)							)




      @slow
      def     a_      (							self							)						->					int:
          UpperCAmelCase									=					FlaxRobertaPreLayerNormModel.from_pretrained('andreasmadsen/efficient_mlm_m0.40'   ,       from_pt=lowercase_							)

          UpperCAmelCase									=					np.array([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]   ,       dtype=jnp.intaa							)
          UpperCAmelCase									=					model(lowercase_							)[0]
          # compare the actual values for a slice.
          UpperCAmelCase									=					np.array(
              [[[0.0_2_0_8, -0.0_3_5_6, 0.0_2_3_7], [-0.1_5_6_9, -0.0_4_1_1, -0.2_6_2_6], [0.1_8_7_9, 0.0_1_2_5, -0.0_0_8_9]]]   ,       dtype=np.floataa							)
          self.assertTrue(np.allclose(output[:, :3, :3]   ,       lowercase_   ,       atol=1E-4							)							)



