


import json
import os
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoConfig, BertConfig, GPTaConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402


A   =			{
    """return_dict""": False,
    """output_hidden_states""": True,
    """output_attentions""": True,
    """torchscript""": True,
    """torch_dtype""": """float16""",
    """use_bfloat16""": True,
    """tf_legacy_loss""": True,
    """pruned_heads""": {"""a""": 1},
    """tie_word_embeddings""": False,
    """is_decoder""": True,
    """cross_attention_hidden_size""": 128,
    """add_cross_attention""": True,
    """tie_encoder_decoder""": True,
    """max_length""": 50,
    """min_length""": 3,
    """do_sample""": True,
    """early_stopping""": True,
    """num_beams""": 3,
    """num_beam_groups""": 3,
    """diversity_penalty""": 0.5,
    """temperature""": 2.0,
    """top_k""": 10,
    """top_p""": 0.7,
    """typical_p""": 0.2,
    """repetition_penalty""": 0.8,
    """length_penalty""": 0.8,
    """no_repeat_ngram_size""": 5,
    """encoder_no_repeat_ngram_size""": 5,
    """bad_words_ids""": [1, 2, 3],
    """num_return_sequences""": 3,
    """chunk_size_feed_forward""": 5,
    """output_scores""": True,
    """return_dict_in_generate""": True,
    """forced_bos_token_id""": 2,
    """forced_eos_token_id""": 3,
    """remove_invalid_values""": True,
    """architectures""": ["""BertModel"""],
    """finetuning_task""": """translation""",
    """id2label""": {0: """label"""},
    """label2id""": {"""label""": """0"""},
    """tokenizer_class""": """BertTokenizerFast""",
    """prefix""": """prefix""",
    """bos_token_id""": 6,
    """pad_token_id""": 7,
    """eos_token_id""": 8,
    """sep_token_id""": 9,
    """decoder_start_token_id""": 10,
    """exponential_decay_length_penalty""": (5, 1.01),
    """suppress_tokens""": [0, 1],
    """begin_suppress_tokens""": 2,
    """task_specific_params""": {"""translation""": """some_params"""},
    """problem_type""": """regression""",
}


@is_staging_test
class      lowercase__	(       unittest.TestCase		):






  @classmethod
  def       _UpperCAmelCase	(       cls   :       str			):




         """simple docstring"""







         UpperCAmelCase__					    =  TOKEN
         HfFolder.save_token(_lowercase			)






  @classmethod
  def       _UpperCAmelCase	(       cls   :       str			):




         """simple docstring"""







         try:
                delete_repo(token=cls._token		,    repo_id="test-config"			)
         except HTTPError:
                pass

         try:
                delete_repo(token=cls._token		,    repo_id="valid_org/test-config-org"			)
         except HTTPError:
                pass

         try:
                delete_repo(token=cls._token		,    repo_id="test-dynamic-config"			)
         except HTTPError:
                pass






  def       _UpperCAmelCase	(       self   :       Optional[Any]			):




         """simple docstring"""







         UpperCAmelCase__					    =  BertConfig(
             vocab_size=99		,    hidden_size=32		,    num_hidden_layers=5		,    num_attention_heads=4		,    intermediate_size=37			)
         config.push_to_hub("test-config"		,    use_auth_token=self._token			)

         UpperCAmelCase__					    =  BertConfig.from_pretrained(F"""{USER}/test-config"""			)
         for k, v in config.to_dict().items():
                if k != "transformers_version":
                       self.assertEqual(_lowercase		,    getattr(_lowercase		,    _lowercase			)			)

        # Reset repo
         delete_repo(token=self._token		,    repo_id="test-config"			)

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
                config.save_pretrained(_lowercase		,    repo_id="test-config"		,    push_to_hub=_lowercase		,    use_auth_token=self._token			)

         UpperCAmelCase__					    =  BertConfig.from_pretrained(F"""{USER}/test-config"""			)
         for k, v in config.to_dict().items():
                if k != "transformers_version":
                       self.assertEqual(_lowercase		,    getattr(_lowercase		,    _lowercase			)			)






  def       _UpperCAmelCase	(       self   :       Dict			):




         """simple docstring"""







         UpperCAmelCase__					    =  BertConfig(
             vocab_size=99		,    hidden_size=32		,    num_hidden_layers=5		,    num_attention_heads=4		,    intermediate_size=37			)
         config.push_to_hub("valid_org/test-config-org"		,    use_auth_token=self._token			)

         UpperCAmelCase__					    =  BertConfig.from_pretrained("valid_org/test-config-org"			)
         for k, v in config.to_dict().items():
                if k != "transformers_version":
                       self.assertEqual(_lowercase		,    getattr(_lowercase		,    _lowercase			)			)

        # Reset repo
         delete_repo(token=self._token		,    repo_id="valid_org/test-config-org"			)

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
                config.save_pretrained(
                    _lowercase		,    repo_id="valid_org/test-config-org"		,    push_to_hub=_lowercase		,    use_auth_token=self._token			)

         UpperCAmelCase__					    =  BertConfig.from_pretrained("valid_org/test-config-org"			)
         for k, v in config.to_dict().items():
                if k != "transformers_version":
                       self.assertEqual(_lowercase		,    getattr(_lowercase		,    _lowercase			)			)






  def       _UpperCAmelCase	(       self   :       Optional[int]			):




         """simple docstring"""







         CustomConfig.register_for_auto_class()
         UpperCAmelCase__					    =  CustomConfig(attribute=42			)

         config.push_to_hub("test-dynamic-config"		,    use_auth_token=self._token			)

         # This has added the proper auto_map field to the config
         self.assertDictEqual(config.auto_map		,    {"AutoConfig": "custom_configuration.CustomConfig"}			)

         UpperCAmelCase__					    =  AutoConfig.from_pretrained(F"""{USER}/test-dynamic-config"""		,    trust_remote_code=_lowercase			)
         # Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module
         self.assertEqual(new_config.__class__.__name__		,    "CustomConfig"			)
         self.assertEqual(new_config.attribute		,    42			)




class      lowercase__	(       unittest.TestCase		):






  def       _UpperCAmelCase	(       self   :       List[Any]			):




         """simple docstring"""







         UpperCAmelCase__					    =  GPTaConfig()

         # attempt to modify each of int/float/bool/str config records and verify they were updated
         UpperCAmelCase__					    =  c.n_embd + 1  # int
         UpperCAmelCase__					    =  c.resid_pdrop + 1.0  # float
         UpperCAmelCase__					    =  not c.scale_attn_weights  # bool
         UpperCAmelCase__					    =  c.summary_type + "foo"  # str
         c.update_from_string(
             F"""n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}"""			)
         self.assertEqual(_lowercase		,    c.n_embd		,    "mismatch for key: n_embd"			)
         self.assertEqual(_lowercase		,    c.resid_pdrop		,    "mismatch for key: resid_pdrop"			)
         self.assertEqual(_lowercase		,    c.scale_attn_weights		,    "mismatch for key: scale_attn_weights"			)
         self.assertEqual(_lowercase		,    c.summary_type		,    "mismatch for key: summary_type"			)






  def       _UpperCAmelCase	(       self   :       Optional[int]			):




         """simple docstring"""







         UpperCAmelCase__					    =  PretrainedConfig()
         UpperCAmelCase__					    =  [key for key in base_config.__dict__ if key not in config_common_kwargs]
         # If this part of the test fails, you have arguments to addin config_common_kwargs above.
         self.assertListEqual(
             _lowercase		,    ["is_encoder_decoder", "_name_or_path", "_commit_hash", "transformers_version"]			)
         UpperCAmelCase__					    =  [key for key, value in config_common_kwargs.items() if value == getattr(_lowercase		,    _lowercase			)]
         if len(_lowercase			) > 0:
                raise ValueError(
                    "The following keys are set with the default values in"
                    " `test_configuration_common.config_common_kwargs` pick another value for them:"
                    F""" {", ".join(_lowercase			)}."""			)






  def       _UpperCAmelCase	(       self   :       int			):




         """simple docstring"""







         with self.assertRaises(_lowercase			):
                # config is in subfolder, the following should not work without specifying the subfolder
                UpperCAmelCase__					    =  BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert-subfolder"			)

         UpperCAmelCase__					    =  BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert-subfolder"		,    subfolder="bert"			)

         self.assertIsNotNone(_lowercase			)






  def       _UpperCAmelCase	(       self   :       Optional[Any]			):




         """simple docstring"""







         UpperCAmelCase__					    =  mock.Mock()
         UpperCAmelCase__					    =  5_00
         UpperCAmelCase__					    =  {}
         UpperCAmelCase__					    =  HTTPError
         UpperCAmelCase__					    =  {}

         # Download this model to make sure it's in the cache.
         UpperCAmelCase__					    =  BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert"			)

         # Under the mock environment we get a 500 error when trying to reach the model.
         with mock.patch("requests.Session.request"		,    return_value=_lowercase			) as mock_head:
                UpperCAmelCase__					    =  BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert"			)
                # This check we did call the fake head request
                mock_head.assert_called()






  def       _UpperCAmelCase	(       self   :       str			):




         """simple docstring"""







         UpperCAmelCase__					    =  BertConfig.from_pretrained(
             "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json"			)






  def       _UpperCAmelCase	(       self   :       str			):




         """simple docstring"""







         UpperCAmelCase__					    =  AutoConfig.from_pretrained("bert-base-cased"			)
         UpperCAmelCase__					    =  ["config.4.0.0.json"]

         with tempfile.TemporaryDirectory() as tmp_dir:
                configuration.save_pretrained(_lowercase			)
                UpperCAmelCase__					    =  2
                json.dump(configuration.to_dict()		,    open(os.path.join(_lowercase		,    "config.4.0.0.json"			)		,    "w"			)			)

                # This should pick the new configuration file as the version of Transformers is > 4.0.0
                UpperCAmelCase__					    =  AutoConfig.from_pretrained(_lowercase			)
                self.assertEqual(new_configuration.hidden_size		,    2			)

                # Will need to be adjusted if we reach v42 and this test is still here.
                # Should pick the old configuration file as the version of Transformers is < 4.42.0
                UpperCAmelCase__					    =  ["config.42.0.0.json"]
                UpperCAmelCase__					    =  7_68
                configuration.save_pretrained(_lowercase			)
                shutil.move(os.path.join(_lowercase		,    "config.4.0.0.json"			)		,    os.path.join(_lowercase		,    "config.42.0.0.json"			)			)
                UpperCAmelCase__					    =  AutoConfig.from_pretrained(_lowercase			)
                self.assertEqual(new_configuration.hidden_size		,    7_68			)






  def       _UpperCAmelCase	(       self   :       Tuple			):




         """simple docstring"""







         UpperCAmelCase__					    =  "hf-internal-testing/test-two-configs"

         import transformers as new_transformers

         UpperCAmelCase__					    =  "v4.0.0"
         UpperCAmelCase__    ,      UpperCAmelCase__					    =  new_transformers.models.auto.AutoConfig.from_pretrained(
             _lowercase		,    return_unused_kwargs=_lowercase			)
         self.assertEqual(new_configuration.hidden_size		,    2			)
         # This checks `_configuration_file` ia not kept in the kwargs by mistake.
         self.assertDictEqual(_lowercase		,    {}			)

         # Testing an older version by monkey-patching the version in the module it's used.
         import transformers as old_transformers

         UpperCAmelCase__					    =  "v3.0.0"
         UpperCAmelCase__					    =  old_transformers.models.auto.AutoConfig.from_pretrained(_lowercase			)
         self.assertEqual(old_configuration.hidden_size		,    7_68			)













'''simple docstring'''



__UpperCamelCase   :							List[Any]         =   """ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"""



def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						bytes     )	->      bytes:

   """simple docstring"""
   # Make sure the supplied data is a bytes-like object
   if not isinstance(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     ):
      __a					  =     f"""a bytes-like object is required, not '{data.__class__.__name__}'"""
      raise TypeError(SCREAMING_SNAKE_CASE__     )

   __a					  =     ''.join(bin(SCREAMING_SNAKE_CASE__     )[2:].zfill(8     ) for byte in data     )

   __a					  =     len(SCREAMING_SNAKE_CASE__     ) % 6 != 0

   if padding_needed:
      # The padding that will be added later
      __a					  =     b'=' * ((6 - len(SCREAMING_SNAKE_CASE__     ) % 6) // 2)

      # Append binary_stream with arbitrary binary digits (0's by default) to make its
      # length a multiple of 6.
      binary_stream += "0" * (6 - len(SCREAMING_SNAKE_CASE__     ) % 6)
   else:
      __a					  =     b''

   # Encode every 6 binary digits to their corresponding Base64 character
   return (
       "".join(
           B64_CHARSET[int(binary_stream[index : index + 6],       2     )]
           for index in range(0,       len(SCREAMING_SNAKE_CASE__     ),       6     )     ).encode()
       + padding
   )



def  __UpperCAmelCase						(					SCREAMING_SNAKE_CASE__:						str     )	->      bytes:

   """simple docstring"""
   # Make sure encoded_data is either a string or a bytes-like object
   if not isinstance(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     ) and not isinstance(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     ):
      __a					  =     (
          'argument should be a bytes-like object or ASCII string, '
          f"""not '{encoded_data.__class__.__name__}'"""
      )
      raise TypeError(SCREAMING_SNAKE_CASE__     )

   # In case encoded_data is a bytes-like object, make sure it contains only
   # ASCII characters so we convert it to a string object
   if isinstance(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__     ):
      try:
         __a					  =     encoded_data.decode('utf-8'     )
      except UnicodeDecodeError:
         raise ValueError('base64 encoded data should only contain ASCII characters'     )

   __a					  =     encoded_data.count('='     )

   # Check if the encoded string contains non base64 characters
   if padding:
      assert all(
          char in B64_CHARSET for char in encoded_data[:-padding]     ), "Invalid base64 character(s) found."
   else:
      assert all(
          char in B64_CHARSET for char in encoded_data     ), "Invalid base64 character(s) found."

   # Check the padding
   assert len(SCREAMING_SNAKE_CASE__     ) % 4 == 0 and padding < 3, "Incorrect padding"

   if padding:
      # Remove padding if there is one
      __a					  =     encoded_data[:-padding]

      __a					  =     ''.join(
          bin(B64_CHARSET.index(SCREAMING_SNAKE_CASE__     )     )[2:].zfill(6     ) for char in encoded_data     )[: -padding * 2]
   else:
      __a					  =     ''.join(
          bin(B64_CHARSET.index(SCREAMING_SNAKE_CASE__     )     )[2:].zfill(6     ) for char in encoded_data     )

   __a					  =     [
       int(binary_stream[index : index + 8],       2     )
       for index in range(0,       len(SCREAMING_SNAKE_CASE__     ),       8     )
   ]

   return bytes(SCREAMING_SNAKE_CASE__     )


if __name__ == "__main__":
 import doctest

 doctest.testmod()