

"""simple docstring"""
import json
import os
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoConfig, BertConfig, GPTaConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402


_A							=  {
    """return_dict""": False,
    """output_hidden_states""": True,
    """output_attentions""": True,
    """torchscript""": True,
    """torch_dtype""": """float16""",
    """use_bfloat16""": True,
    """tf_legacy_loss""": True,
    """pruned_heads""": {"""a""": 1},
    """tie_word_embeddings""": False,
    """is_decoder""": True,
    """cross_attention_hidden_size""": 1_28,
    """add_cross_attention""": True,
    """tie_encoder_decoder""": True,
    """max_length""": 50,
    """min_length""": 3,
    """do_sample""": True,
    """early_stopping""": True,
    """num_beams""": 3,
    """num_beam_groups""": 3,
    """diversity_penalty""": 0.5,
    """temperature""": 2.0,
    """top_k""": 10,
    """top_p""": 0.7,
    """typical_p""": 0.2,
    """repetition_penalty""": 0.8,
    """length_penalty""": 0.8,
    """no_repeat_ngram_size""": 5,
    """encoder_no_repeat_ngram_size""": 5,
    """bad_words_ids""": [1, 2, 3],
    """num_return_sequences""": 3,
    """chunk_size_feed_forward""": 5,
    """output_scores""": True,
    """return_dict_in_generate""": True,
    """forced_bos_token_id""": 2,
    """forced_eos_token_id""": 3,
    """remove_invalid_values""": True,
    """architectures""": ["""BertModel"""],
    """finetuning_task""": """translation""",
    """id2label""": {0: """label"""},
    """label2id""": {"""label""": """0"""},
    """tokenizer_class""": """BertTokenizerFast""",
    """prefix""": """prefix""",
    """bos_token_id""": 6,
    """pad_token_id""": 7,
    """eos_token_id""": 8,
    """sep_token_id""": 9,
    """decoder_start_token_id""": 10,
    """exponential_decay_length_penalty""": (5, 1.01),
    """suppress_tokens""": [0, 1],
    """begin_suppress_tokens""": 2,
    """task_specific_params""": {"""translation""": """some_params"""},
    """problem_type""": """regression""",
}

@is_staging_test
class       lowerCamelCase	(       unittest.TestCase   ):


  '''simple docstring'''







  @classmethod
  def  _a      (cls	):


        """simple docstring"""

        UpperCAmelCase__			:							int							=    TOKEN
        HfFolder.save_token(_lowerCamelCase	)



  @classmethod
  def  _a      (cls	):


        """simple docstring"""

        try:
              delete_repo(token=cls._token							,       repo_id="""test-config"""	)
        except HTTPError:
              pass

        try:
              delete_repo(token=cls._token							,       repo_id="""valid_org/test-config-org"""	)
        except HTTPError:
              pass

        try:
              delete_repo(token=cls._token							,       repo_id="""test-dynamic-config"""	)
        except HTTPError:
              pass



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Tuple							=    BertConfig(
            vocab_size=99							,       hidden_size=32							,       num_hidden_layers=5							,       num_attention_heads=4							,       intermediate_size=37	)
        config.push_to_hub("""test-config"""							,       use_auth_token=self._token	)

        UpperCAmelCase__			:							Optional[Any]							=    BertConfig.from_pretrained(F"""{USER}/test-config"""	)
        for k, v in config.to_dict().items():
              if k != "transformers_version":
                    self.assertEqual(_lowerCamelCase							,       getattr(_lowerCamelCase							,       _lowerCamelCase	)	)

        # Reset repo
        delete_repo(token=self._token							,       repo_id="""test-config"""	)

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
              config.save_pretrained(_lowerCamelCase							,       repo_id="""test-config"""							,       push_to_hub=_lowerCamelCase							,       use_auth_token=self._token	)

        UpperCAmelCase__			:							Any							=    BertConfig.from_pretrained(F"""{USER}/test-config"""	)
        for k, v in config.to_dict().items():
              if k != "transformers_version":
                    self.assertEqual(_lowerCamelCase							,       getattr(_lowerCamelCase							,       _lowerCamelCase	)	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							str							=    BertConfig(
            vocab_size=99							,       hidden_size=32							,       num_hidden_layers=5							,       num_attention_heads=4							,       intermediate_size=37	)
        config.push_to_hub("""valid_org/test-config-org"""							,       use_auth_token=self._token	)

        UpperCAmelCase__			:							Optional[int]							=    BertConfig.from_pretrained("""valid_org/test-config-org"""	)
        for k, v in config.to_dict().items():
              if k != "transformers_version":
                    self.assertEqual(_lowerCamelCase							,       getattr(_lowerCamelCase							,       _lowerCamelCase	)	)

        # Reset repo
        delete_repo(token=self._token							,       repo_id="""valid_org/test-config-org"""	)

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
              config.save_pretrained(
                  _lowerCamelCase							,       repo_id="""valid_org/test-config-org"""							,       push_to_hub=_lowerCamelCase							,       use_auth_token=self._token	)

        UpperCAmelCase__			:							Any							=    BertConfig.from_pretrained("""valid_org/test-config-org"""	)
        for k, v in config.to_dict().items():
              if k != "transformers_version":
                    self.assertEqual(_lowerCamelCase							,       getattr(_lowerCamelCase							,       _lowerCamelCase	)	)




  def  _a      (self	):


        """simple docstring"""

        CustomConfig.register_for_auto_class()
        UpperCAmelCase__			:							List[str]							=    CustomConfig(attribute=42	)

        config.push_to_hub("""test-dynamic-config"""							,       use_auth_token=self._token	)

        # This has added the proper auto_map field to the config
        self.assertDictEqual(config.auto_map							,       {"""AutoConfig""": """custom_configuration.CustomConfig"""}	)

        UpperCAmelCase__			:							List[str]							=    AutoConfig.from_pretrained(F"""{USER}/test-dynamic-config"""							,       trust_remote_code=_lowerCamelCase	)
        # Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module
        self.assertEqual(new_config.__class__.__name__							,       """CustomConfig"""	)
        self.assertEqual(new_config.attribute							,       42	)




class       lowerCamelCase	(       unittest.TestCase   ):


  '''simple docstring'''







  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[int]							=    GPTaConfig()

        # attempt to modify each of int/float/bool/str config records and verify they were updated
        UpperCAmelCase__			:							Optional[Any]							=    c.n_embd + 1  # int
        UpperCAmelCase__			:							Tuple							=    c.resid_pdrop + 1.0  # float
        UpperCAmelCase__			:							int							=    not c.scale_attn_weights  # bool
        UpperCAmelCase__			:							Optional[int]							=    c.summary_type + """foo"""  # str
        c.update_from_string(
            F"""n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}"""	)
        self.assertEqual(_lowerCamelCase							,       c.n_embd							,       """mismatch for key: n_embd"""	)
        self.assertEqual(_lowerCamelCase							,       c.resid_pdrop							,       """mismatch for key: resid_pdrop"""	)
        self.assertEqual(_lowerCamelCase							,       c.scale_attn_weights							,       """mismatch for key: scale_attn_weights"""	)
        self.assertEqual(_lowerCamelCase							,       c.summary_type							,       """mismatch for key: summary_type"""	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[Any]							=    PretrainedConfig()
        UpperCAmelCase__			:							Dict							=    [key for key in base_config.__dict__ if key not in config_common_kwargs]
        # If this part of the test fails, you have arguments to addin config_common_kwargs above.
        self.assertListEqual(
            _lowerCamelCase							,       ["""is_encoder_decoder""", """_name_or_path""", """_commit_hash""", """transformers_version"""]	)
        UpperCAmelCase__			:							int							=    [key for key, value in config_common_kwargs.items() if value == getattr(_lowerCamelCase							,       _lowerCamelCase	)]
        if len(_lowerCamelCase	) > 0:
              raise ValueError(
                  """The following keys are set with the default values in"""
                  """ `test_configuration_common.config_common_kwargs` pick another value for them:"""
                  F""" {', '.join(_lowerCamelCase	)}."""	)



  def  _a      (self	):


        """simple docstring"""

        with self.assertRaises(_lowerCamelCase	):
              # config is in subfolder, the following should not work without specifying the subfolder
              UpperCAmelCase__			:							str							=    BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert-subfolder"""	)

        UpperCAmelCase__			:							Any							=    BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert-subfolder"""							,       subfolder="""bert"""	)

        self.assertIsNotNone(_lowerCamelCase	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    mock.Mock()
        UpperCAmelCase__			:							Optional[Any]							=    500
        UpperCAmelCase__			:							str							=    {}
        UpperCAmelCase__			:							Optional[Any]							=    HTTPError
        UpperCAmelCase__			:							Optional[int]							=    {}

        # Download this model to make sure it's in the cache.
        UpperCAmelCase__			:							Optional[Any]							=    BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert"""	)

        # Under the mock environment we get a 500 error when trying to reach the model.
        with mock.patch("""requests.Session.request"""							,       return_value=_lowerCamelCase	) as mock_head:
              UpperCAmelCase__			:							str							=    BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert"""	)
              # This check we did call the fake head request
              mock_head.assert_called()



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							str							=    BertConfig.from_pretrained(
            """https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json"""	)



  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							Any							=    AutoConfig.from_pretrained("""bert-base-cased"""	)
        UpperCAmelCase__			:							str							=    ["""config.4.0.0.json"""]

        with tempfile.TemporaryDirectory() as tmp_dir:
              configuration.save_pretrained(_lowerCamelCase	)
              UpperCAmelCase__			:							List[Any]							=    2
              json.dump(configuration.to_dict()							,       open(os.path.join(_lowerCamelCase							,       """config.4.0.0.json"""	)							,       """w"""	)	)

              # This should pick the new configuration file as the version of Transformers is > 4.0.0
              UpperCAmelCase__			:							Union[str, Any]							=    AutoConfig.from_pretrained(_lowerCamelCase	)
              self.assertEqual(new_configuration.hidden_size							,       2	)

              # Will need to be adjusted if we reach v42 and this test is still here.
              # Should pick the old configuration file as the version of Transformers is < 4.42.0
              UpperCAmelCase__			:							Optional[Any]							=    ["""config.42.0.0.json"""]
              UpperCAmelCase__			:							int							=    768
              configuration.save_pretrained(_lowerCamelCase	)
              shutil.move(os.path.join(_lowerCamelCase							,       """config.4.0.0.json"""	)							,       os.path.join(_lowerCamelCase							,       """config.42.0.0.json"""	)	)
              UpperCAmelCase__			:							List[str]							=    AutoConfig.from_pretrained(_lowerCamelCase	)
              self.assertEqual(new_configuration.hidden_size							,       768	)




  def  _a      (self	):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    """hf-internal-testing/test-two-configs"""

        import transformers as new_transformers

        UpperCAmelCase__			:							List[Any]							=    """v4.0.0"""
        UpperCAmelCase__					,		UpperCAmelCase__			:							List[Any]							=    new_transformers.models.auto.AutoConfig.from_pretrained(
            _lowerCamelCase							,       return_unused_kwargs=_lowerCamelCase	)
        self.assertEqual(new_configuration.hidden_size							,       2	)
        # This checks `_configuration_file` ia not kept in the kwargs by mistake.
        self.assertDictEqual(_lowerCamelCase							,       {}	)

        # Testing an older version by monkey-patching the version in the module it's used.
        import transformers as old_transformers

        UpperCAmelCase__			:							int							=    """v3.0.0"""
        UpperCAmelCase__			:							Optional[int]							=    old_transformers.models.auto.AutoConfig.from_pretrained(_lowerCamelCase	)
        self.assertEqual(old_configuration.hidden_size							,       768	)




"""simple docstring"""
import warnings
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_A							=  logging.get_logger(__name__)

class       lowerCamelCase	(       lowerCAmelCase__   ):


  '''simple docstring'''







  SCREAMING_SNAKE_CASE					      =  ['input_ids', 'attention_mask']



  def __init__(self							,       _lowerCamelCase="</s>"							,       _lowerCamelCase="<unk>"							,       _lowerCamelCase="<pad>"							,       _lowerCamelCase=125							,       _lowerCamelCase=None							,       **_lowerCamelCase							,       ):


        """simple docstring"""

        if extra_ids > 0 and additional_special_tokens is None:
              UpperCAmelCase__			:							List[str]							=    [F"""<extra_id_{i}>""" for i in range(_lowerCamelCase	)]
        elif extra_ids > 0 and additional_special_tokens is not None:
              # Check that we have the right number of extra_id special tokens
              UpperCAmelCase__			:							int							=    len(set(filter(lambda _lowerCamelCase	: bool("""extra_id""" in str(_lowerCamelCase	)	)							,       _lowerCamelCase	)	)	)
              if extra_tokens != extra_ids:
                    raise ValueError(
                        F"""Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"""
                        """ provided to ByT5Tokenizer. In this case the additional_special_tokens must include the"""
                        """ extra_ids tokens"""	)

        UpperCAmelCase__			:							List[str]							=    AddedToken(_lowerCamelCase							,       lstrip=_lowerCamelCase							,       rstrip=_lowerCamelCase	) if isinstance(_lowerCamelCase							,       _lowerCamelCase	) else pad_token
        UpperCAmelCase__			:							int							=    AddedToken(_lowerCamelCase							,       lstrip=_lowerCamelCase							,       rstrip=_lowerCamelCase	) if isinstance(_lowerCamelCase							,       _lowerCamelCase	) else eos_token
        UpperCAmelCase__			:							Optional[int]							=    AddedToken(_lowerCamelCase							,       lstrip=_lowerCamelCase							,       rstrip=_lowerCamelCase	) if isinstance(_lowerCamelCase							,       _lowerCamelCase	) else unk_token

        super().__init__(
            eos_token=_lowerCamelCase							,       unk_token=_lowerCamelCase							,       pad_token=_lowerCamelCase							,       extra_ids=_lowerCamelCase							,       additional_special_tokens=_lowerCamelCase							,       **_lowerCamelCase							,       )

        UpperCAmelCase__			:							int							=    extra_ids

        UpperCAmelCase__			:							List[str]							=    2**8  # utf is 8 bits

        # define special tokens dict
        UpperCAmelCase__			:							Dict[int, str]							=    {
            self.pad_token: 0,
            self.eos_token: 1,
            self.unk_token: 2,
        }
        UpperCAmelCase__			:							str							=    len(self.special_tokens_encoder	)
        UpperCAmelCase__			:							Union[str, Any]							=    len(_lowerCamelCase	)
        for i, token in enumerate(_lowerCamelCase	):
              UpperCAmelCase__			:							Optional[Any]							=    self.vocab_size + i - n
        UpperCAmelCase__			:							Dict[str, int]							=    {v: k for k, v in self.special_tokens_encoder.items()}



  @property
  def  _a      (self	):


        """simple docstring"""

        return self._utf_vocab_size + self._num_special_tokens + self._extra_ids



  def  _a      (self							,       _lowerCamelCase							,       _lowerCamelCase = None							,       _lowerCamelCase = False	):


        """simple docstring"""

        if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=_lowerCamelCase							,       token_ids_a=_lowerCamelCase							,       already_has_special_tokens=_lowerCamelCase	)

        # normal case: some special tokens
        if token_ids_a is None:
              return ([0] * len(_lowerCamelCase	)) + [1]
        return ([0] * len(_lowerCamelCase	)) + [1] + ([0] * len(_lowerCamelCase	)) + [1]



  def  _a      (self							,       _lowerCamelCase	):


        """simple docstring"""

        if len(_lowerCamelCase	) > 0 and token_ids[-1] == self.eos_token_id:
              warnings.warn(
                  F"""This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"""
                  """ eos tokens being added."""	)
              return token_ids
        else:
              return token_ids + [self.eos_token_id]



  def  _a      (self							,       _lowerCamelCase							,       _lowerCamelCase = None	):


        """simple docstring"""

        UpperCAmelCase__			:							List[str]							=    [self.eos_token_id]

        if token_ids_a is None:
              return len(token_ids_a + eos	) * [0]
        return len(token_ids_a + eos + token_ids_a + eos	) * [0]



  def  _a      (self							,       _lowerCamelCase							,       _lowerCamelCase = None	):


        """simple docstring"""

        UpperCAmelCase__			:							Tuple							=    self._add_eos_if_not_present(_lowerCamelCase	)
        if token_ids_a is None:
              return token_ids_a
        else:
              UpperCAmelCase__			:							List[str]							=    self._add_eos_if_not_present(_lowerCamelCase	)
              return token_ids_a + token_ids_a



  def  _a      (self							,       _lowerCamelCase	):


        """simple docstring"""

        UpperCAmelCase__			:							Optional[Any]							=    [chr(_lowerCamelCase	) for i in text.encode("""utf-8"""	)]
        return tokens



  def  _a      (self							,       _lowerCamelCase	):


        """simple docstring"""

        if token in self.special_tokens_encoder:
              UpperCAmelCase__			:							List[str]							=    self.special_tokens_encoder[token]
        elif token in self.added_tokens_encoder:
              UpperCAmelCase__			:							Tuple							=    self.added_tokens_encoder[token]
        elif len(_lowerCamelCase	) != 1:
              UpperCAmelCase__			:							Dict							=    self.unk_token_id
        else:
              UpperCAmelCase__			:							Tuple							=    ord(_lowerCamelCase	) + self._num_special_tokens
        return token_id



  def  _a      (self							,       _lowerCamelCase	):


        """simple docstring"""

        if index in self.special_tokens_decoder:
              UpperCAmelCase__			:							Optional[Any]							=    self.special_tokens_decoder[index]
        else:
              UpperCAmelCase__			:							Dict							=    chr(index - self._num_special_tokens	)
        return token



  def  _a      (self							,       _lowerCamelCase	):


        """simple docstring"""

        UpperCAmelCase__			:							List[Any]							=    B""""""
        for token in tokens:
              if token in self.special_tokens_decoder:
                    UpperCAmelCase__			:							Tuple							=    self.special_tokens_decoder[token].encode("""utf-8"""	)
              elif token in self.added_tokens_decoder:
                    UpperCAmelCase__			:							Optional[Any]							=    self.special_tokens_decoder[token].encode("""utf-8"""	)
              elif token in self.special_tokens_encoder:
                    UpperCAmelCase__			:							str							=    token.encode("""utf-8"""	)
              elif token in self.added_tokens_encoder:
                    UpperCAmelCase__			:							List[Any]							=    token.encode("""utf-8"""	)
              else:
                    UpperCAmelCase__			:							Optional[int]							=    bytes([ord(_lowerCamelCase	)]	)
              bstring += tok_string
        UpperCAmelCase__			:							int							=    bstring.decode("""utf-8"""							,       errors="""ignore"""	)
        return string




  def  _a      (self							,       _lowerCamelCase							,       _lowerCamelCase = None	):


        """simple docstring"""

        return ()

