



'''simple docstring'''






import json
import os
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoConfig, BertConfig, GPTaConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402


lowerCAmelCase:      List[Any]		 =		{
    'return_dict': False,
    'output_hidden_states': True,
    'output_attentions': True,
    'torchscript': True,
    'torch_dtype': 'float16',
    'use_bfloat16': True,
    'tf_legacy_loss': True,
    'pruned_heads': {'a': 1},
    'tie_word_embeddings': False,
    'is_decoder': True,
    'cross_attention_hidden_size': 1_2_8,
    'add_cross_attention': True,
    'tie_encoder_decoder': True,
    'max_length': 5_0,
    'min_length': 3,
    'do_sample': True,
    'early_stopping': True,
    'num_beams': 3,
    'num_beam_groups': 3,
    'diversity_penalty': 0.5,
    'temperature': 2.0,
    'top_k': 1_0,
    'top_p': 0.7,
    'typical_p': 0.2,
    'repetition_penalty': 0.8,
    'length_penalty': 0.8,
    'no_repeat_ngram_size': 5,
    'encoder_no_repeat_ngram_size': 5,
    'bad_words_ids': [1, 2, 3],
    'num_return_sequences': 3,
    'chunk_size_feed_forward': 5,
    'output_scores': True,
    'return_dict_in_generate': True,
    'forced_bos_token_id': 2,
    'forced_eos_token_id': 3,
    'remove_invalid_values': True,
    'architectures': ['BertModel'],
    'finetuning_task': 'translation',
    'id2label': {0: 'label'},
    'label2id': {'label': '0'},
    'tokenizer_class': 'BertTokenizerFast',
    'prefix': 'prefix',
    'bos_token_id': 6,
    'pad_token_id': 7,
    'eos_token_id': 8,
    'sep_token_id': 9,
    'decoder_start_token_id': 1_0,
    'exponential_decay_length_penalty': (5, 1.01),
    'suppress_tokens': [0, 1],
    'begin_suppress_tokens': 2,
    'task_specific_params': {'translation': 'some_params'},
    'problem_type': 'regression',
}

@is_staging_test
class 							a__(      unittest.TestCase			):







    @classmethod
    def     lowercase_ (   cls				:						Any			):
      a		:      str       =  TOKEN
      HfFolder.save_token(UpperCAmelCase_			)







    @classmethod
    def     lowercase_ (   cls				:						List[str]			):
      try:
        delete_repo(token=cls._token		,    repo_id='test-config'			)
      except HTTPError:
        pass

      try:
        delete_repo(token=cls._token		,    repo_id='valid_org/test-config-org'			)
      except HTTPError:
        pass

      try:
        delete_repo(token=cls._token		,    repo_id='test-dynamic-config'			)
      except HTTPError:
        pass







    def     lowercase_ (   self				:						str			):
      a		:      List[str]       =  BertConfig(
          vocab_size=99		,    hidden_size=32		,    num_hidden_layers=5		,    num_attention_heads=4		,    intermediate_size=37			)
      config.push_to_hub('test-config'		,    use_auth_token=self._token			)

      a		:      List[str]       =  BertConfig.from_pretrained(F"""{USER}/test-config"""			)
      for k, v in config.to_dict().items():
        if k != "transformers_version":
          self.assertEqual(UpperCAmelCase_		,    getattr(UpperCAmelCase_		,    UpperCAmelCase_			)			)

        # Reset repo
      delete_repo(token=self._token		,    repo_id='test-config'			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
        config.save_pretrained(UpperCAmelCase_		,    repo_id='test-config'		,    push_to_hub=UpperCAmelCase_		,    use_auth_token=self._token			)

      a		:      List[Any]       =  BertConfig.from_pretrained(F"""{USER}/test-config"""			)
      for k, v in config.to_dict().items():
        if k != "transformers_version":
          self.assertEqual(UpperCAmelCase_		,    getattr(UpperCAmelCase_		,    UpperCAmelCase_			)			)







    def     lowercase_ (   self				:						Dict			):
      a		:      int       =  BertConfig(
          vocab_size=99		,    hidden_size=32		,    num_hidden_layers=5		,    num_attention_heads=4		,    intermediate_size=37			)
      config.push_to_hub('valid_org/test-config-org'		,    use_auth_token=self._token			)

      a		:      Any       =  BertConfig.from_pretrained('valid_org/test-config-org'			)
      for k, v in config.to_dict().items():
        if k != "transformers_version":
          self.assertEqual(UpperCAmelCase_		,    getattr(UpperCAmelCase_		,    UpperCAmelCase_			)			)

        # Reset repo
      delete_repo(token=self._token		,    repo_id='valid_org/test-config-org'			)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
        config.save_pretrained(
            UpperCAmelCase_		,    repo_id='valid_org/test-config-org'		,    push_to_hub=UpperCAmelCase_		,    use_auth_token=self._token			)

      a		:      int       =  BertConfig.from_pretrained('valid_org/test-config-org'			)
      for k, v in config.to_dict().items():
        if k != "transformers_version":
          self.assertEqual(UpperCAmelCase_		,    getattr(UpperCAmelCase_		,    UpperCAmelCase_			)			)







    def     lowercase_ (   self				:						Any			):
      CustomConfig.register_for_auto_class()
      a		:      Dict       =  CustomConfig(attribute=42			)

      config.push_to_hub('test-dynamic-config'		,    use_auth_token=self._token			)

      # This has added the proper auto_map field to the config
      self.assertDictEqual(config.auto_map		,    {'AutoConfig': 'custom_configuration.CustomConfig'}			)

      a		:      Tuple       =  AutoConfig.from_pretrained(F"""{USER}/test-dynamic-config"""		,    trust_remote_code=UpperCAmelCase_			)
      # Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module
      self.assertEqual(new_config.__class__.__name__		,    'CustomConfig'			)
      self.assertEqual(new_config.attribute		,    42			)

class 							a__(      unittest.TestCase			):







    def     lowercase_ (   self				:						Optional[Any]			):
      a		:      List[str]       =  GPTaConfig()

      # attempt to modify each of int/float/bool/str config records and verify they were updated
      a		:      int       =  c.n_embd + 1  # int
      a		:      List[str]       =  c.resid_pdrop + 1.0  # float
      a		:      List[Any]       =  not c.scale_attn_weights  # bool
      a		:      Dict       =  c.summary_type + 'foo'  # str
      c.update_from_string(
          F"""n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}"""			)
      self.assertEqual(UpperCAmelCase_		,    c.n_embd		,    'mismatch for key: n_embd'			)
      self.assertEqual(UpperCAmelCase_		,    c.resid_pdrop		,    'mismatch for key: resid_pdrop'			)
      self.assertEqual(UpperCAmelCase_		,    c.scale_attn_weights		,    'mismatch for key: scale_attn_weights'			)
      self.assertEqual(UpperCAmelCase_		,    c.summary_type		,    'mismatch for key: summary_type'			)







    def     lowercase_ (   self				:						Tuple			):
      a		:      Any       =  PretrainedConfig()
      a		:      Optional[int]       =  [key for key in base_config.__dict__ if key not in config_common_kwargs]
      # If this part of the test fails, you have arguments to addin config_common_kwargs above.
      self.assertListEqual(
          UpperCAmelCase_		,    ['is_encoder_decoder', '_name_or_path', '_commit_hash', 'transformers_version']			)
      a		:      Union[str, Any]       =  [key for key, value in config_common_kwargs.items() if value == getattr(UpperCAmelCase_		,    UpperCAmelCase_			)]
      if len(UpperCAmelCase_			) > 0:
        raise ValueError(
            'The following keys are set with the default values in'
            ' `test_configuration_common.config_common_kwargs` pick another value for them:'
            F""" {", ".join(UpperCAmelCase_			)}."""			)







    def     lowercase_ (   self				:						Optional[Any]			):
      with self.assertRaises(UpperCAmelCase_			):
        # config is in subfolder, the following should not work without specifying the subfolder
        a		:      int       =  BertConfig.from_pretrained('hf-internal-testing/tiny-random-bert-subfolder'			)

      a		:      int       =  BertConfig.from_pretrained('hf-internal-testing/tiny-random-bert-subfolder'		,    subfolder='bert'			)

      self.assertIsNotNone(UpperCAmelCase_			)







    def     lowercase_ (   self				:						Tuple			):
      # A mock response for an HTTP head request to emulate server down
      a		:      Any       =  mock.Mock()
      a		:      int       =  5_00
      a		:      str       =  {}
      a		:      Optional[Any]       =  HTTPError
      a		:      Optional[int]       =  {}

      # Download this model to make sure it's in the cache.
      a		:      Tuple       =  BertConfig.from_pretrained('hf-internal-testing/tiny-random-bert'			)

      # Under the mock environment we get a 500 error when trying to reach the model.
      with mock.patch('requests.Session.request'		,    return_value=UpperCAmelCase_			) as mock_head:
        a		:      List[str]       =  BertConfig.from_pretrained('hf-internal-testing/tiny-random-bert'			)
        # This check we did call the fake head request
        mock_head.assert_called()







    def     lowercase_ (   self				:						Union[str, Any]			):
      # This test is for deprecated behavior and can be removed in v5
      a		:      int       =  BertConfig.from_pretrained(
          'https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json'			)







    def     lowercase_ (   self				:						str			):
      a		:      List[Any]       =  AutoConfig.from_pretrained('bert-base-cased'			)
      a		:      Tuple       =  ['config.4.0.0.json']

      with tempfile.TemporaryDirectory() as tmp_dir:
        configuration.save_pretrained(UpperCAmelCase_			)
        a		:      List[str]       =  2
        json.dump(configuration.to_dict()		,    open(os.path.join(UpperCAmelCase_		,    'config.4.0.0.json'			)		,    'w'			)			)

        # This should pick the new configuration file as the version of Transformers is > 4.0.0
        a		:      List[str]       =  AutoConfig.from_pretrained(UpperCAmelCase_			)
        self.assertEqual(new_configuration.hidden_size		,    2			)

        # Will need to be adjusted if we reach v42 and this test is still here.
        # Should pick the old configuration file as the version of Transformers is < 4.42.0
        a		:      Dict       =  ['config.42.0.0.json']
        a		:      Optional[Any]       =  7_68
        configuration.save_pretrained(UpperCAmelCase_			)
        shutil.move(os.path.join(UpperCAmelCase_		,    'config.4.0.0.json'			)		,    os.path.join(UpperCAmelCase_		,    'config.42.0.0.json'			)			)
        a		:      Tuple       =  AutoConfig.from_pretrained(UpperCAmelCase_			)
        self.assertEqual(new_configuration.hidden_size		,    7_68			)







    def     lowercase_ (   self				:						List[str]			):
      # This repo has two configuration files, one for v4.0.0 and above with a different hidden size.
      a		:      int       =  'hf-internal-testing/test-two-configs'

      import transformers as new_transformers

      a		:      int       =  'v4.0.0'
      a		:      str       =  new_transformers.models.auto.AutoConfig.from_pretrained(
          UpperCAmelCase_		,    return_unused_kwargs=UpperCAmelCase_			)
      self.assertEqual(new_configuration.hidden_size		,    2			)
      # This checks `_configuration_file` ia not kept in the kwargs by mistake.
      self.assertDictEqual(UpperCAmelCase_		,    {}			)

      # Testing an older version by monkey-patching the version in the module it's used.
      import transformers as old_transformers

      a		:      Optional[Any]       =  'v3.0.0'
      a		:      List[Any]       =  old_transformers.models.auto.AutoConfig.from_pretrained(UpperCAmelCase_			)
      self.assertEqual(old_configuration.hidden_size		,    7_68			)










'''simple docstring'''






from typing import Dict, List, Optional, Union

import numpy as np

from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from .utils import PaddingStrategy, TensorType, is_tf_tensor, is_torch_tensor, logging, to_numpy


lowerCAmelCase:      int		 =		logging.get_logger(__name__)

class 							a__(      lowerCamelCase__			):







    def __init__(   self				:						Union[str, Any]		,    __snake_case				:						int		,    __snake_case				:						int		,    __snake_case				:						float		,    **__snake_case				:						Optional[int]			):
      a		:      Optional[Any]       =  feature_size
      a		:      Tuple       =  sampling_rate
      a		:      str       =  padding_value

      a		:      Any       =  kwargs.pop('padding_side'		,    'right'			)
      a		:      Tuple       =  kwargs.pop('return_attention_mask'		,    __snake_case			)

      super().__init__(**__snake_case			)







    def     lowercase_ (   self				:						Tuple		,    __snake_case				:						Union[
            BatchFeature,
            List[BatchFeature],
            Dict[str, BatchFeature],
            Dict[str, List[BatchFeature]],
            List[Dict[str, BatchFeature]],
        ]		,    __snake_case				:						Union[bool, str, PaddingStrategy] = True		,    __snake_case				:						Optional[int] = None		,    __snake_case				:						bool = False		,    __snake_case				:						Optional[int] = None		,    __snake_case				:						Optional[bool] = None		,    __snake_case				:						Optional[Union[str, TensorType]] = None		,    ):
      # If we have a list of dicts, let's convert it in a dict of lists
      # We do this to allow using this method as a collate_fn function in PyTorch Dataloader
      if isinstance(__snake_case		,    (list, tuple)			) and isinstance(processed_features[0]		,    (dict, BatchFeature)			):
        a		:      Union[str, Any]       =  {
            key: [example[key] for example in processed_features] for key in processed_features[0].keys()
        }

      # The model's main input name, usually `input_values`, has be passed for padding
      if self.model_input_names[0] not in processed_features:
        raise ValueError(
            'You should supply an instance of `transformers.BatchFeature` or list of `transformers.BatchFeature`'
            F""" to this method that includes {self.model_input_names[0]}, but you provided"""
            F""" {list(processed_features.keys()			)}"""			)

      a		:      Tuple       =  processed_features[self.model_input_names[0]]
      a		:      str       =  (
          return_attention_mask if return_attention_mask is not None else self.return_attention_mask
      )

      if len(__snake_case			) == 0:
        if return_attention_mask:
          a		:      Dict       =  []
        return processed_features

      # If we have PyTorch/TF tensors or lists as inputs, we cast them as Numpy arrays
      # and rebuild them afterwards if no return_tensors is specified
      # Note that we lose the specific device the tensor may be on for PyTorch

      a		:      Optional[int]       =  required_input[0]
      if isinstance(__snake_case		,    (list, tuple)			):
        # first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
        a		:      Dict       =  0
        while len(required_input[index]			) == 0:
          index += 1
        if index < len(__snake_case			):
          a		:      Tuple       =  required_input[index][0]

      if return_tensors is None:
        if is_tf_tensor(__snake_case			):
          a		:      Any       =  'tf'
        elif is_torch_tensor(__snake_case			):
          a		:      Optional[Any]       =  'pt'
        elif isinstance(__snake_case		,    (int, float, list, tuple, np.ndarray)			):
          a		:      List[Any]       =  'np'
        else:
          raise ValueError(
              F"""type of {first_element} unknown: {type(__snake_case			)}. """
              'Should be one of a python, numpy, pytorch or tensorflow object.'			)

      for key, value in processed_features.items():
        if isinstance(value[0]		,    (int, float)			):
          a		:      List[Any]       =  to_numpy(__snake_case			)
        else:
          a		:      Tuple       =  [to_numpy(__snake_case			) for v in value]

        # Convert padding_strategy in PaddingStrategy
      a		:      Any       =  self._get_padding_strategies(padding=__snake_case		,    max_length=__snake_case			)

      a		:      int       =  processed_features[self.model_input_names[0]]

      a		:      Dict       =  len(__snake_case			)
      if not all(len(__snake_case			) == batch_size for v in processed_features.values()			):
        raise ValueError('Some items in the output dictionary have a different batch size than others.'			)

      a		:      Optional[int]       =  []
      for i in range(__snake_case			):
        a		:      int       =  {k: v[i] for k, v in processed_features.items()}
        # truncation
        a		:      List[str]       =  self._truncate(
            __snake_case		,    max_length=__snake_case		,    pad_to_multiple_of=__snake_case		,    truncation=__snake_case		,    )
        truncated_inputs.append(__snake_case			)

      if padding_strategy == PaddingStrategy.LONGEST:
        # make sure that `max_length` cannot be longer than the longest truncated length
        a		:      List[str]       =  max(len(input_slice[self.model_input_names[0]]			) for input_slice in truncated_inputs			)
        a		:      Tuple       =  PaddingStrategy.MAX_LENGTH

      a		:      int       =  {}
      for i in range(__snake_case			):
        # padding
        a		:      int       =  self._pad(
            truncated_inputs[i]		,    max_length=__snake_case		,    padding_strategy=__snake_case		,    pad_to_multiple_of=__snake_case		,    return_attention_mask=__snake_case		,    )

        for key, value in outputs.items():
          if key not in batch_outputs:
            a		:      int       =  []
          if value.dtype is np.dtype(np.floataa			):
            a		:      List[str]       =  value.astype(np.floataa			)
          batch_outputs[key].append(__snake_case			)

      return BatchFeature(__snake_case		,    tensor_type=__snake_case			)







    def     lowercase_ (   self				:						List[str]		,    __snake_case				:						Union[Dict[str, np.ndarray], BatchFeature]		,    __snake_case				:						Optional[int] = None		,    __snake_case				:						PaddingStrategy = PaddingStrategy.DO_NOT_PAD		,    __snake_case				:						Optional[int] = None		,    __snake_case				:						Optional[bool] = None		,    ):
      a		:      List[str]       =  processed_features[self.model_input_names[0]]

      if padding_strategy == PaddingStrategy.LONGEST:
        a		:      Any       =  len(__snake_case			)

      if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
        a		:      Dict       =  ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

      a		:      List[Any]       =  padding_strategy != PaddingStrategy.DO_NOT_PAD and len(__snake_case			) < max_length

      if return_attention_mask and "attention_mask" not in processed_features:
        a		:      Union[str, Any]       =  np.ones(len(__snake_case			)		,    dtype=np.intaa			)

      if needs_to_be_padded:
        a		:      Tuple       =  max_length - len(__snake_case			)
        if self.padding_side == "right":
          if return_attention_mask:
            a		:      Any       =  np.pad(
                processed_features['attention_mask']		,    (0, difference)			)
          a		:      Optional[Any]       =  ((0, difference), (0, 0)) if self.feature_size > 1 else (0, difference)
          a		:      Optional[int]       =  np.pad(
              __snake_case		,    __snake_case		,    'constant'		,    constant_values=self.padding_value			)
        elif self.padding_side == "left":
          if return_attention_mask:
            a		:      Optional[int]       =  np.pad(
                processed_features['attention_mask']		,    (difference, 0)			)
          a		:      Optional[Any]       =  ((difference, 0), (0, 0)) if self.feature_size > 1 else (difference, 0)
          a		:      List[Any]       =  np.pad(
              __snake_case		,    __snake_case		,    'constant'		,    constant_values=self.padding_value			)
        else:
          raise ValueError('Invalid padding strategy:' + str(self.padding_side			)			)

      return processed_features







    def     lowercase_ (   self				:						List[str]		,    __snake_case				:						Union[Dict[str, np.ndarray], BatchFeature]		,    __snake_case				:						Optional[int] = None		,    __snake_case				:						Optional[int] = None		,    __snake_case				:						Optional[bool] = None		,    ):
      if not truncation:
        return processed_features
      elif truncation and max_length is None:
        raise ValueError('When setting ``truncation=True``, make sure that ``max_length`` is defined.'			)

      a		:      Dict       =  processed_features[self.model_input_names[0]]

      # find `max_length` that fits `pad_to_multiple_of`
      if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
        a		:      Optional[Any]       =  ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

      a		:      str       =  len(__snake_case			) > max_length

      if needs_to_be_truncated:
        a		:      str       =  processed_features[self.model_input_names[0]][:max_length]
        if "attention_mask" in processed_features:
          a		:      List[str]       =  processed_features['attention_mask'][:max_length]

      return processed_features







    def     lowercase_ (   self				:						Dict		,    __snake_case				:						Optional[Any]=False		,    __snake_case				:						Optional[int]=None			):

      # Get padding strategy
      if padding is not False:
        if padding is True:
          a		:      Union[str, Any]       =  PaddingStrategy.LONGEST  # Default to pad to the longest sequence in the batch
        elif not isinstance(__snake_case		,    __snake_case			):
          a		:      List[Any]       =  PaddingStrategy(__snake_case			)
        elif isinstance(__snake_case		,    __snake_case			):
          a		:      List[str]       =  padding
      else:
        a		:      Dict       =  PaddingStrategy.DO_NOT_PAD

      # Set max length if needed
      if max_length is None:
        if padding_strategy == PaddingStrategy.MAX_LENGTH:
          raise ValueError(
              F"""When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined"""			)

        # Test if we have a padding value
      if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None):
        raise ValueError(
            'Asking to pad but the feature_extractor does not have a padding value. Please select a value to use'
            ' as `padding_value`. For example: `feature_extractor.padding_value = 0.0`.'			)

      return padding_strategy