


"""simple docstring"""


import os
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen, xsplitext
from ..table import array_cast
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
   from .features import FeatureType


__snake_case					,     __snake_case					,     __snake_case    					=					False, False, False

@dataclass
class 				_SCREAMING_SNAKE_CASE				:
   """simple docstring"""



   _a   :       Optional[int]									=    None
   _a   :       bool									=    True
   _a   :       bool									=    True
   _a   :       Optional[str]									=    None
   # Automatically constructed
   _a   :       ClassVar[str]									=    "dict"
   _a   :       ClassVar[Any]									=    pa.struct({'''bytes''': pa.binary(), '''path''': pa.string()}  )
   _a   :       str									=    field(default='''Audio'''     ,   init=__UpperCAmelCase     ,   repr=__UpperCAmelCase  )


   def __call__(						self						)      ->							Union[str, Any]:
    return self.pa_type


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__						)      ->							dict:
    try:
     import soundfile as sf  # soundfile is a dependency of librosa, needed to decode audio files.
    except ImportError as err:
     raise ImportError("""To support encoding audio data, please install 'soundfile'."""						) from err
    if isinstance(lowerCamelCase__ ,     lowerCamelCase__						):
     return {"bytes": None, "path": value}
    elif isinstance(lowerCamelCase__ ,     lowerCamelCase__						):
     return {"bytes": value, "path": None}
    elif "array" in value:
     # convert the audio array to wav bytes
     lowercase__       :       int				=						BytesIO()
     sf.write(lowerCamelCase__ ,     value["""array"""] ,     value["""sampling_rate"""] ,     format="""wav"""						)
     return {"bytes": buffer.getvalue(), "path": None}
    elif value.get("""path"""						) is not None and os.path.isfile(value["""path"""]						):
     # we set "bytes": None to not duplicate the data if they're already available locally
     if value["path"].endswith("""pcm"""						):
      # "PCM" only has raw audio bytes
      if value.get("""sampling_rate"""						) is None:
       # At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
       raise KeyError("""To use PCM files, please specify a 'sampling_rate' in Audio object"""						)
      if value.get("""bytes"""						):
       # If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
       lowercase__       :       Tuple				=						np.frombuffer(value["""bytes"""] ,     dtype=np.intaa						).astype(np.floataa						) / 3_2767
      else:
       lowercase__       :       Union[str, Any]				=						np.memmap(value["""path"""] ,     dtype="""h""" ,     mode="""r"""						).astype(np.floataa						) / 3_2767

      lowercase__       :       Union[str, Any]				=						BytesIO(bytes()						)
      sf.write(lowerCamelCase__ ,     lowerCamelCase__ ,     value["""sampling_rate"""] ,     format="""wav"""						)
      return {"bytes": buffer.getvalue(), "path": None}
     else:
      return {"bytes": None, "path": value.get("""path"""						)}
    elif value.get("""bytes"""						) is not None or value.get("""path"""						) is not None:
     # store the audio bytes, and path is used to infer the audio format using the file extension
     return {"bytes": value.get("""bytes"""						), "path": value.get("""path"""						)}
    else:
     raise ValueError(
         F'''An audio sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.'''						)


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__ ,     lowerCamelCase__ = None						)      ->							dict:
    if not self.decode:
     raise RuntimeError("""Decoding is disabled for this feature. Please use Audio(decode=True) instead."""						)

    lowercase__	,       lowercase__       :       Tuple				=						(value["""path"""], BytesIO(value["""bytes"""]						)) if value["""bytes"""] is not None else (value["""path"""], None)
    if path is None and file is None:
     raise ValueError(F'''An audio sample should have one of \'path\' or \'bytes\' but both are None in {value}.'''						)

    try:
     import librosa
     import soundfile as sf
    except ImportError as err:
     raise ImportError("""To support decoding audio files, please install 'librosa' and 'soundfile'."""						) from err

    lowercase__       :       Dict				=						xsplitext(lowerCamelCase__						)[1][1:].lower() if path is not None else None
    if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
     raise RuntimeError(
         """Decoding 'opus' files requires system library 'libsndfile'>=1.0.31, """
         """You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. """						)
    elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
     raise RuntimeError(
         """Decoding 'mp3' files requires system library 'libsndfile'>=1.1.0, """
         """You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. """						)

    if file is None:
     lowercase__       :       str				=						token_per_repo_id or {}
     lowercase__       :       str				=						path.split("""::"""						)[-1]
     try:
      lowercase__       :       Dict				=						string_to_dict(lowerCamelCase__ ,     config.HUB_DATASETS_URL						)["""repo_id"""]
      lowercase__       :       List[str]				=						token_per_repo_id[repo_id]
     except (ValueError, KeyError):
      lowercase__       :       Union[str, Any]				=						None

     with xopen(lowerCamelCase__ ,     """rb""" ,     use_auth_token=lowerCamelCase__						) as f:
      lowercase__	,       lowercase__       :       int				=						sf.read(lowerCamelCase__						)

    else:
     lowercase__	,       lowercase__       :       Union[str, Any]				=						sf.read(lowerCamelCase__						)

    lowercase__       :       str				=						array.T
    if self.mono:
     lowercase__       :       Optional[int]				=						librosa.to_mono(lowerCamelCase__						)
    if self.sampling_rate and self.sampling_rate != sampling_rate:
     lowercase__       :       List[Any]				=						librosa.resample(lowerCamelCase__ ,     orig_sr=lowerCamelCase__ ,     target_sr=self.sampling_rate						)
     lowercase__       :       str				=						self.sampling_rate

    return {"path": path, "array": array, "sampling_rate": sampling_rate}


   def 					UpperCAmelCase__(						self						)      ->							Union["FeatureType", Dict[str, "FeatureType"]]:
    from .features import Value

    if self.decode:
     raise ValueError("""Cannot flatten a decoded Audio feature."""						)
    return {
        "bytes": Value("""binary"""						),
        "path": Value("""string"""						),
    }


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__						)      ->							pa.StructArray:
    if pa.types.is_string(storage.type						):
     lowercase__       :       Tuple				=						pa.array([None] * len(lowerCamelCase__						) ,     type=pa.binary()						)
     lowercase__       :       Dict				=						pa.StructArray.from_arrays([bytes_array, storage] ,     ["""bytes""", """path"""] ,     mask=storage.is_null()						)
    elif pa.types.is_binary(storage.type						):
     lowercase__       :       Tuple				=						pa.array([None] * len(lowerCamelCase__						) ,     type=pa.string()						)
     lowercase__       :       Union[str, Any]				=						pa.StructArray.from_arrays([storage, path_array] ,     ["""bytes""", """path"""] ,     mask=storage.is_null()						)
    elif pa.types.is_struct(storage.type						) and storage.type.get_all_field_indices("""array"""						):
     lowercase__       :       List[Any]				=						pa.array([Audio().encode_example(lowerCamelCase__						) if x is not None else None for x in storage.to_pylist()]						)
    elif pa.types.is_struct(storage.type						):
     if storage.type.get_field_index("""bytes"""						) >= 0:
      lowercase__       :       List[str]				=						storage.field("""bytes"""						)
     else:
      lowercase__       :       Dict				=						pa.array([None] * len(lowerCamelCase__						) ,     type=pa.binary()						)
     if storage.type.get_field_index("""path"""						) >= 0:
      lowercase__       :       Union[str, Any]				=						storage.field("""path"""						)
     else:
      lowercase__       :       Union[str, Any]				=						pa.array([None] * len(lowerCamelCase__						) ,     type=pa.string()						)
     lowercase__       :       Dict				=						pa.StructArray.from_arrays([bytes_array, path_array] ,     ["""bytes""", """path"""] ,     mask=storage.is_null()						)
    return array_cast(lowerCamelCase__ ,     self.pa_type						)







   def 					UpperCAmelCase__(						self ,     lowerCamelCase__						)      ->							pa.StructArray:

    @no_op_if_value_is_null
    def path_to_bytes(lowerCamelCase__						):
     with xopen(lowerCamelCase__ ,     """rb"""						) as f:
      lowercase__       :       Optional[Any]				=						f.read()
     return bytes_

    lowercase__       :       Optional[Any]				=						pa.array(
        [
            (path_to_bytes(x["""path"""]						) if x["""bytes"""] is None else x["""bytes"""]) if x is not None else None
            for x in storage.to_pylist()
        ] ,     type=pa.binary() ,     )
    lowercase__       :       Any				=						pa.array(
        [os.path.basename(lowerCamelCase__						) if path is not None else None for path in storage.field("""path"""						).to_pylist()] ,     type=pa.string() ,     )
    lowercase__       :       str				=						pa.StructArray.from_arrays([bytes_array, path_array] ,     ["""bytes""", """path"""] ,     mask=bytes_array.is_null()						)
    return array_cast(lowerCamelCase__ ,     self.pa_type						)



"""simple docstring"""


import json
import os
import shutil
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoConfig, BertConfig, GPTaConfig
from transformers.configuration_utils import PretrainedConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402


__snake_case    					=					{
    'return_dict': False,
    'output_hidden_states': True,
    'output_attentions': True,
    'torchscript': True,
    'torch_dtype': 'float16',
    'use_bfloat16': True,
    'tf_legacy_loss': True,
    'pruned_heads': {'a': 1},
    'tie_word_embeddings': False,
    'is_decoder': True,
    'cross_attention_hidden_size': 128,
    'add_cross_attention': True,
    'tie_encoder_decoder': True,
    'max_length': 50,
    'min_length': 3,
    'do_sample': True,
    'early_stopping': True,
    'num_beams': 3,
    'num_beam_groups': 3,
    'diversity_penalty': 0.5,
    'temperature': 2.0,
    'top_k': 10,
    'top_p': 0.7,
    'typical_p': 0.2,
    'repetition_penalty': 0.8,
    'length_penalty': 0.8,
    'no_repeat_ngram_size': 5,
    'encoder_no_repeat_ngram_size': 5,
    'bad_words_ids': [1, 2, 3],
    'num_return_sequences': 3,
    'chunk_size_feed_forward': 5,
    'output_scores': True,
    'return_dict_in_generate': True,
    'forced_bos_token_id': 2,
    'forced_eos_token_id': 3,
    'remove_invalid_values': True,
    'architectures': ['BertModel'],
    'finetuning_task': 'translation',
    'id2label': {0: 'label'},
    'label2id': {'label': '0'},
    'tokenizer_class': 'BertTokenizerFast',
    'prefix': 'prefix',
    'bos_token_id': 6,
    'pad_token_id': 7,
    'eos_token_id': 8,
    'sep_token_id': 9,
    'decoder_start_token_id': 10,
    'exponential_decay_length_penalty': (5, 1.01),
    'suppress_tokens': [0, 1],
    'begin_suppress_tokens': 2,
    'task_specific_params': {'translation': 'some_params'},
    'problem_type': 'regression',
}

@is_staging_test
class 				_SCREAMING_SNAKE_CASE				(							unittest.TestCase  ):
   """simple docstring"""



   @classmethod
   def 					UpperCAmelCase__(						cls						)      ->							Dict:
    lowercase__       :       str				=						TOKEN
    HfFolder.save_token(lowerCamelCase__						)


   @classmethod
   def 					UpperCAmelCase__(						cls						)      ->							Optional[Any]:
    try:
     delete_repo(token=cls._token ,     repo_id="""test-config"""						)
    except HTTPError:
     pass

    try:
     delete_repo(token=cls._token ,     repo_id="""valid_org/test-config-org"""						)
    except HTTPError:
     pass

    try:
     delete_repo(token=cls._token ,     repo_id="""test-dynamic-config"""						)
    except HTTPError:
     pass


   def 					UpperCAmelCase__(						self						)      ->							List[str]:
    lowercase__       :       Union[str, Any]				=						BertConfig(
        vocab_size=99 ,     hidden_size=32 ,     num_hidden_layers=5 ,     num_attention_heads=4 ,     intermediate_size=37						)
    config.push_to_hub("""test-config""" ,     use_auth_token=self._token						)

    lowercase__       :       Union[str, Any]				=						BertConfig.from_pretrained(F'''{USER}/test-config'''						)
    for k, v in config.to_dict().items():
     if k != "transformers_version":
      self.assertEqual(lowerCamelCase__ ,     getattr(lowerCamelCase__ ,     lowerCamelCase__						)						)

        # Reset repo
    delete_repo(token=self._token ,     repo_id="""test-config"""						)

    # Push to hub via save_pretrained
    with tempfile.TemporaryDirectory() as tmp_dir:
     config.save_pretrained(lowerCamelCase__ ,     repo_id="""test-config""" ,     push_to_hub=lowerCamelCase__ ,     use_auth_token=self._token						)

    lowercase__       :       Optional[Any]				=						BertConfig.from_pretrained(F'''{USER}/test-config'''						)
    for k, v in config.to_dict().items():
     if k != "transformers_version":
      self.assertEqual(lowerCamelCase__ ,     getattr(lowerCamelCase__ ,     lowerCamelCase__						)						)


   def 					UpperCAmelCase__(						self						)      ->							Optional[Any]:
    lowercase__       :       int				=						BertConfig(
        vocab_size=99 ,     hidden_size=32 ,     num_hidden_layers=5 ,     num_attention_heads=4 ,     intermediate_size=37						)
    config.push_to_hub("""valid_org/test-config-org""" ,     use_auth_token=self._token						)

    lowercase__       :       List[str]				=						BertConfig.from_pretrained("""valid_org/test-config-org"""						)
    for k, v in config.to_dict().items():
     if k != "transformers_version":
      self.assertEqual(lowerCamelCase__ ,     getattr(lowerCamelCase__ ,     lowerCamelCase__						)						)

        # Reset repo
    delete_repo(token=self._token ,     repo_id="""valid_org/test-config-org"""						)

    # Push to hub via save_pretrained
    with tempfile.TemporaryDirectory() as tmp_dir:
     config.save_pretrained(
         lowerCamelCase__ ,     repo_id="""valid_org/test-config-org""" ,     push_to_hub=lowerCamelCase__ ,     use_auth_token=self._token						)

    lowercase__       :       Optional[Any]				=						BertConfig.from_pretrained("""valid_org/test-config-org"""						)
    for k, v in config.to_dict().items():
     if k != "transformers_version":
      self.assertEqual(lowerCamelCase__ ,     getattr(lowerCamelCase__ ,     lowerCamelCase__						)						)







   def 					UpperCAmelCase__(						self						)      ->							Dict:
    CustomConfig.register_for_auto_class()
    lowercase__       :       int				=						CustomConfig(attribute=42						)

    config.push_to_hub("""test-dynamic-config""" ,     use_auth_token=self._token						)

    # This has added the proper auto_map field to the config
    self.assertDictEqual(config.auto_map ,     {"""AutoConfig""": """custom_configuration.CustomConfig"""}						)

    lowercase__       :       Any				=						AutoConfig.from_pretrained(F'''{USER}/test-dynamic-config''' ,     trust_remote_code=lowerCamelCase__						)
    # Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module
    self.assertEqual(new_config.__class__.__name__ ,     """CustomConfig"""						)
    self.assertEqual(new_config.attribute ,     42						)

class 				_SCREAMING_SNAKE_CASE				(							unittest.TestCase  ):
   """simple docstring"""



   def 					UpperCAmelCase__(						self						)      ->							Tuple:
    lowercase__       :       str				=						GPTaConfig()

    # attempt to modify each of int/float/bool/str config records and verify they were updated
    lowercase__       :       List[str]				=						c.n_embd + 1  # int
    lowercase__       :       Union[str, Any]				=						c.resid_pdrop + 1.0  # float
    lowercase__       :       int				=						not c.scale_attn_weights  # bool
    lowercase__       :       Optional[int]				=						c.summary_type + """foo"""  # str
    c.update_from_string(
        F'''n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}'''						)
    self.assertEqual(lowerCamelCase__ ,     c.n_embd ,     """mismatch for key: n_embd"""						)
    self.assertEqual(lowerCamelCase__ ,     c.resid_pdrop ,     """mismatch for key: resid_pdrop"""						)
    self.assertEqual(lowerCamelCase__ ,     c.scale_attn_weights ,     """mismatch for key: scale_attn_weights"""						)
    self.assertEqual(lowerCamelCase__ ,     c.summary_type ,     """mismatch for key: summary_type"""						)


   def 					UpperCAmelCase__(						self						)      ->							Optional[int]:
    lowercase__       :       List[str]				=						PretrainedConfig()
    lowercase__       :       Union[str, Any]				=						[key for key in base_config.__dict__ if key not in config_common_kwargs]
    # If this part of the test fails, you have arguments to addin config_common_kwargs above.
    self.assertListEqual(
        lowerCamelCase__ ,     ["""is_encoder_decoder""", """_name_or_path""", """_commit_hash""", """transformers_version"""]						)
    lowercase__       :       List[Any]				=						[key for key, value in config_common_kwargs.items() if value == getattr(lowerCamelCase__ ,     lowerCamelCase__						)]
    if len(lowerCamelCase__						) > 0:
     raise ValueError(
         """The following keys are set with the default values in"""
         """ `test_configuration_common.config_common_kwargs` pick another value for them:"""
         F''' {', '.join(lowerCamelCase__						)}.'''						)


   def 					UpperCAmelCase__(						self						)      ->							Optional[Any]:
    with self.assertRaises(lowerCamelCase__						):
     # config is in subfolder, the following should not work without specifying the subfolder
     lowercase__       :       str				=						BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert-subfolder"""						)

    lowercase__       :       Union[str, Any]				=						BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert-subfolder""" ,     subfolder="""bert"""						)

    self.assertIsNotNone(lowerCamelCase__						)


   def 					UpperCAmelCase__(						self						)      ->							Union[str, Any]:
    # A mock response for an HTTP head request to emulate server down
    lowercase__       :       Dict				=						mock.Mock()
    lowercase__       :       str				=						500
    lowercase__       :       int				=						{}
    lowercase__       :       Optional[int]				=						HTTPError
    lowercase__       :       Dict				=						{}

    # Download this model to make sure it's in the cache.
    lowercase__       :       Any				=						BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert"""						)

    # Under the mock environment we get a 500 error when trying to reach the model.
    with mock.patch("""requests.Session.request""" ,     return_value=lowerCamelCase__						) as mock_head:
     lowercase__       :       int				=						BertConfig.from_pretrained("""hf-internal-testing/tiny-random-bert"""						)
     # This check we did call the fake head request
     mock_head.assert_called()


   def 					UpperCAmelCase__(						self						)      ->							int:
    # This test is for deprecated behavior and can be removed in v5
    lowercase__       :       Optional[Any]				=						BertConfig.from_pretrained(
        """https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json"""						)


   def 					UpperCAmelCase__(						self						)      ->							List[Any]:
    lowercase__       :       Optional[int]				=						AutoConfig.from_pretrained("""bert-base-cased"""						)
    lowercase__       :       Union[str, Any]				=						["""config.4.0.0.json"""]

    with tempfile.TemporaryDirectory() as tmp_dir:
     configuration.save_pretrained(lowerCamelCase__						)
     lowercase__       :       Optional[Any]				=						2
     json.dump(configuration.to_dict() ,     open(os.path.join(lowerCamelCase__ ,     """config.4.0.0.json"""						) ,     """w"""						)						)

     # This should pick the new configuration file as the version of Transformers is > 4.0.0
     lowercase__       :       Optional[int]				=						AutoConfig.from_pretrained(lowerCamelCase__						)
     self.assertEqual(new_configuration.hidden_size ,     2						)

     # Will need to be adjusted if we reach v42 and this test is still here.
     # Should pick the old configuration file as the version of Transformers is < 4.42.0
     lowercase__       :       Optional[int]				=						["""config.42.0.0.json"""]
     lowercase__       :       Optional[int]				=						768
     configuration.save_pretrained(lowerCamelCase__						)
     shutil.move(os.path.join(lowerCamelCase__ ,     """config.4.0.0.json"""						) ,     os.path.join(lowerCamelCase__ ,     """config.42.0.0.json"""						)						)
     lowercase__       :       Dict				=						AutoConfig.from_pretrained(lowerCamelCase__						)
     self.assertEqual(new_configuration.hidden_size ,     768						)







   def 					UpperCAmelCase__(						self						)      ->							List[Any]:
    # This repo has two configuration files, one for v4.0.0 and above with a different hidden size.
    lowercase__       :       Any				=						"""hf-internal-testing/test-two-configs"""

    import transformers as new_transformers

    lowercase__       :       Union[str, Any]				=						"""v4.0.0"""
    lowercase__	,       lowercase__       :       int				=						new_transformers.models.auto.AutoConfig.from_pretrained(
        lowerCamelCase__ ,     return_unused_kwargs=lowerCamelCase__						)
    self.assertEqual(new_configuration.hidden_size ,     2						)
    # This checks `_configuration_file` ia not kept in the kwargs by mistake.
    self.assertDictEqual(lowerCamelCase__ ,     {}						)

    # Testing an older version by monkey-patching the version in the module it's used.
    import transformers as old_transformers

    lowercase__       :       str				=						"""v3.0.0"""
    lowercase__       :       int				=						old_transformers.models.auto.AutoConfig.from_pretrained(lowerCamelCase__						)
    self.assertEqual(old_configuration.hidden_size ,     768						)