






import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
   __A					=	"pt"
elif is_tf_available():
   __A					=	"tf"
else:
   __A					=	"jax"



class 				_A			(     UpperCamelCase					,					unittest.TestCase				):
    """simple docstring"""

    lowerCamelCase	:							List[str]				= ByTaTokenizer
    lowerCamelCase	:							Union[str, Any]				= False



    def _a							(				self      :     Dict      )      ->							List[str]:
           super().setUp()
           __UpperCAmelCase     				=ByTaTokenizer()
           tokenizer.save_pretrained(self.tmpdirname      )



    @cached_property
    def _a							(				self      :     int      )      ->							Optional[int]:
           return ByTaTokenizer.from_pretrained("""google/byt5-small"""      )



    def _a							(				self      :     Union[str, Any] ,     **__SCREAMING_SNAKE_CASE      :     Optional[Any]      )      ->							ByTaTokenizer:
           return self.tokenizer_class.from_pretrained(self.tmpdirname ,     **__SCREAMING_SNAKE_CASE      )



    def _a							(				self      :     Dict ,     __SCREAMING_SNAKE_CASE      :     Tuple ,     __SCREAMING_SNAKE_CASE      :     Union[str, Any]=False ,     __SCREAMING_SNAKE_CASE      :     Optional[int]=20 ,     __SCREAMING_SNAKE_CASE      :     Union[str, Any]=5      )      ->							Tuple[str, list]:
           # XXX The default common tokenizer tests assume that every ID is decodable on its own.
           # This assumption is invalid for ByT5 because single bytes might not be
           # valid utf-8 (byte 128 for instance).
           # Here we're overriding the smallest possible method to provide
           # a clean sequence without making the same assumption.

           __UpperCAmelCase     				=[]
           for i in range(len(__SCREAMING_SNAKE_CASE      )      ):
                  try:
                         __UpperCAmelCase     				=tokenizer.decode([i] ,     clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE      )
                  except UnicodeDecodeError:
                         pass
                  toks.append((i, tok)      )

           __UpperCAmelCase     				=list(filter(lambda __SCREAMING_SNAKE_CASE      : re.match(R"""^[ a-zA-Z]+$""" ,     t[1]      ) ,     __SCREAMING_SNAKE_CASE      )      )
           __UpperCAmelCase     				=list(filter(lambda __SCREAMING_SNAKE_CASE      : [t[0]] == tokenizer.encode(t[1] ,     add_special_tokens=__SCREAMING_SNAKE_CASE      ) ,     __SCREAMING_SNAKE_CASE      )      )
           if max_length is not None and len(__SCREAMING_SNAKE_CASE      ) > max_length:
                  __UpperCAmelCase     				=toks[:max_length]
           if min_length is not None and len(__SCREAMING_SNAKE_CASE      ) < min_length and len(__SCREAMING_SNAKE_CASE      ) > 0:
                  while len(__SCREAMING_SNAKE_CASE      ) < min_length:
                         __UpperCAmelCase     				=toks + toks
        # toks_str = [t[1] for t in toks]
           __UpperCAmelCase     				=[t[0] for t in toks]

           # Ensure consistency
           __UpperCAmelCase     				=tokenizer.decode(__SCREAMING_SNAKE_CASE ,     clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE      )
           if " " not in output_txt and len(__SCREAMING_SNAKE_CASE      ) > 1:
                  __UpperCAmelCase     				=(
                      tokenizer.decode([toks_ids[0]] ,     clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE      )
                      + """ """
                      + tokenizer.decode(toks_ids[1:] ,     clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE      )
                  )
           if with_prefix_space:
                  __UpperCAmelCase     				=""" """ + output_txt
           __UpperCAmelCase     				=tokenizer.encode(__SCREAMING_SNAKE_CASE ,     add_special_tokens=__SCREAMING_SNAKE_CASE      )
           return output_txt, output_ids



    def _a							(				self      :     str      )      ->							Optional[int]:
           __UpperCAmelCase     				=self.ta_base_tokenizer
           __UpperCAmelCase     				=tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]      )
           __UpperCAmelCase     				=tokenizer(["""hi""", """I went to the gym""", """"""]      )
           self.assertListEqual(batch_with_eos_added["""input_ids"""] ,     batch_without_eos_added["""input_ids"""]      )



    def _a							(				self      :     Union[str, Any]      )      ->							str:
           __UpperCAmelCase     				=self.ta_base_tokenizer
           __UpperCAmelCase     				="""Unicode €."""
           __UpperCAmelCase     				=tokenizer(__SCREAMING_SNAKE_CASE      )
           __UpperCAmelCase     				=[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
           self.assertEqual(encoded["""input_ids"""] ,     __SCREAMING_SNAKE_CASE      )

           # decoding
           __UpperCAmelCase     				=tokenizer.decode(__SCREAMING_SNAKE_CASE      )
           self.assertEqual(__SCREAMING_SNAKE_CASE ,     """Unicode €.</s>"""      )

           __UpperCAmelCase     				=tokenizer("""e è é ê ë"""      )
           __UpperCAmelCase     				=[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
           self.assertEqual(encoded["""input_ids"""] ,     __SCREAMING_SNAKE_CASE      )
           # decoding
           __UpperCAmelCase     				=tokenizer.decode(__SCREAMING_SNAKE_CASE      )
           self.assertEqual(__SCREAMING_SNAKE_CASE ,     """e è é ê ë</s>"""      )

           # encode/decode, but with `encode` instead of `__call__`
           self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""      )      ) ,     """e è é ê ë</s>"""      )



    def _a							(				self      :     Any      )      ->							Union[str, Any]:
           __UpperCAmelCase     				=self.ta_base_tokenizer
           __UpperCAmelCase     				=["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
           # fmt: off
           __UpperCAmelCase     				=[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
           # fmt: on
           __UpperCAmelCase     				=tokenizer(__SCREAMING_SNAKE_CASE ,     padding=__SCREAMING_SNAKE_CASE ,     return_tensors=__SCREAMING_SNAKE_CASE      )
           self.assertIsInstance(__SCREAMING_SNAKE_CASE ,     __SCREAMING_SNAKE_CASE      )

           if FRAMEWORK != "jax":
                  __UpperCAmelCase     				=list(batch.input_ids.numpy()[0]      )
           else:
                  __UpperCAmelCase     				=list(batch.input_ids.tolist()[0]      )

           self.assertListEqual(__SCREAMING_SNAKE_CASE ,     __SCREAMING_SNAKE_CASE      )

           self.assertEqual((2, 37) ,     batch.input_ids.shape      )
           self.assertEqual((2, 37) ,     batch.attention_mask.shape      )



    def _a							(				self      :     int      )      ->							List[Any]:
           __UpperCAmelCase     				=self.ta_base_tokenizer
           __UpperCAmelCase     				=["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
           __UpperCAmelCase     				=tokenizer(__SCREAMING_SNAKE_CASE ,     padding=__SCREAMING_SNAKE_CASE ,     return_tensors=__SCREAMING_SNAKE_CASE      )
           # check if input_ids are returned and no decoder_input_ids
           self.assertIn("""input_ids""" ,     __SCREAMING_SNAKE_CASE      )
           self.assertIn("""attention_mask""" ,     __SCREAMING_SNAKE_CASE      )
           self.assertNotIn("""decoder_input_ids""" ,     __SCREAMING_SNAKE_CASE      )
           self.assertNotIn("""decoder_attention_mask""" ,     __SCREAMING_SNAKE_CASE      )



    def _a							(				self      :     Optional[Any]      )      ->							Optional[Any]:
           __UpperCAmelCase     				=self.ta_base_tokenizer
           __UpperCAmelCase     				=[
               """Summary of the text.""",
               """Another summary.""",
           ]
           __UpperCAmelCase     				=tokenizer(
               text_target=__SCREAMING_SNAKE_CASE ,     max_length=32 ,     padding="""max_length""" ,     truncation=__SCREAMING_SNAKE_CASE ,     return_tensors=__SCREAMING_SNAKE_CASE      )
           self.assertEqual(32 ,     targets["""input_ids"""].shape[1]      )



    def _a							(				self      :     int      )      ->							Tuple:
           __UpperCAmelCase     				=self.ta_base_tokenizer
           __UpperCAmelCase     				=["""A long paragraph for summarization. </s>"""]
           __UpperCAmelCase     				=["""Summary of the text. </s>"""]
           # fmt: off
           __UpperCAmelCase     				=[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
           __UpperCAmelCase     				=[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
           # fmt: on

           __UpperCAmelCase     				=tokenizer(__SCREAMING_SNAKE_CASE ,     text_target=__SCREAMING_SNAKE_CASE      )

           self.assertEqual(__SCREAMING_SNAKE_CASE ,     batch["""input_ids"""][0]      )
           self.assertEqual(__SCREAMING_SNAKE_CASE ,     batch["""labels"""][0]      )



    def _a							(				self      :     Union[str, Any]      )      ->							List[str]:
           # safety check on max_len default value so we are sure the test works
           __UpperCAmelCase     				=self.get_tokenizers()
           for tokenizer in tokenizers:
                  with self.subTest(f'''{tokenizer.__class__.__name__}'''      ):
                         self.assertNotEqual(tokenizer.model_max_length ,     42      )

        # Now let's start the test
           __UpperCAmelCase     				=self.get_tokenizers()
           for tokenizer in tokenizers:
                  with self.subTest(f'''{tokenizer.__class__.__name__}'''      ):
                         # Isolate this from the other tests because we save additional tokens/etc
                         __UpperCAmelCase     				=tempfile.mkdtemp()

                         __UpperCAmelCase     				=""" He is very happy, UNwant\u00E9d,running"""
                         __UpperCAmelCase     				=tokenizer.encode(__SCREAMING_SNAKE_CASE ,     add_special_tokens=__SCREAMING_SNAKE_CASE      )
                         tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE      )

                         __UpperCAmelCase     				=tokenizer.__class__.from_pretrained(__SCREAMING_SNAKE_CASE      )
                         __UpperCAmelCase     				=after_tokenizer.encode(__SCREAMING_SNAKE_CASE ,     add_special_tokens=__SCREAMING_SNAKE_CASE      )
                         self.assertListEqual(__SCREAMING_SNAKE_CASE ,     __SCREAMING_SNAKE_CASE      )

                         shutil.rmtree(__SCREAMING_SNAKE_CASE      )

           __UpperCAmelCase     				=self.get_tokenizers(model_max_length=42      )
           for tokenizer in tokenizers:
                  with self.subTest(f'''{tokenizer.__class__.__name__}'''      ):
                         # Isolate this from the other tests because we save additional tokens/etc
                         __UpperCAmelCase     				=tempfile.mkdtemp()

                         __UpperCAmelCase     				=""" He is very happy, UNwant\u00E9d,running"""
                         tokenizer.add_tokens(["""bim""", """bambam"""]      )
                         __UpperCAmelCase     				=tokenizer.additional_special_tokens
                         additional_special_tokens.append("""new_additional_special_token"""      )
                         tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}      )
                         __UpperCAmelCase     				=tokenizer.encode(__SCREAMING_SNAKE_CASE ,     add_special_tokens=__SCREAMING_SNAKE_CASE      )
                         tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE      )

                         __UpperCAmelCase     				=tokenizer.__class__.from_pretrained(__SCREAMING_SNAKE_CASE      )
                         __UpperCAmelCase     				=after_tokenizer.encode(__SCREAMING_SNAKE_CASE ,     add_special_tokens=__SCREAMING_SNAKE_CASE      )
                         self.assertListEqual(__SCREAMING_SNAKE_CASE ,     __SCREAMING_SNAKE_CASE      )
                         self.assertIn("""new_additional_special_token""" ,     after_tokenizer.additional_special_tokens      )
                         self.assertEqual(after_tokenizer.model_max_length ,     42      )

                         __UpperCAmelCase     				=tokenizer.__class__.from_pretrained(__SCREAMING_SNAKE_CASE ,     model_max_length=43      )
                         self.assertEqual(tokenizer.model_max_length ,     43      )

                         shutil.rmtree(__SCREAMING_SNAKE_CASE      )



    def _a							(				self      :     int      )      ->							Any:
           __UpperCAmelCase     				=[]
           if self.test_slow_tokenizer:
                  tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())      )

           if self.test_rust_tokenizer:
                  tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())      )

           for tokenizer_class, tokenizer_utils in tokenizer_list:
                  with tempfile.TemporaryDirectory() as tmp_dir:
                         tokenizer_utils.save_pretrained(__SCREAMING_SNAKE_CASE      )

                         with open(os.path.join(__SCREAMING_SNAKE_CASE ,     """special_tokens_map.json"""      ) ,     encoding="""utf-8"""      ) as json_file:
                                __UpperCAmelCase     				=json.load(__SCREAMING_SNAKE_CASE      )

                         with open(os.path.join(__SCREAMING_SNAKE_CASE ,     """tokenizer_config.json"""      ) ,     encoding="""utf-8"""      ) as json_file:
                                __UpperCAmelCase     				=json.load(__SCREAMING_SNAKE_CASE      )

                         __UpperCAmelCase     				=[f'''<extra_id_{i}>''' for i in range(125      )]

                         __UpperCAmelCase     				=added_tokens_extra_ids + [
                             """an_additional_special_token"""
                         ]
                         __UpperCAmelCase     				=added_tokens_extra_ids + [
                             """an_additional_special_token"""
                         ]

                         with open(os.path.join(__SCREAMING_SNAKE_CASE ,     """special_tokens_map.json"""      ) ,     """w""" ,     encoding="""utf-8"""      ) as outfile:
                                json.dump(__SCREAMING_SNAKE_CASE ,     __SCREAMING_SNAKE_CASE      )
                         with open(os.path.join(__SCREAMING_SNAKE_CASE ,     """tokenizer_config.json"""      ) ,     """w""" ,     encoding="""utf-8"""      ) as outfile:
                                json.dump(__SCREAMING_SNAKE_CASE ,     __SCREAMING_SNAKE_CASE      )

                         # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                         # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                         # "special_tokens_map.json" files
                         __UpperCAmelCase     				=tokenizer_class.from_pretrained(
                             __SCREAMING_SNAKE_CASE ,     )
                         self.assertIn(
                             """an_additional_special_token""" ,     tokenizer_without_change_in_init.additional_special_tokens      )
                         # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                         self.assertEqual(
                             ["""an_additional_special_token"""] ,     tokenizer_without_change_in_init.convert_ids_to_tokens(
                                 tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]      )      ) ,     )

                         # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                         __UpperCAmelCase     				=added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token""" ,     lstrip=__SCREAMING_SNAKE_CASE      )]
                         __UpperCAmelCase     				=tokenizer_class.from_pretrained(
                             __SCREAMING_SNAKE_CASE ,     additional_special_tokens=__SCREAMING_SNAKE_CASE ,     )

                         self.assertIn("""a_new_additional_special_token""" ,     tokenizer.additional_special_tokens      )
                         self.assertEqual(
                             ["""a_new_additional_special_token"""] ,     tokenizer.convert_ids_to_tokens(
                                 tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]      )      ) ,     )



    def _a							(				self      :     str      )      ->							List[Any]:
           __UpperCAmelCase     				=[]
           if self.test_slow_tokenizer:
                  tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())      )

           if self.test_rust_tokenizer:
                  tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())      )

           for tokenizer_class, tokenizer_utils in tokenizer_list:
                  with tempfile.TemporaryDirectory() as tmp_dir:
                         tokenizer_utils.save_pretrained(__SCREAMING_SNAKE_CASE      )

                         __UpperCAmelCase     				=tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE      )

                         self.assertTrue(tokenizer.decode([255]      ) == """"""      )



    def _a							(				self      :     Union[str, Any]      )      ->							Optional[Any]:
           pass



    def _a							(				self      :     Optional[Any]      )      ->							List[Any]:
           pass



    def _a							(				self      :     Optional[Any]      )      ->							int:
           pass



    def _a							(				self      :     List[Any]      )      ->							int:
           pass



    def _a							(				self      :     Any      )      ->							Optional[int]:
           # The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
           # and special added tokens as tokens
           __UpperCAmelCase     				=self.get_tokenizers(fast=__SCREAMING_SNAKE_CASE ,     do_lower_case=__SCREAMING_SNAKE_CASE      )
           for tokenizer in tokenizers:
                  with self.subTest(f'''{tokenizer.__class__.__name__}'''      ):
                         __UpperCAmelCase     				=["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
                         __UpperCAmelCase     				=tokenizer.convert_tokens_to_string(__SCREAMING_SNAKE_CASE      )

                         self.assertIsInstance(__SCREAMING_SNAKE_CASE ,     __SCREAMING_SNAKE_CASE      )



    def _a							(				self      :     List[Any]      )      ->							Any:
           __UpperCAmelCase     				=self.get_tokenizers()
           for tokenizer in tokenizers:
                  with self.subTest(f'''{tokenizer.__class__.__name__}'''      ):
                         __UpperCAmelCase     				=[
                             """bos_token""",
                             """eos_token""",
                             """unk_token""",
                             """sep_token""",
                             """pad_token""",
                             """cls_token""",
                             """mask_token""",
                         ]

                         __UpperCAmelCase     				=0
                         __UpperCAmelCase     				=tokenizer.convert_ids_to_tokens(
                             __SCREAMING_SNAKE_CASE ,     skip_special_tokens=__SCREAMING_SNAKE_CASE      )

                         for attr in attributes_list:
                                setattr(__SCREAMING_SNAKE_CASE ,     attr + """_id""" ,     __SCREAMING_SNAKE_CASE      )
                                self.assertEqual(getattr(__SCREAMING_SNAKE_CASE ,     __SCREAMING_SNAKE_CASE      ) ,     __SCREAMING_SNAKE_CASE      )
                                self.assertEqual(getattr(__SCREAMING_SNAKE_CASE ,     attr + """_id"""      ) ,     __SCREAMING_SNAKE_CASE      )

                                setattr(__SCREAMING_SNAKE_CASE ,     attr + """_id""" ,     __SCREAMING_SNAKE_CASE      )
                                self.assertEqual(getattr(__SCREAMING_SNAKE_CASE ,     __SCREAMING_SNAKE_CASE      ) ,     __SCREAMING_SNAKE_CASE      )
                                self.assertEqual(getattr(__SCREAMING_SNAKE_CASE ,     attr + """_id"""      ) ,     __SCREAMING_SNAKE_CASE      )

                         setattr(__SCREAMING_SNAKE_CASE ,     """additional_special_tokens_ids""" ,     []      )
                         self.assertListEqual(getattr(__SCREAMING_SNAKE_CASE ,     """additional_special_tokens"""      ) ,     []      )
                         self.assertListEqual(getattr(__SCREAMING_SNAKE_CASE ,     """additional_special_tokens_ids"""      ) ,     []      )

                         setattr(__SCREAMING_SNAKE_CASE ,     """additional_special_tokens_ids""" ,     [token_id_to_test_setters]      )
                         self.assertListEqual(getattr(__SCREAMING_SNAKE_CASE ,     """additional_special_tokens"""      ) ,     [token_to_test_setters]      )
                         self.assertListEqual(getattr(__SCREAMING_SNAKE_CASE ,     """additional_special_tokens_ids"""      ) ,     [token_id_to_test_setters]      )






import os
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen, xsplitext
from ..table import array_cast
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
							from .features import FeatureType


lowerCamelCase_ ,							lowerCamelCase_ ,							lowerCamelCase_    :			Dict    =	False, False, False

@dataclass
class 							_UpperCamelCase					:






			'''simple docstring'''

			__UpperCamelCase   :    Optional[int]          =			None
			__UpperCamelCase   :    bool          =			True
			__UpperCamelCase   :    bool          =			True
			__UpperCamelCase   :    Optional[str]          =			None
			# Automatically constructed
			__UpperCamelCase   :    ClassVar[str]          =			"dict"
			__UpperCamelCase   :    ClassVar[Any]          =			pa.struct({"""bytes""": pa.binary(), """path""": pa.string()}	)
			__UpperCamelCase   :    str          =			field(default="""Audio"""  ,							init=_A  ,							repr=_A	)

			def __call__(							self			:							List[Any]   ):
							return self.pa_type

			def    lowerCAmelCase__						(							self			:							Optional[int]    ,						snake_case_			:							Union[str, bytes, dict]   ):
							try:
											import soundfile as sf  # soundfile is a dependency of librosa, needed to decode audio files.
							except ImportError as err:
											raise ImportError("""To support encoding audio data, please install 'soundfile'."""   ) from err
							if isinstance(snake_case_    ,						snake_case_   ):
											return {"bytes": None, "path": value}
							elif isinstance(snake_case_    ,						snake_case_   ):
											return {"bytes": value, "path": None}
							elif "array" in value:
											# convert the audio array to wav bytes
											UpperCamelCase_:				List[Any]						=    BytesIO()
											sf.write(snake_case_    ,						value["""array"""]    ,						value["""sampling_rate"""]    ,						format="""wav"""   )
											return {"bytes": buffer.getvalue(), "path": None}
							elif value.get("""path"""   ) is not None and os.path.isfile(value["""path"""]   ):
											# we set "bytes": None to not duplicate the data if they're already available locally
											if value["path"].endswith("""pcm"""   ):
															# "PCM" only has raw audio bytes
															if value.get("""sampling_rate"""   ) is None:
																			# At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
																			raise KeyError("""To use PCM files, please specify a 'sampling_rate' in Audio object"""   )
															if value.get("""bytes"""   ):
																			# If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
																			UpperCamelCase_:				str						=    np.frombuffer(value["""bytes"""]    ,						dtype=np.intaa   ).astype(np.floataa   ) / 3_2767
															else:
																			UpperCamelCase_:				Dict						=    np.memmap(value["""path"""]    ,						dtype="""h"""    ,						mode="""r"""   ).astype(np.floataa   ) / 3_2767

															UpperCamelCase_:				Dict						=    BytesIO(bytes()   )
															sf.write(snake_case_    ,						snake_case_    ,						value["""sampling_rate"""]    ,						format="""wav"""   )
															return {"bytes": buffer.getvalue(), "path": None}
											else:
															return {"bytes": None, "path": value.get("""path"""   )}
							elif value.get("""bytes"""   ) is not None or value.get("""path"""   ) is not None:
											# store the audio bytes, and path is used to infer the audio format using the file extension
											return {"bytes": value.get("""bytes"""   ), "path": value.get("""path"""   )}
							else:
											raise ValueError(
											    f'''An audio sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.'''   )

			def    lowerCAmelCase__						(							self			:							List[str]    ,						snake_case_			:							dict    ,						snake_case_			:							Optional[Dict[str, Union[str, bool, None]]] = None   ):
							if not self.decode:
											raise RuntimeError("""Decoding is disabled for this feature. Please use Audio(decode=True) instead."""   )

							UpperCamelCase_, UpperCamelCase_:				Any						=    (value["""path"""], BytesIO(value["""bytes"""]   )) if value["""bytes"""] is not None else (value["""path"""], None)
							if path is None and file is None:
											raise ValueError(f'''An audio sample should have one of \'path\' or \'bytes\' but both are None in {value}.'''   )

							try:
											import librosa
											import soundfile as sf
							except ImportError as err:
											raise ImportError("""To support decoding audio files, please install 'librosa' and 'soundfile'."""   ) from err

							UpperCamelCase_:				Optional[Any]						=    xsplitext(snake_case_   )[1][1:].lower() if path is not None else None
							if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
											raise RuntimeError(
											    """Decoding 'opus' files requires system library 'libsndfile'>=1.0.31, """
											    """You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. """   )
							elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
											raise RuntimeError(
											    """Decoding 'mp3' files requires system library 'libsndfile'>=1.1.0, """
											    """You can try to update `soundfile` python library: `pip install \"soundfile>=0.12.1\"`. """   )

							if file is None:
											UpperCamelCase_:				Dict						=    token_per_repo_id or {}
											UpperCamelCase_:				str						=    path.split("""::"""   )[-1]
											try:
															UpperCamelCase_:				Any						=    string_to_dict(snake_case_    ,						config.HUB_DATASETS_URL   )["""repo_id"""]
															UpperCamelCase_:				List[str]						=    token_per_repo_id[repo_id]
											except (ValueError, KeyError):
															UpperCamelCase_:				Any						=    None

											with xopen(snake_case_    ,						"""rb"""    ,						use_auth_token=snake_case_   ) as f:
															UpperCamelCase_, UpperCamelCase_:				List[Any]						=    sf.read(snake_case_   )

							else:
											UpperCamelCase_, UpperCamelCase_:				Union[str, Any]						=    sf.read(snake_case_   )

							UpperCamelCase_:				List[str]						=    array.T
							if self.mono:
											UpperCamelCase_:				int						=    librosa.to_mono(snake_case_   )
							if self.sampling_rate and self.sampling_rate != sampling_rate:
											UpperCamelCase_:				Union[str, Any]						=    librosa.resample(snake_case_    ,						orig_sr=snake_case_    ,						target_sr=self.sampling_rate   )
											UpperCamelCase_:				Dict						=    self.sampling_rate

							return {"path": path, "array": array, "sampling_rate": sampling_rate}

			def    lowerCAmelCase__						(							self			:							Tuple   ):
							from .features import Value

							if self.decode:
											raise ValueError("""Cannot flatten a decoded Audio feature."""   )
							return {
							    "bytes": Value("""binary"""   ),
							    "path": Value("""string"""   ),
							}

			def    lowerCAmelCase__						(							self			:							Any    ,						snake_case_			:							Union[pa.StringArray, pa.StructArray]   ):
							if pa.types.is_string(storage.type   ):
											UpperCamelCase_:				Optional[Any]						=    pa.array([None] * len(snake_case_   )    ,						type=pa.binary()   )
											UpperCamelCase_:				List[str]						=    pa.StructArray.from_arrays([bytes_array, storage]    ,						["""bytes""", """path"""]    ,						mask=storage.is_null()   )
							elif pa.types.is_binary(storage.type   ):
											UpperCamelCase_:				str						=    pa.array([None] * len(snake_case_   )    ,						type=pa.string()   )
											UpperCamelCase_:				List[Any]						=    pa.StructArray.from_arrays([storage, path_array]    ,						["""bytes""", """path"""]    ,						mask=storage.is_null()   )
							elif pa.types.is_struct(storage.type   ) and storage.type.get_all_field_indices("""array"""   ):
											UpperCamelCase_:				Union[str, Any]						=    pa.array([Audio().encode_example(snake_case_   ) if x is not None else None for x in storage.to_pylist()]   )
							elif pa.types.is_struct(storage.type   ):
											if storage.type.get_field_index("""bytes"""   ) >= 0:
															UpperCamelCase_:				Optional[Any]						=    storage.field("""bytes"""   )
											else:
															UpperCamelCase_:				str						=    pa.array([None] * len(snake_case_   )    ,						type=pa.binary()   )
											if storage.type.get_field_index("""path"""   ) >= 0:
															UpperCamelCase_:				List[str]						=    storage.field("""path"""   )
											else:
															UpperCamelCase_:				int						=    pa.array([None] * len(snake_case_   )    ,						type=pa.string()   )
											UpperCamelCase_:				Dict						=    pa.StructArray.from_arrays([bytes_array, path_array]    ,						["""bytes""", """path"""]    ,						mask=storage.is_null()   )
							return array_cast(snake_case_    ,						self.pa_type   )




			def    lowerCAmelCase__						(							self			:							Optional[Any]    ,						snake_case_			:							pa.StructArray   ):

							@no_op_if_value_is_null
							def path_to_bytes(snake_case_			:							Optional[int]   ):
											with xopen(snake_case_    ,						"""rb"""   ) as f:
															UpperCamelCase_:				Optional[int]						=    f.read()
											return bytes_

							UpperCamelCase_:				int						=    pa.array(
							    [
							        (path_to_bytes(x["""path"""]   ) if x["""bytes"""] is None else x["""bytes"""]) if x is not None else None
							        for x in storage.to_pylist()
							    ]    ,						type=pa.binary()    ,						)
							UpperCamelCase_:				Union[str, Any]						=    pa.array(
							    [os.path.basename(snake_case_   ) if path is not None else None for path in storage.field("""path"""   ).to_pylist()]    ,						type=pa.string()    ,						)
							UpperCamelCase_:				Optional[int]						=    pa.StructArray.from_arrays([bytes_array, path_array]    ,						["""bytes""", """path"""]    ,						mask=bytes_array.is_null()   )
							return array_cast(snake_case_    ,						self.pa_type   )






