






"""simple docstring"""







import os
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen, xsplitext
from ..table import array_cast
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
	from .features import FeatureType


__a       , __a       , __a       :	Optional[Any]             = False, False, False







@dataclass
class        _SCREAMING_SNAKE_CASE						:





							"""simple docstring"""





							_SCREAMING_SNAKE_CASE    =None
							_SCREAMING_SNAKE_CASE    =True
							_SCREAMING_SNAKE_CASE    =True
							_SCREAMING_SNAKE_CASE    =None
							# Automatically constructed
							_SCREAMING_SNAKE_CASE    ="dict"
							_SCREAMING_SNAKE_CASE    =pa.struct({'bytes': pa.binary(), 'path': pa.string()}							)
							_SCREAMING_SNAKE_CASE    =field(default='Audio'       , init=__snake_case       , repr=__snake_case							)




							def __call__(   self:     List[str]    ):

												'''simple docstring'''

												return self.pa_type




							def  lowercase		(   self:     List[str]				,		__A:     Union[str, bytes, dict]    ):

												'''simple docstring'''

												try:
																	import soundfile as sf  # soundfile is a dependency of librosa, needed to decode audio files.
												except ImportError as err:
																	raise ImportError('''To support encoding audio data, please install \'soundfile\'.'''    ) from err
												if isinstance(__A				,		__A    ):
																	return {"bytes": None, "path": value}
												elif isinstance(__A				,		__A    ):
																	return {"bytes": value, "path": None}
												elif "array" in value:
																	# convert the audio array to wav bytes
																	a__          =  BytesIO()
																	sf.write(__A				,		value['''array''']				,		value['''sampling_rate''']				,		format='''wav'''    )
																	return {"bytes": buffer.getvalue(), "path": None}
												elif value.get('''path'''    ) is not None and os.path.isfile(value['''path''']    ):
																	# we set "bytes": None to not duplicate the data if they're already available locally
																	if value["path"].endswith('''pcm'''    ):
																						# "PCM" only has raw audio bytes
																						if value.get('''sampling_rate'''    ) is None:
																											# At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
																											raise KeyError('''To use PCM files, please specify a \'sampling_rate\' in Audio object'''    )
																						if value.get('''bytes'''    ):
																											# If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
																											a__          =  np.frombuffer(value['''bytes''']				,		dtype=np.intaa    ).astype(np.floataa    ) / 32767
																						else:
																											a__          =  np.memmap(value['''path''']				,		dtype='''h'''				,		mode='''r'''    ).astype(np.floataa    ) / 32767

																						a__          =  BytesIO(bytes()    )
																						sf.write(__A				,		__A				,		value['''sampling_rate''']				,		format='''wav'''    )
																						return {"bytes": buffer.getvalue(), "path": None}
																	else:
																						return {"bytes": None, "path": value.get('''path'''    )}
												elif value.get('''bytes'''    ) is not None or value.get('''path'''    ) is not None:
																	# store the audio bytes, and path is used to infer the audio format using the file extension
																	return {"bytes": value.get('''bytes'''    ), "path": value.get('''path'''    )}
												else:
																	raise ValueError(
																	    F'An audio sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.'    )




							def  lowercase		(   self:     str				,		__A:     dict				,		__A:     Optional[Dict[str, Union[str, bool, None]]] = None    ):

												'''simple docstring'''

												if not self.decode:
																	raise RuntimeError('''Decoding is disabled for this feature. Please use Audio(decode=True) instead.'''    )

												a__       ,a__          =  (value['''path'''], BytesIO(value['''bytes''']    )) if value['''bytes'''] is not None else (value['''path'''], None)
												if path is None and file is None:
																	raise ValueError(F'An audio sample should have one of \'path\' or \'bytes\' but both are None in {value}.'    )

												try:
																	import librosa
																	import soundfile as sf
												except ImportError as err:
																	raise ImportError('''To support decoding audio files, please install \'librosa\' and \'soundfile\'.'''    ) from err

												a__          =  xsplitext(__A    )[1][1:].lower() if path is not None else None
												if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
																	raise RuntimeError(
																	    '''Decoding \'opus\' files requires system library \'libsndfile\'>=1.0.31, '''
																	    '''You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. '''    )
												elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
																	raise RuntimeError(
																	    '''Decoding \'mp3\' files requires system library \'libsndfile\'>=1.1.0, '''
																	    '''You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. '''    )

												if file is None:
																	a__          =  token_per_repo_id or {}
																	a__          =  path.split('''::'''    )[-1]
																	try:
																						a__          =  string_to_dict(__A				,		config.HUB_DATASETS_URL    )['''repo_id''']
																						a__          =  token_per_repo_id[repo_id]
																	except (ValueError, KeyError):
																						a__          =  None

																	with xopen(__A				,		'''rb'''				,		use_auth_token=__A    ) as f:
																						a__       ,a__          =  sf.read(__A    )

												else:
																	a__       ,a__          =  sf.read(__A    )

												a__          =  array.T
												if self.mono:
																	a__          =  librosa.to_mono(__A    )
												if self.sampling_rate and self.sampling_rate != sampling_rate:
																	a__          =  librosa.resample(__A				,		orig_sr=__A				,		target_sr=self.sampling_rate    )
																	a__          =  self.sampling_rate

												return {"path": path, "array": array, "sampling_rate": sampling_rate}




							def  lowercase		(   self:     Any    ):

												'''simple docstring'''

												from .features import Value

												if self.decode:
																	raise ValueError('''Cannot flatten a decoded Audio feature.'''    )
												return {
												    "bytes": Value('''binary'''    ),
												    "path": Value('''string'''    ),
												}




							def  lowercase		(   self:     Dict				,		__A:     Union[pa.StringArray, pa.StructArray]    ):

												'''simple docstring'''

												if pa.types.is_string(storage.type    ):
																	a__          =  pa.array([None] * len(__A    )				,		type=pa.binary()    )
																	a__          =  pa.StructArray.from_arrays([bytes_array, storage]				,		['''bytes''', '''path''']				,		mask=storage.is_null()    )
												elif pa.types.is_binary(storage.type    ):
																	a__          =  pa.array([None] * len(__A    )				,		type=pa.string()    )
																	a__          =  pa.StructArray.from_arrays([storage, path_array]				,		['''bytes''', '''path''']				,		mask=storage.is_null()    )
												elif pa.types.is_struct(storage.type    ) and storage.type.get_all_field_indices('''array'''    ):
																	a__          =  pa.array([Audio().encode_example(__A    ) if x is not None else None for x in storage.to_pylist()]    )
												elif pa.types.is_struct(storage.type    ):
																	if storage.type.get_field_index('''bytes'''    ) >= 0:
																						a__          =  storage.field('''bytes'''    )
																	else:
																						a__          =  pa.array([None] * len(__A    )				,		type=pa.binary()    )
																	if storage.type.get_field_index('''path'''    ) >= 0:
																						a__          =  storage.field('''path'''    )
																	else:
																						a__          =  pa.array([None] * len(__A    )				,		type=pa.string()    )
																	a__          =  pa.StructArray.from_arrays([bytes_array, path_array]				,		['''bytes''', '''path''']				,		mask=storage.is_null()    )
												return array_cast(__A				,		self.pa_type    )





							def  lowercase		(   self:     List[Any]				,		__A:     pa.StructArray    ):

												'''simple docstring'''

												@no_op_if_value_is_null
												def path_to_bytes(__A:     str    ):
																	with xopen(__A				,		'''rb'''    ) as f:
																						a__          =  f.read()
																	return bytes_

												a__          =  pa.array(
												    [
												        (path_to_bytes(x['''path''']    ) if x['''bytes'''] is None else x['''bytes''']) if x is not None else None
												        for x in storage.to_pylist()
												    ]				,		type=pa.binary()				,		)
												a__          =  pa.array(
												    [os.path.basename(__A    ) if path is not None else None for path in storage.field('''path'''    ).to_pylist()]				,		type=pa.string()				,		)
												a__          =  pa.StructArray.from_arrays([bytes_array, path_array]				,		['''bytes''', '''path''']				,		mask=bytes_array.is_null()    )
												return array_cast(__A				,		self.pa_type    )










"""simple docstring"""







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__a       :	str             = logging.get_logger(__name__)

__a       :	Dict             = {
    'xlm-mlm-en-2048': 'https://huggingface.co/xlm-mlm-en-2048/resolve/main/config.json',
    'xlm-mlm-ende-1024': 'https://huggingface.co/xlm-mlm-ende-1024/resolve/main/config.json',
    'xlm-mlm-enfr-1024': 'https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/config.json',
    'xlm-mlm-enro-1024': 'https://huggingface.co/xlm-mlm-enro-1024/resolve/main/config.json',
    'xlm-mlm-tlm-xnli15-1024': 'https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/config.json',
    'xlm-mlm-xnli15-1024': 'https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/config.json',
    'xlm-clm-enfr-1024': 'https://huggingface.co/xlm-clm-enfr-1024/resolve/main/config.json',
    'xlm-clm-ende-1024': 'https://huggingface.co/xlm-clm-ende-1024/resolve/main/config.json',
    'xlm-mlm-17-1280': 'https://huggingface.co/xlm-mlm-17-1280/resolve/main/config.json',
    'xlm-mlm-100-1280': 'https://huggingface.co/xlm-mlm-100-1280/resolve/main/config.json',
}







class        _SCREAMING_SNAKE_CASE						(  __snake_case							):





							"""simple docstring"""





							_SCREAMING_SNAKE_CASE    ='xlm'
							_SCREAMING_SNAKE_CASE    ={
							    'hidden_size': 'emb_dim',
							    'num_attention_heads': 'n_heads',
							    'num_hidden_layers': 'n_layers',
							    'n_words': 'vocab_size',  # For backward compatibility
							}




							def __init__(   self:     Optional[Any]				,		__A:     str=30145				,		__A:     Union[str, Any]=2048				,		__A:     Union[str, Any]=12				,		__A:     List[Any]=16				,		__A:     List[Any]=0.1				,		__A:     int=0.1				,		__A:     str=True				,		__A:     int=False				,		__A:     Optional[int]=False				,		__A:     Union[str, Any]=False				,		__A:     List[Any]=1				,		__A:     Dict=True				,		__A:     List[Any]=512				,		__A:     Optional[int]=2048**-0.5				,		__A:     Tuple=1e-12				,		__A:     Optional[int]=0.0_2				,		__A:     Optional[int]=0				,		__A:     Union[str, Any]=1				,		__A:     List[Any]=2				,		__A:     Union[str, Any]=3				,		__A:     Tuple=5				,		__A:     Tuple=True				,		__A:     Dict="first"				,		__A:     Optional[int]=True				,		__A:     int=None				,		__A:     Optional[Any]=True				,		__A:     Dict=0.1				,		__A:     Tuple=5				,		__A:     Union[str, Any]=5				,		__A:     Tuple=0				,		__A:     List[str]=0				,		__A:     int=2				,		__A:     Optional[int]=0				,		**__A:     Union[str, Any]				,		):

												'''simple docstring'''

												a__          =  vocab_size
												a__          =  emb_dim
												a__          =  n_layers
												a__          =  n_heads
												a__          =  dropout
												a__          =  attention_dropout
												a__          =  gelu_activation
												a__          =  sinusoidal_embeddings
												a__          =  causal
												a__          =  asm
												a__          =  n_langs
												a__          =  use_lang_emb
												a__          =  layer_norm_eps
												a__          =  bos_index
												a__          =  eos_index
												a__          =  pad_index
												a__          =  unk_index
												a__          =  mask_index
												a__          =  is_encoder
												a__          =  max_position_embeddings
												a__          =  embed_init_std
												a__          =  init_std
												a__          =  summary_type
												a__          =  summary_use_proj
												a__          =  summary_activation
												a__          =  summary_proj_to_labels
												a__          =  summary_first_dropout
												a__          =  start_n_top
												a__          =  end_n_top
												a__          =  mask_token_id
												a__          =  lang_id

												if "n_words" in kwargs:
																	a__          =  kwargs['''n_words''']

												super().__init__(pad_token_id=__A				,		bos_token_id=__A				,		**__A    )







class        _SCREAMING_SNAKE_CASE						(  __snake_case							):





							"""simple docstring"""





							@property
							def  lowercase		(   self:     int    ):

												'''simple docstring'''

												if self.task == "multiple-choice":
																	a__          =  {0: '''batch''', 1: '''choice''', 2: '''sequence'''}
												else:
																	a__          =  {0: '''batch''', 1: '''sequence'''}
												return OrderedDict(
												    [
												        ('''input_ids''', dynamic_axis),
												        ('''attention_mask''', dynamic_axis),
												        ('''token_type_ids''', dynamic_axis),
												    ]    )


