
import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Tuple       )  ->    str:
							lowerCamelCase_							  =					np.inf

							def set_batch_size(UpperCAmelCase__				:      List[str]       ) -> None:
														nonlocal batch_size
														if isinstance(_lowercase			,				_lowercase       ):
																					lowerCamelCase_							  =					min(_lowercase			,				config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS       )
														elif isinstance(_lowercase			,				_lowercase       ):
																					lowerCamelCase_							  =					min(_lowercase			,				config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS       )
														elif isinstance(_lowercase			,				_lowercase       ) and feature.dtype == "binary":
																					lowerCamelCase_							  =					min(_lowercase			,				config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS       )

							_visit(_lowercase			,				_lowercase       )

							return None if batch_size is np.inf else batch_size





class       __A(				UpperCamelCase_       ):

					def __init__(	self					:     Tuple   ,					__UpperCamelCase					:     NestedDataStructureLike[PathLike]   ,					__UpperCamelCase					:     Optional[NamedSplit] = None   ,					__UpperCamelCase					:     Optional[Features] = None   ,					__UpperCamelCase					:     str = None   ,					__UpperCamelCase					:     bool = False   ,					__UpperCamelCase					:     bool = False   ,					__UpperCamelCase					:     Optional[int] = None   ,					**__UpperCamelCase					:     List[str]   ,					):
												super().__init__(
												    UpperCamelCase__   ,					split=UpperCamelCase__   ,					features=UpperCamelCase__   ,					cache_dir=UpperCamelCase__   ,					keep_in_memory=UpperCamelCase__   ,					streaming=UpperCamelCase__   ,					num_proc=UpperCamelCase__   ,					**UpperCamelCase__   ,					)
												lowerCamelCase_							  =					path_or_paths if isinstance(UpperCamelCase__   ,					UpperCamelCase__   ) else {self.split: path_or_paths}
												lowerCamelCase_							  =					_PACKAGED_DATASETS_MODULES['''parquet'''][1]
												lowerCamelCase_							  =					Parquet(
												    cache_dir=UpperCamelCase__   ,					data_files=UpperCamelCase__   ,					features=UpperCamelCase__   ,					hash=UpperCamelCase__   ,					**UpperCamelCase__   ,					)



					def    lowercase__      (	self					:     str   ):





												if self.streaming:
																			lowerCamelCase_							  =					self.builder.as_streaming_dataset(split=self.split   )
												# Build regular (map-style) dataset
												else:
																			lowerCamelCase_							  =					None
																			lowerCamelCase_							  =					None
																			lowerCamelCase_							  =					None
																			lowerCamelCase_							  =					None

																			self.builder.download_and_prepare(
																			    download_config=UpperCamelCase__   ,					download_mode=UpperCamelCase__   ,					verification_mode=UpperCamelCase__   ,					base_path=UpperCamelCase__   ,					num_proc=self.num_proc   ,					)
																			lowerCamelCase_							  =					self.builder.as_dataset(
																			    split=self.split   ,					verification_mode=UpperCamelCase__   ,					in_memory=self.keep_in_memory   )
												return dataset





class       __A:

					def __init__(	self					:     List[str]   ,					__UpperCamelCase					:     Dataset   ,					__UpperCamelCase					:     Union[PathLike, BinaryIO]   ,					__UpperCamelCase					:     Optional[int] = None   ,					**__UpperCamelCase					:     Union[str, Any]   ,					):
												lowerCamelCase_							  =					dataset
												lowerCamelCase_							  =					path_or_buf
												lowerCamelCase_							  =					batch_size or get_writer_batch_size(dataset.features   )
												lowerCamelCase_							  =					parquet_writer_kwargs

					def    lowercase__      (	self					:     Any   ):
												lowerCamelCase_							  =					self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

												if isinstance(self.path_or_buf   ,					(str, bytes, os.PathLike)   ):
																			with open(self.path_or_buf   ,					"""wb+"""   ) as buffer:
																										lowerCamelCase_							  =					self._write(file_obj=UpperCamelCase__   ,					batch_size=UpperCamelCase__   ,					**self.parquet_writer_kwargs   )
												else:
																			lowerCamelCase_							  =					self._write(file_obj=self.path_or_buf   ,					batch_size=UpperCamelCase__   ,					**self.parquet_writer_kwargs   )
												return written



					def    lowercase__      (	self					:     Tuple   ,					__UpperCamelCase					:     BinaryIO   ,					__UpperCamelCase					:     int   ,					**__UpperCamelCase					:     List[Any]   ):
												lowerCamelCase_							  =					0
												lowerCamelCase_							  =					parquet_writer_kwargs.pop("""path_or_buf"""   ,					UpperCamelCase__   )
												lowerCamelCase_							  =					self.dataset.features.arrow_schema

												lowerCamelCase_							  =					pq.ParquetWriter(UpperCamelCase__   ,					schema=UpperCamelCase__   ,					**UpperCamelCase__   )

												for offset in logging.tqdm(
												    range(0   ,					len(self.dataset   )   ,					UpperCamelCase__   )   ,					unit="""ba"""   ,					disable=not logging.is_progress_bar_enabled()   ,					desc="""Creating parquet from Arrow format"""   ,					):
																			lowerCamelCase_							  =					query_table(
																			    table=self.dataset._data   ,					key=slice(UpperCamelCase__   ,					offset + batch_size   )   ,					indices=self.dataset._indices if self.dataset._indices is not None else None   ,					)
																			writer.write_table(UpperCamelCase__   )
																			written += batch.nbytes
												writer.close()
												return written







import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase        =  logging.get_logger(__name__)

lowercase        =  {
    '''facebook/wav2vec2-base-960h''': '''https://huggingface.co/facebook/wav2vec2-base-960h/resolve/main/config.json''',
    # See all Wav2Vec2 models at https://huggingface.co/models?filter=wav2vec2
}





class       __A(				UpperCAmelCase       ):
					SCREAMING_SNAKE_CASE			    =    '''wav2vec2'''

					def __init__(	self					:     Tuple   ,					__UpperCamelCase					:     Optional[int]=3_2   ,					__UpperCamelCase					:     List[str]=7_6_8   ,					__UpperCamelCase					:     Tuple=1_2   ,					__UpperCamelCase					:     str=1_2   ,					__UpperCamelCase					:     Any=3_0_7_2   ,					__UpperCamelCase					:     Tuple="gelu"   ,					__UpperCamelCase					:     Tuple=0.1   ,					__UpperCamelCase					:     Any=0.1   ,					__UpperCamelCase					:     List[str]=0.1   ,					__UpperCamelCase					:     List[Any]=0.0   ,					__UpperCamelCase					:     str=0.0   ,					__UpperCamelCase					:     str=0.1   ,					__UpperCamelCase					:     List[Any]=0.1   ,					__UpperCamelCase					:     Dict=0.02   ,					__UpperCamelCase					:     int=1E-5   ,					__UpperCamelCase					:     Dict="group"   ,					__UpperCamelCase					:     Tuple="gelu"   ,					__UpperCamelCase					:     Optional[int]=(5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2)   ,					__UpperCamelCase					:     Union[str, Any]=(5, 2, 2, 2, 2, 2, 2)   ,					__UpperCamelCase					:     int=(1_0, 3, 3, 3, 3, 2, 2)   ,					__UpperCamelCase					:     List[Any]=False   ,					__UpperCamelCase					:     Tuple=1_2_8   ,					__UpperCamelCase					:     Any=1_6   ,					__UpperCamelCase					:     List[str]=False   ,					__UpperCamelCase					:     List[str]=True   ,					__UpperCamelCase					:     Dict=0.05   ,					__UpperCamelCase					:     Union[str, Any]=1_0   ,					__UpperCamelCase					:     int=2   ,					__UpperCamelCase					:     List[Any]=0.0   ,					__UpperCamelCase					:     List[Any]=1_0   ,					__UpperCamelCase					:     List[str]=0   ,					__UpperCamelCase					:     Optional[Any]=3_2_0   ,					__UpperCamelCase					:     Optional[int]=2   ,					__UpperCamelCase					:     List[Any]=0.1   ,					__UpperCamelCase					:     int=1_0_0   ,					__UpperCamelCase					:     int=2_5_6   ,					__UpperCamelCase					:     str=2_5_6   ,					__UpperCamelCase					:     Dict=0.1   ,					__UpperCamelCase					:     Union[str, Any]="sum"   ,					__UpperCamelCase					:     Optional[int]=False   ,					__UpperCamelCase					:     Optional[int]=False   ,					__UpperCamelCase					:     Dict=2_5_6   ,					__UpperCamelCase					:     List[Any]=(5_1_2, 5_1_2, 5_1_2, 5_1_2, 1_5_0_0)   ,					__UpperCamelCase					:     int=(5, 3, 3, 1, 1)   ,					__UpperCamelCase					:     Optional[Any]=(1, 2, 3, 1, 1)   ,					__UpperCamelCase					:     Dict=5_1_2   ,					__UpperCamelCase					:     Any=0   ,					__UpperCamelCase					:     Union[str, Any]=1   ,					__UpperCamelCase					:     List[str]=2   ,					__UpperCamelCase					:     List[str]=False   ,					__UpperCamelCase					:     int=3   ,					__UpperCamelCase					:     Dict=2   ,					__UpperCamelCase					:     Optional[Any]=3   ,					__UpperCamelCase					:     Optional[int]=None   ,					__UpperCamelCase					:     Tuple=None   ,					**__UpperCamelCase					:     Tuple   ,					):
												super().__init__(**__UpperCamelCase   ,					pad_token_id=__UpperCamelCase   ,					bos_token_id=__UpperCamelCase   ,					eos_token_id=__UpperCamelCase   )
												lowerCamelCase_							  =					hidden_size
												lowerCamelCase_							  =					feat_extract_norm
												lowerCamelCase_							  =					feat_extract_activation
												lowerCamelCase_							  =					list(__UpperCamelCase   )
												lowerCamelCase_							  =					list(__UpperCamelCase   )
												lowerCamelCase_							  =					list(__UpperCamelCase   )
												lowerCamelCase_							  =					conv_bias
												lowerCamelCase_							  =					num_conv_pos_embeddings
												lowerCamelCase_							  =					num_conv_pos_embedding_groups
												lowerCamelCase_							  =					len(self.conv_dim   )
												lowerCamelCase_							  =					num_hidden_layers
												lowerCamelCase_							  =					intermediate_size
												lowerCamelCase_							  =					hidden_act
												lowerCamelCase_							  =					num_attention_heads
												lowerCamelCase_							  =					hidden_dropout
												lowerCamelCase_							  =					attention_dropout
												lowerCamelCase_							  =					activation_dropout
												lowerCamelCase_							  =					feat_proj_dropout
												lowerCamelCase_							  =					final_dropout
												lowerCamelCase_							  =					layerdrop
												lowerCamelCase_							  =					layer_norm_eps
												lowerCamelCase_							  =					initializer_range
												lowerCamelCase_							  =					vocab_size
												lowerCamelCase_							  =					do_stable_layer_norm
												lowerCamelCase_							  =					use_weighted_layer_sum

												if (
												    (len(self.conv_stride   ) != self.num_feat_extract_layers)
												    or (len(self.conv_kernel   ) != self.num_feat_extract_layers)
												    or (len(self.conv_dim   ) != self.num_feat_extract_layers)
												):
																			raise ValueError(
																			    """Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="""
																			    """ `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="""
																			    F''' {len(self.conv_dim   )}`, `len(config.conv_stride) = {len(self.conv_stride   )}`,'''
																			    F''' `len(config.conv_kernel) = {len(self.conv_kernel   )}`.'''   )

												# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
												lowerCamelCase_							  =					apply_spec_augment
												lowerCamelCase_							  =					mask_time_prob
												lowerCamelCase_							  =					mask_time_length
												lowerCamelCase_							  =					mask_time_min_masks
												lowerCamelCase_							  =					mask_feature_prob
												lowerCamelCase_							  =					mask_feature_length
												lowerCamelCase_							  =					mask_feature_min_masks

												# parameters for pretraining with codevector quantized representations
												lowerCamelCase_							  =					num_codevectors_per_group
												lowerCamelCase_							  =					num_codevector_groups
												lowerCamelCase_							  =					contrastive_logits_temperature
												lowerCamelCase_							  =					feat_quantizer_dropout
												lowerCamelCase_							  =					num_negatives
												lowerCamelCase_							  =					codevector_dim
												lowerCamelCase_							  =					proj_codevector_dim
												lowerCamelCase_							  =					diversity_loss_weight

												# ctc loss
												lowerCamelCase_							  =					ctc_loss_reduction
												lowerCamelCase_							  =					ctc_zero_infinity

												# adapter
												lowerCamelCase_							  =					add_adapter
												lowerCamelCase_							  =					adapter_kernel_size
												lowerCamelCase_							  =					adapter_stride
												lowerCamelCase_							  =					num_adapter_layers
												lowerCamelCase_							  =					output_hidden_size or hidden_size
												lowerCamelCase_							  =					adapter_attn_dim

												# SequenceClassification-specific parameter. Feel free to ignore for other classes.
												lowerCamelCase_							  =					classifier_proj_size

												# XVector-specific parameters. Feel free to ignore for other classes.
												lowerCamelCase_							  =					list(__UpperCamelCase   )
												lowerCamelCase_							  =					list(__UpperCamelCase   )
												lowerCamelCase_							  =					list(__UpperCamelCase   )
												lowerCamelCase_							  =					xvector_output_dim



					@property
					def    lowercase__      (	self					:     Dict   ):
												return functools.reduce(operator.mul   ,					self.conv_stride   ,					1   )





