




import itertools
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast
from datasets.utils.py_utils import Literal


a_   		=     datasets.utils.logging.get_logger(__name__)

a_   		=     ['names', 'prefix']
a_   		=     ['warn_bad_lines', 'error_bad_lines', 'mangle_dupe_cols']
a_   		=     ['encoding_errors', 'on_bad_lines']
a_   		=     ['date_format']

@dataclass
class  _lowerCamelCase						(			datasets.BuilderConfig   ):

      """simple docstring"""

      lowerCAmelCase__			: str    =					","
      lowerCAmelCase__			: Optional[str]    =					None
      lowerCAmelCase__			: Optional[Union[int, List[int], str]]    =					"infer"
      lowerCAmelCase__			: Optional[List[str]]    =					None
      lowerCAmelCase__			: Optional[List[str]]    =					None
      lowerCAmelCase__			: Optional[Union[int, str, List[int], List[str]]]    =					None
      lowerCAmelCase__			: Optional[Union[List[int], List[str]]]    =					None
      lowerCAmelCase__			: Optional[str]    =					None
      lowerCAmelCase__			: bool    =					True
      lowerCAmelCase__			: Optional[Literal["c", "python", "pyarrow"]]    =					None
      lowerCAmelCase__			: Dict[Union[int, str], Callable[[Any], Any]]    =					None
      lowerCAmelCase__			: Optional[list]    =					None
      lowerCAmelCase__			: Optional[list]    =					None
      lowerCAmelCase__			: bool    =					False
      lowerCAmelCase__			: Optional[Union[int, List[int]]]    =					None
      lowerCAmelCase__			: Optional[int]    =					None
      lowerCAmelCase__			: Optional[Union[str, List[str]]]    =					None
      lowerCAmelCase__			: bool    =					True
      lowerCAmelCase__			: bool    =					True
      lowerCAmelCase__			: bool    =					False
      lowerCAmelCase__			: bool    =					True
      lowerCAmelCase__			: Optional[str]    =					None
      lowerCAmelCase__			: str    =					"."
      lowerCAmelCase__			: Optional[str]    =					None
      lowerCAmelCase__			: str    =					'"'
      lowerCAmelCase__			: int    =					0
      lowerCAmelCase__			: Optional[str]    =					None
      lowerCAmelCase__			: Optional[str]    =					None
      lowerCAmelCase__			: Optional[str]    =					None
      lowerCAmelCase__			: Optional[str]    =					None
      lowerCAmelCase__			: bool    =					True
      lowerCAmelCase__			: bool    =					True
      lowerCAmelCase__			: int    =					0
      lowerCAmelCase__			: bool    =					True
      lowerCAmelCase__			: bool    =					False
      lowerCAmelCase__			: Optional[str]    =					None
      lowerCAmelCase__			: int    =					1_00_00
      lowerCAmelCase__			: Optional[datasets.Features]    =					None
      lowerCAmelCase__			: Optional[str]    =					"strict"
      lowerCAmelCase__			: Literal["error", "warn", "skip"]    =					"error"
      lowerCAmelCase__			: Optional[str]    =					None







      def       snake_case	(     self					: int				):
          if self.delimiter is not None:
              __UpperCamelCase  		=						self.delimiter
          if self.column_names is not None:
              __UpperCamelCase  		=						self.column_names







      @property
      def       snake_case	(     self					: List[Any]				):
          __UpperCamelCase  		=						{
              """sep""": self.sep,
              """header""": self.header,
              """names""": self.names,
              """index_col""": self.index_col,
              """usecols""": self.usecols,
              """prefix""": self.prefix,
              """mangle_dupe_cols""": self.mangle_dupe_cols,
              """engine""": self.engine,
              """converters""": self.converters,
              """true_values""": self.true_values,
              """false_values""": self.false_values,
              """skipinitialspace""": self.skipinitialspace,
              """skiprows""": self.skiprows,
              """nrows""": self.nrows,
              """na_values""": self.na_values,
              """keep_default_na""": self.keep_default_na,
              """na_filter""": self.na_filter,
              """verbose""": self.verbose,
              """skip_blank_lines""": self.skip_blank_lines,
              """thousands""": self.thousands,
              """decimal""": self.decimal,
              """lineterminator""": self.lineterminator,
              """quotechar""": self.quotechar,
              """quoting""": self.quoting,
              """escapechar""": self.escapechar,
              """comment""": self.comment,
              """encoding""": self.encoding,
              """dialect""": self.dialect,
              """error_bad_lines""": self.error_bad_lines,
              """warn_bad_lines""": self.warn_bad_lines,
              """skipfooter""": self.skipfooter,
              """doublequote""": self.doublequote,
              """memory_map""": self.memory_map,
              """float_precision""": self.float_precision,
              """chunksize""": self.chunksize,
              """encoding_errors""": self.encoding_errors,
              """on_bad_lines""": self.on_bad_lines,
              """date_format""": self.date_format,
          }

          # some kwargs must not be passed if they don't have a default value
          # some others are deprecated and we can also not pass them if they are the default value
          for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
              if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig() ,       A__				):
                  del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 2.0 new arguments
          if not (datasets.config.PANDAS_VERSION.major >= 2):
              for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
                  del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 1.3 new arguments
          if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
              for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
                  del pd_read_csv_kwargs[pd_read_csv_parameter]

          return pd_read_csv_kwargs







class  _lowerCamelCase						(			datasets.ArrowBasedBuilder   ):

      """simple docstring"""

      lowerCAmelCase__			: Tuple    =					CsvConfig







      def       snake_case	(     self					: Optional[int]				):
          return datasets.DatasetInfo(features=self.config.features				)







      def       snake_case	(     self					: Any ,       snake_case					: Optional[int]				):
          if not self.config.data_files:
              raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"				)
          __UpperCamelCase  		=						dl_manager.download_and_extract(self.config.data_files				)
          if isinstance(A__ ,       (str, list, tuple)				):
              __UpperCamelCase  		=						data_files
              if isinstance(A__ ,       A__				):
                  __UpperCamelCase  		=						[files]
              __UpperCamelCase  		=						[dl_manager.iter_files(A__				) for file in files]
              return [datasets.SplitGenerator(name=datasets.Split.TRAIN ,       gen_kwargs={'''files''': files}				)]
          __UpperCamelCase  		=						[]
          for split_name, files in data_files.items():
              if isinstance(A__ ,       A__				):
                  __UpperCamelCase  		=						[files]
              __UpperCamelCase  		=						[dl_manager.iter_files(A__				) for file in files]
              splits.append(datasets.SplitGenerator(name=A__ ,       gen_kwargs={'''files''': files}				)				)
          return splits







      def       snake_case	(     self					: Optional[int] ,       snake_case					: Optional[int]				):
          if self.config.features is not None:
              __UpperCamelCase  		=						self.config.features.arrow_schema
              if all(not require_storage_cast(A__				) for feature in self.config.features.values()				):
                  # cheaper cast
                  __UpperCamelCase  		=						pa.Table.from_arrays([pa_table[field.name] for field in schema] ,       schema=A__				)
              else:
                  # more expensive cast; allows str <-> int/float or str to Audio for example
                  __UpperCamelCase  		=						table_cast(A__ ,       A__				)
          return pa_table







      def       snake_case	(     self					: Tuple ,       snake_case					: List[str]				):
          __UpperCamelCase  		=						self.config.features.arrow_schema if self.config.features else None
          # dtype allows reading an int column as str
          __UpperCamelCase  		=						(
              {
                  name: dtype.to_pandas_dtype() if not require_storage_cast(A__				) else object
                  for name, dtype, feature in zip(schema.names ,       schema.types ,       self.config.features.values()				)
              }
              if schema is not None
              else None
          )
          for file_idx, file in enumerate(itertools.chain.from_iterable(A__				)				):
              __UpperCamelCase  		=						pd.read_csv(A__ ,       iterator=A__ ,       dtype=A__ ,       **self.config.pd_read_csv_kwargs				)
              try:
                  for batch_idx, df in enumerate(A__				):
                      __UpperCamelCase  		=						pa.Table.from_pandas(A__				)
                      # Uncomment for debugging (will print the Arrow table size and elements)
                      # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                      # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                      yield (file_idx, batch_idx), self._cast_table(A__				)
              except ValueError as e:
                  logger.error(F"Failed to read file '{file}' with error {type(A__				)}: {e}"				)
                  raise








import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_   		=     logging.get_logger(__name__)

a_   		=     {
    "microsoft/unispeech-large-1500h-cv": (
        "https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json"
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}

class  _lowerCamelCase						(			_SCREAMING_SNAKE_CASE   ):

      """simple docstring"""

      lowerCAmelCase__			: Dict    =					"unispeech"







      def __init__(     self					: str ,       snake_case					: Union[str, Any]=32 ,       snake_case					: Optional[Any]=768 ,       snake_case					: Dict=12 ,       snake_case					: Tuple=12 ,       snake_case					: Optional[Any]=3072 ,       snake_case					: Any="gelu" ,       snake_case					: Dict=0.1 ,       snake_case					: Tuple=0.1 ,       snake_case					: str=0.1 ,       snake_case					: Optional[int]=0.0 ,       snake_case					: Any=0.0 ,       snake_case					: Optional[Any]=0.1 ,       snake_case					: List[Any]=0.1 ,       snake_case					: Optional[int]=0.02 ,       snake_case					: List[str]=1E-5 ,       snake_case					: str="group" ,       snake_case					: List[Any]="gelu" ,       snake_case					: Union[str, Any]=(512, 512, 512, 512, 512, 512, 512) ,       snake_case					: List[Any]=(5, 2, 2, 2, 2, 2, 2) ,       snake_case					: Union[str, Any]=(10, 3, 3, 3, 3, 2, 2) ,       snake_case					: Tuple=False ,       snake_case					: Optional[int]=128 ,       snake_case					: List[str]=16 ,       snake_case					: List[str]=False ,       snake_case					: Dict=True ,       snake_case					: Optional[Any]=0.05 ,       snake_case					: Optional[Any]=10 ,       snake_case					: Union[str, Any]=2 ,       snake_case					: List[str]=0.0 ,       snake_case					: str=10 ,       snake_case					: int=0 ,       snake_case					: Tuple=320 ,       snake_case					: Any=2 ,       snake_case					: List[str]=0.1 ,       snake_case					: Optional[Any]=100 ,       snake_case					: List[Any]=256 ,       snake_case					: Union[str, Any]=256 ,       snake_case					: Any=0.1 ,       snake_case					: str="mean" ,       snake_case					: Union[str, Any]=False ,       snake_case					: str=False ,       snake_case					: Union[str, Any]=256 ,       snake_case					: Optional[Any]=80 ,       snake_case					: str=0 ,       snake_case					: int=1 ,       snake_case					: int=2 ,       snake_case					: Dict=0.5 ,       **snake_case					: Optional[int] ,       ):
          super().__init__(**snake_case ,       pad_token_id=snake_case ,       bos_token_id=snake_case ,       eos_token_id=snake_case				)
          __UpperCamelCase  		=						hidden_size
          __UpperCamelCase  		=						feat_extract_norm
          __UpperCamelCase  		=						feat_extract_activation
          __UpperCamelCase  		=						list(snake_case				)
          __UpperCamelCase  		=						list(snake_case				)
          __UpperCamelCase  		=						list(snake_case				)
          __UpperCamelCase  		=						conv_bias
          __UpperCamelCase  		=						num_conv_pos_embeddings
          __UpperCamelCase  		=						num_conv_pos_embedding_groups
          __UpperCamelCase  		=						len(self.conv_dim				)
          __UpperCamelCase  		=						num_hidden_layers
          __UpperCamelCase  		=						intermediate_size
          __UpperCamelCase  		=						hidden_act
          __UpperCamelCase  		=						num_attention_heads
          __UpperCamelCase  		=						hidden_dropout
          __UpperCamelCase  		=						attention_dropout
          __UpperCamelCase  		=						activation_dropout
          __UpperCamelCase  		=						feat_proj_dropout
          __UpperCamelCase  		=						final_dropout
          __UpperCamelCase  		=						layerdrop
          __UpperCamelCase  		=						layer_norm_eps
          __UpperCamelCase  		=						initializer_range
          __UpperCamelCase  		=						num_ctc_classes
          __UpperCamelCase  		=						vocab_size
          __UpperCamelCase  		=						do_stable_layer_norm
          __UpperCamelCase  		=						use_weighted_layer_sum
          __UpperCamelCase  		=						classifier_proj_size

          if (
              (len(self.conv_stride				) != self.num_feat_extract_layers)
              or (len(self.conv_kernel				) != self.num_feat_extract_layers)
              or (len(self.conv_dim				) != self.num_feat_extract_layers)
          ):
              raise ValueError(
                  '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
                  ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
                  F" {len(self.conv_dim				)}`, `len(config.conv_stride) = {len(self.conv_stride				)}`,"
                  F" `len(config.conv_kernel) = {len(self.conv_kernel				)}`."				)

          # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
          __UpperCamelCase  		=						apply_spec_augment
          __UpperCamelCase  		=						mask_time_prob
          __UpperCamelCase  		=						mask_time_length
          __UpperCamelCase  		=						mask_time_min_masks
          __UpperCamelCase  		=						mask_feature_prob
          __UpperCamelCase  		=						mask_feature_length
          __UpperCamelCase  		=						mask_feature_min_masks

          # parameters for pretraining with codevector quantized representations
          __UpperCamelCase  		=						num_codevectors_per_group
          __UpperCamelCase  		=						num_codevector_groups
          __UpperCamelCase  		=						contrastive_logits_temperature
          __UpperCamelCase  		=						feat_quantizer_dropout
          __UpperCamelCase  		=						num_negatives
          __UpperCamelCase  		=						codevector_dim
          __UpperCamelCase  		=						proj_codevector_dim
          __UpperCamelCase  		=						diversity_loss_weight

          # ctc loss
          __UpperCamelCase  		=						ctc_loss_reduction
          __UpperCamelCase  		=						ctc_zero_infinity

          # pretraining loss
          __UpperCamelCase  		=						replace_prob







      @property
      def       snake_case	(     self					: Dict				):
          return functools.reduce(operator.mul ,       self.conv_stride ,       1				)






