import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_			    =							logging.get_logger(__name__)

UpperCAmelCase_			    =							{
    '''asapp/sew-tiny-100k''': '''https://huggingface.co/asapp/sew-tiny-100k/resolve/main/config.json''',
    # See all SEW models at https://huggingface.co/models?filter=sew
}




class  __magic_name__  (		__a    ):





   """simple docstring"""




   lowerCAmelCase      :    List[Any]  			=   '''sew'''



   def __init__(	self					:  str		,  _lowercase					:  Tuple=32		,  _lowercase					:  Dict=768		,  _lowercase					:  Optional[int]=12		,  _lowercase					:  List[Any]=12		,  _lowercase					:  List[Any]=3_072		,  _lowercase					:  Optional[Any]=2		,  _lowercase					:  Union[str, Any]="gelu"		,  _lowercase					:  Dict=0.1		,  _lowercase					:  Optional[int]=0.1		,  _lowercase					:  str=0.1		,  _lowercase					:  Any=0.0		,  _lowercase					:  Optional[int]=0.1		,  _lowercase					:  str=0.1		,  _lowercase					:  Any=0.02		,  _lowercase					:  Any=1E-5		,  _lowercase					:  Tuple="group"		,  _lowercase					:  int="gelu"		,  _lowercase					:  Union[str, Any]=(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512)		,  _lowercase					:  List[Any]=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)		,  _lowercase					:  Optional[int]=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)		,  _lowercase					:  List[str]=False		,  _lowercase					:  Tuple=128		,  _lowercase					:  int=16		,  _lowercase					:  Union[str, Any]=True		,  _lowercase					:  List[str]=0.05		,  _lowercase					:  Tuple=10		,  _lowercase					:  Union[str, Any]=2		,  _lowercase					:  int=0.0		,  _lowercase					:  List[str]=10		,  _lowercase					:  Tuple=0		,  _lowercase					:  Tuple="mean"		,  _lowercase					:  Dict=False		,  _lowercase					:  List[Any]=False		,  _lowercase					:  Any=256		,  _lowercase					:  Optional[int]=0		,  _lowercase					:  Dict=1		,  _lowercase					:  Tuple=2		,  **_lowercase					:  List[Any]		,  ):



    """simple docstring"""



    super().__init__(**_lowercase		,  pad_token_id=_lowercase		,  bos_token_id=_lowercase		,  eos_token_id=_lowercase							)
    _UpperCamelCase:     Union[str, Any]							= hidden_size
    _UpperCamelCase:     Union[str, Any]							= feat_extract_norm
    _UpperCamelCase:     Optional[Any]							= feat_extract_activation
    _UpperCamelCase:     int							= list(_lowercase							)
    _UpperCamelCase:     Union[str, Any]							= list(_lowercase							)
    _UpperCamelCase:     Union[str, Any]							= list(_lowercase							)
    _UpperCamelCase:     str							= conv_bias
    _UpperCamelCase:     List[str]							= num_conv_pos_embeddings
    _UpperCamelCase:     int							= num_conv_pos_embedding_groups
    _UpperCamelCase:     Tuple							= len(self.conv_dim							)
    _UpperCamelCase:     Optional[Any]							= num_hidden_layers
    _UpperCamelCase:     List[str]							= intermediate_size
    _UpperCamelCase:     List[Any]							= squeeze_factor
    _UpperCamelCase:     Optional[Any]							= hidden_act
    _UpperCamelCase:     Optional[int]							= num_attention_heads
    _UpperCamelCase:     Optional[Any]							= hidden_dropout
    _UpperCamelCase:     Tuple							= attention_dropout
    _UpperCamelCase:     Union[str, Any]							= activation_dropout
    _UpperCamelCase:     List[str]							= feat_proj_dropout
    _UpperCamelCase:     List[Any]							= final_dropout
    _UpperCamelCase:     Optional[int]							= layerdrop
    _UpperCamelCase:     Optional[Any]							= layer_norm_eps
    _UpperCamelCase:     str							= initializer_range
    _UpperCamelCase:     int							= vocab_size

    if (
        (len(self.conv_stride							) != self.num_feat_extract_layers)
        or (len(self.conv_kernel							) != self.num_feat_extract_layers)
        or (len(self.conv_dim							) != self.num_feat_extract_layers)
    ):
     raise ValueError(
         '''Configuration for convolutional layers is incorrect.'''
         '''It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,'''
         f"""but is `len(config.conv_dim) = {len(self.conv_dim							)}`, `len(config.conv_stride)"""
         f"""= {len(self.conv_stride							)}`, `len(config.conv_kernel) = {len(self.conv_kernel							)}`."""							)

    # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
    _UpperCamelCase:     Optional[int]							= apply_spec_augment
    _UpperCamelCase:     Any							= mask_time_prob
    _UpperCamelCase:     Tuple							= mask_time_length
    _UpperCamelCase:     Union[str, Any]							= mask_time_min_masks
    _UpperCamelCase:     Optional[Any]							= mask_feature_prob
    _UpperCamelCase:     List[Any]							= mask_feature_length
    _UpperCamelCase:     List[str]							= mask_feature_min_masks

    # ctc loss
    _UpperCamelCase:     Tuple							= ctc_loss_reduction
    _UpperCamelCase:     List[str]							= ctc_zero_infinity

    # sequence classification
    _UpperCamelCase:     List[Any]							= use_weighted_layer_sum
    _UpperCamelCase:     Optional[Any]							= classifier_proj_size



   @property
   def   lowerCAmelCase						(	self					:  Dict							):



    """simple docstring"""



    return functools.reduce(operator.mul		,  self.conv_stride		,  1							)
from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)




def       lowerCAmelCase_     (		lowercase:							Optional[Any]  ,					lowercase:							Tuple		)    ->				Any:







 '''simple docstring'''
 _UpperCamelCase:     Union[str, Any]							= []
 for part_id in partition_order:
  _UpperCamelCase:     int							= df.where(F"""SPARK_PARTITION_ID() = {part_id}"""		).collect()
  for row_idx, row in enumerate(lowercase		):
   expected_row_ids_and_row_dicts.append((F"""{part_id}_{row_idx}""", row.asDict())		)
 return expected_row_ids_and_row_dicts




@require_not_windows
@require_dill_gt_0_3_2
def       lowerCAmelCase_     (		)    ->				List[str]:







 '''simple docstring'''
 _UpperCamelCase:     Optional[int]							= pyspark.sql.SparkSession.builder.master('''local[*]'''		).appName('''pyspark'''		).getOrCreate()
 _UpperCamelCase:     int							= spark.range(100		).repartition(1		)
 _UpperCamelCase:     int							= Spark(lowercase		)
 # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
 # that each partition can hold 2 rows.
 spark_builder._repartition_df_if_needed(max_shard_size=16		)
 # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
 assert spark_builder.df.rdd.getNumPartitions() == 50




@require_not_windows
@require_dill_gt_0_3_2
def       lowerCAmelCase_     (		)    ->				List[str]:







 '''simple docstring'''
 _UpperCamelCase:     List[Any]							= pyspark.sql.SparkSession.builder.master('''local[*]'''		).appName('''pyspark'''		).getOrCreate()
 _UpperCamelCase:     Optional[Any]							= spark.range(10		).repartition(2		)
 _UpperCamelCase:     int							= [1, 0]
 _UpperCamelCase:     Any							= _generate_iterable_examples(lowercase  ,					lowercase		)  # Reverse the partitions.
 _UpperCamelCase:     List[str]							= _get_expected_row_ids_and_row_dicts_for_partition_order(lowercase  ,					lowercase		)

 for i, (row_id, row_dict) in enumerate(generate_fn()		):
  _UpperCamelCase	,							_UpperCamelCase:     List[Any]							= expected_row_ids_and_row_dicts[i]
  assert row_id == expected_row_id
  assert row_dict == expected_row_dict




@require_not_windows
@require_dill_gt_0_3_2
def       lowerCAmelCase_     (		)    ->				Any:







 '''simple docstring'''
 _UpperCamelCase:     Optional[Any]							= pyspark.sql.SparkSession.builder.master('''local[*]'''		).appName('''pyspark'''		).getOrCreate()
 _UpperCamelCase:     Any							= spark.range(10		).repartition(1		)
 _UpperCamelCase:     int							= SparkExamplesIterable(lowercase		)
 assert it.n_shards == 1
 for i, (row_id, row_dict) in enumerate(lowercase		):
  assert row_id == F"""0_{i}"""
  assert row_dict == {"id": i}




@require_not_windows
@require_dill_gt_0_3_2
def       lowerCAmelCase_     (		)    ->				Union[str, Any]:







 '''simple docstring'''
 _UpperCamelCase:     Any							= pyspark.sql.SparkSession.builder.master('''local[*]'''		).appName('''pyspark'''		).getOrCreate()
 _UpperCamelCase:     str							= spark.range(30		).repartition(3		)
 # Mock the generator so that shuffle reverses the partition indices.
 with patch('''numpy.random.Generator'''		) as generator_mock:
  _UpperCamelCase:     Union[str, Any]							= lambda lowercase		: x.reverse()
  _UpperCamelCase:     List[str]							= _get_expected_row_ids_and_row_dicts_for_partition_order(lowercase  ,					[2, 1, 0]		)

  _UpperCamelCase:     Union[str, Any]							= SparkExamplesIterable(lowercase		).shuffle_data_sources(lowercase		)
  assert shuffled_it.n_shards == 3
  for i, (row_id, row_dict) in enumerate(lowercase		):
   _UpperCamelCase	,							_UpperCamelCase:     List[Any]							= expected_row_ids_and_row_dicts[i]
   assert row_id == expected_row_id
   assert row_dict == expected_row_dict




@require_not_windows
@require_dill_gt_0_3_2
def       lowerCAmelCase_     (		)    ->				Union[str, Any]:







 '''simple docstring'''
 _UpperCamelCase:     str							= pyspark.sql.SparkSession.builder.master('''local[*]'''		).appName('''pyspark'''		).getOrCreate()
 _UpperCamelCase:     Tuple							= spark.range(20		).repartition(4		)

 # Partitions 0 and 2
 _UpperCamelCase:     List[Any]							= SparkExamplesIterable(lowercase		).shard_data_sources(worker_id=0  ,					num_workers=2		)
 assert shard_it_a.n_shards == 2
 _UpperCamelCase:     Dict							= _get_expected_row_ids_and_row_dicts_for_partition_order(lowercase  ,					[0, 2]		)
 for i, (row_id, row_dict) in enumerate(lowercase		):
  _UpperCamelCase	,							_UpperCamelCase:     Optional[int]							= expected_row_ids_and_row_dicts_a[i]
  assert row_id == expected_row_id
  assert row_dict == expected_row_dict

 # Partitions 1 and 3
 _UpperCamelCase:     str							= SparkExamplesIterable(lowercase		).shard_data_sources(worker_id=1  ,					num_workers=2		)
 assert shard_it_a.n_shards == 2
 _UpperCamelCase:     str							= _get_expected_row_ids_and_row_dicts_for_partition_order(lowercase  ,					[1, 3]		)
 for i, (row_id, row_dict) in enumerate(lowercase		):
  _UpperCamelCase	,							_UpperCamelCase:     str							= expected_row_ids_and_row_dicts_a[i]
  assert row_id == expected_row_id
  assert row_dict == expected_row_dict





@require_not_windows
@require_dill_gt_0_3_2
def       lowerCAmelCase_     (		)    ->				Optional[int]:







 '''simple docstring'''
 _UpperCamelCase:     Any							= pyspark.sql.SparkSession.builder.master('''local[*]'''		).appName('''pyspark'''		).getOrCreate()
 _UpperCamelCase:     Union[str, Any]							= spark.range(100		).repartition(1		)
 _UpperCamelCase:     Optional[Any]							= Spark(lowercase		)
 # Choose a small max_shard_size for maximum partitioning.
 spark_builder._repartition_df_if_needed(max_shard_size=1		)
 # The new number of partitions should not be greater than the number of rows.
 assert spark_builder.df.rdd.getNumPartitions() == 100