from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)



def 			__SCREAMING_SNAKE_CASE						(      __UpperCamelCase					: List[Any]				,  __UpperCamelCase					: Dict			)							->      Any:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__    							=   []
  for part_id in partition_order:
    SCREAMING_SNAKE_CASE__    							=   df.where(f"""SPARK_PARTITION_ID() = {part_id}"""			).collect()
    for row_idx, row in enumerate(__UpperCamelCase			):
      expected_row_ids_and_row_dicts.append((f"""{part_id}_{row_idx}""", row.asDict())			)
  return expected_row_ids_and_row_dicts



@require_not_windows
@require_dill_gt_0_3_2
def 			__SCREAMING_SNAKE_CASE						(      )							->      str:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__    							=   pyspark.sql.SparkSession.builder.master("""local[*]"""			).appName("""pyspark"""			).getOrCreate()
  SCREAMING_SNAKE_CASE__    							=   spark.range(1_00			).repartition(1			)
  SCREAMING_SNAKE_CASE__    							=   Spark(__UpperCamelCase			)
  # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
  # that each partition can hold 2 rows.
  spark_builder._repartition_df_if_needed(max_shard_size=16			)
  # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
  assert spark_builder.df.rdd.getNumPartitions() == 50



@require_not_windows
@require_dill_gt_0_3_2
def 			__SCREAMING_SNAKE_CASE						(      )							->      Tuple:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__    							=   pyspark.sql.SparkSession.builder.master("""local[*]"""			).appName("""pyspark"""			).getOrCreate()
  SCREAMING_SNAKE_CASE__    							=   spark.range(10			).repartition(2			)
  SCREAMING_SNAKE_CASE__    							=   [1, 0]
  SCREAMING_SNAKE_CASE__    							=   _generate_iterable_examples(__UpperCamelCase				,  __UpperCamelCase			)  # Reverse the partitions.
  SCREAMING_SNAKE_CASE__    							=   _get_expected_row_ids_and_row_dicts_for_partition_order(__UpperCamelCase				,  __UpperCamelCase			)

  for i, (row_id, row_dict) in enumerate(generate_fn()			):
    SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__    							=   expected_row_ids_and_row_dicts[i]
    assert row_id == expected_row_id
    assert row_dict == expected_row_dict



@require_not_windows
@require_dill_gt_0_3_2
def 			__SCREAMING_SNAKE_CASE						(      )							->      Tuple:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__    							=   pyspark.sql.SparkSession.builder.master("""local[*]"""			).appName("""pyspark"""			).getOrCreate()
  SCREAMING_SNAKE_CASE__    							=   spark.range(10			).repartition(1			)
  SCREAMING_SNAKE_CASE__    							=   SparkExamplesIterable(__UpperCamelCase			)
  assert it.n_shards == 1
  for i, (row_id, row_dict) in enumerate(__UpperCamelCase			):
    assert row_id == f"""0_{i}"""
    assert row_dict == {"id": i}



@require_not_windows
@require_dill_gt_0_3_2
def 			__SCREAMING_SNAKE_CASE						(      )							->      Dict:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__    							=   pyspark.sql.SparkSession.builder.master("""local[*]"""			).appName("""pyspark"""			).getOrCreate()
  SCREAMING_SNAKE_CASE__    							=   spark.range(30			).repartition(3			)
  # Mock the generator so that shuffle reverses the partition indices.
  with patch("""numpy.random.Generator"""			) as generator_mock:
    SCREAMING_SNAKE_CASE__    							=   lambda __UpperCamelCase			: x.reverse()
    SCREAMING_SNAKE_CASE__    							=   _get_expected_row_ids_and_row_dicts_for_partition_order(__UpperCamelCase				,  [2, 1, 0]			)

    SCREAMING_SNAKE_CASE__    							=   SparkExamplesIterable(__UpperCamelCase			).shuffle_data_sources(__UpperCamelCase			)
    assert shuffled_it.n_shards == 3
    for i, (row_id, row_dict) in enumerate(__UpperCamelCase			):
      SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__    							=   expected_row_ids_and_row_dicts[i]
      assert row_id == expected_row_id
      assert row_dict == expected_row_dict



@require_not_windows
@require_dill_gt_0_3_2
def 			__SCREAMING_SNAKE_CASE						(      )							->      int:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__    							=   pyspark.sql.SparkSession.builder.master("""local[*]"""			).appName("""pyspark"""			).getOrCreate()
  SCREAMING_SNAKE_CASE__    							=   spark.range(20			).repartition(4			)

  # Partitions 0 and 2
  SCREAMING_SNAKE_CASE__    							=   SparkExamplesIterable(__UpperCamelCase			).shard_data_sources(worker_id=0				,  num_workers=2			)
  assert shard_it_a.n_shards == 2
  SCREAMING_SNAKE_CASE__    							=   _get_expected_row_ids_and_row_dicts_for_partition_order(__UpperCamelCase				,  [0, 2]			)
  for i, (row_id, row_dict) in enumerate(__UpperCamelCase			):
    SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__    							=   expected_row_ids_and_row_dicts_a[i]
    assert row_id == expected_row_id
    assert row_dict == expected_row_dict

  # Partitions 1 and 3
  SCREAMING_SNAKE_CASE__    							=   SparkExamplesIterable(__UpperCamelCase			).shard_data_sources(worker_id=1				,  num_workers=2			)
  assert shard_it_a.n_shards == 2
  SCREAMING_SNAKE_CASE__    							=   _get_expected_row_ids_and_row_dicts_for_partition_order(__UpperCamelCase				,  [1, 3]			)
  for i, (row_id, row_dict) in enumerate(__UpperCamelCase			):
    SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__    							=   expected_row_ids_and_row_dicts_a[i]
    assert row_id == expected_row_id
    assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def 			__SCREAMING_SNAKE_CASE						(      )							->      Dict:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__    							=   pyspark.sql.SparkSession.builder.master("""local[*]"""			).appName("""pyspark"""			).getOrCreate()
  SCREAMING_SNAKE_CASE__    							=   spark.range(1_00			).repartition(1			)
  SCREAMING_SNAKE_CASE__    							=   Spark(__UpperCamelCase			)
  # Choose a small max_shard_size for maximum partitioning.
  spark_builder._repartition_df_if_needed(max_shard_size=1			)
  # The new number of partitions should not be greater than the number of rows.
  assert spark_builder.df.rdd.getNumPartitions() == 1_00

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase    :				List[Any]							     =	logging.get_logger(__name__)

__lowerCamelCase    :				str							     =	{
    '''facebook/timesformer''': '''https://huggingface.co/facebook/timesformer/resolve/main/config.json''',
}







class __snake_case       (		lowerCamelCase_	):
   lowerCAmelCase_														=							"timesformer"




   def __init__(					self    :       Any ,			_lowercase    :       Tuple=2_24 ,			_lowercase    :       Tuple=16 ,			_lowercase    :       List[Any]=3 ,			_lowercase    :       str=8 ,			_lowercase    :       Optional[Any]=7_68 ,			_lowercase    :       List[str]=12 ,			_lowercase    :       List[str]=12 ,			_lowercase    :       Dict=30_72 ,			_lowercase    :       List[str]="gelu" ,			_lowercase    :       List[str]=0.0 ,			_lowercase    :       List[str]=0.0 ,			_lowercase    :       Tuple=0.02 ,			_lowercase    :       List[Any]=1E-6 ,			_lowercase    :       List[str]=True ,			_lowercase    :       Dict="divided_space_time" ,			_lowercase    :       Union[str, Any]=0 ,			**_lowercase    :       Union[str, Any] ,			):



     """simple docstring"""






     super().__init__(**_lowercase   )

     SCREAMING_SNAKE_CASE__    							=   image_size
     SCREAMING_SNAKE_CASE__    							=   patch_size
     SCREAMING_SNAKE_CASE__    							=   num_channels
     SCREAMING_SNAKE_CASE__    							=   num_frames

     SCREAMING_SNAKE_CASE__    							=   hidden_size
     SCREAMING_SNAKE_CASE__    							=   num_hidden_layers
     SCREAMING_SNAKE_CASE__    							=   num_attention_heads
     SCREAMING_SNAKE_CASE__    							=   intermediate_size
     SCREAMING_SNAKE_CASE__    							=   hidden_act
     SCREAMING_SNAKE_CASE__    							=   hidden_dropout_prob
     SCREAMING_SNAKE_CASE__    							=   attention_probs_dropout_prob
     SCREAMING_SNAKE_CASE__    							=   initializer_range
     SCREAMING_SNAKE_CASE__    							=   layer_norm_eps
     SCREAMING_SNAKE_CASE__    							=   qkv_bias

     SCREAMING_SNAKE_CASE__    							=   attention_type
     SCREAMING_SNAKE_CASE__    							=   drop_path_rate
