


'''simple docstring'''



from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)



def    __UpperCAmelCase	(				SCREAMING_SNAKE_CASE__:	List[Any],					SCREAMING_SNAKE_CASE__:	Any    )  ->			Optional[int]:

   """simple docstring"""
   __a				     =    []
   for part_id in partition_order:
      __a				     =    df.where(f"""SPARK_PARTITION_ID() = {part_id}"""    ).collect()
      for row_idx, row in enumerate(SCREAMING_SNAKE_CASE__    ):
         expected_row_ids_and_row_dicts.append((f"""{part_id}_{row_idx}""", row.asDict())    )
   return expected_row_ids_and_row_dicts



@require_not_windows
@require_dill_gt_0_3_2
def    __UpperCAmelCase	(				)  ->			Optional[int]:

   """simple docstring"""
   __a				     =    pyspark.sql.SparkSession.builder.master('local[*]'    ).appName('pyspark'    ).getOrCreate()
   __a				     =    spark.range(100    ).repartition(1    )
   __a				     =    Spark(SCREAMING_SNAKE_CASE__    )
   # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
   # that each partition can hold 2 rows.
   spark_builder._repartition_df_if_needed(max_shard_size=16    )
   # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
   assert spark_builder.df.rdd.getNumPartitions() == 50



@require_not_windows
@require_dill_gt_0_3_2
def    __UpperCAmelCase	(				)  ->			Optional[int]:

   """simple docstring"""
   __a				     =    pyspark.sql.SparkSession.builder.master('local[*]'    ).appName('pyspark'    ).getOrCreate()
   __a				     =    spark.range(10    ).repartition(2    )
   __a				     =    [1, 0]
   __a				     =    _generate_iterable_examples(SCREAMING_SNAKE_CASE__,					SCREAMING_SNAKE_CASE__    )  # Reverse the partitions.
   __a				     =    _get_expected_row_ids_and_row_dicts_for_partition_order(SCREAMING_SNAKE_CASE__,					SCREAMING_SNAKE_CASE__    )

   for i, (row_id, row_dict) in enumerate(generate_fn()    ):
      __a     ,			__a				     =    expected_row_ids_and_row_dicts[i]
      assert row_id == expected_row_id
      assert row_dict == expected_row_dict



@require_not_windows
@require_dill_gt_0_3_2
def    __UpperCAmelCase	(				)  ->			Dict:

   """simple docstring"""
   __a				     =    pyspark.sql.SparkSession.builder.master('local[*]'    ).appName('pyspark'    ).getOrCreate()
   __a				     =    spark.range(10    ).repartition(1    )
   __a				     =    SparkExamplesIterable(SCREAMING_SNAKE_CASE__    )
   assert it.n_shards == 1
   for i, (row_id, row_dict) in enumerate(SCREAMING_SNAKE_CASE__    ):
      assert row_id == f"""0_{i}"""
      assert row_dict == {"id": i}



@require_not_windows
@require_dill_gt_0_3_2
def    __UpperCAmelCase	(				)  ->			Optional[int]:

   """simple docstring"""
   __a				     =    pyspark.sql.SparkSession.builder.master('local[*]'    ).appName('pyspark'    ).getOrCreate()
   __a				     =    spark.range(30    ).repartition(3    )
   # Mock the generator so that shuffle reverses the partition indices.
   with patch('numpy.random.Generator'    ) as generator_mock:
      __a				     =    lambda SCREAMING_SNAKE_CASE__    : x.reverse()
      __a				     =    _get_expected_row_ids_and_row_dicts_for_partition_order(SCREAMING_SNAKE_CASE__,					[2, 1, 0]    )

      __a				     =    SparkExamplesIterable(SCREAMING_SNAKE_CASE__    ).shuffle_data_sources(SCREAMING_SNAKE_CASE__    )
      assert shuffled_it.n_shards == 3
      for i, (row_id, row_dict) in enumerate(SCREAMING_SNAKE_CASE__    ):
         __a     ,			__a				     =    expected_row_ids_and_row_dicts[i]
         assert row_id == expected_row_id
         assert row_dict == expected_row_dict



@require_not_windows
@require_dill_gt_0_3_2
def    __UpperCAmelCase	(				)  ->			Tuple:

   """simple docstring"""
   __a				     =    pyspark.sql.SparkSession.builder.master('local[*]'    ).appName('pyspark'    ).getOrCreate()
   __a				     =    spark.range(20    ).repartition(4    )

   # Partitions 0 and 2
   __a				     =    SparkExamplesIterable(SCREAMING_SNAKE_CASE__    ).shard_data_sources(worker_id=0,					num_workers=2    )
   assert shard_it_a.n_shards == 2
   __a				     =    _get_expected_row_ids_and_row_dicts_for_partition_order(SCREAMING_SNAKE_CASE__,					[0, 2]    )
   for i, (row_id, row_dict) in enumerate(SCREAMING_SNAKE_CASE__    ):
      __a     ,			__a				     =    expected_row_ids_and_row_dicts_a[i]
      assert row_id == expected_row_id
      assert row_dict == expected_row_dict

   # Partitions 1 and 3
   __a				     =    SparkExamplesIterable(SCREAMING_SNAKE_CASE__    ).shard_data_sources(worker_id=1,					num_workers=2    )
   assert shard_it_a.n_shards == 2
   __a				     =    _get_expected_row_ids_and_row_dicts_for_partition_order(SCREAMING_SNAKE_CASE__,					[1, 3]    )
   for i, (row_id, row_dict) in enumerate(SCREAMING_SNAKE_CASE__    ):
      __a     ,			__a				     =    expected_row_ids_and_row_dicts_a[i]
      assert row_id == expected_row_id
      assert row_dict == expected_row_dict



@require_not_windows
@require_dill_gt_0_3_2
def    __UpperCAmelCase	(				)  ->			List[Any]:

   """simple docstring"""
   __a				     =    pyspark.sql.SparkSession.builder.master('local[*]'    ).appName('pyspark'    ).getOrCreate()
   __a				     =    spark.range(100    ).repartition(1    )
   __a				     =    Spark(SCREAMING_SNAKE_CASE__    )
   # Choose a small max_shard_size for maximum partitioning.
   spark_builder._repartition_df_if_needed(max_shard_size=1    )
   # The new number of partitions should not be greater than the number of rows.
   assert spark_builder.df.rdd.getNumPartitions() == 100







'''simple docstring'''



def    __UpperCAmelCase	(				SCREAMING_SNAKE_CASE__:	int,					SCREAMING_SNAKE_CASE__:	bool = False    )  ->			bool:

   """simple docstring"""
   if n == 2:
      return True
   if not n % 2 or n < 2:
      return False
   if n > 5 and n % 10 not in (1, 3, 7, 9):  # can quickly check last digit
      return False
   if n > 3317044064679887385961981 and not allow_probable:
      raise ValueError(
          'Warning: upper bound of deterministic test is exceeded. '
          'Pass allow_probable=True to allow probabilistic test. '
          'A return value of True indicates a probable prime.'    )
   # array bounds provided by analysis
   __a				     =    [
       2047,
       1373653,
       25326001,
       3215031751,
       2152302898747,
       3474749660383,
       341550071728321,
       1,
       3825123056546413051,
       1,
       1,
       318665857834031151167461,
       3317044064679887385961981,
   ]

   __a				     =    [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41]
   for idx, _p in enumerate(SCREAMING_SNAKE_CASE__,					1    ):
      if n < _p:
         # then we have our last prime to check
         __a				     =    primes[:idx]
         break
   __a     ,			__a				     =    n - 1, 0
   # break up n -1 into a power of 2 (s) and
   # remaining odd component
   # essentially, solve for d * 2 ** s == n - 1
   while d % 2 == 0:
      d //= 2
      s += 1
   for prime in plist:
      __a				     =    False
      for r in range(SCREAMING_SNAKE_CASE__    ):
         __a				     =    pow(SCREAMING_SNAKE_CASE__,					d * 2**r,					SCREAMING_SNAKE_CASE__    )
         # see article for analysis explanation for m
         if (r == 0 and m == 1) or ((m + 1) % n == 0):
            __a				     =    True
            # this loop will not determine compositeness
            break
      if pr:
         continue
      # if pr is False, then the above loop never evaluated to true,
      # and the n MUST be composite
      return False
   return True



def    __UpperCAmelCase	(				)  ->			None:

   """simple docstring"""
   assert not miller_rabin(561    )
   assert miller_rabin(563    )
   # 2047

   assert not miller_rabin(838201    )
   assert miller_rabin(838207    )
   # 1_373_653

   assert not miller_rabin(17316001    )
   assert miller_rabin(17316017    )
   # 25_326_001

   assert not miller_rabin(3078386641    )
   assert miller_rabin(3078386653    )
   # 3_215_031_751

   assert not miller_rabin(1713045574801    )
   assert miller_rabin(1713045574819    )
   # 2_152_302_898_747

   assert not miller_rabin(2779799728307    )
   assert miller_rabin(2779799728327    )
   # 3_474_749_660_383

   assert not miller_rabin(113850023909441    )
   assert miller_rabin(113850023909527    )
   # 341_550_071_728_321

   assert not miller_rabin(1275041018848804351    )
   assert miller_rabin(1275041018848804391    )
   # 3_825_123_056_546_413_051

   assert not miller_rabin(79666464458507787791867    )
   assert miller_rabin(79666464458507787791951    )
   # 318_665_857_834_031_151_167_461

   assert not miller_rabin(552840677446647897660333    )
   assert miller_rabin(552840677446647897660359    )
   # 3_317_044_064_679_887_385_961_981
   # upper limit for probabilistic test


if __name__ == "__main__":
 test_miller_rabin()