


'''simple docstring'''



from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)




def 			_a			(		_lowercase      :			str    ,      _lowercase      :			Tuple						):





     '''simple docstring'''






     __UpperCAmelCase			:				List[str]													=    []
     for part_id in partition_order:
          __UpperCAmelCase			:				Optional[Any]													=    df.where(F'SPARK_PARTITION_ID() = {part_id}'						).collect()
          for row_idx, row in enumerate(_lowercase						):
               expected_row_ids_and_row_dicts.append((F'{part_id}_{row_idx}', row.asDict())						)
     return expected_row_ids_and_row_dicts




@require_not_windows
@require_dill_gt_0_3_2
def 			_a			(		):





     '''simple docstring'''






     __UpperCAmelCase			:				Any													=    pyspark.sql.SparkSession.builder.master('''local[*]'''						).appName('''pyspark'''						).getOrCreate()
     __UpperCAmelCase			:				List[Any]													=    spark.range(100						).repartition(1						)
     __UpperCAmelCase			:				Dict													=    Spark(_lowercase						)
     # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
     # that each partition can hold 2 rows.
     spark_builder._repartition_df_if_needed(max_shard_size=16						)
     # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
     assert spark_builder.df.rdd.getNumPartitions() == 50




@require_not_windows
@require_dill_gt_0_3_2
def 			_a			(		):





     '''simple docstring'''






     __UpperCAmelCase			:				Optional[int]													=    pyspark.sql.SparkSession.builder.master('''local[*]'''						).appName('''pyspark'''						).getOrCreate()
     __UpperCAmelCase			:				Union[str, Any]													=    spark.range(10						).repartition(2						)
     __UpperCAmelCase			:				List[Any]													=    [1, 0]
     __UpperCAmelCase			:				Optional[int]													=    _generate_iterable_examples(_lowercase    ,      _lowercase						)  # Reverse the partitions.
     __UpperCAmelCase			:				str													=    _get_expected_row_ids_and_row_dicts_for_partition_order(_lowercase    ,      _lowercase						)

     for i, (row_id, row_dict) in enumerate(generate_fn()						):
          __UpperCAmelCase							,     __UpperCAmelCase			:				str													=    expected_row_ids_and_row_dicts[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict




@require_not_windows
@require_dill_gt_0_3_2
def 			_a			(		):





     '''simple docstring'''






     __UpperCAmelCase			:				Optional[Any]													=    pyspark.sql.SparkSession.builder.master('''local[*]'''						).appName('''pyspark'''						).getOrCreate()
     __UpperCAmelCase			:				List[Any]													=    spark.range(10						).repartition(1						)
     __UpperCAmelCase			:				Optional[Any]													=    SparkExamplesIterable(_lowercase						)
     assert it.n_shards == 1
     for i, (row_id, row_dict) in enumerate(_lowercase						):
          assert row_id == F'0_{i}'
          assert row_dict == {"id": i}




@require_not_windows
@require_dill_gt_0_3_2
def 			_a			(		):





     '''simple docstring'''






     __UpperCAmelCase			:				Optional[Any]													=    pyspark.sql.SparkSession.builder.master('''local[*]'''						).appName('''pyspark'''						).getOrCreate()
     __UpperCAmelCase			:				Optional[int]													=    spark.range(30						).repartition(3						)
     # Mock the generator so that shuffle reverses the partition indices.
     with patch('''numpy.random.Generator'''						) as generator_mock:
          __UpperCAmelCase			:				Optional[int]													=    lambda _lowercase						: x.reverse()
          __UpperCAmelCase			:				List[str]													=    _get_expected_row_ids_and_row_dicts_for_partition_order(_lowercase    ,      [2, 1, 0]						)

          __UpperCAmelCase			:				List[str]													=    SparkExamplesIterable(_lowercase						).shuffle_data_sources(_lowercase						)
          assert shuffled_it.n_shards == 3
          for i, (row_id, row_dict) in enumerate(_lowercase						):
               __UpperCAmelCase							,     __UpperCAmelCase			:				Tuple													=    expected_row_ids_and_row_dicts[i]
               assert row_id == expected_row_id
               assert row_dict == expected_row_dict




@require_not_windows
@require_dill_gt_0_3_2
def 			_a			(		):





     '''simple docstring'''






     __UpperCAmelCase			:				Optional[int]													=    pyspark.sql.SparkSession.builder.master('''local[*]'''						).appName('''pyspark'''						).getOrCreate()
     __UpperCAmelCase			:				Tuple													=    spark.range(20						).repartition(4						)

     # Partitions 0 and 2
     __UpperCAmelCase			:				Optional[Any]													=    SparkExamplesIterable(_lowercase						).shard_data_sources(worker_id=0    ,      num_workers=2						)
     assert shard_it_a.n_shards == 2
     __UpperCAmelCase			:				Optional[int]													=    _get_expected_row_ids_and_row_dicts_for_partition_order(_lowercase    ,      [0, 2]						)
     for i, (row_id, row_dict) in enumerate(_lowercase						):
          __UpperCAmelCase							,     __UpperCAmelCase			:				Optional[Any]													=    expected_row_ids_and_row_dicts_a[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict

     # Partitions 1 and 3
     __UpperCAmelCase			:				Union[str, Any]													=    SparkExamplesIterable(_lowercase						).shard_data_sources(worker_id=1    ,      num_workers=2						)
     assert shard_it_a.n_shards == 2
     __UpperCAmelCase			:				str													=    _get_expected_row_ids_and_row_dicts_for_partition_order(_lowercase    ,      [1, 3]						)
     for i, (row_id, row_dict) in enumerate(_lowercase						):
          __UpperCAmelCase							,     __UpperCAmelCase			:				List[Any]													=    expected_row_ids_and_row_dicts_a[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict




@require_not_windows
@require_dill_gt_0_3_2
def 			_a			(		):





     '''simple docstring'''






     __UpperCAmelCase			:				Optional[Any]													=    pyspark.sql.SparkSession.builder.master('''local[*]'''						).appName('''pyspark'''						).getOrCreate()
     __UpperCAmelCase			:				Any													=    spark.range(100						).repartition(1						)
     __UpperCAmelCase			:				Any													=    Spark(_lowercase						)
     # Choose a small max_shard_size for maximum partitioning.
     spark_builder._repartition_df_if_needed(max_shard_size=1						)
     # The new number of partitions should not be greater than the number of rows.
     assert spark_builder.df.rdd.getNumPartitions() == 100



'''simple docstring'''



def 			_a			(		_lowercase      :			int = 50000000						):





     '''simple docstring'''






     __UpperCAmelCase			:				List[str]													=    set()
     __UpperCAmelCase			:				Tuple													=    int((limit - 24) ** (1 / 2)						)

     __UpperCAmelCase			:				Tuple													=    set(range(3    ,      prime_square_limit + 1    ,      2						)						)
     primes.add(2						)
     for p in range(3    ,      prime_square_limit + 1    ,      2						):
          if p not in primes:
               continue
          primes.difference_update(set(range(p * p    ,      prime_square_limit + 1    ,      _lowercase						)						)						)

     for primea in primes:
          __UpperCAmelCase			:				Any													=    primea * primea
          for primea in primes:
               __UpperCAmelCase			:				Any													=    primea * primea * primea
               if square + cube >= limit - 16:
                    break
               for primea in primes:
                    __UpperCAmelCase			:				Optional[Any]													=    primea * primea * primea * primea
                    __UpperCAmelCase			:				Union[str, Any]													=    square + cube + tetr
                    if total >= limit:
                         break
                    ret.add(_lowercase						)

     return len(_lowercase						)


if __name__ == "__main__":
      print(f"""{solution() = }""")