





from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)





def 				__lowerCAmelCase(       _SCREAMING_SNAKE_CASE							,     _SCREAMING_SNAKE_CASE				)							-> List[str]:




    """simple docstring"""




    _A     =							[]
    for part_id in partition_order:
        _A     =							df.where(F"SPARK_PARTITION_ID() = {part_id}"				).collect()
        for row_idx, row in enumerate(_SCREAMING_SNAKE_CASE				):
            expected_row_ids_and_row_dicts.append((F"{part_id}_{row_idx}", row.asDict())				)
    return expected_row_ids_and_row_dicts





@require_not_windows
@require_dill_gt_0_3_2
def 				__lowerCAmelCase(       )							-> str:




    """simple docstring"""




    _A     =							pyspark.sql.SparkSession.builder.master('local[*]'				).appName('pyspark'				).getOrCreate()
    _A     =							spark.range(100				).repartition(1				)
    _A     =							Spark(_SCREAMING_SNAKE_CASE				)
    # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
    # that each partition can hold 2 rows.
    spark_builder._repartition_df_if_needed(max_shard_size=16				)
    # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
    assert spark_builder.df.rdd.getNumPartitions() == 50





@require_not_windows
@require_dill_gt_0_3_2
def 				__lowerCAmelCase(       )							-> int:




    """simple docstring"""




    _A     =							pyspark.sql.SparkSession.builder.master('local[*]'				).appName('pyspark'				).getOrCreate()
    _A     =							spark.range(10				).repartition(2				)
    _A     =							[1, 0]
    _A     =							_generate_iterable_examples(_SCREAMING_SNAKE_CASE							,     _SCREAMING_SNAKE_CASE				)  # Reverse the partitions.
    _A     =							_get_expected_row_ids_and_row_dicts_for_partition_order(_SCREAMING_SNAKE_CASE							,     _SCREAMING_SNAKE_CASE				)

    for i, (row_id, row_dict) in enumerate(generate_fn()				):
        _A,			_A     =							expected_row_ids_and_row_dicts[i]
        assert row_id == expected_row_id
        assert row_dict == expected_row_dict





@require_not_windows
@require_dill_gt_0_3_2
def 				__lowerCAmelCase(       )							-> Optional[int]:




    """simple docstring"""




    _A     =							pyspark.sql.SparkSession.builder.master('local[*]'				).appName('pyspark'				).getOrCreate()
    _A     =							spark.range(10				).repartition(1				)
    _A     =							SparkExamplesIterable(_SCREAMING_SNAKE_CASE				)
    assert it.n_shards == 1
    for i, (row_id, row_dict) in enumerate(_SCREAMING_SNAKE_CASE				):
        assert row_id == F"0_{i}"
        assert row_dict == {"id": i}





@require_not_windows
@require_dill_gt_0_3_2
def 				__lowerCAmelCase(       )							-> Any:




    """simple docstring"""




    _A     =							pyspark.sql.SparkSession.builder.master('local[*]'				).appName('pyspark'				).getOrCreate()
    _A     =							spark.range(30				).repartition(3				)
    # Mock the generator so that shuffle reverses the partition indices.
    with patch('numpy.random.Generator'				) as generator_mock:
        _A     =							lambda _SCREAMING_SNAKE_CASE				: x.reverse()
        _A     =							_get_expected_row_ids_and_row_dicts_for_partition_order(_SCREAMING_SNAKE_CASE							,     [2, 1, 0]				)

        _A     =							SparkExamplesIterable(_SCREAMING_SNAKE_CASE				).shuffle_data_sources(_SCREAMING_SNAKE_CASE				)
        assert shuffled_it.n_shards == 3
        for i, (row_id, row_dict) in enumerate(_SCREAMING_SNAKE_CASE				):
            _A,			_A     =							expected_row_ids_and_row_dicts[i]
            assert row_id == expected_row_id
            assert row_dict == expected_row_dict





@require_not_windows
@require_dill_gt_0_3_2
def 				__lowerCAmelCase(       )							-> List[str]:




    """simple docstring"""




    _A     =							pyspark.sql.SparkSession.builder.master('local[*]'				).appName('pyspark'				).getOrCreate()
    _A     =							spark.range(20				).repartition(4				)

    # Partitions 0 and 2
    _A     =							SparkExamplesIterable(_SCREAMING_SNAKE_CASE				).shard_data_sources(worker_id=0							,     num_workers=2				)
    assert shard_it_a.n_shards == 2
    _A     =							_get_expected_row_ids_and_row_dicts_for_partition_order(_SCREAMING_SNAKE_CASE							,     [0, 2]				)
    for i, (row_id, row_dict) in enumerate(_SCREAMING_SNAKE_CASE				):
        _A,			_A     =							expected_row_ids_and_row_dicts_a[i]
        assert row_id == expected_row_id
        assert row_dict == expected_row_dict

    # Partitions 1 and 3
    _A     =							SparkExamplesIterable(_SCREAMING_SNAKE_CASE				).shard_data_sources(worker_id=1							,     num_workers=2				)
    assert shard_it_a.n_shards == 2
    _A     =							_get_expected_row_ids_and_row_dicts_for_partition_order(_SCREAMING_SNAKE_CASE							,     [1, 3]				)
    for i, (row_id, row_dict) in enumerate(_SCREAMING_SNAKE_CASE				):
        _A,			_A     =							expected_row_ids_and_row_dicts_a[i]
        assert row_id == expected_row_id
        assert row_dict == expected_row_dict





@require_not_windows
@require_dill_gt_0_3_2
def 				__lowerCAmelCase(       )							-> int:




    """simple docstring"""




    _A     =							pyspark.sql.SparkSession.builder.master('local[*]'				).appName('pyspark'				).getOrCreate()
    _A     =							spark.range(100				).repartition(1				)
    _A     =							Spark(_SCREAMING_SNAKE_CASE				)
    # Choose a small max_shard_size for maximum partitioning.
    spark_builder._repartition_df_if_needed(max_shard_size=1				)
    # The new number of partitions should not be greater than the number of rows.
    assert spark_builder.df.rdd.getNumPartitions() == 100












def 				__lowerCAmelCase(       _SCREAMING_SNAKE_CASE							,     _SCREAMING_SNAKE_CASE				)							-> int:




    """simple docstring"""




    return int((input_a, input_a).count(0				) != 0				)





def 				__lowerCAmelCase(       )							-> None:




    """simple docstring"""




    assert nand_gate(0							,     0				) == 1
    assert nand_gate(0							,     1				) == 1
    assert nand_gate(1							,     0				) == 1
    assert nand_gate(1							,     1				) == 0


if __name__ == "__main__":
 print(nand_gate(0, 0))
 print(nand_gate(0, 1))
 print(nand_gate(1, 0))
 print(nand_gate(1, 1))





