


"""simple docstring"""






from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)
def       A_   (   _lowerCAmelCase     :					Union[str, Any],     _lowerCAmelCase     :					Dict       ):






     """simple docstring"""

     _a    	=				[]
     for part_id in partition_order:
          _a    	=				df.where(f'SPARK_PARTITION_ID() = {part_id}'       ).collect()
          for row_idx, row in enumerate(_lowerCAmelCase       ):
               expected_row_ids_and_row_dicts.append((f'{part_id}_{row_idx}', row.asDict())       )
     return expected_row_ids_and_row_dicts
@require_not_windows
@require_dill_gt_0_3_2
def       A_   (   ):






     """simple docstring"""

     _a    	=				pyspark.sql.SparkSession.builder.master('''local[*]'''       ).appName('''pyspark'''       ).getOrCreate()
     _a    	=				spark.range(1_00       ).repartition(1       )
     _a    	=				Spark(_lowerCAmelCase       )
     # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
     # that each partition can hold 2 rows.
     spark_builder._repartition_df_if_needed(max_shard_size=16       )
     # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
     assert spark_builder.df.rdd.getNumPartitions() == 50
@require_not_windows
@require_dill_gt_0_3_2
def       A_   (   ):






     """simple docstring"""

     _a    	=				pyspark.sql.SparkSession.builder.master('''local[*]'''       ).appName('''pyspark'''       ).getOrCreate()
     _a    	=				spark.range(10       ).repartition(2       )
     _a    	=				[1, 0]
     _a    	=				_generate_iterable_examples(_lowerCAmelCase,     _lowerCAmelCase       )  # Reverse the partitions.
     _a    	=				_get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase,     _lowerCAmelCase       )

     for i, (row_id, row_dict) in enumerate(generate_fn()       ):
          _a	,      _a    	=				expected_row_ids_and_row_dicts[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict
@require_not_windows
@require_dill_gt_0_3_2
def       A_   (   ):






     """simple docstring"""

     _a    	=				pyspark.sql.SparkSession.builder.master('''local[*]'''       ).appName('''pyspark'''       ).getOrCreate()
     _a    	=				spark.range(10       ).repartition(1       )
     _a    	=				SparkExamplesIterable(_lowerCAmelCase       )
     assert it.n_shards == 1
     for i, (row_id, row_dict) in enumerate(_lowerCAmelCase       ):
          assert row_id == f'0_{i}'
          assert row_dict == {"id": i}
@require_not_windows
@require_dill_gt_0_3_2
def       A_   (   ):






     """simple docstring"""

     _a    	=				pyspark.sql.SparkSession.builder.master('''local[*]'''       ).appName('''pyspark'''       ).getOrCreate()
     _a    	=				spark.range(30       ).repartition(3       )
     # Mock the generator so that shuffle reverses the partition indices.
     with patch('''numpy.random.Generator'''       ) as generator_mock:
          _a    	=				lambda _lowerCAmelCase       : x.reverse()
          _a    	=				_get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase,     [2, 1, 0]       )

          _a    	=				SparkExamplesIterable(_lowerCAmelCase       ).shuffle_data_sources(_lowerCAmelCase       )
          assert shuffled_it.n_shards == 3
          for i, (row_id, row_dict) in enumerate(_lowerCAmelCase       ):
               _a	,      _a    	=				expected_row_ids_and_row_dicts[i]
               assert row_id == expected_row_id
               assert row_dict == expected_row_dict
@require_not_windows
@require_dill_gt_0_3_2
def       A_   (   ):






     """simple docstring"""

     _a    	=				pyspark.sql.SparkSession.builder.master('''local[*]'''       ).appName('''pyspark'''       ).getOrCreate()
     _a    	=				spark.range(20       ).repartition(4       )

     # Partitions 0 and 2
     _a    	=				SparkExamplesIterable(_lowerCAmelCase       ).shard_data_sources(worker_id=0,     num_workers=2       )
     assert shard_it_a.n_shards == 2
     _a    	=				_get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase,     [0, 2]       )
     for i, (row_id, row_dict) in enumerate(_lowerCAmelCase       ):
          _a	,      _a    	=				expected_row_ids_and_row_dicts_a[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict

     # Partitions 1 and 3
     _a    	=				SparkExamplesIterable(_lowerCAmelCase       ).shard_data_sources(worker_id=1,     num_workers=2       )
     assert shard_it_a.n_shards == 2
     _a    	=				_get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase,     [1, 3]       )
     for i, (row_id, row_dict) in enumerate(_lowerCAmelCase       ):
          _a	,      _a    	=				expected_row_ids_and_row_dicts_a[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict

@require_not_windows
@require_dill_gt_0_3_2
def       A_   (   ):






     """simple docstring"""

     _a    	=				pyspark.sql.SparkSession.builder.master('''local[*]'''       ).appName('''pyspark'''       ).getOrCreate()
     _a    	=				spark.range(1_00       ).repartition(1       )
     _a    	=				Spark(_lowerCAmelCase       )
     # Choose a small max_shard_size for maximum partitioning.
     spark_builder._repartition_df_if_needed(max_shard_size=1       )
     # The new number of partitions should not be greater than the number of rows.
     assert spark_builder.df.rdd.getNumPartitions() == 1_00



"""simple docstring"""






def       A_   (   _lowerCAmelCase     :					list,     _lowerCAmelCase     :					list,     _lowerCAmelCase     :					int       ):






     """simple docstring"""

     _a    	=				len(_lowerCAmelCase       )
     _a    	=				[[0] * n for i in range(_lowerCAmelCase       )]
     for i in range(_lowerCAmelCase       ):
          _a    	=				y_points[i]

     for i in range(2,     _lowerCAmelCase       ):
          for j in range(_lowerCAmelCase,     _lowerCAmelCase       ):
               _a    	=				(
                   (xa - x_points[j - i + 1]) * q[j][i - 1]
                   - (xa - x_points[j]) * q[j - 1][i - 1]
               ) / (x_points[j] - x_points[j - i + 1])

     return [q[n - 1][n - 1], q]


if __name__ == "__main__":
   import doctest

   doctest.testmod()