
'''simple docstring'''
from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)







def 		A_(     A						:		Any		,     A						:		str):
							UpperCamelCase   			=     []
							for part_id in partition_order:
														UpperCamelCase   			=     df.where(f'''SPARK_PARTITION_ID() = {part_id}''').collect()
														for row_idx, row in enumerate(A):
																					expected_row_ids_and_row_dicts.append((f'''{part_id}_{row_idx}''', row.asDict()))
							return expected_row_ids_and_row_dicts







@require_not_windows
@require_dill_gt_0_3_2
def 		A_(     ):
							UpperCamelCase   			=     pyspark.sql.SparkSession.builder.master('local[*]').appName('pyspark').getOrCreate()
							UpperCamelCase   			=     spark.range(100).repartition(1)
							UpperCamelCase   			=     Spark(A)
							# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
							# that each partition can hold 2 rows.
							spark_builder._repartition_df_if_needed(max_shard_size=16)
							# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
							assert spark_builder.df.rdd.getNumPartitions() == 50







@require_not_windows
@require_dill_gt_0_3_2
def 		A_(     ):
							UpperCamelCase   			=     pyspark.sql.SparkSession.builder.master('local[*]').appName('pyspark').getOrCreate()
							UpperCamelCase   			=     spark.range(10).repartition(2)
							UpperCamelCase   			=     [1, 0]
							UpperCamelCase   			=     _generate_iterable_examples(A		,     A)  # Reverse the partitions.
							UpperCamelCase   			=     _get_expected_row_ids_and_row_dicts_for_partition_order(A		,     A)

							for i, (row_id, row_dict) in enumerate(generate_fn()):
														UpperCamelCase			,	UpperCamelCase   			=     expected_row_ids_and_row_dicts[i]
														assert row_id == expected_row_id
														assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def 		A_(     ):
							UpperCamelCase   			=     pyspark.sql.SparkSession.builder.master('local[*]').appName('pyspark').getOrCreate()
							UpperCamelCase   			=     spark.range(10).repartition(1)
							UpperCamelCase   			=     SparkExamplesIterable(A)
							assert it.n_shards == 1
							for i, (row_id, row_dict) in enumerate(A):
														assert row_id == f'''0_{i}'''
														assert row_dict == {"id": i}







@require_not_windows
@require_dill_gt_0_3_2
def 		A_(     ):
							UpperCamelCase   			=     pyspark.sql.SparkSession.builder.master('local[*]').appName('pyspark').getOrCreate()
							UpperCamelCase   			=     spark.range(30).repartition(3)
							# Mock the generator so that shuffle reverses the partition indices.
							with patch('numpy.random.Generator') as generator_mock:
														UpperCamelCase   			=     lambda A: x.reverse()
														UpperCamelCase   			=     _get_expected_row_ids_and_row_dicts_for_partition_order(A		,     [2, 1, 0])

														UpperCamelCase   			=     SparkExamplesIterable(A).shuffle_data_sources(A)
														assert shuffled_it.n_shards == 3
														for i, (row_id, row_dict) in enumerate(A):
																					UpperCamelCase			,	UpperCamelCase   			=     expected_row_ids_and_row_dicts[i]
																					assert row_id == expected_row_id
																					assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def 		A_(     ):
							UpperCamelCase   			=     pyspark.sql.SparkSession.builder.master('local[*]').appName('pyspark').getOrCreate()
							UpperCamelCase   			=     spark.range(20).repartition(4)

							# Partitions 0 and 2
							UpperCamelCase   			=     SparkExamplesIterable(A).shard_data_sources(worker_id=0		,     num_workers=2)
							assert shard_it_a.n_shards == 2
							UpperCamelCase   			=     _get_expected_row_ids_and_row_dicts_for_partition_order(A		,     [0, 2])
							for i, (row_id, row_dict) in enumerate(A):
														UpperCamelCase			,	UpperCamelCase   			=     expected_row_ids_and_row_dicts_a[i]
														assert row_id == expected_row_id
														assert row_dict == expected_row_dict

							# Partitions 1 and 3
							UpperCamelCase   			=     SparkExamplesIterable(A).shard_data_sources(worker_id=1		,     num_workers=2)
							assert shard_it_a.n_shards == 2
							UpperCamelCase   			=     _get_expected_row_ids_and_row_dicts_for_partition_order(A		,     [1, 3])
							for i, (row_id, row_dict) in enumerate(A):
														UpperCamelCase			,	UpperCamelCase   			=     expected_row_ids_and_row_dicts_a[i]
														assert row_id == expected_row_id
														assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def 		A_(     ):
							UpperCamelCase   			=     pyspark.sql.SparkSession.builder.master('local[*]').appName('pyspark').getOrCreate()
							UpperCamelCase   			=     spark.range(100).repartition(1)
							UpperCamelCase   			=     Spark(A)
							# Choose a small max_shard_size for maximum partitioning.
							spark_builder._repartition_df_if_needed(max_shard_size=1)
							# The new number of partitions should not be greater than the number of rows.
							assert spark_builder.df.rdd.getNumPartitions() == 100






'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


lowerCAmelCase					:    Optional[Any]           = {
    'configuration_falcon': ['FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP', 'FalconConfig'],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCAmelCase					:    str           = [
						    'FALCON_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'FalconForCausalLM',
						    'FalconModel',
						    'FalconPreTrainedModel',
						    'FalconForSequenceClassification',
						    'FalconForTokenClassification',
						    'FalconForQuestionAnswering',
						]


if TYPE_CHECKING:
						from .configuration_falcon import FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP, FalconConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_falcon import (
												    FALCON_PRETRAINED_MODEL_ARCHIVE_LIST,
												    FalconForCausalLM,
												    FalconForQuestionAnswering,
												    FalconForSequenceClassification,
												    FalconForTokenClassification,
												    FalconModel,
												    FalconPreTrainedModel,
												)


else:
						import sys

						lowerCAmelCase					:    Tuple           = _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




