





from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)





def 						lowerCamelCase__     (     UpperCamelCase__			:       List[Any]       ,    UpperCamelCase__			:       List[str]						)			-> int:



					'''simple docstring'''




					_snake_case									=				[]
					for part_id in partition_order:
										_snake_case									=				df.where(F'''SPARK_PARTITION_ID() = {part_id}'''						).collect()
										for row_idx, row in enumerate(_lowerCAmelCase						):
															expected_row_ids_and_row_dicts.append((F'''{part_id}_{row_idx}''', row.asDict())						)
					return expected_row_ids_and_row_dicts





@require_not_windows
@require_dill_gt_0_3_2
def 						lowerCamelCase__     (     )			-> List[str]:



					'''simple docstring'''




					_snake_case									=				pyspark.sql.SparkSession.builder.master('local[*]'						).appName('pyspark'						).getOrCreate()
					_snake_case									=				spark.range(100						).repartition(1						)
					_snake_case									=				Spark(_lowerCAmelCase						)
					# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
					# that each partition can hold 2 rows.
					spark_builder._repartition_df_if_needed(max_shard_size=16						)
					# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
					assert spark_builder.df.rdd.getNumPartitions() == 50





@require_not_windows
@require_dill_gt_0_3_2
def 						lowerCamelCase__     (     )			-> Union[str, Any]:



					'''simple docstring'''




					_snake_case									=				pyspark.sql.SparkSession.builder.master('local[*]'						).appName('pyspark'						).getOrCreate()
					_snake_case									=				spark.range(10						).repartition(2						)
					_snake_case									=				[1, 0]
					_snake_case									=				_generate_iterable_examples(_lowerCAmelCase       ,    _lowerCAmelCase						)  # Reverse the partitions.
					_snake_case									=				_get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase       ,    _lowerCAmelCase						)

					for i, (row_id, row_dict) in enumerate(generate_fn()						):
										_snake_case									=				expected_row_ids_and_row_dicts[i]
										assert row_id == expected_row_id
										assert row_dict == expected_row_dict





@require_not_windows
@require_dill_gt_0_3_2
def 						lowerCamelCase__     (     )			-> Any:



					'''simple docstring'''




					_snake_case									=				pyspark.sql.SparkSession.builder.master('local[*]'						).appName('pyspark'						).getOrCreate()
					_snake_case									=				spark.range(10						).repartition(1						)
					_snake_case									=				SparkExamplesIterable(_lowerCAmelCase						)
					assert it.n_shards == 1
					for i, (row_id, row_dict) in enumerate(_lowerCAmelCase						):
										assert row_id == F'''0_{i}'''
										assert row_dict == {"id": i}





@require_not_windows
@require_dill_gt_0_3_2
def 						lowerCamelCase__     (     )			-> Optional[Any]:



					'''simple docstring'''




					_snake_case									=				pyspark.sql.SparkSession.builder.master('local[*]'						).appName('pyspark'						).getOrCreate()
					_snake_case									=				spark.range(30						).repartition(3						)
					# Mock the generator so that shuffle reverses the partition indices.
					with patch('numpy.random.Generator'						) as generator_mock:
										_snake_case									=				lambda UpperCamelCase__						: x.reverse()
										_snake_case									=				_get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase       ,    [2, 1, 0]						)

										_snake_case									=				SparkExamplesIterable(_lowerCAmelCase						).shuffle_data_sources(_lowerCAmelCase						)
										assert shuffled_it.n_shards == 3
										for i, (row_id, row_dict) in enumerate(_lowerCAmelCase						):
															_snake_case									=				expected_row_ids_and_row_dicts[i]
															assert row_id == expected_row_id
															assert row_dict == expected_row_dict





@require_not_windows
@require_dill_gt_0_3_2
def 						lowerCamelCase__     (     )			-> Dict:



					'''simple docstring'''




					_snake_case									=				pyspark.sql.SparkSession.builder.master('local[*]'						).appName('pyspark'						).getOrCreate()
					_snake_case									=				spark.range(20						).repartition(4						)

					# Partitions 0 and 2
					_snake_case									=				SparkExamplesIterable(_lowerCAmelCase						).shard_data_sources(worker_id=0       ,    num_workers=2						)
					assert shard_it_a.n_shards == 2
					_snake_case									=				_get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase       ,    [0, 2]						)
					for i, (row_id, row_dict) in enumerate(_lowerCAmelCase						):
										_snake_case									=				expected_row_ids_and_row_dicts_a[i]
										assert row_id == expected_row_id
										assert row_dict == expected_row_dict

					# Partitions 1 and 3
					_snake_case									=				SparkExamplesIterable(_lowerCAmelCase						).shard_data_sources(worker_id=1       ,    num_workers=2						)
					assert shard_it_a.n_shards == 2
					_snake_case									=				_get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase       ,    [1, 3]						)
					for i, (row_id, row_dict) in enumerate(_lowerCAmelCase						):
										_snake_case									=				expected_row_ids_and_row_dicts_a[i]
										assert row_id == expected_row_id
										assert row_dict == expected_row_dict





@require_not_windows
@require_dill_gt_0_3_2
def 						lowerCamelCase__     (     )			-> Any:



					'''simple docstring'''




					_snake_case									=				pyspark.sql.SparkSession.builder.master('local[*]'						).appName('pyspark'						).getOrCreate()
					_snake_case									=				spark.range(100						).repartition(1						)
					_snake_case									=				Spark(_lowerCAmelCase						)
					# Choose a small max_shard_size for maximum partitioning.
					spark_builder._repartition_df_if_needed(max_shard_size=1						)
					# The new number of partitions should not be greater than the number of rows.
					assert spark_builder.df.rdd.getNumPartitions() == 100










UpperCAmelCase_      							=     range(2, 20 + 1)
UpperCAmelCase_      							=     [10**k for k in range(ks[-1] + 1)]
UpperCAmelCase_      							=     {}





def 						lowerCamelCase__     (     UpperCamelCase__			:       Optional[Any]       ,    UpperCamelCase__			:       List[Any]       ,    UpperCamelCase__			:       Union[str, Any]       ,    UpperCamelCase__			:       str						)			-> List[str]:



					'''simple docstring'''




					_snake_case									=				sum(a_i[j] for j in range(UpperCamelCase__       ,    len(UpperCamelCase__						)						)						)
					_snake_case									=				sum(a_i[j] * base[j] for j in range(min(len(UpperCamelCase__						)       ,    UpperCamelCase__						)						)						)

					_snake_case							,     _snake_case									=				0, 0
					_snake_case									=				n - i

					_snake_case									=				memo.get(UpperCamelCase__						)

					if sub_memo is not None:
										_snake_case									=				sub_memo.get(UpperCamelCase__						)

										if jumps is not None and len(UpperCamelCase__						) > 0:
															# find and make the largest jump without going over
															_snake_case									=				-1
															for _k in range(len(UpperCamelCase__						) - 1       ,    -1       ,    -1						):
																				if jumps[_k][2] <= k and jumps[_k][1] <= max_dn:
																									_snake_case									=				_k
																									break

															if max_jump >= 0:
																				_snake_case							,     _snake_case							,     _snake_case									=				jumps[max_jump]
																				# since the difference between jumps is cached, add c
																				_snake_case									=				diff + c
																				for j in range(min(UpperCamelCase__       ,    len(UpperCamelCase__						)						)						):
																									_snake_case							,     _snake_case									=				divmod(UpperCamelCase__       ,    10						)
																				if new_c > 0:
																									add(UpperCamelCase__       ,    UpperCamelCase__       ,    UpperCamelCase__						)

										else:
															_snake_case									=				[]
					else:
										_snake_case									=				{c: []}
										_snake_case									=				sub_memo

					if dn >= max_dn or c + diff >= base[k]:
										return diff, dn

					if k > ks[0]:
										while True:
															# keep doing smaller jumps
															_snake_case							,     _snake_case									=				next_term(UpperCamelCase__       ,    k - 1       ,    i + dn       ,    UpperCamelCase__						)
															diff += _diff
															dn += terms_jumped

															if dn >= max_dn or c + diff >= base[k]:
																				break
					else:
										# would be too small a jump, just compute sequential terms instead
										_snake_case							,     _snake_case									=				compute(UpperCamelCase__       ,    UpperCamelCase__       ,    i + dn       ,    UpperCamelCase__						)
										diff += _diff
										dn += terms_jumped

					_snake_case									=				sub_memo[c]

					# keep jumps sorted by # of terms skipped
					_snake_case									=				0
					while j < len(UpperCamelCase__						):
										if jumps[j][1] > dn:
															break
										j += 1

					# cache the jump for this value digitsum(b) and c
					sub_memo[c].insert(UpperCamelCase__       ,    (diff, dn, k)						)
					return (diff, dn)





def 						lowerCamelCase__     (     UpperCamelCase__			:       str       ,    UpperCamelCase__			:       str       ,    UpperCamelCase__			:       List[str]       ,    UpperCamelCase__			:       Union[str, Any]						)			-> Optional[Any]:



					'''simple docstring'''




					if i >= n:
										return 0, i
					if k > len(UpperCamelCase__						):
										a_i.extend([0 for _ in range(k - len(UpperCamelCase__						)						)]						)

					# note: a_i -> b * 10^k + c
					# ds_b -> digitsum(b)
					# ds_c -> digitsum(c)
					_snake_case									=				i
					_snake_case							,     _snake_case							,     _snake_case									=				0, 0, 0
					for j in range(len(UpperCamelCase__						)						):
										if j >= k:
															ds_b += a_i[j]
										else:
															ds_c += a_i[j]

					while i < n:
										i += 1
										_snake_case									=				ds_c + ds_b
										diff += addend
										_snake_case									=				0
										for j in range(UpperCamelCase__						):
															_snake_case									=				a_i[j] + addend
															_snake_case							,     _snake_case									=				divmod(UpperCamelCase__       ,    10						)

															ds_c += a_i[j]

										if addend > 0:
															break

					if addend > 0:
										add(UpperCamelCase__       ,    UpperCamelCase__       ,    UpperCamelCase__						)
					return diff, i - start_i





def 						lowerCamelCase__     (     UpperCamelCase__			:       Any       ,    UpperCamelCase__			:       Union[str, Any]       ,    UpperCamelCase__			:       Optional[Any]						)			-> Any:



					'''simple docstring'''




					for j in range(UpperCamelCase__       ,    len(UpperCamelCase__						)						):
										_snake_case									=				digits[j] + addend
										if s >= 10:
															_snake_case							,     _snake_case									=				divmod(UpperCamelCase__       ,    10						)
															_snake_case									=				addend // 10 + quotient
										else:
															_snake_case									=				s
															_snake_case									=				addend // 10

										if addend == 0:
															break

					while addend > 0:
										_snake_case							,     _snake_case									=				divmod(UpperCamelCase__       ,    10						)
										digits.append(UpperCamelCase__						)





def 						lowerCamelCase__     (     UpperCamelCase__			:       int = 10**15						)			-> int:



					'''simple docstring'''




					_snake_case									=				[1]
					_snake_case									=				1
					_snake_case									=				0
					while True:
										_snake_case							,     _snake_case									=				next_term(UpperCamelCase__       ,    20       ,    i + dn       ,    UpperCamelCase__						)
										dn += terms_jumped
										if dn == n - i:
															break

					_snake_case									=				0
					for j in range(len(UpperCamelCase__						)						):
										a_n += digits[j] * 10**j
					return a_n


if __name__ == "__main__":
					print(F"{solution() = }")



