





"""simple docstring"""





from typing import Optional

from .. import Features, NamedSplit
from ..packaged_modules.text.text import Text
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader





class        _lowerCamelCase      (       _lowercase ):


						def __init__(self     :   List[str]   ,							__a     :   NestedDataStructureLike[PathLike]   ,							__a     :   Optional[NamedSplit] = None   ,							__a     :   Optional[Features] = None   ,							__a     :   str = None   ,							__a     :   bool = False   ,							__a     :   bool = False   ,							__a     :   Optional[int] = None   ,							**__a     :   Optional[Any]   ,							) ->   Any:
								super().__init__(
								    __UpperCamelCase   ,							split=__UpperCamelCase   ,							features=__UpperCamelCase   ,							cache_dir=__UpperCamelCase   ,							keep_in_memory=__UpperCamelCase   ,							streaming=__UpperCamelCase   ,							num_proc=__UpperCamelCase   ,							**__UpperCamelCase   ,							)
								UpperCamelCase   =	path_or_paths if isinstance(__UpperCamelCase   ,							__UpperCamelCase		) else {self.split: path_or_paths}
								UpperCamelCase   =	Text(
								    cache_dir=__UpperCamelCase   ,							data_files=__UpperCamelCase   ,							features=__UpperCamelCase   ,							**__UpperCamelCase   ,							)




						def     snake_case_   (self     :   Optional[Any]		) ->   str:
								# Build iterable dataset
								if self.streaming:
										UpperCamelCase   =	self.builder.as_streaming_dataset(split=self.split		)
								# Build regular (map-style) dataset
								else:
										UpperCamelCase   =	None
										UpperCamelCase   =	None
										UpperCamelCase   =	None
										UpperCamelCase   =	None

										self.builder.download_and_prepare(
										    download_config=__UpperCamelCase   ,							download_mode=__UpperCamelCase   ,							verification_mode=__UpperCamelCase   ,							base_path=__UpperCamelCase   ,							num_proc=self.num_proc   ,							)
										UpperCamelCase   =	self.builder.as_dataset(
										    split=self.split   ,							verification_mode=__UpperCamelCase   ,							in_memory=self.keep_in_memory		)
								return dataset













"""simple docstring"""





import pytest

from datasets.parallel import ParallelBackendConfig, parallel_backend
from datasets.utils.py_utils import map_nested

from .utils import require_dill_gt_0_3_2, require_joblibspark, require_not_windows


def       a__					(					_SCREAMING_SNAKE_CASE    ):  # picklable for multiprocessing




		"""simple docstring"""


		return i + 1


@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
def       a__					(					):




		"""simple docstring"""


		with parallel_backend("spark"    ):
				assert ParallelBackendConfig.backend_name == "spark"

		UpperCamelCase   =	[1, 2, 3]
		with pytest.raises(_SCREAMING_SNAKE_CASE    ):
				with parallel_backend("unsupported backend"    ):
						map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=2    )

		with pytest.raises(_SCREAMING_SNAKE_CASE    ):
				with parallel_backend("unsupported backend"    ):
						map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=-1    )






@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
@pytest.mark.parametrize("num_proc"  ,	[2, -1]    )
def       a__					(					_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		UpperCamelCase   =	[1, 2]
		UpperCamelCase   =	{"a": 1, "b": 2}
		UpperCamelCase   =	{"a": [1, 2], "b": [3, 4]}
		UpperCamelCase   =	{"a": {"1": 1}, "b": 2}
		UpperCamelCase   =	{"a": 1, "b": 2, "c": 3, "d": 4}
		UpperCamelCase   =	[2, 3]
		UpperCamelCase   =	{"a": 2, "b": 3}
		UpperCamelCase   =	{"a": [2, 3], "b": [4, 5]}
		UpperCamelCase   =	{"a": {"1": 2}, "b": 3}
		UpperCamelCase   =	{"a": 2, "b": 3, "c": 4, "d": 5}

		with parallel_backend("spark"    ):
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa






