





"""simple docstring"""





import json
import os
import tempfile

import transformers

import datasets
from utils import generate_example_dataset, get_duration


lowerCAmelCase__     			=			500_000

lowerCAmelCase__			,     lowerCAmelCase__     			=			os.path.split(__file__)
lowerCAmelCase__     			=			os.path.join(RESULTS_BASEPATH, '''results''', RESULTS_FILENAME.replace('''.py''', '''.json'''))


@get_duration
def       a__					(					_SCREAMING_SNAKE_CASE  ,	**_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		UpperCamelCase   =	dataset.map(**_SCREAMING_SNAKE_CASE    )


@get_duration
def       a__					(					_SCREAMING_SNAKE_CASE  ,	**_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		UpperCamelCase   =	dataset.filter(**_SCREAMING_SNAKE_CASE    )


def       a__					(					):




		"""simple docstring"""


		UpperCamelCase   =	{"num examples": SPEED_TEST_N_EXAMPLES}
		with tempfile.TemporaryDirectory() as tmp_dir:
				UpperCamelCase   =	datasets.Features({"text": datasets.Value("string"    ), "numbers": datasets.Value("float32"    )}    )
				UpperCamelCase   =	generate_example_dataset(
				    os.path.join(_SCREAMING_SNAKE_CASE  ,	"dataset.arrow"    )  ,	_SCREAMING_SNAKE_CASE  ,	num_examples=_SCREAMING_SNAKE_CASE    )

				UpperCamelCase   =	transformers.AutoTokenizer.from_pretrained("bert-base-cased"  ,	use_fast=_SCREAMING_SNAKE_CASE    )

				def tokenize(_SCREAMING_SNAKE_CASE    ):
						return tokenizer(examples["text"]    )

				UpperCamelCase   =	map(_SCREAMING_SNAKE_CASE    )

				UpperCamelCase   =	map(_SCREAMING_SNAKE_CASE  ,	batched=_SCREAMING_SNAKE_CASE    )

				UpperCamelCase   =	map(_SCREAMING_SNAKE_CASE  ,	function=lambda _SCREAMING_SNAKE_CASE    : None  ,	batched=_SCREAMING_SNAKE_CASE    )

				with dataset.formatted_as(type="numpy"    ):
						UpperCamelCase   =	map(_SCREAMING_SNAKE_CASE  ,	function=lambda _SCREAMING_SNAKE_CASE    : None  ,	batched=_SCREAMING_SNAKE_CASE    )

				with dataset.formatted_as(type="pandas"    ):
						UpperCamelCase   =	map(_SCREAMING_SNAKE_CASE  ,	function=lambda _SCREAMING_SNAKE_CASE    : None  ,	batched=_SCREAMING_SNAKE_CASE    )

				with dataset.formatted_as(type="torch"  ,	columns="numbers"    ):
						UpperCamelCase   =	map(_SCREAMING_SNAKE_CASE  ,	function=lambda _SCREAMING_SNAKE_CASE    : None  ,	batched=_SCREAMING_SNAKE_CASE    )

				with dataset.formatted_as(type="tensorflow"  ,	columns="numbers"    ):
						UpperCamelCase   =	map(_SCREAMING_SNAKE_CASE  ,	function=lambda _SCREAMING_SNAKE_CASE    : None  ,	batched=_SCREAMING_SNAKE_CASE    )

				UpperCamelCase   =	map(_SCREAMING_SNAKE_CASE  ,	function=_SCREAMING_SNAKE_CASE  ,	batched=_SCREAMING_SNAKE_CASE    )

				UpperCamelCase   =	filter(_SCREAMING_SNAKE_CASE    )

				# Activate later when tokenizer support batched inputs
				# with dataset.formatted_as(type='numpy'):
				#     times[func.__name__ + " fast-tokenizer batched numpy"] = func(dataset, function=tokenize, batched=True)

		with open(_SCREAMING_SNAKE_CASE  ,	"wb"    ) as f:
				f.write(json.dumps(_SCREAMING_SNAKE_CASE    ).encode("utf-8"    )    )


if __name__ == "__main__":  # useful to run the profiler
						benchmark_map_filter()













"""simple docstring"""





import pytest

from datasets.parallel import ParallelBackendConfig, parallel_backend
from datasets.utils.py_utils import map_nested

from .utils import require_dill_gt_0_3_2, require_joblibspark, require_not_windows


def       a__					(					_SCREAMING_SNAKE_CASE    ):  # picklable for multiprocessing




		"""simple docstring"""


		return i + 1


@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
def       a__					(					):




		"""simple docstring"""


		with parallel_backend("spark"    ):
				assert ParallelBackendConfig.backend_name == "spark"

		UpperCamelCase   =	[1, 2, 3]
		with pytest.raises(_SCREAMING_SNAKE_CASE    ):
				with parallel_backend("unsupported backend"    ):
						map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=2    )

		with pytest.raises(_SCREAMING_SNAKE_CASE    ):
				with parallel_backend("unsupported backend"    ):
						map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=-1    )






@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
@pytest.mark.parametrize("num_proc"  ,	[2, -1]    )
def       a__					(					_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		UpperCamelCase   =	[1, 2]
		UpperCamelCase   =	{"a": 1, "b": 2}
		UpperCamelCase   =	{"a": [1, 2], "b": [3, 4]}
		UpperCamelCase   =	{"a": {"1": 1}, "b": 2}
		UpperCamelCase   =	{"a": 1, "b": 2, "c": 3, "d": 4}
		UpperCamelCase   =	[2, 3]
		UpperCamelCase   =	{"a": 2, "b": 3}
		UpperCamelCase   =	{"a": [2, 3], "b": [4, 5]}
		UpperCamelCase   =	{"a": {"1": 2}, "b": 3}
		UpperCamelCase   =	{"a": 2, "b": 3, "c": 4, "d": 5}

		with parallel_backend("spark"    ):
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
				assert map_nested(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	num_proc=_SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa






