




import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch







def      lowerCamelCase_						(			UpperCamelCase__							: int )	->			Optional[int]:  # picklable for multiprocessing





				"""simple docstring"""


				return x.sum()







def      lowerCamelCase_						(			UpperCamelCase__							: Optional[Any] )	->			List[Any]:  # picklable for multiprocessing





				"""simple docstring"""


				return i + 1







@dataclass
class        __lowerCAmelCase					:

			"""simple docstring"""







			snake_case_  						=			42
			snake_case_  						=			42







class        __lowerCAmelCase					(			__magic_name__					):

			"""simple docstring"""







			def  lowercase_       ( self				)       -> Tuple:



							'''simple docstring'''
							__lowerCamelCase									=       {}
							__lowerCamelCase									=       []
							__lowerCamelCase									=       1
							__lowerCamelCase									=       [1, 2]
							__lowerCamelCase									=       {'a': 1, 'b': 2}
							__lowerCamelCase									=       {'a': [1, 2], 'b': [3, 4]}
							__lowerCamelCase									=       {'a': {'1': 1}, 'b': 2}
							__lowerCamelCase									=       {'a': 1, 'b': 2, 'c': 3, 'd': 4}
							__lowerCamelCase									=       {}
							__lowerCamelCase									=       []
							__lowerCamelCase									=       2
							__lowerCamelCase									=       [2, 3]
							__lowerCamelCase									=       {'a': 2, 'b': 3}
							__lowerCamelCase									=       {'a': [2, 3], 'b': [4, 5]}
							__lowerCamelCase									=       {'a': {'1': 2}, 'b': 3}
							__lowerCamelCase									=       {'a': 2, 'b': 3, 'c': 4, 'd': 5}
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__				)     ,		lowerCamelCase__				)

							__lowerCamelCase									=       2
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__     ,		num_proc=lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__     ,		num_proc=lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__     ,		num_proc=lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__     ,		num_proc=lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__     ,		num_proc=lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__     ,		num_proc=lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__     ,		num_proc=lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__     ,		num_proc=lowerCamelCase__				)     ,		lowerCamelCase__				)

							__lowerCamelCase									=       {'a': np.eye(2				), 'b': np.zeros(3				), 'c': np.ones(2				)}
							__lowerCamelCase									=       {'a': 2, 'b': 0, 'c': 2}
							__lowerCamelCase									=       {
							    'a': np.eye(2				).astype(lowerCamelCase__				),
							    'b': np.zeros(3				).astype(lowerCamelCase__				),
							    'c': np.ones(2				).astype(lowerCamelCase__				),
							}
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__     ,		map_numpy=lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(
							    {k: v.tolist() for k, v in map_nested(lowerCamelCase__     ,		lowerCamelCase__     ,		map_numpy=lowerCamelCase__				).items()}     ,		{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}     ,		)
							self.assertEqual(map_nested(lowerCamelCase__     ,		lowerCamelCase__     ,		map_numpy=lowerCamelCase__     ,		num_proc=lowerCamelCase__				)     ,		lowerCamelCase__				)
							self.assertEqual(
							    {k: v.tolist() for k, v in map_nested(lowerCamelCase__     ,		lowerCamelCase__     ,		map_numpy=lowerCamelCase__     ,		num_proc=lowerCamelCase__				).items()}     ,		{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}     ,		)
							with self.assertRaises(lowerCamelCase__				):  # can't pickle a local lambda
											map_nested(lambda lowerCamelCase__				: x + 1     ,		lowerCamelCase__     ,		num_proc=lowerCamelCase__				)


			def  lowercase_       ( self				)       -> Tuple:



							'''simple docstring'''
							__lowerCamelCase									=       {'a': 1, 'b': 2}
							__lowerCamelCase									=       {'a': 3, 'b': 4}
							__lowerCamelCase									=       {'a': 5, 'b': 6}
							__lowerCamelCase									=       sorted([('a', (1, 3, 5)), ('b', (2, 4, 6))]				)
							self.assertEqual(sorted(zip_dict(lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__				)				)     ,		lowerCamelCase__				)






			def  lowercase_       ( self				)       -> Union[str, Any]:



							'''simple docstring'''







							class        __lowerCAmelCase					:

										"""simple docstring"""







										snake_case_  						=			'''bar'''




							__lowerCamelCase									=       Foo()
							self.assertEqual(foo.my_attr     ,		'bar'				)
							with temporary_assignment(lowerCamelCase__     ,		'my_attr'     ,		'BAR'				):
											self.assertEqual(foo.my_attr     ,		'BAR'				)
							self.assertEqual(foo.my_attr     ,		'bar'				)







@pytest.mark.parametrize(
    'iterable_length, num_proc, expected_num_proc'					,	[
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]					,	)
def      lowerCamelCase_						(			UpperCamelCase__							: Any					,	UpperCamelCase__							: Dict					,	UpperCamelCase__							: List[Any] )	->			Any:





				"""simple docstring"""


				with patch('datasets.utils.py_utils._single_map_nested' ) as mock_single_map_nested, patch(
				    'datasets.parallel.parallel.Pool' ) as mock_multiprocessing_pool:
								__lowerCamelCase									=       {F"""{i}""": i for i in range(UpperCamelCase__ )}
								__lowerCamelCase									=       map_nested(lambda UpperCamelCase__ : x + 10					,	UpperCamelCase__					,	num_proc=UpperCamelCase__					,	parallel_min_length=16 )
								if expected_num_proc == 1:
												assert mock_single_map_nested.called
												assert not mock_multiprocessing_pool.called
								else:
												assert not mock_single_map_nested.called
												assert mock_multiprocessing_pool.called
												assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc







class        __lowerCAmelCase					(			__magic_name__					):

			"""simple docstring"""







			@require_tf
			def  lowercase_       ( self				)       -> Dict:



							'''simple docstring'''
							import tensorflow as tf
							from tensorflow.keras import layers

							__lowerCamelCase									=       layers.Dense(2				)

							def gen_random_output():
											__lowerCamelCase									=       tf.random.uniform((1, 3)				)
											return model(lowerCamelCase__				).numpy()

							with temp_seed(42     ,		set_tensorflow=lowerCamelCase__				):
											__lowerCamelCase									=       gen_random_output()
							with temp_seed(42     ,		set_tensorflow=lowerCamelCase__				):
											__lowerCamelCase									=       gen_random_output()
							__lowerCamelCase									=       gen_random_output()

							np.testing.assert_equal(lowerCamelCase__     ,		lowerCamelCase__				)
							self.assertGreater(np.abs(outa - outa				).sum()     ,		0				)


			@require_torch
			def  lowercase_       ( self				)       -> List[str]:



							'''simple docstring'''
							import torch

							def gen_random_output():
											__lowerCamelCase									=       torch.nn.Linear(3     ,		2				)
											__lowerCamelCase									=       torch.rand(1     ,		3				)
											return model(lowerCamelCase__				).detach().numpy()

							with temp_seed(42     ,		set_pytorch=lowerCamelCase__				):
											__lowerCamelCase									=       gen_random_output()
							with temp_seed(42     ,		set_pytorch=lowerCamelCase__				):
											__lowerCamelCase									=       gen_random_output()
							__lowerCamelCase									=       gen_random_output()

							np.testing.assert_equal(lowerCamelCase__     ,		lowerCamelCase__				)
							self.assertGreater(np.abs(outa - outa				).sum()     ,		0				)






			def  lowercase_       ( self				)       -> Tuple:



							'''simple docstring'''
							def gen_random_output():
											return np.random.rand(1     ,		3				)

							with temp_seed(42				):
											__lowerCamelCase									=       gen_random_output()
							with temp_seed(42				):
											__lowerCamelCase									=       gen_random_output()
							__lowerCamelCase									=       gen_random_output()

							np.testing.assert_equal(lowerCamelCase__     ,		lowerCamelCase__				)
							self.assertGreater(np.abs(outa - outa				).sum()     ,		0				)







@pytest.mark.parametrize('input_data'					,	[{}] )
def      lowerCamelCase_						(			UpperCamelCase__							: Optional[int] )	->			Optional[int]:





				"""simple docstring"""


				__lowerCamelCase									=       NestedDataStructure(UpperCamelCase__ ).data
				assert output_data == input_data







@pytest.mark.parametrize(
    'data, expected_output'					,	[
        ({}, []),
        ([], []),
        ('foo', ['foo']),
        (['foo', 'bar'], ['foo', 'bar']),
        ([['foo', 'bar']], ['foo', 'bar']),
        ([[['foo'], ['bar']]], ['foo', 'bar']),
        ([[['foo'], 'bar']], ['foo', 'bar']),
        ({'a': 1, 'b': 2}, [1, 2]),
        ({'a': [1, 2], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[1, 2]], 'b': [[3, 4]]}, [1, 2, 3, 4]),
        ({'a': [[1, 2]], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [[[3], [4]]]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [[3, 4]]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [3, [4]]}, [1, 2, 3, 4]),
        ({'a': {'1': 1}, 'b': 2}, [1, 2]),
        ({'a': {'1': [1]}, 'b': 2}, [1, 2]),
        ({'a': {'1': [1]}, 'b': [2]}, [1, 2]),
    ]					,	)
def      lowerCamelCase_						(			UpperCamelCase__							: Tuple					,	UpperCamelCase__							: Union[str, Any] )	->			Any:





				"""simple docstring"""


				__lowerCamelCase									=       NestedDataStructure(UpperCamelCase__ ).flatten()
				assert output == expected_output







def      lowerCamelCase_						(			)	->			List[Any]:





				"""simple docstring"""


				__lowerCamelCase									=       A(x=1					,	y='foobar' )
				__lowerCamelCase									=       {'x': 1, 'y': 'foobar'}
				assert asdict(UpperCamelCase__ ) == expected_output

				__lowerCamelCase									=       {'a': {'b': A(x=10					,	y='foo' )}, 'c': [A(x=20					,	y='bar' )]}
				__lowerCamelCase									=       {'a': {'b': {'x': 10, 'y': 'foo'}}, 'c': [{'x': 20, 'y': 'bar'}]}
				assert asdict(UpperCamelCase__ ) == expected_output

				with pytest.raises(UpperCamelCase__ ):
								asdict([1, A(x=10					,	y='foo' )] )







def      lowerCamelCase_						(			UpperCamelCase__							: str )	->			Tuple:





				"""simple docstring"""


				return text.split()







def      lowerCamelCase_						(			UpperCamelCase__							: List[str] )	->			Optional[int]:





				"""simple docstring"""


				yield (time.time(), content)
				time.sleep(2 )
				yield (time.time(), content)







def      lowerCamelCase_						(			)	->			List[Any]:





				"""simple docstring"""


				with Pool(2 ) as pool:
								__lowerCamelCase									=       list(iflatmap_unordered(UpperCamelCase__					,	_split_text					,	kwargs_iterable=[{'text': 'hello there'}] * 10 ) )
								assert out.count('hello' ) == 10
								assert out.count('there' ) == 10
								assert len(UpperCamelCase__ ) == 20

				# check multiprocess from pathos (uses dill for pickling)
				with multiprocess.Pool(2 ) as pool:
								__lowerCamelCase									=       list(iflatmap_unordered(UpperCamelCase__					,	_split_text					,	kwargs_iterable=[{'text': 'hello there'}] * 10 ) )
								assert out.count('hello' ) == 10
								assert out.count('there' ) == 10
								assert len(UpperCamelCase__ ) == 20

				# check that we get items as fast as possible
				with Pool(2 ) as pool:
								__lowerCamelCase									=       []
								for yield_time, content in iflatmap_unordered(
								    UpperCamelCase__					,	_aseconds_generator_of_aitems_with_timing					,	kwargs_iterable=[{'content': 'a'}, {'content': 'b'}] ):
												assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
												out.append(UpperCamelCase__ )
								assert out.count('a' ) == 2
								assert out.count('b' ) == 2
								assert len(UpperCamelCase__ ) == 4








import importlib.metadata
import operator
import re
import sys
from typing import Optional

from packaging import version


__A								=       {
    "<": operator.lt,
    "<=": operator.le,
    "==": operator.eq,
    "!=": operator.ne,
    ">=": operator.ge,
    ">": operator.gt,
}







def      lowerCamelCase_						(			UpperCamelCase__							: List[str]					,	UpperCamelCase__							: Dict					,	UpperCamelCase__							: Dict					,	UpperCamelCase__							: Any					,	UpperCamelCase__							: Dict					,	UpperCamelCase__							: int )	->			List[Any]:





				"""simple docstring"""


				if got_ver is None or want_ver is None:
								raise ValueError(
								    F"""Unable to compare versions for {requirement}: need={want_ver} found={got_ver}. This is unusual. Consider"""
								    F""" reinstalling {pkg}.""" )
				if not ops[op](version.parse(UpperCamelCase__ )					,	version.parse(UpperCamelCase__ ) ):
								raise ImportError(
								    F"""{requirement} is required for a normal functioning of this module, but found {pkg}=={got_ver}.{hint}""" )







def      lowerCamelCase_						(			UpperCamelCase__							: str					,	UpperCamelCase__							: Optional[str] = None )	->			None:





				"""simple docstring"""


				__lowerCamelCase									=       F"""\n{hint}""" if hint is not None else ''

				# non-versioned check
				if re.match(R'^[\w_\-\d]+$'					,	UpperCamelCase__ ):
								__lowerCamelCase			,		__lowerCamelCase			,		__lowerCamelCase									=       requirement, None, None
				else:
								__lowerCamelCase									=       re.findall(R'^([^!=<>\s]+)([\s!=<>]{1,2}.+)'					,	UpperCamelCase__ )
								if not match:
												raise ValueError(
												    'requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23, but'
												    F""" got {requirement}""" )
								__lowerCamelCase			,		__lowerCamelCase									=       match[0]
								__lowerCamelCase									=       want_full.split(',' )  # there could be multiple requirements
								__lowerCamelCase									=       {}
								for w in want_range:
												__lowerCamelCase									=       re.findall(R'^([\s!=<>]{1,2})(.+)'					,	UpperCamelCase__ )
												if not match:
																raise ValueError(
																    'requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23,'
																    F""" but got {requirement}""" )
												__lowerCamelCase			,		__lowerCamelCase									=       match[0]
												__lowerCamelCase									=       want_ver
												if op not in ops:
																raise ValueError(F"""{requirement}: need one of {list(ops.keys() )}, but got {op}""" )

    # special case
				if pkg == "python":
								__lowerCamelCase									=       '.'.join([str(UpperCamelCase__ ) for x in sys.version_info[:3]] )
								for op, want_ver in wanted.items():
												_compare_versions(UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__ )
								return

				# check if any version is installed
				try:
								__lowerCamelCase									=       importlib.metadata.version(UpperCamelCase__ )
				except importlib.metadata.PackageNotFoundError:
								raise importlib.metadata.PackageNotFoundError(
								    F"""The '{requirement}' distribution was not found and is required by this application. {hint}""" )

				# check that the right version is installed if version number or a range was provided
				if want_ver is not None:
								for op, want_ver in wanted.items():
												_compare_versions(UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__ )







def      lowerCamelCase_						(			UpperCamelCase__							: Optional[int] )	->			Optional[int]:





				"""simple docstring"""


				__lowerCamelCase									=       'Try: pip install transformers -U or pip install -e \'.[dev]\' if you\'re working with git main'
				return require_version(UpperCamelCase__					,	UpperCamelCase__ )


