

import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch
def 							__SCREAMING_SNAKE_CASE     (   a__    :   str       )					->				List[str]:  # picklable for multiprocessing
	return x.sum()
def 							__SCREAMING_SNAKE_CASE     (   a__    :   Union[str, Any]       )					->				Union[str, Any]:  # picklable for multiprocessing
	return i + 1






@dataclass
class      lowerCamelCase_      :
	_lowercase		:   Optional[int]     					=				42
	_lowercase		:   Union[str, Any]     					=				42






class      lowerCamelCase_      (	snake_case_	):




	def        lowerCAmelCase_       (     self						:     List[Any]							):
		__A :			int		    =			{}
		__A :			Union[str, Any]		    =			[]
		__A :			Optional[Any]		    =			1
		__A :			List[Any]		    =			[1, 2]
		__A :			Dict		    =			{"""a""": 1, """b""": 2}
		__A :			Dict		    =			{"""a""": [1, 2], """b""": [3, 4]}
		__A :			Optional[Any]		    =			{"""a""": {"""1""": 1}, """b""": 2}
		__A :			Optional[Any]		    =			{"""a""": 1, """b""": 2, """c""": 3, """d""": 4}
		__A :			Tuple		    =			{}
		__A :			Any		    =			[]
		__A :			Union[str, Any]		    =			2
		__A :			List[Any]		    =			[2, 3]
		__A :			Optional[Any]		    =			{"""a""": 2, """b""": 3}
		__A :			Dict		    =			{"""a""": [2, 3], """b""": [4, 5]}
		__A :			List[Any]		    =			{"""a""": {"""1""": 2}, """b""": 3}
		__A :			Union[str, Any]		    =			{"""a""": 2, """b""": 3, """c""": 4, """d""": 5}
		self.assertEqual(map_nested(A_					,      A_							)					,      A_							)
		self.assertEqual(map_nested(A_					,      A_							)					,      A_							)
		self.assertEqual(map_nested(A_					,      A_							)					,      A_							)
		self.assertEqual(map_nested(A_					,      A_							)					,      A_							)
		self.assertEqual(map_nested(A_					,      A_							)					,      A_							)
		self.assertEqual(map_nested(A_					,      A_							)					,      A_							)
		self.assertEqual(map_nested(A_					,      A_							)					,      A_							)
		self.assertEqual(map_nested(A_					,      A_							)					,      A_							)

		__A :			Dict		    =			2
		self.assertEqual(map_nested(A_					,      A_					,      num_proc=A_							)					,      A_							)
		self.assertEqual(map_nested(A_					,      A_					,      num_proc=A_							)					,      A_							)
		self.assertEqual(map_nested(A_					,      A_					,      num_proc=A_							)					,      A_							)
		self.assertEqual(map_nested(A_					,      A_					,      num_proc=A_							)					,      A_							)
		self.assertEqual(map_nested(A_					,      A_					,      num_proc=A_							)					,      A_							)
		self.assertEqual(map_nested(A_					,      A_					,      num_proc=A_							)					,      A_							)
		self.assertEqual(map_nested(A_					,      A_					,      num_proc=A_							)					,      A_							)
		self.assertEqual(map_nested(A_					,      A_					,      num_proc=A_							)					,      A_							)

		__A :			Any		    =			{"""a""": np.eye(2							), """b""": np.zeros(3							), """c""": np.ones(2							)}
		__A :			Any		    =			{"""a""": 2, """b""": 0, """c""": 2}
		__A :			List[Any]		    =			{
		    """a""": np.eye(2							).astype(A_							),
		    """b""": np.zeros(3							).astype(A_							),
		    """c""": np.ones(2							).astype(A_							),
		}
		self.assertEqual(map_nested(A_					,      A_					,      map_numpy=A_							)					,      A_							)
		self.assertEqual(
		    {k: v.tolist() for k, v in map_nested(A_					,      A_					,      map_numpy=A_							).items()}					,      {k: v.tolist() for k, v in expected_map_nested_sna_int.items()}					,      )
		self.assertEqual(map_nested(A_					,      A_					,      map_numpy=A_					,      num_proc=A_							)					,      A_							)
		self.assertEqual(
		    {k: v.tolist() for k, v in map_nested(A_					,      A_					,      map_numpy=A_					,      num_proc=A_							).items()}					,      {k: v.tolist() for k, v in expected_map_nested_sna_int.items()}					,      )
		with self.assertRaises(A_							):  # can't pickle a local lambda
			map_nested(lambda __A							: x + 1					,      A_					,      num_proc=A_							)




	def        lowerCAmelCase_       (     self						:     Tuple							):
		__A :			Dict		    =			{"""a""": 1, """b""": 2}
		__A :			int		    =			{"""a""": 3, """b""": 4}
		__A :			int		    =			{"""a""": 5, """b""": 6}
		__A :			List[str]		    =			sorted([("""a""", (1, 3, 5)), ("""b""", (2, 4, 6))]							)
		self.assertEqual(sorted(zip_dict(A_					,      A_					,      A_							)							)					,      A_							)




	def        lowerCAmelCase_       (     self						:     str							):






		class      lowerCamelCase_      :
			_lowercase		:   List[str]     					=				'''bar'''






		__A :			Union[str, Any]		    =			Foo()
		self.assertEqual(foo.my_attr					,      """bar"""							)
		with temporary_assignment(A_					,      """my_attr"""					,      """BAR"""							):
			self.assertEqual(foo.my_attr					,      """BAR"""							)
		self.assertEqual(foo.my_attr					,      """bar"""							)






@pytest.mark.parametrize(
    """iterable_length, num_proc, expected_num_proc"""			,[
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]			,)
def 							__SCREAMING_SNAKE_CASE     (   a__    :   Optional[Any]			,a__    :   str			,a__    :   Optional[Any]       )					->				Dict:
	with patch("""datasets.utils.py_utils._single_map_nested"""       ) as mock_single_map_nested, patch(
	    """datasets.parallel.parallel.Pool"""       ) as mock_multiprocessing_pool:
		__A :			List[Any]		    =			{f"""{i}""": i for i in range(a__       )}
		__A :			List[Any]		    =			map_nested(lambda a__       : x + 10			,a__			,num_proc=a__			,parallel_min_length=16       )
		if expected_num_proc == 1:
			assert mock_single_map_nested.called
			assert not mock_multiprocessing_pool.called
		else:
			assert not mock_single_map_nested.called
			assert mock_multiprocessing_pool.called
			assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc






class      lowerCamelCase_      (	snake_case_	):




	@require_tf
	def        lowerCAmelCase_       (     self						:     Tuple							):
		import tensorflow as tf
		from tensorflow.keras import layers

		__A :			str		    =			layers.Dense(2							)

		def gen_random_output():
			__A :			int		    =			tf.random.uniform((1, 3)							)
			return model(A_							).numpy()

		with temp_seed(42					,      set_tensorflow=A_							):
			__A :			Optional[int]		    =			gen_random_output()
		with temp_seed(42					,      set_tensorflow=A_							):
			__A :			List[Any]		    =			gen_random_output()
		__A :			int		    =			gen_random_output()

		np.testing.assert_equal(A_					,      A_							)
		self.assertGreater(np.abs(outa - outa							).sum()					,      0							)




	@require_torch
	def        lowerCAmelCase_       (     self						:     List[Any]							):
		import torch

		def gen_random_output():
			__A :			Optional[int]		    =			torch.nn.Linear(3					,      2							)
			__A :			List[str]		    =			torch.rand(1					,      3							)
			return model(A_							).detach().numpy()

		with temp_seed(42					,      set_pytorch=A_							):
			__A :			Union[str, Any]		    =			gen_random_output()
		with temp_seed(42					,      set_pytorch=A_							):
			__A :			Tuple		    =			gen_random_output()
		__A :			Union[str, Any]		    =			gen_random_output()

		np.testing.assert_equal(A_					,      A_							)
		self.assertGreater(np.abs(outa - outa							).sum()					,      0							)




	def        lowerCAmelCase_       (     self						:     List[str]							):






		def gen_random_output():
			return np.random.rand(1					,      3							)

		with temp_seed(42							):
			__A :			List[Any]		    =			gen_random_output()
		with temp_seed(42							):
			__A :			List[Any]		    =			gen_random_output()
		__A :			Optional[Any]		    =			gen_random_output()

		np.testing.assert_equal(A_					,      A_							)
		self.assertGreater(np.abs(outa - outa							).sum()					,      0							)






@pytest.mark.parametrize("""input_data"""			,[{}]       )
def 							__SCREAMING_SNAKE_CASE     (   a__    :   Any       )					->				Optional[Any]:
	__A :			int		    =			NestedDataStructure(a__       ).data
	assert output_data == input_data
@pytest.mark.parametrize(
    """data, expected_output"""			,[
        ({}, []),
        ([], []),
        ("""foo""", ["""foo"""]),
        (["""foo""", """bar"""], ["""foo""", """bar"""]),
        ([["""foo""", """bar"""]], ["""foo""", """bar"""]),
        ([[["""foo"""], ["""bar"""]]], ["""foo""", """bar"""]),
        ([[["""foo"""], """bar"""]], ["""foo""", """bar"""]),
        ({"""a""": 1, """b""": 2}, [1, 2]),
        ({"""a""": [1, 2], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[1, 2]], """b""": [[3, 4]]}, [1, 2, 3, 4]),
        ({"""a""": [[1, 2]], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [[[3], [4]]]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [[3, 4]]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [3, 4]}, [1, 2, 3, 4]),
        ({"""a""": [[[1], [2]]], """b""": [3, [4]]}, [1, 2, 3, 4]),
        ({"""a""": {"""1""": 1}, """b""": 2}, [1, 2]),
        ({"""a""": {"""1""": [1]}, """b""": 2}, [1, 2]),
        ({"""a""": {"""1""": [1]}, """b""": [2]}, [1, 2]),
    ]			,)
def 							__SCREAMING_SNAKE_CASE     (   a__    :   Optional[Any]			,a__    :   Optional[int]       )					->				int:
	__A :			Optional[int]		    =			NestedDataStructure(a__       ).flatten()
	assert output == expected_output
def 							__SCREAMING_SNAKE_CASE     (   )					->				Union[str, Any]:
	__A :			str		    =			A(x=1			,y="""foobar"""       )
	__A :			Optional[Any]		    =			{"""x""": 1, """y""": """foobar"""}
	assert asdict(a__       ) == expected_output

	__A :			Tuple		    =			{"""a""": {"""b""": A(x=10			,y="""foo"""       )}, """c""": [A(x=20			,y="""bar"""       )]}
	__A :			List[str]		    =			{"""a""": {"""b""": {"""x""": 10, """y""": """foo"""}}, """c""": [{"""x""": 20, """y""": """bar"""}]}
	assert asdict(a__       ) == expected_output

	with pytest.raises(a__       ):
		asdict([1, A(x=10			,y="""foo"""       )]       )
def 							__SCREAMING_SNAKE_CASE     (   a__    :   str       )					->				List[Any]:
	return text.split()
def 							__SCREAMING_SNAKE_CASE     (   a__    :   Optional[int]       )					->				Optional[Any]:
	yield (time.time(), content)
	time.sleep(2       )
	yield (time.time(), content)







def 							__SCREAMING_SNAKE_CASE     (   )					->				List[str]:
	with Pool(2       ) as pool:
		__A :			Any		    =			list(iflatmap_unordered(a__			,_split_text			,kwargs_iterable=[{"""text""": """hello there"""}] * 10       )       )
		assert out.count("""hello"""       ) == 10
		assert out.count("""there"""       ) == 10
		assert len(a__       ) == 20

	# check multiprocess from pathos (uses dill for pickling)
	with multiprocess.Pool(2       ) as pool:
		__A :			Optional[Any]		    =			list(iflatmap_unordered(a__			,_split_text			,kwargs_iterable=[{"""text""": """hello there"""}] * 10       )       )
		assert out.count("""hello"""       ) == 10
		assert out.count("""there"""       ) == 10
		assert len(a__       ) == 20

	# check that we get items as fast as possible
	with Pool(2       ) as pool:
		__A :			Tuple		    =			[]
		for yield_time, content in iflatmap_unordered(
		    a__			,_aseconds_generator_of_aitems_with_timing			,kwargs_iterable=[{"""content""": """a"""}, {"""content""": """b"""}]       ):
			assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
			out.append(a__       )
		assert out.count("""a"""       ) == 2
		assert out.count("""b"""       ) == 2
		assert len(a__       ) == 4







'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


lowerCAmelCase					:    Optional[Any]           = {
    'configuration_falcon': ['FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP', 'FalconConfig'],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCAmelCase					:    str           = [
						    'FALCON_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'FalconForCausalLM',
						    'FalconModel',
						    'FalconPreTrainedModel',
						    'FalconForSequenceClassification',
						    'FalconForTokenClassification',
						    'FalconForQuestionAnswering',
						]


if TYPE_CHECKING:
						from .configuration_falcon import FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP, FalconConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_falcon import (
												    FALCON_PRETRAINED_MODEL_ARCHIVE_LIST,
												    FalconForCausalLM,
												    FalconForQuestionAnswering,
												    FalconForSequenceClassification,
												    FalconForTokenClassification,
												    FalconModel,
												    FalconPreTrainedModel,
												)


else:
						import sys

						lowerCAmelCase					:    Tuple           = _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




