






'''simple docstring'''







import contextlib
from multiprocessing import Pool, RLock

from tqdm.auto import tqdm

from ..utils import experimental, logging


UpperCamelCase     :   Optional[Any]	      =		logging.get_logger(__name__)




class 		UpperCamelCase__     :




		'''simple docstring'''





		_UpperCamelCase						=       None



@experimental
def     A__      (					__lowerCAmelCase		:      str ,     __lowerCAmelCase		:      Any ,     __lowerCAmelCase		:      Union[str, Any] ,     __lowerCAmelCase		:      Optional[int] ,     __lowerCAmelCase		:      Optional[Any] ,     __lowerCAmelCase		:      Optional[Any] ,     __lowerCAmelCase		:      Tuple    ):
						if ParallelBackendConfig.backend_name is None:
												return _map_with_multiprocessing_pool(
												    __lowerCAmelCase ,     __lowerCAmelCase ,     __lowerCAmelCase ,     __lowerCAmelCase ,     __lowerCAmelCase ,     __lowerCAmelCase ,     __lowerCAmelCase    )

						return _map_with_joblib(__lowerCAmelCase ,     __lowerCAmelCase ,     __lowerCAmelCase ,     __lowerCAmelCase ,     __lowerCAmelCase ,     __lowerCAmelCase ,     __lowerCAmelCase    )
def     A__      (					__lowerCAmelCase		:      int ,     __lowerCAmelCase		:      List[str] ,     __lowerCAmelCase		:      Tuple ,     __lowerCAmelCase		:      List[str] ,     __lowerCAmelCase		:      int ,     __lowerCAmelCase		:      str ,     __lowerCAmelCase		:      List[str]    ):
						lowerCamelCase__								=	num_proc if num_proc <= len(__lowerCAmelCase    ) else len(__lowerCAmelCase    )
						lowerCamelCase__								=	[]  # We organize the splits ourselve (contiguous splits)
						for index in range(__lowerCAmelCase    ):
												lowerCamelCase__								=	len(__lowerCAmelCase    ) // num_proc
												lowerCamelCase__								=	len(__lowerCAmelCase    ) % num_proc
												lowerCamelCase__								=	div * index + min(__lowerCAmelCase ,     __lowerCAmelCase    )
												lowerCamelCase__								=	start + div + (1 if index < mod else 0)
												split_kwds.append((function, iterable[start:end], types, index, disable_tqdm, desc)    )

						if len(__lowerCAmelCase    ) != sum(len(i[1]    ) for i in split_kwds    ):
												raise ValueError(
												    F'''Error dividing inputs iterable among processes. '''
												    F'''Total number of objects {len(__lowerCAmelCase    )}, '''
												    F'''length: {sum(len(i[1]    ) for i in split_kwds    )}'''    )

						logger.info(
						    F'''Spawning {num_proc} processes for {len(__lowerCAmelCase    )} objects in slices of {[len(i[1]    ) for i in split_kwds]}'''    )
						lowerCamelCase__							,      lowerCamelCase__								=	None, None
						if not disable_tqdm:
												lowerCamelCase__							,      lowerCamelCase__								=	(RLock(),), tqdm.set_lock
						with Pool(__lowerCAmelCase ,     initargs=__lowerCAmelCase ,     initializer=__lowerCAmelCase    ) as pool:
												lowerCamelCase__								=	pool.map(__lowerCAmelCase ,     __lowerCAmelCase    )
						logger.info(F'''Finished {num_proc} processes'''    )
						lowerCamelCase__								=	[obj for proc_res in mapped for obj in proc_res]
						logger.info(F'''Unpacked {len(__lowerCAmelCase    )} objects'''    )

						return mapped
def     A__      (					__lowerCAmelCase		:      str ,     __lowerCAmelCase		:      str ,     __lowerCAmelCase		:      Tuple ,     __lowerCAmelCase		:      Tuple ,     __lowerCAmelCase		:      str ,     __lowerCAmelCase		:      Dict ,     __lowerCAmelCase		:      Optional[int]    ):
						# progress bar is not yet supported for _map_with_joblib, because tqdm couldn't accurately be applied to joblib,
						# and it requires monkey-patching joblib internal classes which is subject to change
						import joblib

						with joblib.parallel_backend(ParallelBackendConfig.backend_name ,     n_jobs=__lowerCAmelCase    ):
												return joblib.Parallel()(
												    joblib.delayed(__lowerCAmelCase    )((function, obj, types, None, True, None)    ) for obj in iterable    )





@experimental
@contextlib.contextmanager
def     A__      (					__lowerCAmelCase		:      str    ):
						lowerCamelCase__								=	backend_name

						if backend_name == "spark":
												from joblibspark import register_spark

												register_spark()

												# TODO: call create_cache_and_write_probe if "download" in steps
												# TODO: raise NotImplementedError when Dataset.map etc is called

						try:
												yield
						finally:
												lowerCamelCase__								=	None





'''simple docstring'''






import os

import pytest

from datasets import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
)


__A   :    int											= pytest.mark.integration







@pytest.mark.parametrize("""path"""				,       ["""paws""", """csv"""]   )
def 							UpperCamelCase_     (       A__   :  Union[str, Any]				,       A__   :  Tuple   ):
  '''simple docstring'''





  inspect_dataset(A__				,       A__   )
  lowerCAmelCase_     : Any     =  path + """.py"""
  assert script_name in os.listdir(A__   )
  assert "__pycache__" not in os.listdir(A__   )







@pytest.mark.filterwarnings("""ignore:inspect_metric is deprecated:FutureWarning"""   )
@pytest.mark.filterwarnings("""ignore:metric_module_factory is deprecated:FutureWarning"""   )
@pytest.mark.parametrize("""path"""				,       ["""accuracy"""]   )
def 							UpperCamelCase_     (       A__   :  Tuple				,       A__   :  str   ):
  '''simple docstring'''





  inspect_metric(A__				,       A__   )
  lowerCAmelCase_     : Any     =  path + """.py"""
  assert script_name in os.listdir(A__   )
  assert "__pycache__" not in os.listdir(A__   )







@pytest.mark.parametrize(
    """path, config_name, expected_splits"""				,       [
        ("""squad""", """plain_text""", ["""train""", """validation"""]),
        ("""dalle-mini/wit""", """dalle-mini--wit""", ["""train"""]),
        ("""paws""", """labeled_final""", ["""train""", """test""", """validation"""]),
    ]				,       )
def 							UpperCamelCase_     (       A__   :  List[str]				,       A__   :  List[str]				,       A__   :  Optional[Any]   ):
  '''simple docstring'''





  lowerCAmelCase_     : List[Any]     =  get_dataset_config_info(A__				,       config_name=A__   )
  assert info.config_name == config_name
  assert list(info.splits.keys()   ) == expected_splits







@pytest.mark.parametrize(
    """path, config_name, expected_exception"""				,       [
        ("""paws""", None, ValueError),
    ]				,       )
def 							UpperCamelCase_     (       A__   :  List[str]				,       A__   :  List[str]				,       A__   :  int   ):
  '''simple docstring'''





  with pytest.raises(A__   ):
    get_dataset_config_info(A__				,       config_name=A__   )







@pytest.mark.parametrize(
    """path, expected"""				,       [
        ("""squad""", """plain_text"""),
        ("""acronym_identification""", """default"""),
        ("""lhoestq/squad""", """plain_text"""),
        ("""lhoestq/test""", """default"""),
        ("""lhoestq/demo1""", """lhoestq--demo1"""),
        ("""dalle-mini/wit""", """dalle-mini--wit"""),
    ]				,       )
def 							UpperCamelCase_     (       A__   :  str				,       A__   :  List[str]   ):
  '''simple docstring'''





  lowerCAmelCase_     : Dict     =  get_dataset_config_names(A__   )
  assert expected in config_names







@pytest.mark.parametrize(
    """path, expected_configs, expected_splits_in_first_config"""				,       [
        ("""squad""", ["""plain_text"""], ["""train""", """validation"""]),
        ("""dalle-mini/wit""", ["""dalle-mini--wit"""], ["""train"""]),
        ("""paws""", ["""labeled_final""", """labeled_swap""", """unlabeled_final"""], ["""train""", """test""", """validation"""]),
    ]				,       )
def 							UpperCamelCase_     (       A__   :  List[Any]				,       A__   :  Dict				,       A__   :  Optional[int]   ):
  '''simple docstring'''





  lowerCAmelCase_     : Tuple     =  get_dataset_infos(A__   )
  assert list(infos.keys()   ) == expected_configs
  lowerCAmelCase_     : Optional[int]     =  expected_configs[0]
  assert expected_config in infos
  lowerCAmelCase_     : int     =  infos[expected_config]
  assert info.config_name == expected_config
  assert list(info.splits.keys()   ) == expected_splits_in_first_config







@pytest.mark.parametrize(
    """path, expected_config, expected_splits"""				,       [
        ("""squad""", """plain_text""", ["""train""", """validation"""]),
        ("""dalle-mini/wit""", """dalle-mini--wit""", ["""train"""]),
        ("""paws""", """labeled_final""", ["""train""", """test""", """validation"""]),
    ]				,       )
def 							UpperCamelCase_     (       A__   :  List[str]				,       A__   :  Optional[Any]				,       A__   :  List[Any]   ):
  '''simple docstring'''





  lowerCAmelCase_     : Dict     =  get_dataset_infos(A__   )
  assert expected_config in infos
  lowerCAmelCase_     : List[Any]     =  infos[expected_config]
  assert info.config_name == expected_config
  assert list(info.splits.keys()   ) == expected_splits







@pytest.mark.parametrize(
    """path, config_name, expected_exception"""				,       [
        ("""paws""", None, ValueError),
    ]				,       )
def 							UpperCamelCase_     (       A__   :  List[str]				,       A__   :  Any				,       A__   :  Dict   ):
  '''simple docstring'''





  with pytest.raises(A__   ):
    get_dataset_split_names(A__				,       config_name=A__   )



