




from typing import List

import numpy as np





def 				A__ (snake_case   :		dict				)     ->						int:
					__UpperCamelCase					:	Union[str, Any]             =      {key: len(snake_case				) for key, value in gen_kwargs.items() if isinstance(snake_case		,	snake_case				)}
					if len(set(lists_lengths.values()				)				) > 1:
										raise RuntimeError(
										    (
										        """Sharding is ambiguous for this dataset: """
										        + """we found several data sources lists of different lengths, and we don't know over which list we should parallelize:\n"""
										        + """\n""".join(F'''\t- key {key} has length {length}''' for key, length in lists_lengths.items()				)
										        + """\nTo fix this, check the 'gen_kwargs' and make sure to use lists only for data sources, """
										        + """and use tuples otherwise. In the end there should only be one single list, or several lists with the same length."""
										    )				)
					__UpperCamelCase					:	int             =      max(lists_lengths.values()		,	default=0				)
					return max(1		,	snake_case				)





def 				A__ (snake_case   :		int		,	snake_case   :		int				)     ->						List[range]:
					__UpperCamelCase					:	str             =      []
					for group_idx in range(snake_case				):
										__UpperCamelCase					:	str             =      num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs))
										if num_shards_to_add == 0:
															break
										__UpperCamelCase					:	int             =      shards_indices_per_group[-1].stop if shards_indices_per_group else 0
										__UpperCamelCase					:	int             =      range(snake_case		,	start + num_shards_to_add				)
										shards_indices_per_group.append(snake_case				)
					return shards_indices_per_group





def 				A__ (snake_case   :		dict		,	snake_case   :		int				)     ->						List[dict]:
					__UpperCamelCase					:	str             =      _number_of_shards_in_gen_kwargs(snake_case				)
					if num_shards == 1:
										return [dict(snake_case				)]
					else:
										__UpperCamelCase					:	Any             =      _distribute_shards(num_shards=snake_case		,	max_num_jobs=snake_case				)
										return [
										    {
										        key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]]
										        if isinstance(snake_case		,	snake_case				)
										        else value
										        for key, value in gen_kwargs.items()
										    }
										    for group_idx in range(len(snake_case				)				)
										]





def 				A__ (snake_case   :		List[dict]				)     ->						dict:
					return {
					    key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]]
					    if isinstance(gen_kwargs_list[0][key]		,	snake_case				)
					    else gen_kwargs_list[0][key]
					    for key in gen_kwargs_list[0]
					}







def 				A__ (snake_case   :		np.random.Generator		,	snake_case   :		dict				)     ->						dict:
					__UpperCamelCase					:	int             =      {len(snake_case				) for value in gen_kwargs.values() if isinstance(snake_case		,	snake_case				)}
					__UpperCamelCase					:	int             =      {}
					for size in list_sizes:
										__UpperCamelCase					:	Dict             =      list(range(snake_case				)				)
										rng.shuffle(indices_per_size[size]				)
					# Now let's copy the gen_kwargs and shuffle the lists based on their sizes
					__UpperCamelCase					:	Dict             =      dict(snake_case				)
					for key, value in shuffled_kwargs.items():
										if isinstance(snake_case		,	snake_case				):
															__UpperCamelCase					:	List[str]             =      [value[i] for i in indices_per_size[len(snake_case				)]]
					return shuffled_kwargs












import logging
import os
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


a__		     =     {
    '''debug''': logging.DEBUG,
    '''info''': logging.INFO,
    '''warning''': logging.WARNING,
    '''error''': logging.ERROR,
    '''critical''': logging.CRITICAL,
}

a__		     =     logging.WARNING





def 				A__ ()     ->						Any:
					__UpperCamelCase					:	Union[str, Any]             =      os.getenv("""DATASETS_VERBOSITY"""		,	snake_case				)
					if env_level_str:
										if env_level_str in log_levels:
															return log_levels[env_level_str]
										else:
															logging.getLogger().warning(
															    F'''Unknown option DATASETS_VERBOSITY={env_level_str}, '''
															    F'''has to be one of: { ", ".join(log_levels.keys()				) }'''				)
					return _default_log_level





def 				A__ ()     ->						str:
					return __name__.split("""."""				)[0]





def 				A__ ()     ->						logging.Logger:
					return logging.getLogger(_get_library_name()				)





def 				A__ ()     ->						None:
					# Apply our default configuration to the library root logger.
					__UpperCamelCase					:	Any             =      _get_library_root_logger()
					library_root_logger.setLevel(_get_default_logging_level()				)





def 				A__ ()     ->						None:
					__UpperCamelCase					:	List[Any]             =      _get_library_root_logger()
					library_root_logger.setLevel(logging.NOTSET				)





def 				A__ (snake_case   :		Optional[str] = None				)     ->						logging.Logger:
					if name is None:
										__UpperCamelCase					:	Optional[int]             =      _get_library_name()
					return logging.getLogger(snake_case				)





def 				A__ ()     ->						int:
					return _get_library_root_logger().getEffectiveLevel()





def 				A__ (snake_case   :		int				)     ->						None:
					_get_library_root_logger().setLevel(snake_case				)





def 				A__ ()     ->						Dict:
					return set_verbosity(snake_case				)





def 				A__ ()     ->						Any:
					return set_verbosity(snake_case				)





def 				A__ ()     ->						Dict:
					return set_verbosity(snake_case				)





def 				A__ ()     ->						str:
					return set_verbosity(snake_case				)





def 				A__ ()     ->						None:
					__UpperCamelCase					:	str             =      False





def 				A__ ()     ->						None:
					__UpperCamelCase					:	List[str]             =      True


# Configure the library root logger at the module level (singleton-like)
_configure_library_root_logger()


class    SCREAMING_SNAKE_CASE_				:



	"""simple docstring"""


	def __init__(					self		:						Optional[int]					,     *lowerCAmelCase		:						str					,     **lowerCAmelCase		:						List[str]	)	->       Union[str, Any]:  # pylint: disable=unused-argument







						"""simple docstring"""







						__UpperCamelCase					:	List[str]             =      args[0] if args else None






	def __iter__(					self		:						List[Any]	)	->       List[Any]:







						"""simple docstring"""







						return iter(self._iterator	)






	def __getattr__(					self		:						int					,     lowerCAmelCase		:						List[str]	)	->       Dict:







						"""simple docstring"""







						def empty_fn(*lowerCAmelCase		:						List[Any]					,     **lowerCAmelCase		:						Dict	):  # pylint: disable=unused-argument
											return

						return empty_fn






	def __enter__(					self		:						Tuple	)	->       Any:







						"""simple docstring"""







						return self






	def __exit__(					self		:						str					,     lowerCAmelCase		:						Dict					,     lowerCAmelCase		:						Dict					,     lowerCAmelCase		:						List[str]	)	->       Tuple:







						"""simple docstring"""







						return






a__		     =     True


class    SCREAMING_SNAKE_CASE_				:



	"""simple docstring"""


	def __call__(					self		:						str					,     *lowerCAmelCase		:						List[Any]					,     lowerCAmelCase		:						Dict=False					,     **lowerCAmelCase		:						Tuple	)	->       Optional[Any]:







						"""simple docstring"""







						if _tqdm_active and not disable:
											return tqdm_lib.tqdm(*lowerCAmelCase					,     **lowerCAmelCase	)
						else:
											return EmptyTqdm(*lowerCAmelCase					,     **lowerCAmelCase	)






	def 					lowerCamelCase__     (					self		:						Any					,     *lowerCAmelCase		:						List[str]					,     **lowerCAmelCase		:						str	)	->       Dict:







						"""simple docstring"""







						__UpperCamelCase					:	Any             =      None
						if _tqdm_active:
											return tqdm_lib.tqdm.set_lock(*lowerCAmelCase					,     **lowerCAmelCase	)






	def 					lowerCamelCase__     (					self		:						Dict	)	->       Union[str, Any]:







						"""simple docstring"""







						if _tqdm_active:
											return tqdm_lib.tqdm.get_lock()






a__		     =     _tqdm_cls()





def 				A__ ()     ->						bool:
					global _tqdm_active
					return bool(_tqdm_active				)





def 				A__ ()     ->						List[str]:
					global _tqdm_active
					__UpperCamelCase					:	Tuple             =      True







def 				A__ ()     ->						int:
					global _tqdm_active
					__UpperCamelCase					:	Any             =      False






