

'''simple docstring'''





import contextlib
from multiprocessing import Pool, RLock

from tqdm.auto import tqdm

from ..utils import experimental, logging


UpperCamelCase_    						=       logging.get_logger(__name__)





class 		__SCREAMING_SNAKE_CASE							:
				lowerCamelCase_		       =    None






@experimental
def 				_lowerCAmelCase			(   __magic_name__						:     Union[str, Any]   , __magic_name__						:     Tuple   , __magic_name__						:     List[Any]   , __magic_name__						:     Any   , __magic_name__						:     List[str]   , __magic_name__						:     int   , __magic_name__						:     List[str]						)				->  Any:
						if ParallelBackendConfig.backend_name is None:
												return _map_with_multiprocessing_pool(
												    __magic_name__   , __magic_name__   , __magic_name__   , __magic_name__   , __magic_name__   , __magic_name__   , __magic_name__						)

						return _map_with_joblib(__magic_name__   , __magic_name__   , __magic_name__   , __magic_name__   , __magic_name__   , __magic_name__   , __magic_name__						)






def 				_lowerCAmelCase			(   __magic_name__						:     str   , __magic_name__						:     Optional[int]   , __magic_name__						:     List[str]   , __magic_name__						:     Any   , __magic_name__						:     List[Any]   , __magic_name__						:     Optional[Any]   , __magic_name__						:     Optional[Any]						)				->  int:
						lowercase :							Optional[int]    		=num_proc if num_proc <= len(__magic_name__						) else len(__magic_name__						)
						lowercase :							Any    		=[]  # We organize the splits ourselve (contiguous splits)
						for index in range(__magic_name__						):
												lowercase :							Tuple    		=len(__magic_name__						) // num_proc
												lowercase :							Any    		=len(__magic_name__						) % num_proc
												lowercase :							Tuple    		=div * index + min(__magic_name__   , __magic_name__						)
												lowercase :							Optional[Any]    		=start + div + (1 if index < mod else 0)
												split_kwds.append((function, iterable[start:end], types, index, disable_tqdm, desc)						)

						if len(__magic_name__						) != sum(len(i[1]						) for i in split_kwds						):
												raise ValueError(
												    f'''Error dividing inputs iterable among processes. '''
												    f'''Total number of objects {len(__magic_name__						)}, '''
												    f'''length: {sum(len(i[1]						) for i in split_kwds						)}'''						)

						logger.info(
						    f'''Spawning {num_proc} processes for {len(__magic_name__						)} objects in slices of {[len(i[1]						) for i in split_kwds]}'''						)
						lowercase		,	lowercase :							Union[str, Any]    		=None, None
						if not disable_tqdm:
												lowercase		,	lowercase :							Tuple    		=(RLock(),), tqdm.set_lock
						with Pool(__magic_name__   , initargs=__magic_name__   , initializer=__magic_name__						) as pool:
												lowercase :							Any    		=pool.map(__magic_name__   , __magic_name__						)
						logger.info(f'''Finished {num_proc} processes'''						)
						lowercase :							Dict    		=[obj for proc_res in mapped for obj in proc_res]
						logger.info(f'''Unpacked {len(__magic_name__						)} objects'''						)

						return mapped






def 				_lowerCAmelCase			(   __magic_name__						:     List[Any]   , __magic_name__						:     int   , __magic_name__						:     Tuple   , __magic_name__						:     Tuple   , __magic_name__						:     List[Any]   , __magic_name__						:     Any   , __magic_name__						:     Any						)				->  str:
						# progress bar is not yet supported for _map_with_joblib, because tqdm couldn't accurately be applied to joblib,
						# and it requires monkey-patching joblib internal classes which is subject to change
						import joblib

						with joblib.parallel_backend(ParallelBackendConfig.backend_name   , n_jobs=__magic_name__						):
												return joblib.Parallel()(
												    joblib.delayed(__magic_name__						)((function, obj, types, None, True, None)						) for obj in iterable						)







@experimental
@contextlib.contextmanager
def 				_lowerCAmelCase			(   __magic_name__						:     str						)				->  Union[str, Any]:
						lowercase :							int    		=backend_name

						if backend_name == "spark":
												from joblibspark import register_spark

												register_spark()

												# TODO: call create_cache_and_write_probe if "download" in steps
												# TODO: raise NotImplementedError when Dataset.map etc is called

						try:
												yield
						finally:
												lowercase :							Tuple    		=None








'''simple docstring'''





from typing import TYPE_CHECKING

from ...utils import _LazyModule


UpperCamelCase_    						=       {"""tokenization_byt5""": ["""ByT5Tokenizer"""]}


if TYPE_CHECKING:
							from .tokenization_byta import ByTaTokenizer
else:
							import sys

							UpperCamelCase_    						=       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





