
'''simple docstring'''
# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

from .config import config_command_parser
from .config_args import default_config_file, load_config_from_file  # noqa: F401
from .default import default_command_parser
from .update import update_command_parser

def 						UpperCamelCase_     (	__SCREAMING_SNAKE_CASE=None       ):
    """simple docstring"""






    snake_case_ :    Union[str, Any]     			=						argparse.ArgumentParser(add_help=__SCREAMING_SNAKE_CASE,		allow_abbrev=__SCREAMING_SNAKE_CASE       )
    # The main config parser
    snake_case_ :    Union[str, Any]     			=						config_command_parser(__SCREAMING_SNAKE_CASE       )
    # The subparser to add commands to
    snake_case_ :    Dict     			=						config_parser.add_subparsers(title="subcommands",		dest="subcommand"       )

    # Then add other parsers with the parent parser
    default_command_parser(__SCREAMING_SNAKE_CASE,		parents=[parent_parser]       )
    update_command_parser(__SCREAMING_SNAKE_CASE,		parents=[parent_parser]       )

    return config_parser

def 						UpperCamelCase_     (	):
    """simple docstring"""






    snake_case_ :    Dict     			=						get_config_parser()
    snake_case_ :    Union[str, Any]     			=						config_parser.parse_args()

    if not hasattr(__SCREAMING_SNAKE_CASE,		"func"       ):
        config_parser.print_help()
        exit(1       )

    # Run
    args.func(__SCREAMING_SNAKE_CASE       )


if __name__ == "__main__":
  main()






'''simple docstring'''
import contextlib
from multiprocessing import Pool, RLock

from tqdm.auto import tqdm

from ..utils import experimental, logging


a_											=						logging.get_logger(__name__)






class    UpperCAmelCase_				:
 UpperCAmelCase_		    =						None


@experimental
def 						UpperCamelCase_     (	__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       ):
    """simple docstring"""






    if ParallelBackendConfig.backend_name is None:
        return _map_with_multiprocessing_pool(
            __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       )

    return _map_with_joblib(__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       )

def 						UpperCamelCase_     (	__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       ):
    """simple docstring"""






    snake_case_ :    str     			=						num_proc if num_proc <= len(__SCREAMING_SNAKE_CASE       ) else len(__SCREAMING_SNAKE_CASE       )
    snake_case_ :    Optional[int]     			=						[]  # We organize the splits ourselve (contiguous splits)
    for index in range(__SCREAMING_SNAKE_CASE       ):
        snake_case_ :    List[str]     			=						len(__SCREAMING_SNAKE_CASE       ) // num_proc
        snake_case_ :    Optional[Any]     			=						len(__SCREAMING_SNAKE_CASE       ) % num_proc
        snake_case_ :    Optional[Any]     			=						div * index + min(__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       )
        snake_case_ :    Union[str, Any]     			=						start + div + (1 if index < mod else 0)
        split_kwds.append((function, iterable[start:end], types, index, disable_tqdm, desc)       )

    if len(__SCREAMING_SNAKE_CASE       ) != sum(len(i[1]       ) for i in split_kwds       ):
        raise ValueError(
            f'Error dividing inputs iterable among processes. '
            f'Total number of objects {len(__SCREAMING_SNAKE_CASE       )}, '
            f'length: {sum(len(i[1]       ) for i in split_kwds       )}'       )

    logger.info(
        f'Spawning {num_proc} processes for {len(__SCREAMING_SNAKE_CASE       )} objects in slices of {[len(i[1]       ) for i in split_kwds]}'       )
    snake_case_			, snake_case_ :    Tuple     			=						None, None
    if not disable_tqdm:
        snake_case_			, snake_case_ :    int     			=						(RLock(),), tqdm.set_lock
    with Pool(__SCREAMING_SNAKE_CASE,		initargs=__SCREAMING_SNAKE_CASE,		initializer=__SCREAMING_SNAKE_CASE       ) as pool:
        snake_case_ :    Union[str, Any]     			=						pool.map(__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       )
    logger.info(f'Finished {num_proc} processes'       )
    snake_case_ :    str     			=						[obj for proc_res in mapped for obj in proc_res]
    logger.info(f'Unpacked {len(__SCREAMING_SNAKE_CASE       )} objects'       )

    return mapped

def 						UpperCamelCase_     (	__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE       ):
    """simple docstring"""






    import joblib

    with joblib.parallel_backend(ParallelBackendConfig.backend_name,		n_jobs=__SCREAMING_SNAKE_CASE       ):
        return joblib.Parallel()(
            joblib.delayed(__SCREAMING_SNAKE_CASE       )((function, obj, types, None, True, None)       ) for obj in iterable       )


@experimental
@contextlib.contextmanager
def 						UpperCamelCase_     (	__SCREAMING_SNAKE_CASE       ):
    """simple docstring"""






    snake_case_ :    Union[str, Any]     			=						backend_name

    if backend_name == "spark":
        from joblibspark import register_spark

        register_spark()

        # TODO: call create_cache_and_write_probe if "download" in steps
        # TODO: raise NotImplementedError when Dataset.map etc is called

    try:
        yield
    finally:
        snake_case_ :    Any     			=						None




