





import os
import unittest

from huggingface_hub.utils import are_progress_bars_disabled

import transformers.models.bart.tokenization_bart
from transformers import logging
from transformers.testing_utils import CaptureLogger, mockenv, mockenv_context
from transformers.utils.logging import disable_progress_bar, enable_progress_bar

class      lowerCAmelCase							(      unittest.TestCase    ):
 def   A_			(   self						:   int			)    ->			Union[str, Any]:
     lowerCamelCase__					:   Dict      				=	logging.get_logger()

     # the current default level is logging.WARNING
     lowerCamelCase__					:   Tuple      				=	logging.get_verbosity()

     logging.set_verbosity_error()
     self.assertEqual(logger.getEffectiveLevel()    ,  logging.get_verbosity()			)

     logging.set_verbosity_warning()
     self.assertEqual(logger.getEffectiveLevel()    ,  logging.get_verbosity()			)

     logging.set_verbosity_info()
     self.assertEqual(logger.getEffectiveLevel()    ,  logging.get_verbosity()			)

     logging.set_verbosity_debug()
     self.assertEqual(logger.getEffectiveLevel()    ,  logging.get_verbosity()			)

     # restore to the original level
     logging.set_verbosity(UpperCAmelCase			)
 def   A_			(   self						:   str			)    ->			int:
     lowerCamelCase__					:   Any      				=	logging.get_verbosity()

     lowerCamelCase__					:   Any      				=	logging.get_logger('transformers.models.bart.tokenization_bart'			)
     lowerCamelCase__					:   Optional[int]      				=	'Testing 1, 2, 3'

     # should be able to log warnings (if default settings weren't overridden by `pytest --log-level-all`)
     if level_origin <= logging.WARNING:
         with CaptureLogger(UpperCAmelCase			) as cl:
             logger.warning(UpperCAmelCase			)
         self.assertEqual(cl.out    ,  msg + '\n'			)

     # this is setting the level for all of `transformers.*` loggers
     logging.set_verbosity_error()

     # should not be able to log warnings
     with CaptureLogger(UpperCAmelCase			) as cl:
         logger.warning(UpperCAmelCase			)
     self.assertEqual(cl.out    ,  ''			)

     # should be able to log warnings again
     logging.set_verbosity_warning()
     with CaptureLogger(UpperCAmelCase			) as cl:
         logger.warning(UpperCAmelCase			)
     self.assertEqual(cl.out    ,  msg + '\n'			)

     # restore to the original level
     logging.set_verbosity(UpperCAmelCase			)
 @mockenv(TRANSFORMERS_VERBOSITY='error'			)
 def   A_			(   self						:   List[str]			)    ->			List[str]:
     # reset for the env var to take effect, next time some logger call is made
     transformers.utils.logging._reset_library_root_logger()
     # this action activates the env var
     lowerCamelCase__					:   Optional[int]      				=	logging.get_logger('transformers.models.bart.tokenization_bart'			)

     lowerCamelCase__					:   Optional[int]      				=	os.getenv('TRANSFORMERS_VERBOSITY'    ,  UpperCAmelCase			)
     lowerCamelCase__					:   Any      				=	logging.log_levels[env_level_str]

     lowerCamelCase__					:   Dict      				=	logging.get_verbosity()
     self.assertEqual(
         UpperCAmelCase    ,  UpperCAmelCase    ,  F"""TRANSFORMERS_VERBOSITY={env_level_str}/{env_level}, but internal verbosity is {current_level}"""    ,  )

     # restore to the original level
     lowerCamelCase__					:   Tuple      				=	''
     transformers.utils.logging._reset_library_root_logger()
 @mockenv(TRANSFORMERS_VERBOSITY='super-error'			)
 def   A_			(   self						:   str			)    ->			int:
     # reset for the env var to take effect, next time some logger call is made
     transformers.utils.logging._reset_library_root_logger()
     lowerCamelCase__					:   Any      				=	logging.logging.getLogger()
     with CaptureLogger(UpperCAmelCase			) as cl:
         # this action activates the env var
         logging.get_logger('transformers.models.bart.tokenization_bart'			)
     self.assertIn('Unknown option TRANSFORMERS_VERBOSITY=super-error'    ,  cl.out			)

     # no need to restore as nothing was changed







 def   A_			(   self						:   int			)    ->			str:
     # testing `logger.warning_advice()`
     transformers.utils.logging._reset_library_root_logger()

     lowerCamelCase__					:   List[Any]      				=	logging.get_logger('transformers.models.bart.tokenization_bart'			)
     lowerCamelCase__					:   str      				=	'Testing 1, 2, 3'

     with mockenv_context(TRANSFORMERS_NO_ADVISORY_WARNINGS='1'			):
         # nothing should be logged as env var disables this method
         with CaptureLogger(UpperCAmelCase			) as cl:
             logger.warning_advice(UpperCAmelCase			)
         self.assertEqual(cl.out    ,  ''			)

     with mockenv_context(TRANSFORMERS_NO_ADVISORY_WARNINGS=''			):
         # should log normally as TRANSFORMERS_NO_ADVISORY_WARNINGS is unset
         with CaptureLogger(UpperCAmelCase			) as cl:
             logger.warning_advice(UpperCAmelCase			)
         self.assertEqual(cl.out    ,  msg + '\n'			)


def  SCREAMING_SNAKE_CASE   ( )    ->  Dict:
    disable_progress_bar()
    assert are_progress_bars_disabled()

    enable_progress_bar()
    assert not are_progress_bars_disabled()












import contextlib
from multiprocessing import Pool, RLock

from tqdm.auto import tqdm

from ..utils import experimental, logging


_UpperCAmelCase       : Union[str, Any]			     =       logging.get_logger(__name__)

class      lowerCAmelCase							:
 UpperCAmelCase__									=	None


@experimental
def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase				)    ->  Any:
    if ParallelBackendConfig.backend_name is None:
        return _map_with_multiprocessing_pool(
            _UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase				)

    return _map_with_joblib(_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase				)


def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase				)    ->  Any:
    lowerCamelCase__					:   int      				=	num_proc if num_proc <= len(_UpperCAmelCase				) else len(_UpperCAmelCase				)
    lowerCamelCase__					:   Optional[int]      				=	[]  # We organize the splits ourselve (contiguous splits)
    for index in range(_UpperCAmelCase				):
        lowerCamelCase__					:   str      				=	len(_UpperCAmelCase				) // num_proc
        lowerCamelCase__					:   List[str]      				=	len(_UpperCAmelCase				) % num_proc
        lowerCamelCase__					:   Tuple      				=	div * index + min(_UpperCAmelCase		,			_UpperCAmelCase				)
        lowerCamelCase__					:   int      				=	start + div + (1 if index < mod else 0)
        split_kwds.append((function, iterable[start:end], types, index, disable_tqdm, desc)				)

    if len(_UpperCAmelCase				) != sum(len(i[1]				) for i in split_kwds				):
        raise ValueError(
            F"""Error dividing inputs iterable among processes. """
            F"""Total number of objects {len(_UpperCAmelCase				)}, """
            F"""length: {sum(len(i[1]				) for i in split_kwds				)}"""				)

    logger.info(
        F"""Spawning {num_proc} processes for {len(_UpperCAmelCase				)} objects in slices of {[len(i[1]				) for i in split_kwds]}"""				)
    lowerCamelCase__				,							lowerCamelCase__					:   List[str]      				=	None, None
    if not disable_tqdm:
        lowerCamelCase__				,							lowerCamelCase__					:   Union[str, Any]      				=	(RLock(),), tqdm.set_lock
    with Pool(_UpperCAmelCase		,			initargs=_UpperCAmelCase		,			initializer=_UpperCAmelCase				) as pool:
        lowerCamelCase__					:   Any      				=	pool.map(_UpperCAmelCase		,			_UpperCAmelCase				)
    logger.info(F"""Finished {num_proc} processes"""				)
    lowerCamelCase__					:   Union[str, Any]      				=	[obj for proc_res in mapped for obj in proc_res]
    logger.info(F"""Unpacked {len(_UpperCAmelCase				)} objects"""				)

    return mapped


def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase		,			_UpperCAmelCase				)    ->  int:
    # progress bar is not yet supported for _map_with_joblib, because tqdm couldn't accurately be applied to joblib,
    # and it requires monkey-patching joblib internal classes which is subject to change
    import joblib

    with joblib.parallel_backend(ParallelBackendConfig.backend_name		,			n_jobs=_UpperCAmelCase				):
        return joblib.Parallel()(
            joblib.delayed(_UpperCAmelCase				)((function, obj, types, None, True, None)				) for obj in iterable				)


@experimental
@contextlib.contextmanager
def  SCREAMING_SNAKE_CASE   ( _UpperCAmelCase				)    ->  Tuple:
    lowerCamelCase__					:   List[str]      				=	backend_name

    if backend_name == "spark":
        from joblibspark import register_spark

        register_spark()

        # TODO: call create_cache_and_write_probe if "download" in steps
        # TODO: raise NotImplementedError when Dataset.map etc is called

    try:
        yield
    finally:
        lowerCamelCase__					:   str      				=	None





