
"""simple docstring"""
import functools
import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

import huggingface_hub.utils as hf_hub_utils
from tqdm import auto as tqdm_lib


lowercase_       :						Tuple     					=	threading.Lock()
lowercase_       :						Optional[logging.Handler]     					=	None

lowercase_       :						Any     					=	{
    '''debug''': logging.DEBUG,
    '''info''': logging.INFO,
    '''warning''': logging.WARNING,
    '''error''': logging.ERROR,
    '''critical''': logging.CRITICAL,
}

lowercase_       :						str     					=	logging.WARNING

lowercase_       :						int     					=	True





def    _lowerCAmelCase    (     )  ->    List[str]:
  _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = os.getenv("TRANSFORMERS_VERBOSITY",						lowerCamelCase__				)
  if env_level_str:
    if env_level_str in log_levels:
      return log_levels[env_level_str]
    else:
      logging.getLogger().warning(
          f'''Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, '''
          f'''has to be one of: { ", ".join(log_levels.keys()				) }'''				)
  return _default_log_level





def    _lowerCAmelCase    (     )  ->    str:
  return __name__.split("."				)[0]





def    _lowerCAmelCase    (     )  ->    logging.Logger:
  return logging.getLogger(_get_library_name()				)





def    _lowerCAmelCase    (     )  ->    None:
  global _default_handler

  with _lock:
    if _default_handler:
      # This library has already configured the library root logger.
      return
    _SCREAMING_SNAKE_CASE   :				Tuple							    = logging.StreamHandler()  # Set sys.stderr as stream.
    _SCREAMING_SNAKE_CASE   :				str							    = sys.stderr.flush

    # Apply our default configuration to the library root logger.
    _SCREAMING_SNAKE_CASE   :				Dict							    = _get_library_root_logger()
    library_root_logger.addHandler(_default_handler				)
    library_root_logger.setLevel(_get_default_logging_level()				)
    _SCREAMING_SNAKE_CASE   :				Any							    = False





def    _lowerCAmelCase    (     )  ->    None:
  global _default_handler

  with _lock:
    if not _default_handler:
      return

    _SCREAMING_SNAKE_CASE   :				int							    = _get_library_root_logger()
    library_root_logger.removeHandler(_default_handler				)
    library_root_logger.setLevel(logging.NOTSET				)
    _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = None





def    _lowerCAmelCase    (     )  ->    Any:
  return log_levels





def    _lowerCAmelCase    (     lowerCamelCase__							:							Optional[str] = None				)  ->    logging.Logger:

  if name is None:
    _SCREAMING_SNAKE_CASE   :				Optional[int]							    = _get_library_name()

  _configure_library_root_logger()
  return logging.getLogger(lowerCamelCase__				)





def    _lowerCAmelCase    (     )  ->    int:
  _configure_library_root_logger()
  return _get_library_root_logger().getEffectiveLevel()





def    _lowerCAmelCase    (     lowerCamelCase__							:							int				)  ->    None:
  _configure_library_root_logger()
  _get_library_root_logger().setLevel(lowerCamelCase__				)





def    _lowerCAmelCase    (     )  ->    Any:
  return set_verbosity(lowerCamelCase__				)





def    _lowerCAmelCase    (     )  ->    Optional[Any]:
  return set_verbosity(lowerCamelCase__				)





def    _lowerCAmelCase    (     )  ->    str:
  return set_verbosity(lowerCamelCase__				)





def    _lowerCAmelCase    (     )  ->    Optional[Any]:
  return set_verbosity(lowerCamelCase__				)





def    _lowerCAmelCase    (     )  ->    None:
  _configure_library_root_logger()

  assert _default_handler is not None
  _get_library_root_logger().removeHandler(_default_handler				)





def    _lowerCAmelCase    (     )  ->    None:
  _configure_library_root_logger()

  assert _default_handler is not None
  _get_library_root_logger().addHandler(_default_handler				)





def    _lowerCAmelCase    (     lowerCamelCase__							:							logging.Handler				)  ->    None:
  _configure_library_root_logger()

  assert handler is not None
  _get_library_root_logger().addHandler(lowerCamelCase__				)





def    _lowerCAmelCase    (     lowerCamelCase__							:							logging.Handler				)  ->    None:
  _configure_library_root_logger()

  assert handler is not None and handler not in _get_library_root_logger().handlers
  _get_library_root_logger().removeHandler(lowerCamelCase__				)





def    _lowerCAmelCase    (     )  ->    None:
  _configure_library_root_logger()
  _SCREAMING_SNAKE_CASE   :				int							    = False





def    _lowerCAmelCase    (     )  ->    None:
  _configure_library_root_logger()
  _SCREAMING_SNAKE_CASE   :				Dict							    = True





def    _lowerCAmelCase    (     )  ->    None:
  _SCREAMING_SNAKE_CASE   :				Optional[int]							    = _get_library_root_logger().handlers

  for handler in handlers:
    _SCREAMING_SNAKE_CASE   :				List[str]							    = logging.Formatter("[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s"				)
    handler.setFormatter(lowerCamelCase__				)





def    _lowerCAmelCase    (     )  ->    None:
  _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = _get_library_root_logger().handlers

  for handler in handlers:
    handler.setFormatter(lowerCamelCase__				)





def    _lowerCAmelCase    (     self							:							Any,						*lowerCamelCase__							:							List[Any],						**lowerCamelCase__							:							Optional[Any]				)  ->    Dict:
  _SCREAMING_SNAKE_CASE   :				List[Any]							    = os.getenv("TRANSFORMERS_NO_ADVISORY_WARNINGS",						lowerCamelCase__				)
  if no_advisory_warnings:
    return
  self.warning(*lowerCamelCase__,						**lowerCamelCase__				)


lowercase_       :						Union[str, Any]     					=	warning_advice





@functools.lru_cache(lowerCamelCase__				)
def    _lowerCAmelCase    (     self							:							List[str],						*lowerCamelCase__							:							Optional[Any],						**lowerCamelCase__							:							str				)  ->    List[str]:
  self.warning(*lowerCamelCase__,						**lowerCamelCase__				)


lowercase_       :						Any     					=	warning_once
class 			UpperCamelCase      :
     def __init__(     self  ,     *snake_case__  ,     **snake_case__				):  # pylint: disable=unused-argument




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Any							    = args[0] if args else None
     def __iter__(     self				):




       """simple docstring"""


       return iter(self._iterator				)
     def __getattr__(     self  ,     snake_case__				):




       """simple docstring"""


       def empty_fn(*snake_case__  ,     **snake_case__				):  # pylint: disable=unused-argument
         return

       return empty_fn
     def __enter__(     self				):




       """simple docstring"""


       return self




     def __exit__(     self  ,     snake_case__  ,     snake_case__  ,     snake_case__				):




       """simple docstring"""


       return
class 			UpperCamelCase      :
     def __call__(     self  ,     *snake_case__  ,     **snake_case__				):




       """simple docstring"""


       if _tqdm_active:
         return tqdm_lib.tqdm(*snake_case__  ,     **snake_case__				)
       else:
         return EmptyTqdm(*snake_case__  ,     **snake_case__				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     *snake_case__  ,     **snake_case__				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = None
       if _tqdm_active:
         return tqdm_lib.tqdm.set_lock(*snake_case__  ,     **snake_case__				)




     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       if _tqdm_active:
         return tqdm_lib.tqdm.get_lock()
lowercase_       :						List[Any]     					=	_tqdm_cls()





def    _lowerCAmelCase    (     )  ->    bool:
  global _tqdm_active
  return bool(_tqdm_active				)





def    _lowerCAmelCase    (     )  ->    int:
  global _tqdm_active
  _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = True
  hf_hub_utils.enable_progress_bars()





def    _lowerCAmelCase    (     )  ->    Any:
  global _tqdm_active
  _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = False
  hf_hub_utils.disable_progress_bars()





"""simple docstring"""
import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 			UpperCamelCase      (							__SCREAMING_SNAKE_CASE      ,       unittest.TestCase  ):
     A__      =			None
     A__      =			BloomTokenizerFast
     A__      =			BloomTokenizerFast
     A__      =			True
     A__      =			False
     A__      =			"""tokenizer_file"""
     A__      =			{"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       super().setUp()
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = BloomTokenizerFast.from_pretrained("bigscience/tokenizer"				)
       tokenizer.save_pretrained(self.tmpdirname				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     **snake_case__				):




       """simple docstring"""


       kwargs.update(self.special_tokens_map				)
       return BloomTokenizerFast.from_pretrained(self.tmpdirname  ,     **snake_case__				)
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				int							    = self.get_rust_tokenizer()

       _SCREAMING_SNAKE_CASE   :				Tuple							    = ["The quick brown fox</s>", "jumps over the lazy dog</s>"]
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = [[2175, 23714, 73173, 144252, 2], [77, 132619, 3478, 368, 109586, 35433, 2]]

       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = tokenizer.batch_encode_plus(snake_case__				)["input_ids"]
       self.assertListEqual(snake_case__  ,     snake_case__				)

       _SCREAMING_SNAKE_CASE   :				Tuple							    = tokenizer.batch_decode(snake_case__				)
       self.assertListEqual(snake_case__  ,     snake_case__				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__=6				):




       """simple docstring"""


       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
           _SCREAMING_SNAKE_CASE   :				Tuple							    = self.rust_tokenizer_class.from_pretrained(snake_case__  ,     **snake_case__				)
           # tokenizer_r.pad_token = None # Hotfixing padding = None
           # Simple input
           _SCREAMING_SNAKE_CASE   :				str							    = "This is a simple input"
           _SCREAMING_SNAKE_CASE   :				Tuple							    = ["This is a simple input 1", "This is a simple input 2"]
           _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = ("This is a simple input", "This is a pair")
           _SCREAMING_SNAKE_CASE   :				List[str]							    = [
               ("This is a simple input 1", "This is a simple input 2"),
               ("This is a simple pair 1", "This is a simple pair 2"),
           ]

           # Simple input tests
           try:
             tokenizer_r.encode(snake_case__  ,     max_length=snake_case__				)
             tokenizer_r.encode_plus(snake_case__  ,     max_length=snake_case__				)

             tokenizer_r.batch_encode_plus(snake_case__  ,     max_length=snake_case__				)
             tokenizer_r.encode(snake_case__  ,     max_length=snake_case__				)
             tokenizer_r.batch_encode_plus(snake_case__  ,     max_length=snake_case__				)
           except ValueError:
             self.fail("Bloom Tokenizer should be able to deal with padding"				)

           _SCREAMING_SNAKE_CASE   :				Tuple							    = None  # Hotfixing padding = None
           self.assertRaises(snake_case__  ,     tokenizer_r.encode  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"				)

           # Simple input
           self.assertRaises(snake_case__  ,     tokenizer_r.encode_plus  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"				)

           # Simple input
           self.assertRaises(
               snake_case__  ,     tokenizer_r.batch_encode_plus  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"  ,     )

           # Pair input
           self.assertRaises(snake_case__  ,     tokenizer_r.encode  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"				)

           # Pair input
           self.assertRaises(snake_case__  ,     tokenizer_r.encode_plus  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"				)

           # Pair input
           self.assertRaises(
               snake_case__  ,     tokenizer_r.batch_encode_plus  ,     snake_case__  ,     max_length=snake_case__  ,     padding="max_length"  ,     )
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = self.get_rust_tokenizer()
       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = load_dataset("xnli"  ,     "all_languages"  ,     split="test"  ,     streaming=snake_case__				)

       _SCREAMING_SNAKE_CASE   :				Tuple							    = next(iter(snake_case__				)				)["premise"]  # pick up one data
       _SCREAMING_SNAKE_CASE   :				int							    = list(sample_data.values()				)

       _SCREAMING_SNAKE_CASE   :				List[str]							    = list(map(tokenizer.encode  ,     snake_case__				)				)
       _SCREAMING_SNAKE_CASE   :				int							    = [tokenizer.decode(snake_case__  ,     clean_up_tokenization_spaces=snake_case__				) for x in output_tokens]
       self.assertListEqual(snake_case__  ,     snake_case__				)




     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map				)  ,     1				)
       self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()				)[0]				)  ,     1				)



