





import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin

class   A							(						__lowercase		,   unittest.TestCase    ):
      _snake_case				  =CanineTokenizer
      _snake_case				  =False



      def 						lowerCAmelCase__     (						self:    Optional[Any]							)    ->     List[str]:
         '''simple docstring'''

         super().setUp()
         UpperCAmelCase_           =CanineTokenizer()
         tokenizer.save_pretrained(self.tmpdirname							)



      @cached_property
      def 						lowerCAmelCase__     (						self:    Optional[int]							)    ->     List[str]:
         '''simple docstring'''

         return CanineTokenizer.from_pretrained("google/canine-s"							)



      def 						lowerCAmelCase__     (						self:    Union[str, Any]		,	**_lowerCAmelCase:    List[Any]							)    ->     CanineTokenizer:
         '''simple docstring'''

         UpperCAmelCase_           =self.tokenizer_class.from_pretrained(self.tmpdirname		,	**_lowerCAmelCase							)
         UpperCAmelCase_           =1024
         return tokenizer



      @require_torch
      def 						lowerCAmelCase__     (						self:    int							)    ->     Optional[int]:
         '''simple docstring'''

         UpperCAmelCase_           =self.canine_tokenizer
         UpperCAmelCase_           =["Life is like a box of chocolates.", "You never know what you're gonna get."]
         # fmt: off
         UpperCAmelCase_           =[5_7344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 5_7345, 0, 0, 0, 0]
         # fmt: on
         UpperCAmelCase_           =tokenizer(_lowerCAmelCase		,	padding=_lowerCAmelCase		,	return_tensors="pt"							)
         self.assertIsInstance(_lowerCAmelCase		,	_lowerCAmelCase							)

         UpperCAmelCase_           =list(batch.input_ids.numpy()[0]							)

         self.assertListEqual(_lowerCAmelCase		,	_lowerCAmelCase							)

         self.assertEqual((2, 39)		,	batch.input_ids.shape							)
         self.assertEqual((2, 39)		,	batch.attention_mask.shape							)



      @require_torch
      def 						lowerCAmelCase__     (						self:    int							)    ->     str:
         '''simple docstring'''

         UpperCAmelCase_           =self.canine_tokenizer
         UpperCAmelCase_           =["Once there was a man.", "He wrote a test in HuggingFace Tranformers."]
         UpperCAmelCase_           =tokenizer(_lowerCAmelCase		,	padding=_lowerCAmelCase		,	return_tensors="pt"							)
         # check if input_ids, attention_mask and token_type_ids are returned
         self.assertIn("input_ids"		,	_lowerCAmelCase							)
         self.assertIn("attention_mask"		,	_lowerCAmelCase							)
         self.assertIn("token_type_ids"		,	_lowerCAmelCase							)



      @require_torch
      def 						lowerCAmelCase__     (						self:    str							)    ->     Any:
         '''simple docstring'''

         UpperCAmelCase_           =self.canine_tokenizer
         UpperCAmelCase_           =[
             "What's the weater?",
             "It's about 25 degrees.",
         ]
         UpperCAmelCase_           =tokenizer(
             text_target=_lowerCAmelCase		,	max_length=32		,	padding="max_length"		,	truncation=_lowerCAmelCase		,	return_tensors="pt"							)
         self.assertEqual(32		,	targets["input_ids"].shape[1]							)



      def 						lowerCAmelCase__     (						self:    Optional[int]							)    ->     Tuple:
         '''simple docstring'''

         UpperCAmelCase_           =self.get_tokenizers()
         for tokenizer in tokenizers:
            with self.subTest(F'{tokenizer.__class__.__name__}'							):
               self.assertNotEqual(tokenizer.model_max_length		,	42							)

        # Now let's start the test
         UpperCAmelCase_           =self.get_tokenizers()
         for tokenizer in tokenizers:
            with self.subTest(F'{tokenizer.__class__.__name__}'							):
               # Isolate this from the other tests because we save additional tokens/etc
               UpperCAmelCase_           =tempfile.mkdtemp()

               UpperCAmelCase_           =" He is very happy, UNwant\u00E9d,running"
               UpperCAmelCase_           =tokenizer.encode(_lowerCAmelCase		,	add_special_tokens=_lowerCAmelCase							)
               tokenizer.save_pretrained(_lowerCAmelCase							)

               UpperCAmelCase_           =tokenizer.__class__.from_pretrained(_lowerCAmelCase							)
               UpperCAmelCase_           =after_tokenizer.encode(_lowerCAmelCase		,	add_special_tokens=_lowerCAmelCase							)
               self.assertListEqual(_lowerCAmelCase		,	_lowerCAmelCase							)

               shutil.rmtree(_lowerCAmelCase							)

         UpperCAmelCase_           =self.get_tokenizers(model_max_length=42							)
         for tokenizer in tokenizers:
            with self.subTest(F'{tokenizer.__class__.__name__}'							):
               # Isolate this from the other tests because we save additional tokens/etc
               UpperCAmelCase_           =tempfile.mkdtemp()

               UpperCAmelCase_           =" He is very happy, UNwant\u00E9d,running"

               UpperCAmelCase_           =tokenizer.additional_special_tokens

               # We can add a new special token for Canine as follows:
               UpperCAmelCase_           =chr(0xe0_07							)
               additional_special_tokens.append(_lowerCAmelCase							)
               tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}							)
               UpperCAmelCase_           =tokenizer.encode(_lowerCAmelCase		,	add_special_tokens=_lowerCAmelCase							)
               tokenizer.save_pretrained(_lowerCAmelCase							)

               UpperCAmelCase_           =tokenizer.__class__.from_pretrained(_lowerCAmelCase							)
               UpperCAmelCase_           =after_tokenizer.encode(_lowerCAmelCase		,	add_special_tokens=_lowerCAmelCase							)
               self.assertListEqual(_lowerCAmelCase		,	_lowerCAmelCase							)
               self.assertIn(_lowerCAmelCase		,	after_tokenizer.additional_special_tokens							)
               self.assertEqual(after_tokenizer.model_max_length		,	42							)

               UpperCAmelCase_           =tokenizer.__class__.from_pretrained(_lowerCAmelCase		,	model_max_length=43							)
               self.assertEqual(tokenizer.model_max_length		,	43							)

               shutil.rmtree(_lowerCAmelCase							)



      def 						lowerCAmelCase__     (						self:    int							)    ->     Tuple:
         '''simple docstring'''

         UpperCAmelCase_           =self.get_tokenizers(do_lower_case=_lowerCAmelCase							)
         for tokenizer in tokenizers:
            with self.subTest(F'{tokenizer.__class__.__name__}'							):
               UpperCAmelCase_    ,							UpperCAmelCase_           =self.get_clean_sequence(_lowerCAmelCase							)

               # a special token for Canine can be defined as follows:
               UpperCAmelCase_           =0xe0_05
               UpperCAmelCase_           =chr(_lowerCAmelCase							)

               tokenizer.add_special_tokens({"cls_token": special_token}							)
               UpperCAmelCase_           =tokenizer.encode(_lowerCAmelCase		,	add_special_tokens=_lowerCAmelCase							)
               self.assertEqual(len(_lowerCAmelCase							)		,	1							)

               UpperCAmelCase_           =tokenizer.decode(ids + encoded_special_token		,	clean_up_tokenization_spaces=_lowerCAmelCase							)
               UpperCAmelCase_           =tokenizer.encode(_lowerCAmelCase		,	add_special_tokens=_lowerCAmelCase							)

               UpperCAmelCase_           =tokenizer.encode(_lowerCAmelCase		,	add_special_tokens=_lowerCAmelCase							)
               UpperCAmelCase_           =tokenizer.encode(_lowerCAmelCase		,	add_special_tokens=_lowerCAmelCase							)
               self.assertEqual(_lowerCAmelCase		,	input_encoded + special_token_id							)

               UpperCAmelCase_           =tokenizer.decode(_lowerCAmelCase		,	skip_special_tokens=_lowerCAmelCase							)
               self.assertTrue(special_token not in decoded							)



      def 						lowerCAmelCase__     (						self:    Any							)    ->     Any:
         '''simple docstring'''

         UpperCAmelCase_           =self.get_tokenizers(do_lower_case=_lowerCAmelCase							)
         for tokenizer in tokenizers:
            with self.subTest(F'{tokenizer.__class__.__name__}'							):
               UpperCAmelCase_           =chr(0xe0_05							)
               UpperCAmelCase_           =chr(0xe0_06							)

               # `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
               tokenizer.add_tokens([SPECIAL_TOKEN_1]		,	special_tokens=_lowerCAmelCase							)
               # `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
               # which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
               tokenizer.add_special_tokens({"additional_special_tokens": [SPECIAL_TOKEN_2]}							)

               UpperCAmelCase_           =tokenizer.tokenize(_lowerCAmelCase							)
               UpperCAmelCase_           =tokenizer.tokenize(_lowerCAmelCase							)

               self.assertEqual(len(_lowerCAmelCase							)		,	1							)
               self.assertEqual(len(_lowerCAmelCase							)		,	1							)
               self.assertEqual(token_a[0]		,	_lowerCAmelCase							)
               self.assertEqual(token_a[0]		,	_lowerCAmelCase							)



      @require_tokenizers
      def 						lowerCAmelCase__     (						self:    Optional[Any]							)    ->     Optional[Any]:
         '''simple docstring'''

         UpperCAmelCase_           =self.get_tokenizers(do_lower_case=_lowerCAmelCase							)
         for tokenizer in tokenizers:
            with self.subTest(F'{tokenizer.__class__.__name__}'							):
               # a special token for Canine can be defined as follows:
               UpperCAmelCase_           =0xe0_06
               UpperCAmelCase_           =chr(_lowerCAmelCase							)

               UpperCAmelCase_           =AddedToken(_lowerCAmelCase		,	lstrip=_lowerCAmelCase							)
               tokenizer.add_special_tokens({"additional_special_tokens": [new_token]}							)

               with tempfile.TemporaryDirectory() as tmp_dir_name:
                  tokenizer.save_pretrained(_lowerCAmelCase							)
                  tokenizer.from_pretrained(_lowerCAmelCase							)



      def 						lowerCAmelCase__     (						self:    Any							)    ->     List[Any]:
         '''simple docstring'''

         UpperCAmelCase_           =[]
         if self.test_slow_tokenizer:
            tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())							)

         if self.test_rust_tokenizer:
            tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())							)

         for tokenizer_class, tokenizer_utils in tokenizer_list:
            with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer_utils.save_pretrained(_lowerCAmelCase							)

               with open(os.path.join(_lowerCAmelCase		,	"special_tokens_map.json"							)		,	encoding="utf-8"							) as json_file:
                  UpperCAmelCase_           =json.load(_lowerCAmelCase							)

               with open(os.path.join(_lowerCAmelCase		,	"tokenizer_config.json"							)		,	encoding="utf-8"							) as json_file:
                  UpperCAmelCase_           =json.load(_lowerCAmelCase							)

               # a special token for Canine can be defined as follows:
               UpperCAmelCase_           =0xe0_06
               UpperCAmelCase_           =chr(_lowerCAmelCase							)

               UpperCAmelCase_           =[new_token_a]
               UpperCAmelCase_           =[new_token_a]

               with open(os.path.join(_lowerCAmelCase		,	"special_tokens_map.json"							)		,	"w"		,	encoding="utf-8"							) as outfile:
                  json.dump(_lowerCAmelCase		,	_lowerCAmelCase							)
               with open(os.path.join(_lowerCAmelCase		,	"tokenizer_config.json"							)		,	"w"		,	encoding="utf-8"							) as outfile:
                  json.dump(_lowerCAmelCase		,	_lowerCAmelCase							)

               # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
               # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
               # "special_tokens_map.json" files
               UpperCAmelCase_           =tokenizer_class.from_pretrained(_lowerCAmelCase		,	extra_ids=0							)
               self.assertIn(_lowerCAmelCase		,	tokenizer_without_change_in_init.additional_special_tokens							)
               # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
               self.assertEqual(
                   [new_token_a]		,	tokenizer_without_change_in_init.convert_ids_to_tokens(
                       tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]							)							)		,	)

               UpperCAmelCase_           =0xe0_07
               UpperCAmelCase_           =chr(_lowerCAmelCase							)
               # Now we test that we can change the value of additional_special_tokens in the from_pretrained
               UpperCAmelCase_           =[AddedToken(_lowerCAmelCase		,	lstrip=_lowerCAmelCase							)]
               UpperCAmelCase_           =tokenizer_class.from_pretrained(
                   _lowerCAmelCase		,	additional_special_tokens=_lowerCAmelCase		,	extra_ids=0							)

               self.assertIn(_lowerCAmelCase		,	tokenizer.additional_special_tokens							)
               # self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
               self.assertEqual(
                   [new_token_a]		,	tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]							)							)							)



      @require_tokenizers
      def 						lowerCAmelCase__     (						self:    Optional[Any]							)    ->     Optional[Any]:
         '''simple docstring'''

         UpperCAmelCase_           =self.get_tokenizers(do_lower_case=_lowerCAmelCase							)
         for tokenizer in tokenizers:
            with self.subTest(F'{tokenizer.__class__.__name__}'							):
               UpperCAmelCase_           ="hello world"
               if self.space_between_special_tokens:
                  UpperCAmelCase_           ="[CLS] hello world [SEP]"
               else:
                  UpperCAmelCase_           =input
               UpperCAmelCase_           =tokenizer.encode(_lowerCAmelCase		,	add_special_tokens=_lowerCAmelCase							)
               UpperCAmelCase_           =tokenizer.decode(_lowerCAmelCase		,	spaces_between_special_tokens=self.space_between_special_tokens							)
               self.assertIn(_lowerCAmelCase		,	[output, output.lower()]							)



      def 						lowerCAmelCase__     (						self:    List[str]							)    ->     Optional[Any]:
         '''simple docstring'''

         UpperCAmelCase_           =self.get_tokenizers()
         for tokenizer in tokenizers:
            with self.subTest(F'{tokenizer.__class__.__name__}'							):
               UpperCAmelCase_           =[
                   "bos_token",
                   "eos_token",
                   "unk_token",
                   "sep_token",
                   "pad_token",
                   "cls_token",
                   "mask_token",
               ]

         UpperCAmelCase_           ="a"
         UpperCAmelCase_           =ord(_lowerCAmelCase							)

         for attr in attributes_list:
            setattr(_lowerCAmelCase		,	attr + "_id"		,	_lowerCAmelCase							)
            self.assertEqual(getattr(_lowerCAmelCase		,	_lowerCAmelCase							)		,	_lowerCAmelCase							)
            self.assertEqual(getattr(_lowerCAmelCase		,	attr + "_id"							)		,	_lowerCAmelCase							)

            setattr(_lowerCAmelCase		,	attr + "_id"		,	_lowerCAmelCase							)
            self.assertEqual(getattr(_lowerCAmelCase		,	_lowerCAmelCase							)		,	_lowerCAmelCase							)
            self.assertEqual(getattr(_lowerCAmelCase		,	attr + "_id"							)		,	_lowerCAmelCase							)

         setattr(_lowerCAmelCase		,	"additional_special_tokens_ids"		,	[]							)
         self.assertListEqual(getattr(_lowerCAmelCase		,	"additional_special_tokens"							)		,	[]							)
         self.assertListEqual(getattr(_lowerCAmelCase		,	"additional_special_tokens_ids"							)		,	[]							)

         UpperCAmelCase_           =0xe0_06
         UpperCAmelCase_           =chr(_lowerCAmelCase							)
         setattr(_lowerCAmelCase		,	"additional_special_tokens_ids"		,	[additional_special_token_id]							)
         self.assertListEqual(getattr(_lowerCAmelCase		,	"additional_special_tokens"							)		,	[additional_special_token]							)
         self.assertListEqual(getattr(_lowerCAmelCase		,	"additional_special_tokens_ids"							)		,	[additional_special_token_id]							)



      def 						lowerCAmelCase__     (						self:    List[str]							)    ->     List[str]:
         '''simple docstring'''

         pass



      def 						lowerCAmelCase__     (						self:    Dict							)    ->     List[str]:
         '''simple docstring'''

         pass



      def 						lowerCAmelCase__     (						self:    Union[str, Any]							)    ->     Dict:
         '''simple docstring'''

         pass



      def 						lowerCAmelCase__     (						self:    Optional[Any]							)    ->     Union[str, Any]:
         '''simple docstring'''

         pass



      def 						lowerCAmelCase__     (						self:    Any							)    ->     List[Any]:
         '''simple docstring'''

         pass



      def 						lowerCAmelCase__     (						self:    List[Any]							)    ->     List[str]:
         '''simple docstring'''

         pass



      def 						lowerCAmelCase__     (						self:    Tuple							)    ->     Union[str, Any]:
         '''simple docstring'''

         pass






      def 						lowerCAmelCase__     (						self:    str							)    ->     str:
         '''simple docstring'''

         pass






import logging
import os
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


lowerCamelCase  ={
    "debug": logging.DEBUG,
    "info": logging.INFO,
    "warning": logging.WARNING,
    "error": logging.ERROR,
    "critical": logging.CRITICAL,
}

lowerCamelCase  =logging.WARNING


def 				SCREAMING_SNAKE_CASE_   (      ):
      UpperCamelCase__     : int											=					os.getenv('''DATASETS_VERBOSITY'''			,		UpperCamelCase__		)
      if env_level_str:
            if env_level_str in log_levels:
                  return log_levels[env_level_str]
            else:
                  logging.getLogger().warning(
                      f'''Unknown option DATASETS_VERBOSITY={env_level_str}, '''
                      f'''has to be one of: { ", ".join(log_levels.keys()		) }'''		)
      return _default_log_level


def 				SCREAMING_SNAKE_CASE_   (      ):
      return __name__.split('''.'''		)[0]


def 				SCREAMING_SNAKE_CASE_   (      ):
      return logging.getLogger(_get_library_name()		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      # Apply our default configuration to the library root logger.
      UpperCamelCase__     : List[Any]											=					_get_library_root_logger()
      library_root_logger.setLevel(_get_default_logging_level()		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      UpperCamelCase__     : List[Any]											=					_get_library_root_logger()
      library_root_logger.setLevel(logging.NOTSET		)


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__ = None		):
      if name is None:
            UpperCamelCase__     : Union[str, Any]											=					_get_library_name()
      return logging.getLogger(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      return _get_library_root_logger().getEffectiveLevel()


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__		):
      _get_library_root_logger().setLevel(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      return set_verbosity(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      return set_verbosity(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      return set_verbosity(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      return set_verbosity(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      UpperCamelCase__     : str											=					False


def 				SCREAMING_SNAKE_CASE_   (      ):
      UpperCamelCase__     : List[Any]											=					True


# Configure the library root logger at the module level (singleton-like)
_configure_library_root_logger()




class 					_lowerCamelCase			:





     """simple docstring"""







     def __init__(				self    ,    *__SCREAMING_SNAKE_CASE    ,    **__SCREAMING_SNAKE_CASE	)	-> int:  # pylint: disable=unused-argument




           """simple docstring"""







           UpperCamelCase__     : Dict											=					args[0] if args else None






     def __iter__(				self	)	-> Dict:




           """simple docstring"""







           return iter(self._iterator	)






     def __getattr__(				self    ,    __SCREAMING_SNAKE_CASE	)	-> Tuple:




           """simple docstring"""







           def empty_fn(*__SCREAMING_SNAKE_CASE    ,    **__SCREAMING_SNAKE_CASE	):  # pylint: disable=unused-argument
                 return

           return empty_fn






     def __enter__(				self	)	-> Optional[int]:




           """simple docstring"""







           return self






     def __exit__(				self    ,    __SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE	)	-> Dict:




           """simple docstring"""







           return
lowerCamelCase  =True




class 					_lowerCamelCase			:





     """simple docstring"""







     def __call__(				self    ,    *__SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE=False    ,    **__SCREAMING_SNAKE_CASE	)	-> int:




           """simple docstring"""







           if _tqdm_active and not disable:
                 return tqdm_lib.tqdm(*__SCREAMING_SNAKE_CASE    ,    **__SCREAMING_SNAKE_CASE	)
           else:
                 return EmptyTqdm(*__SCREAMING_SNAKE_CASE    ,    **__SCREAMING_SNAKE_CASE	)






     def  __SCREAMING_SNAKE_CASE      (				self    ,    *__SCREAMING_SNAKE_CASE    ,    **__SCREAMING_SNAKE_CASE	)	-> Tuple:




           """simple docstring"""







           UpperCamelCase__     : Any											=					None
           if _tqdm_active:
                 return tqdm_lib.tqdm.set_lock(*__SCREAMING_SNAKE_CASE    ,    **__SCREAMING_SNAKE_CASE	)






     def  __SCREAMING_SNAKE_CASE      (				self	)	-> Tuple:




           """simple docstring"""







           if _tqdm_active:
                 return tqdm_lib.tqdm.get_lock()
lowerCamelCase  =_tqdm_cls()


def 				SCREAMING_SNAKE_CASE_   (      ):
      global _tqdm_active
      return bool(_tqdm_active		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      global _tqdm_active
      UpperCamelCase__     : Dict											=					True


def 				SCREAMING_SNAKE_CASE_   (      ):
      global _tqdm_active
      UpperCamelCase__     : Union[str, Any]											=					False


