'''simple docstring'''




import logging
import os
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


__UpperCAmelCase					={
    "debug": logging.DEBUG,
    "info": logging.INFO,
    "warning": logging.WARNING,
    "error": logging.ERROR,
    "critical": logging.CRITICAL,
}

__UpperCAmelCase					=logging.WARNING






def __lowerCAmelCase					(				)      ->       str:
    __lowerCamelCase            =    os.getenv('''DATASETS_VERBOSITY'''							,    snake_case__							)
    if env_level_str:
        if env_level_str in log_levels:
            return log_levels[env_level_str]
        else:
            logging.getLogger().warning(
                f"""Unknown option DATASETS_VERBOSITY={env_level_str}, """
                f"""has to be one of: { ', '.join(log_levels.keys()							) }"""							)
    return _default_log_level






def __lowerCAmelCase					(				)      ->       Optional[Any]:
    return __name__.split('''.'''							)[0]






def __lowerCAmelCase					(				)      ->       List[Any]:
    return logging.getLogger(_get_library_name()							)






def __lowerCAmelCase					(				)      ->       Optional[int]:
    __lowerCamelCase            =    _get_library_root_logger()
    library_root_logger.setLevel(_get_default_logging_level()							)






def __lowerCAmelCase					(				)      ->       List[Any]:
    __lowerCamelCase            =    _get_library_root_logger()
    library_root_logger.setLevel(logging.NOTSET							)






def __lowerCAmelCase					(				UpperCamelCase__ = None							)      ->       Any:


    if name is None:
        __lowerCamelCase            =    _get_library_name()
    return logging.getLogger(snake_case__							)






def __lowerCAmelCase					(				)      ->       List[Any]:
    return _get_library_root_logger().getEffectiveLevel()






def __lowerCAmelCase					(				UpperCamelCase__							)      ->       Union[str, Any]:
    _get_library_root_logger().setLevel(snake_case__							)






def __lowerCAmelCase					(				)      ->       int:
    return set_verbosity(snake_case__							)






def __lowerCAmelCase					(				)      ->       str:
    return set_verbosity(snake_case__							)






def __lowerCAmelCase					(				)      ->       Optional[int]:
    return set_verbosity(snake_case__							)






def __lowerCAmelCase					(				)      ->       int:
    return set_verbosity(snake_case__							)






def __lowerCAmelCase					(				)      ->       Tuple:
    __lowerCamelCase            =    False






def __lowerCAmelCase					(				)      ->       int:
    __lowerCamelCase            =    True


# Configure the library root logger at the module level (singleton-like)
_configure_library_root_logger()




class        a__ :






     def __init__(					self						:						List[Any]    ,							*a						:						Tuple    ,							**a						:						str       ):  # pylint: disable=unused-argument





         """simple docstring"""



         __lowerCamelCase            =    args[0] if args else None






     def __iter__(					self						:						Dict       ):





         """simple docstring"""



         return iter(self._iterator       )






     def __getattr__(					self						:						int    ,							a						:						str       ):





         """simple docstring"""



         def empty_fn(*a						:						Dict    ,							**a						:						Optional[Any]       ):  # pylint: disable=unused-argument
             return

         return empty_fn






     def __enter__(					self						:						int       ):





         """simple docstring"""



         return self






     def __exit__(					self						:						List[str]    ,							a						:						Dict    ,							a						:						List[Any]    ,							a						:						Dict       ):





         """simple docstring"""



         return
__UpperCAmelCase					=True




class        a__ :






     def __call__(					self						:						Tuple    ,							*a						:						Optional[Any]    ,							a						:						Any=False    ,							**a						:						Optional[int]       ):





         """simple docstring"""



         if _tqdm_active and not disable:
             return tqdm_lib.tqdm(*__UpperCAmelCase    ,							**__UpperCAmelCase       )
         else:
             return EmptyTqdm(*__UpperCAmelCase    ,							**__UpperCAmelCase       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						Any    ,							*a						:						List[Any]    ,							**a						:						Optional[Any]       ):





         """simple docstring"""



         __lowerCamelCase            =    None
         if _tqdm_active:
             return tqdm_lib.tqdm.set_lock(*__UpperCAmelCase    ,							**__UpperCAmelCase       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						Optional[int]       ):





         """simple docstring"""



         if _tqdm_active:
             return tqdm_lib.tqdm.get_lock()
__UpperCAmelCase					=_tqdm_cls()






def __lowerCAmelCase					(				)      ->       List[Any]:
    global _tqdm_active
    return bool(_tqdm_active							)






def __lowerCAmelCase					(				)      ->       List[str]:
    global _tqdm_active
    __lowerCamelCase            =    True






def __lowerCAmelCase					(				)      ->       Optional[int]:
    global _tqdm_active
    __lowerCamelCase            =    False



"""simple docstring"""




import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin





class  lowerCamelCase    (A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		List[str]       		=				PhobertTokenizer
			lowerCamelCase__ :		Union[str, Any]       		=				False

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Any:
						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						SCREAMING_SNAKE_CASE__										=   ["""T@@""", """i""", """I""", """R@@""", """r""", """e@@"""]
						SCREAMING_SNAKE_CASE__										=   dict(zip(__UpperCAmelCase     ,					range(len(__UpperCAmelCase							)							)							)							)
						SCREAMING_SNAKE_CASE__										=   ["""#version: 0.2""", """l à</w>"""]
						SCREAMING_SNAKE_CASE__										=   {"""unk_token""": """<unk>"""}

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""vocab_file"""]							)
						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""merges_file"""]							)

						with open(self.vocab_file     ,					"""w"""     ,					encoding="""utf-8"""							) as fp:
									for token in vocab_tokens:
												fp.write(F"""{token} {vocab_tokens[token]}\n"""							)
						with open(self.merges_file     ,					"""w"""     ,					encoding="""utf-8"""							) as fp:
									fp.write("""\n""".join(__UpperCAmelCase							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]     ,					**__UpperCAmelCase					:    int							) ->     Optional[int]:
						kwargs.update(self.special_tokens_map							)
						return PhobertTokenizer.from_pretrained(self.tmpdirname     ,					**__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict     ,					__UpperCAmelCase					:    int							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   """Tôi là VinAI Research"""
						SCREAMING_SNAKE_CASE__										=   """T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"""
						return input_text, output_text





			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   PhobertTokenizer(self.vocab_file     ,					self.merges_file     ,					**self.special_tokens_map							)
						SCREAMING_SNAKE_CASE__										=   """Tôi là VinAI Research"""
						SCREAMING_SNAKE_CASE__										=   """T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h""".split()
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						print(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokens + [tokenizer.unk_token]

						SCREAMING_SNAKE_CASE__										=   [4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					__UpperCAmelCase							)


