






'''simple docstring'''


from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase_				: str    	=							logging.get_logger(__name__)


lowerCamelCase_				: Union[str, Any]    	=							{
    '''nielsr/canine-s''': 2048,
}

# Unicode defines 1,114,112 total “codepoints”
lowerCamelCase_				: str    	=							1114112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
lowerCamelCase_				: List[str]    	=							0

lowerCamelCase_				: Optional[Any]    	=							0Xe_0_0_0
lowerCamelCase_				: Optional[Any]    	=							0Xe_0_0_1
lowerCamelCase_				: Optional[int]    	=							0Xe_0_0_2
lowerCamelCase_				: List[Any]    	=							0Xe_0_0_3
lowerCamelCase_				: Any    	=							0Xe_0_0_4

# Maps special codepoints to human-readable names.
lowerCamelCase_				: Dict[int, str]    	=							{
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
lowerCamelCase_				: Dict[str, int]    	=							{name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}
class    _SCREAMING_SNAKE_CASE							(		SCREAMING_SNAKE_CASE	):


			'''simple docstring'''





			__a							:							Optional[int]					     =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






			def __init__(   self							:						Tuple ,      lowercase							:						Optional[int]=chr(lowercase			) ,      lowercase							:						Dict=chr(lowercase			) ,      lowercase							:						Optional[int]=chr(lowercase			) ,      lowercase							:						List[Any]=chr(lowercase			) ,      lowercase							:						List[Any]=chr(lowercase			) ,      lowercase							:						Optional[Any]=chr(lowercase			) ,      lowercase							:						Tuple=False ,      lowercase							:						int=2_0_4_8 ,      **lowercase							:						Dict ,      )							->    str:

									'''simple docstring'''







									UpperCamelCase__         = AddedToken(lowercase ,      lstrip=lowercase ,      rstrip=lowercase			) if isinstance(lowercase ,      lowercase			) else bos_token
									UpperCamelCase__         = AddedToken(lowercase ,      lstrip=lowercase ,      rstrip=lowercase			) if isinstance(lowercase ,      lowercase			) else eos_token
									UpperCamelCase__         = AddedToken(lowercase ,      lstrip=lowercase ,      rstrip=lowercase			) if isinstance(lowercase ,      lowercase			) else sep_token
									UpperCamelCase__         = AddedToken(lowercase ,      lstrip=lowercase ,      rstrip=lowercase			) if isinstance(lowercase ,      lowercase			) else cls_token
									UpperCamelCase__         = AddedToken(lowercase ,      lstrip=lowercase ,      rstrip=lowercase			) if isinstance(lowercase ,      lowercase			) else pad_token

									# Mask token behave like a normal word, i.e. include the space before it
									UpperCamelCase__         = AddedToken(lowercase ,      lstrip=lowercase ,      rstrip=lowercase			) if isinstance(lowercase ,      lowercase			) else mask_token

									super().__init__(
									    bos_token=lowercase ,      eos_token=lowercase ,      sep_token=lowercase ,      cls_token=lowercase ,      pad_token=lowercase ,      mask_token=lowercase ,      add_prefix_space=lowercase ,      model_max_length=lowercase ,      **lowercase ,      )

									# Creates a mapping for looking up the IDs of special symbols.
									UpperCamelCase__         = {}
									for codepoint, name in SPECIAL_CODEPOINTS.items():
															UpperCamelCase__         = codepoint

									# Creates a mapping for looking up the string forms of special symbol IDs.
									UpperCamelCase__         = {
									    codepoint: name for name, codepoint in self._special_codepoints.items()
									}

									UpperCamelCase__         = UNICODE_VOCAB_SIZE
									UpperCamelCase__         = len(self._special_codepoints			)






			@property
			def 				A      (   self							:						Any			)							->    int:

									'''simple docstring'''







									return self._unicode_vocab_size






			def 				A      (   self							:						Union[str, Any] ,      lowercase							:						str			)							->    List[str]:

									'''simple docstring'''







									return list(lowercase			)






			def 				A      (   self							:						Optional[Any] ,      lowercase							:						str			)							->    int:

									'''simple docstring'''







									try:
															return ord(lowercase			)
									except TypeError:
															raise ValueError(f"invalid token: '{token}'"			)






			def 				A      (   self							:						str ,      lowercase							:						int			)							->    str:

									'''simple docstring'''







									try:
															if index in SPECIAL_CODEPOINTS:
																					return SPECIAL_CODEPOINTS[index]
															return chr(lowercase			)
									except TypeError:
															raise ValueError(f"invalid id: {index}"			)






			def 				A      (   self							:						Any ,      lowercase							:						Union[str, Any]			)							->    Optional[Any]:

									'''simple docstring'''







									return "".join(lowercase			)






			def 				A      (   self							:						Optional[Any] ,      lowercase							:						List[int] ,      lowercase							:						Optional[List[int]] = None			)							->    List[int]:

									'''simple docstring'''







									UpperCamelCase__         = [self.sep_token_id]
									UpperCamelCase__         = [self.cls_token_id]

									UpperCamelCase__         = cls + token_ids_a + sep
									if token_ids_a is not None:
															result += token_ids_a + sep
									return result






			def 				A      (   self							:						List[str] ,      lowercase							:						List[int] ,      lowercase							:						Optional[List[int]] = None ,      lowercase							:						bool = False			)							->    List[int]:

									'''simple docstring'''







									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowercase ,      token_ids_a=lowercase ,      already_has_special_tokens=lowercase			)

									UpperCamelCase__         = [1] + ([0] * len(lowercase			)) + [1]
									if token_ids_a is not None:
															result += ([0] * len(lowercase			)) + [1]
									return result






			def 				A      (   self							:						Dict ,      lowercase							:						List[int] ,      lowercase							:						Optional[List[int]] = None			)							->    List[int]:

									'''simple docstring'''







									UpperCamelCase__         = [self.sep_token_id]
									UpperCamelCase__         = [self.cls_token_id]

									UpperCamelCase__         = len(cls + token_ids_a + sep			) * [0]
									if token_ids_a is not None:
															result += len(token_ids_a + sep			) * [1]
									return result







			def 				A      (   self							:						Tuple ,      lowercase							:						str ,      lowercase							:						Optional[str] = None			)							->    List[Any]:

									'''simple docstring'''







									return ()








'''simple docstring'''


import logging
import os
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


lowerCamelCase_				: Any    	=							{
    '''debug''': logging.DEBUG,
    '''info''': logging.INFO,
    '''warning''': logging.WARNING,
    '''error''': logging.ERROR,
    '''critical''': logging.CRITICAL,
}

lowerCamelCase_				: Optional[int]    	=							logging.WARNING







def 	__magic_name__( ):
						'''simple docstring'''





						UpperCamelCase__         = os.getenv("""DATASETS_VERBOSITY"""						,      _A					)
						if env_level_str:
												if env_level_str in log_levels:
																		return log_levels[env_level_str]
												else:
																		logging.getLogger().warning(
																		    f"Unknown option DATASETS_VERBOSITY={env_level_str}, "
																		    f"has to be one of: { ', '.join(log_levels.keys()					) }"					)
						return _default_log_level







def 	__magic_name__( ):
						'''simple docstring'''





						return __name__.split("""."""					)[0]







def 	__magic_name__( ):
						'''simple docstring'''





						return logging.getLogger(_get_library_name()					)







def 	__magic_name__( ):
						'''simple docstring'''





						UpperCamelCase__         = _get_library_root_logger()
						library_root_logger.setLevel(_get_default_logging_level()					)







def 	__magic_name__( ):
						'''simple docstring'''





						UpperCamelCase__         = _get_library_root_logger()
						library_root_logger.setLevel(logging.NOTSET					)







def 	__magic_name__( _A = None					):
						'''simple docstring'''





						if name is None:
												UpperCamelCase__         = _get_library_name()
						return logging.getLogger(_A					)







def 	__magic_name__( ):
						'''simple docstring'''





						return _get_library_root_logger().getEffectiveLevel()







def 	__magic_name__( _A					):
						'''simple docstring'''





						_get_library_root_logger().setLevel(_A					)







def 	__magic_name__( ):
						'''simple docstring'''





						return set_verbosity(_A					)







def 	__magic_name__( ):
						'''simple docstring'''





						return set_verbosity(_A					)







def 	__magic_name__( ):
						'''simple docstring'''





						return set_verbosity(_A					)







def 	__magic_name__( ):
						'''simple docstring'''





						return set_verbosity(_A					)







def 	__magic_name__( ):
						'''simple docstring'''





						UpperCamelCase__         = False







def 	__magic_name__( ):
						'''simple docstring'''





						UpperCamelCase__         = True


# Configure the library root logger at the module level (singleton-like)
_configure_library_root_logger()
class    _SCREAMING_SNAKE_CASE							:


			'''simple docstring'''





			def __init__(   self							:						int ,      *lowercase							:						Any ,      **lowercase							:						int			)							->    List[Any]:  # pylint: disable=unused-argument

									'''simple docstring'''







									UpperCamelCase__         = args[0] if args else None






			def __iter__(   self							:						str			)							->    List[str]:

									'''simple docstring'''







									return iter(self._iterator			)






			def __getattr__(   self							:						Optional[int] ,      lowercase							:						Tuple			)							->    int:

									'''simple docstring'''







									def empty_fn(*lowercase							:						List[Any] ,      **lowercase							:						Dict			):  # pylint: disable=unused-argument
															return

									return empty_fn






			def __enter__(   self							:						Dict			)							->    Tuple:

									'''simple docstring'''







									return self







			def __exit__(   self							:						Tuple ,      lowercase							:						Optional[Any] ,      lowercase							:						Any ,      lowercase							:						int			)							->    Any:

									'''simple docstring'''







									return
lowerCamelCase_				: str    	=							True
class    _SCREAMING_SNAKE_CASE							:


			'''simple docstring'''





			def __call__(   self							:						List[Any] ,      *lowercase							:						str ,      lowercase							:						Optional[int]=False ,      **lowercase							:						Optional[int]			)							->    Optional[int]:

									'''simple docstring'''







									if _tqdm_active and not disable:
															return tqdm_lib.tqdm(*lowercase ,      **lowercase			)
									else:
															return EmptyTqdm(*lowercase ,      **lowercase			)






			def 				A      (   self							:						Optional[Any] ,      *lowercase							:						Tuple ,      **lowercase							:						Any			)							->    Any:

									'''simple docstring'''







									UpperCamelCase__         = None
									if _tqdm_active:
															return tqdm_lib.tqdm.set_lock(*lowercase ,      **lowercase			)







			def 				A      (   self							:						Optional[Any]			)							->    Optional[Any]:

									'''simple docstring'''







									if _tqdm_active:
															return tqdm_lib.tqdm.get_lock()
lowerCamelCase_				: int    	=							_tqdm_cls()







def 	__magic_name__( ):
						'''simple docstring'''





						global _tqdm_active
						return bool(_tqdm_active					)







def 	__magic_name__( ):
						'''simple docstring'''





						global _tqdm_active
						UpperCamelCase__         = True







def 	__magic_name__( ):
						'''simple docstring'''





						global _tqdm_active
						UpperCamelCase__         = False
